nixpkgs/pkgs/top-level/release-haskell.nix
Silvan Mosberger fbbe972898 Parallel GH actions workflow for Nixpkgs eval
Motivated by ofborg struggling [1] and its evaluations taking too long,
inspired by Jörg's initial PR [2]
and Adam's previous attempt to parallelise Nixpkgs evaluation [3],
this PR contains initial work to relief ofborg from its evaluation duty
by using GitHub Actions to evaluate Nixpkgs.

For now this doesn't take care of all of what ofborg does, such as
requesting appropriate reviewers or labeling mass rebuilds, but this can
be follow-up work.

[1]: https://discourse.nixos.org/t/infrastructure-announcement-the-future-of-ofborg-your-help-needed/56025?u=infinisil
[2]: https://github.com/NixOS/nixpkgs/pull/352808
[3]: https://github.com/NixOS/nixpkgs/pull/269403

Co-Authored-By: Jörg Thalheim <joerg@thalheim.io>
Co-Authored-By: Adam Joseph <adam@westernsemico.com>
2024-11-20 10:35:56 +01:00

731 lines
21 KiB
Nix

/*
This is the Hydra jobset for the `haskell-updates` branch in Nixpkgs.
You can see the status of this jobset at
https://hydra.nixos.org/jobset/nixpkgs/haskell-updates.
To debug this expression you can use `hydra-eval-jobs` from
`pkgs.hydra` which prints the jobset description
to `stdout`:
$ hydra-eval-jobs -I . pkgs/top-level/release-haskell.nix
*/
{ supportedSystems ? import ../../ci/supportedSystems.nix }:
let
releaseLib = import ./release-lib.nix {
inherit supportedSystems;
};
inherit (releaseLib)
lib
mapTestOn
packagePlatforms
pkgs
;
# Helper function which traverses a (nested) set
# of derivations produced by mapTestOn and flattens
# it to a list of derivations suitable to be passed
# to `releaseTools.aggregate` as constituents.
# Removes all non derivations from the input jobList.
#
# accumulateDerivations :: [ Either Derivation AttrSet ] -> [ Derivation ]
#
# > accumulateDerivations [ drv1 "string" { foo = drv2; bar = { baz = drv3; }; } ]
# [ drv1 drv2 drv3 ]
accumulateDerivations = jobList:
lib.concatMap (
attrs:
if lib.isDerivation attrs
then [ attrs ]
else lib.optionals (lib.isAttrs attrs) (accumulateDerivations (lib.attrValues attrs))
) jobList;
# names of all subsets of `pkgs.haskell.packages`
#
# compilerNames looks like the following:
#
# ```
# {
# ghc810 = "ghc810";
# ghc8107Binary = "ghc8107Binary";
# ghc8107 = "ghc8107";
# ghc924 = "ghc924";
# ...
# }
# ```
compilerNames = lib.mapAttrs (name: _: name) pkgs.haskell.packages;
# list of all compilers to test specific packages on
released = with compilerNames; [
ghc8107
ghc902
ghc925
ghc926
ghc927
ghc928
ghc945
ghc946
ghc947
ghc948
ghc963
ghc964
ghc965
ghc966
ghc981
ghc982
ghc983
ghc9101
];
# packagePlatforms applied to `haskell.packages.*`
#
# This returns an attr set that looks like the following, where each Haskell
# package in the compiler attr set has its list of supported platforms as its
# value.
#
# ```
# {
# ghc810 = {
# conduit = [ ... ];
# lens = [ "i686-cygwin" "x86_64-cygwin" ... "x86_64-windows" "i686-windows" ]
# ...
# };
# ghc902 = { ... };
# ...
# }
# ```
compilerPlatforms = lib.mapAttrs
(_: v: packagePlatforms v)
pkgs.haskell.packages;
# This function lets you specify specific packages
# which are to be tested on a list of specific GHC
# versions and returns a job set for all specified
# combinations.
#
# You can call versionedCompilerJobs like the following:
#
# ```
# versionedCompilerJobs {
# ghc-tags = ["ghc902" "ghc924"];
# }
# ```
#
# This would produce an output like the following:
#
# ```
# {
# haskell.packages = {
# ghc884 = {};
# ghc810 = {};
# ghc902 = {
# ghc-tags = {
# aarch64-darwin = <derivation...>;
# aarch64-linux = <derivation...>;
# ...
# };
# };
# ghc924 = {
# ghc-tags = { ... };
# };
# ...
# };
# }
# ```
versionedCompilerJobs = config: mapTestOn {
haskell.packages =
let
# Mapping function that takes an attrset of jobs, and
# removes all jobs that are not specified in config.
#
# For example, imagine a call to onlyConfigJobs like:
#
# ```
# onlyConfigJobs
# "ghc902"
# {
# conduit = [ ... ];
# lens = [ "i686-cygwin" "x86_64-cygwin" ... "x86_64-windows" "i686-windows" ];
# }
# ```
#
# onlyConfigJobs pulls out only those jobs that are specified in config.
#
# For instance, if config is `{ lens = [ "ghc902" ]; }`, then the above
# example call to onlyConfigJobs will return:
#
# ```
# { lens = [ "i686-cygwin" "x86_64-cygwin" ... "x86_64-windows" "i686-windows" ]; }
# ```
#
# If config is `{ lens = [ "ghc8107" ]; }`, then the above example call
# to onlyConfigJobs returns `{}`.
#
# onlyConfigJobs will also remove all platforms from a job that are not
# supported by the GHC it is compiled with.
onlyConfigJobs = ghc: jobs:
let
configFilteredJobset =
lib.filterAttrs
(jobName: platforms: lib.elem ghc (config."${jobName}" or []))
jobs;
# Remove platforms from each job that are not supported by GHC.
# This is important so that we don't build jobs for platforms
# where GHC can't be compiled.
jobsetWithGHCPlatforms =
lib.mapAttrs
(_: platforms: lib.intersectLists jobs.ghc platforms)
configFilteredJobset;
in
jobsetWithGHCPlatforms;
in
lib.mapAttrs onlyConfigJobs compilerPlatforms;
};
# hydra jobs for `pkgs` of which we import a subset of
pkgsPlatforms = packagePlatforms pkgs;
# names of packages in an attribute set that are maintained
maintainedPkgNames = set: builtins.attrNames
(lib.filterAttrs (
_: v: builtins.length (v.meta.maintainers or []) > 0
) set);
recursiveUpdateMany = builtins.foldl' lib.recursiveUpdate {};
# Remove multiple elements from a list at once.
#
# removeMany
# :: [a] -- list of elements to remove
# -> [a] -- list of elements from which to remove
# -> [a]
#
# > removeMany ["aarch64-linux" "x86_64-darwin"] ["aarch64-linux" "x86_64-darwin" "x86_64-linux"]
# ["x86_64-linux"]
removeMany = itemsToRemove: list: lib.foldr lib.remove list itemsToRemove;
# Recursively remove platforms from the values in an attribute set.
#
# removePlatforms
# :: [String]
# -> AttrSet
# -> AttrSet
#
# > attrSet = {
# foo = ["aarch64-linux" "x86_64-darwin" "x86_64-linux"];
# bar.baz = ["aarch64-linux" "x86_64-linux"];
# bar.quux = ["aarch64-linux" "x86_64-darwin"];
# }
# > removePlatforms ["aarch64-linux" "x86_64-darwin"] attrSet
# {
# foo = ["x86_64-linux"];
# bar = {
# baz = ["x86_64-linux"];
# quux = [];
# };
# }
removePlatforms = platformsToRemove: packageSet:
lib.mapAttrsRecursive
(_: val:
if lib.isList val
then removeMany platformsToRemove val
else val
)
packageSet;
jobs = recursiveUpdateMany [
(mapTestOn {
haskellPackages = packagePlatforms pkgs.haskellPackages;
haskell.compiler = packagePlatforms pkgs.haskell.compiler // (lib.genAttrs [
"ghcjs"
"ghcjs810"
] (ghcjsName: {
# We can't build ghcjs itself, since it exceeds 3GB (Hydra's output limit) due
# to the size of its bundled libs. We can however save users a bit of compile
# time by building the bootstrap ghcjs on Hydra. For this reason, we overwrite
# the ghcjs attributes in haskell.compiler with a reference to the bootstrap
# ghcjs attribute in their bootstrap package set (exposed via passthru) which
# would otherwise be ignored by Hydra.
bootGhcjs = (packagePlatforms pkgs.haskell.compiler.${ghcjsName}.passthru).bootGhcjs;
}));
tests.haskell = packagePlatforms pkgs.tests.haskell;
nixosTests = {
inherit (packagePlatforms pkgs.nixosTests)
agda
xmonad
xmonad-xdg-autostart
;
};
agdaPackages = packagePlatforms pkgs.agdaPackages;
# top-level packages that depend on haskellPackages
inherit (pkgsPlatforms)
agda
alex
arion
bench
blucontrol
cabal-install
cabal2nix
cachix
# carp broken on 2024-04-09
changelog-d
cornelis
cedille
client-ip-echo
darcs
dconf2nix
dhall
dhall-bash
dhall-docs
dhall-lsp-server
dhall-json
dhall-nix
dhall-nixpkgs
dhall-yaml
diagrams-builder
echidna
elm2nix
emanote
fffuu
futhark
ghcid
git-annex
git-brunch
gitit
glirc
hadolint
happy
haskell-ci
haskell-language-server
hasura-graphql-engine
hci
hercules-ci-agent
hinit
hedgewars
hledger
hledger-check-fancyassertions
hledger-iadd
hledger-interest
hledger-ui
hledger-web
hlint
hpack
hscolour
icepeak
ihaskell
jacinda
jl
json2yaml
koka
krank
lambdabot
lhs2tex
madlang
matterhorn
mkjson
mueval
naproche
niv
nix-delegate
nix-deploy
nix-diff
nix-linter
nix-output-monitor
nix-script
nix-tree
nixfmt
nixfmt-classic
nixfmt-rfc-style
nota
nvfetcher
oama
ormolu
pakcs
pandoc
place-cursor-at
pinboard-notes-backup
pretty-simple
purenix
shake
shellcheck
shellcheck-minimal
sourceAndTags
spacecookie
spago
specup
splot
stack
stack2nix
stutter
stylish-haskell
taffybar
tamarin-prover
termonad
tldr-hs
tweet-hs
update-nix-fetchgit
uusi
uqm
uuagc
# vaultenv: broken by connection on 2024-03-16
wstunnel
xmobar
xmonadctl
xmonad-with-packages
yi
zsh-git-prompt
;
# Members of the elmPackages set that are Haskell derivations
elmPackages = {
inherit (pkgsPlatforms.elmPackages)
elm
elm-format
elm-instrument
elmi-to-json
;
};
# GHCs linked to musl.
pkgsMusl =
removePlatforms
[
# pkgsMusl is compiled natively with musl. It is not
# cross-compiled (unlike pkgsStatic). We can only
# natively bootstrap GHC with musl on x86_64-linux because
# upstream doesn't provide a musl bindist for aarch64.
"aarch64-linux"
# musl only supports linux, not darwin.
"x86_64-darwin"
"aarch64-darwin"
]
{
haskell.compiler = lib.recursiveUpdate
(packagePlatforms pkgs.pkgsMusl.haskell.compiler)
{
# remove musl ghc865Binary since it is known to be broken and
# causes an evaluation error on darwin.
ghc865Binary = {};
ghcjs = {};
ghcjs810 = {};
};
# Get some cache going for MUSL-enabled GHC.
haskellPackages =
{
inherit (packagePlatforms pkgs.pkgsMusl.haskellPackages)
hello
lens
random
;
};
};
# Test some statically linked packages to catch regressions
# and get some cache going for static compilation with GHC.
# Use native-bignum to avoid GMP linking problems (LGPL)
pkgsStatic =
removePlatforms
[
"aarch64-linux" # times out on Hydra
# Static doesn't work on darwin
"x86_64-darwin"
"aarch64-darwin"
] {
haskellPackages = {
inherit (packagePlatforms pkgs.pkgsStatic.haskellPackages)
hello
lens
random
QuickCheck
cabal2nix
terminfo # isn't bundled for cross
xhtml # isn't bundled for cross
;
};
haskell.packages.native-bignum.ghc948 = {
inherit (packagePlatforms pkgs.pkgsStatic.haskell.packages.native-bignum.ghc948)
hello
lens
random
QuickCheck
cabal2nix
terminfo # isn't bundled for cross
xhtml # isn't bundled for cross
;
};
haskell.packages.native-bignum.ghc983 = {
inherit (packagePlatforms pkgs.pkgsStatic.haskell.packages.native-bignum.ghc983)
hello
random
QuickCheck
terminfo # isn't bundled for cross
;
};
};
pkgsCross = {
ghcjs =
removePlatforms
[
# Hydra output size of 3GB is exceeded
"aarch64-linux"
]
{
haskellPackages = {
inherit (packagePlatforms pkgs.pkgsCross.ghcjs.haskellPackages)
ghc
hello
microlens
;
};
haskell.packages.ghc98 = {
inherit (packagePlatforms pkgs.pkgsCross.ghcjs.haskell.packages.ghc98)
ghc
hello
microlens
;
};
haskell.packages.ghcHEAD = {
inherit (packagePlatforms pkgs.pkgsCross.ghcjs.haskell.packages.ghcHEAD)
ghc
hello
microlens
;
};
};
riscv64 = {
# Cross compilation of GHC
haskell.compiler = {
inherit (packagePlatforms pkgs.pkgsCross.riscv64.haskell.compiler)
# Our oldest GHC which still uses its own expression. 8.10.7 can
# theoretically be used to chain bootstrap all GHCs on riscv64
# which doesn't have official bindists.
ghc8107
# Latest GHC we are able to cross-compile.
ghc948
;
};
};
aarch64-multiplatform = {
# Cross compilation of GHC
haskell.compiler = {
inherit (packagePlatforms pkgs.pkgsCross.aarch64-multiplatform.haskell.compiler)
# Uses a separate expression and LLVM backend for aarch64.
ghc8107
# Latest GHC we are able to cross-compile. Uses NCG backend.
ghc948
;
};
};
};
})
(versionedCompilerJobs {
# Packages which should be checked on more than the
# default GHC version. This list can be used to test
# the state of the package set with newer compilers
# and to confirm that critical packages for the
# package sets (like Cabal, jailbreak-cabal) are
# working as expected.
cabal-install = lib.subtractLists [
# It is recommended to use pkgs.cabal-install instead of cabal-install
# from the package sets. Due to (transitively) requiring recent versions
# of core packages, it is not always reasonable to get cabal-install to
# work with older compilers.
compilerNames.ghc8107
compilerNames.ghc902
compilerNames.ghc925
compilerNames.ghc926
compilerNames.ghc927
compilerNames.ghc928
compilerNames.ghc945
compilerNames.ghc946
compilerNames.ghc947
compilerNames.ghc948
compilerNames.ghc9101
] released;
Cabal_3_10_3_0 = released;
Cabal_3_12_1_0 = released;
Cabal_3_14_0_0 = released;
cabal2nix = lib.subtractLists [
compilerNames.ghc9101
] released;
cabal2nix-unstable = lib.subtractLists [
compilerNames.ghc9101
] released;
funcmp = released;
haskell-language-server = lib.subtractLists [
# Support ceased as of 2.3.0.0
compilerNames.ghc8107
# Support ceased as of 2.5.0.0
compilerNames.ghc902
] released;
hoogle = lib.subtractLists [
] released;
hlint = lib.subtractLists [
compilerNames.ghc902
compilerNames.ghc9101
] released;
hpack = lib.subtractLists [
compilerNames.ghc9101
] released;
hsdns = released;
jailbreak-cabal = released;
language-nix = lib.subtractLists [
compilerNames.ghc9101
] released;
nix-paths = released;
titlecase = lib.subtractLists [
compilerNames.ghc9101
] released;
ghc-api-compat = [
compilerNames.ghc8107
compilerNames.ghc902
];
ghc-bignum = [
compilerNames.ghc8107
];
ghc-lib = lib.subtractLists [
compilerNames.ghc9101
] released;
ghc-lib-parser = lib.subtractLists [
compilerNames.ghc9101
] released;
ghc-lib-parser-ex = lib.subtractLists [
compilerNames.ghc9101
] released;
ghc-source-gen = [
# Feel free to remove these as they break,
compilerNames.ghc8107
compilerNames.ghc902
compilerNames.ghc928
];
ghc-tags = lib.subtractLists [
compilerNames.ghc9101
] released;
hashable = lib.subtractLists [
compilerNames.ghc9101
] released;
primitive = lib.subtractLists [
compilerNames.ghc9101
] released;
weeder = lib.subtractLists [
compilerNames.ghc9101
] released;
})
{
mergeable = pkgs.releaseTools.aggregate {
name = "haskell-updates-mergeable";
meta = {
description = ''
Critical haskell packages that should work at all times,
serves as minimum requirement for an update merge
'';
maintainers = lib.teams.haskell.members;
};
constituents =
accumulateDerivations [
# haskell specific tests
jobs.tests.haskell
# important top-level packages
jobs.cabal-install
jobs.cabal2nix
jobs.cachix
jobs.darcs
jobs.haskell-language-server
jobs.hledger
jobs.hledger-ui
jobs.hpack
jobs.niv
jobs.pandoc
jobs.stack
jobs.stylish-haskell
jobs.shellcheck
# important haskell (library) packages
jobs.haskellPackages.cabal-plan
jobs.haskellPackages.distribution-nixpkgs
jobs.haskellPackages.hackage-db
jobs.haskellPackages.xmonad
jobs.haskellPackages.xmonad-contrib
# haskell packages maintained by @peti
# imported from the old hydra jobset
jobs.haskellPackages.hopenssl
jobs.haskellPackages.hsemail
jobs.haskellPackages.hsyslog
];
};
maintained = pkgs.releaseTools.aggregate {
name = "maintained-haskell-packages";
meta = {
description = "Aggregate jobset of all haskell packages with a maintainer";
maintainers = lib.teams.haskell.members;
};
constituents = accumulateDerivations
(builtins.map
(name: jobs.haskellPackages."${name}")
(maintainedPkgNames pkgs.haskellPackages));
};
muslGHCs = pkgs.releaseTools.aggregate {
name = "haskell-pkgsMusl-ghcs";
meta = {
description = "GHCs built with musl";
maintainers = with lib.maintainers; [
nh2
];
};
constituents = accumulateDerivations [
jobs.pkgsMusl.haskell.compiler.ghc8107Binary
jobs.pkgsMusl.haskell.compiler.ghc8107
jobs.pkgsMusl.haskell.compiler.ghc902
jobs.pkgsMusl.haskell.compiler.ghc925
jobs.pkgsMusl.haskell.compiler.ghc926
jobs.pkgsMusl.haskell.compiler.ghc927
jobs.pkgsMusl.haskell.compiler.ghc928
jobs.pkgsMusl.haskell.compiler.ghcHEAD
jobs.pkgsMusl.haskell.compiler.integer-simple.ghc8107
jobs.pkgsMusl.haskell.compiler.native-bignum.ghc902
jobs.pkgsMusl.haskell.compiler.native-bignum.ghc925
jobs.pkgsMusl.haskell.compiler.native-bignum.ghc926
jobs.pkgsMusl.haskell.compiler.native-bignum.ghc927
jobs.pkgsMusl.haskell.compiler.native-bignum.ghc928
jobs.pkgsMusl.haskell.compiler.native-bignum.ghcHEAD
];
};
staticHaskellPackages = pkgs.releaseTools.aggregate {
name = "static-haskell-packages";
meta = {
description = "Static haskell builds using the pkgsStatic infrastructure";
maintainers = [
lib.maintainers.sternenseemann
lib.maintainers.rnhmjoj
];
};
constituents = accumulateDerivations [
jobs.pkgsStatic.haskell.packages.native-bignum.ghc948 # non-hadrian
jobs.pkgsStatic.haskellPackages
jobs.pkgsStatic.haskell.packages.native-bignum.ghc983
];
};
}
];
in jobs