mirror of
https://github.com/NixOS/nixpkgs.git
synced 2025-02-21 11:34:13 +00:00

Due to Nix bug <https://github.com/NixOS/nix/issues/11503>, `builtins.filterSource` and chroot stores interact in a confusing and broken way that breaks `lib.fileset`. This means that uses of the API inside Nixpkgs keep breaking the NixOS installer, blocking the channel. The resulting error messages are inscrutable (they look like “the installer test is trying to download `curl`…?” and eventually bottom out in a derivation that has the wrong `outPath` because of the chroot store causing an incorrect `lib.fileset` result). Whenever this happens, someone (well, in practice K900 or I) has to bisect the change that introduced it and remove the use of `lib.fileset`. This has happened at least three times in the past four months (I believe I might actually be missing one here, but these are the ones I remember and could easily dig up): * <https://github.com/NixOS/nixpkgs/pull/340046> * <https://github.com/NixOS/nixpkgs/pull/352491> * <https://github.com/NixOS/nixpkgs/pull/369459> The options I see here are: 1. Forbid use of `lib.fileset` within Nixpkgs until the Nix bug is fixed. This is the approach taken here. External users of Nixpkgs can continue to use the API as normal, but using it from within something that affects any release jobset `outPath`s will cause an evaluation failure with a hopefully‐helpful error message. 2. Forbid `lib.fileset` and also all of the other library APIs that use `builtins.filterSource`. I’m happy to do this, but so far none of those have broken the installer, so I decided to start small and worry about the others if they end up causing a problem in practice. 3. Forbid `builtins.filterSource` directly. This is hard and would require more invasive `builtins.scopedImport` crimes to do at evaluation time. I think this would realistically have to be done in something like nixpkgs-vet instead and I didn’t have much luck shoehorning a check like this into that codebase when I tried. 4. Fix the Nix bug. This would be great! But also it doesn’t seem to be happening any time soon, it seems difficult to fix in a way that doesn’t subtly break compatibility with the previous semantics, and arguably the fix would need backporting all the way back to 2.3 given our minimum version policy. 5. Do nothing; have people continue to innocuously use `lib.fileset` throughout Nixpkgs, breaking the installer whenever one of them sneaks in to that closure, causing the channel to be blocked and requiring expensive bisections to narrow down the inscrutable test failure to the package using `lib.fileset`, which then needs moving back off it. This sucks for the people who keep having to track it down, holds back important channel bumps, and the criteria for when it’s okay to use `lib.fileset` are not realistically possible to teach to all contributors. I'd be happy to work on (2) as an alternative; (3) would be difficult and seems like overkill, (4) is not really something I trust myself to do and wouldn’t address the immediate problem, and (5) isn’t sustainable. I think that the current approach here is the best trade‐off for now, as `lib.fileset` seems to be the only prominent user of the `builtins.filterSource` API that works with full store paths, exposing it to the Nix bug. It’s unfortunate to lose the nice API, but since we can’t rely on it to produce correct results and the channels keep getting blocked as a result, I don’t think we really have an alternative right now.
212 lines
6.9 KiB
Nix
212 lines
6.9 KiB
Nix
{ supportedSystems
|
||
, system ? builtins.currentSystem
|
||
, packageSet ? (import ../..)
|
||
, scrubJobs ? true
|
||
, # Attributes passed to nixpkgs. Don't build packages marked as unfree.
|
||
nixpkgsArgs ? {
|
||
config = { allowUnfree = false; inHydra = true; };
|
||
__allowFileset = false;
|
||
}
|
||
}:
|
||
|
||
let
|
||
lib = import ../../lib;
|
||
|
||
inherit (lib)
|
||
addMetaAttrs
|
||
any
|
||
derivations
|
||
filter
|
||
flip
|
||
genAttrs
|
||
getAttrFromPath
|
||
hydraJob
|
||
id
|
||
isDerivation
|
||
lists
|
||
maintainers
|
||
mapAttrs
|
||
mapAttrs'
|
||
mapAttrsRecursive
|
||
matchAttrs
|
||
meta
|
||
nameValuePair
|
||
platforms
|
||
recursiveUpdate
|
||
subtractLists
|
||
systems
|
||
;
|
||
|
||
pkgs = packageSet (recursiveUpdate { inherit system; config.allowUnsupportedSystem = true; } nixpkgsArgs);
|
||
|
||
hydraJob' = if scrubJobs then hydraJob else id;
|
||
|
||
|
||
/* !!! Hack: poor man's memoisation function. Necessary to prevent
|
||
Nixpkgs from being evaluated again and again for every
|
||
job/platform pair. */
|
||
mkPkgsFor = crossSystem: let
|
||
packageSet' = args: packageSet (args // { inherit crossSystem; } // nixpkgsArgs);
|
||
|
||
pkgs_x86_64_linux = packageSet' { system = "x86_64-linux"; };
|
||
pkgs_i686_linux = packageSet' { system = "i686-linux"; };
|
||
pkgs_aarch64_linux = packageSet' { system = "aarch64-linux"; };
|
||
pkgs_riscv64_linux = packageSet' { system = "riscv64-linux"; };
|
||
pkgs_aarch64_darwin = packageSet' { system = "aarch64-darwin"; };
|
||
pkgs_armv6l_linux = packageSet' { system = "armv6l-linux"; };
|
||
pkgs_armv7l_linux = packageSet' { system = "armv7l-linux"; };
|
||
pkgs_x86_64_darwin = packageSet' { system = "x86_64-darwin"; };
|
||
pkgs_x86_64_freebsd = packageSet' { system = "x86_64-freebsd"; };
|
||
pkgs_i686_freebsd = packageSet' { system = "i686-freebsd"; };
|
||
pkgs_i686_cygwin = packageSet' { system = "i686-cygwin"; };
|
||
pkgs_x86_64_cygwin = packageSet' { system = "x86_64-cygwin"; };
|
||
|
||
in system:
|
||
if system == "x86_64-linux" then pkgs_x86_64_linux
|
||
else if system == "i686-linux" then pkgs_i686_linux
|
||
else if system == "aarch64-linux" then pkgs_aarch64_linux
|
||
else if system == "riscv64-linux" then pkgs_riscv64_linux
|
||
else if system == "aarch64-darwin" then pkgs_aarch64_darwin
|
||
else if system == "armv6l-linux" then pkgs_armv6l_linux
|
||
else if system == "armv7l-linux" then pkgs_armv7l_linux
|
||
else if system == "x86_64-darwin" then pkgs_x86_64_darwin
|
||
else if system == "x86_64-freebsd" then pkgs_x86_64_freebsd
|
||
else if system == "i686-freebsd" then pkgs_i686_freebsd
|
||
else if system == "i686-cygwin" then pkgs_i686_cygwin
|
||
else if system == "x86_64-cygwin" then pkgs_x86_64_cygwin
|
||
else abort "unsupported system type: ${system}";
|
||
|
||
pkgsFor = pkgsForCross null;
|
||
|
||
|
||
# More poor man's memoisation
|
||
pkgsForCross = let
|
||
examplesByConfig = flip mapAttrs'
|
||
systems.examples
|
||
(_: crossSystem: nameValuePair crossSystem.config {
|
||
inherit crossSystem;
|
||
pkgsFor = mkPkgsFor crossSystem;
|
||
});
|
||
native = mkPkgsFor null;
|
||
in crossSystem: let
|
||
candidate = examplesByConfig.${crossSystem.config} or null;
|
||
in if crossSystem == null
|
||
then native
|
||
else if candidate != null && matchAttrs crossSystem candidate.crossSystem
|
||
then candidate.pkgsFor
|
||
else mkPkgsFor crossSystem; # uncached fallback
|
||
|
||
|
||
# Given a list of 'meta.platforms'-style patterns, return the sublist of
|
||
# `supportedSystems` containing systems that matches at least one of the given
|
||
# patterns.
|
||
#
|
||
# This is written in a funny way so that we only elaborate the systems once.
|
||
supportedMatches = let
|
||
supportedPlatforms = map
|
||
(system: systems.elaborate { inherit system; })
|
||
supportedSystems;
|
||
in metaPatterns: let
|
||
anyMatch = platform:
|
||
any (meta.platformMatch platform) metaPatterns;
|
||
matchingPlatforms = filter anyMatch supportedPlatforms;
|
||
in map ({ system, ...}: system) matchingPlatforms;
|
||
|
||
|
||
assertTrue = bool:
|
||
if bool
|
||
then pkgs.runCommand "evaluated-to-true" {} "touch $out"
|
||
else pkgs.runCommand "evaluated-to-false" {} "false";
|
||
|
||
|
||
/* The working or failing mails for cross builds will be sent only to
|
||
the following maintainers, as most package maintainers will not be
|
||
interested in the result of cross building a package. */
|
||
crossMaintainers = [ ];
|
||
|
||
|
||
# Generate attributes for all supported systems.
|
||
forAllSystems = genAttrs supportedSystems;
|
||
|
||
|
||
# Generate attributes for all systems matching at least one of the given
|
||
# patterns
|
||
forMatchingSystems = metaPatterns: genAttrs (supportedMatches metaPatterns);
|
||
|
||
|
||
/* Build a package on the given set of platforms. The function `f'
|
||
is called for each supported platform with Nixpkgs for that
|
||
platform as an argument . We return an attribute set containing
|
||
a derivation for each supported platform, i.e. ‘{ x86_64-linux =
|
||
f pkgs_x86_64_linux; i686-linux = f pkgs_i686_linux; ... }’. */
|
||
testOn = testOnCross null;
|
||
|
||
|
||
/* Similar to the testOn function, but with an additional
|
||
'crossSystem' parameter for packageSet', defining the target
|
||
platform for cross builds. */
|
||
testOnCross = crossSystem: metaPatterns: f: forMatchingSystems metaPatterns
|
||
(system: hydraJob' (f (pkgsForCross crossSystem system)));
|
||
|
||
|
||
/* Given a nested set where the leaf nodes are lists of platforms,
|
||
map each leaf node to `testOn [platforms...] (pkgs:
|
||
pkgs.<attrPath>)'. */
|
||
mapTestOn = _mapTestOnHelper id null;
|
||
|
||
|
||
_mapTestOnHelper = f: crossSystem: mapAttrsRecursive
|
||
(path: metaPatterns: testOnCross crossSystem metaPatterns
|
||
(pkgs: f (getAttrFromPath path pkgs)));
|
||
|
||
/* Similar to the testOn function, but with an additional 'crossSystem'
|
||
* parameter for packageSet', defining the target platform for cross builds,
|
||
* and triggering the build of the host derivation. */
|
||
mapTestOnCross = _mapTestOnHelper
|
||
(addMetaAttrs { maintainers = crossMaintainers; });
|
||
|
||
|
||
/* Recursive for packages and apply a function to them */
|
||
recursiveMapPackages = f: mapAttrs (name: value:
|
||
if isDerivation value then
|
||
f value
|
||
else if value.recurseForDerivations or false || value.recurseForRelease or false then
|
||
recursiveMapPackages f value
|
||
else
|
||
[]
|
||
);
|
||
|
||
/* Gets the list of Hydra platforms for a derivation */
|
||
getPlatforms = drv:
|
||
drv.meta.hydraPlatforms
|
||
or (subtractLists (drv.meta.badPlatforms or [])
|
||
(drv.meta.platforms or supportedSystems));
|
||
|
||
/* Recursively map a (nested) set of derivations to an isomorphic
|
||
set of meta.platforms values. */
|
||
packagePlatforms = recursiveMapPackages getPlatforms;
|
||
|
||
in {
|
||
/* Common platform groups on which to test packages. */
|
||
inherit (platforms) unix linux darwin cygwin all;
|
||
|
||
inherit
|
||
assertTrue
|
||
forAllSystems
|
||
forMatchingSystems
|
||
hydraJob'
|
||
lib
|
||
mapTestOn
|
||
mapTestOnCross
|
||
recursiveMapPackages
|
||
getPlatforms
|
||
packagePlatforms
|
||
pkgs
|
||
pkgsFor
|
||
pkgsForCross
|
||
supportedMatches
|
||
testOn
|
||
testOnCross
|
||
;
|
||
}
|