mirror of
https://github.com/NixOS/nixpkgs.git
synced 2025-04-27 22:39:33 +00:00
ci/eval: re-implement compare in nix
This commit is contained in:
Gaetan Lepage
parent
69590ca7f4
commit
f94b4bd945
@ -1,164 +0,0 @@
|
||||
# Turns
|
||||
#
|
||||
# {
|
||||
# "hello.aarch64-linux": "a",
|
||||
# "hello.x86_64-linux": "b",
|
||||
# "hello.aarch64-darwin": "c",
|
||||
# "hello.x86_64-darwin": "d"
|
||||
# }
|
||||
#
|
||||
# into
|
||||
#
|
||||
# {
|
||||
# "hello": {
|
||||
# "linux": {
|
||||
# "aarch64": "a",
|
||||
# "x86_64": "b"
|
||||
# },
|
||||
# "darwin": {
|
||||
# "aarch64": "c",
|
||||
# "x86_64": "d"
|
||||
# }
|
||||
# }
|
||||
# }
|
||||
#
|
||||
# while filtering out any attribute paths that don't match this pattern
|
||||
def expand_system:
|
||||
to_entries
|
||||
| map(
|
||||
.key |= split(".")
|
||||
| select(.key | length > 1)
|
||||
| .double = (.key[-1] | split("-"))
|
||||
| select(.double | length == 2)
|
||||
)
|
||||
| group_by(.key[0:-1])
|
||||
| map(
|
||||
{
|
||||
key: .[0].key[0:-1] | join("."),
|
||||
value:
|
||||
group_by(.double[1])
|
||||
| map(
|
||||
{
|
||||
key: .[0].double[1],
|
||||
value: map(.key = .double[0]) | from_entries
|
||||
}
|
||||
)
|
||||
| from_entries
|
||||
})
|
||||
| from_entries
|
||||
;
|
||||
|
||||
# Transposes
|
||||
#
|
||||
# {
|
||||
# "a": [ "x", "y" ],
|
||||
# "b": [ "x" ],
|
||||
# }
|
||||
#
|
||||
# into
|
||||
#
|
||||
# {
|
||||
# "x": [ "a", "b" ],
|
||||
# "y": [ "a" ]
|
||||
# }
|
||||
def transpose:
|
||||
[
|
||||
to_entries[]
|
||||
| {
|
||||
key: .key,
|
||||
value: .value[]
|
||||
}
|
||||
]
|
||||
| group_by(.value)
|
||||
| map({
|
||||
key: .[0].value,
|
||||
value: map(.key)
|
||||
})
|
||||
| from_entries
|
||||
;
|
||||
|
||||
# Computes the key difference for two objects:
|
||||
# {
|
||||
# added: [ <keys only in the second object> ],
|
||||
# removed: [ <keys only in the first object> ],
|
||||
# changed: [ <keys with different values between the two objects> ],
|
||||
# }
|
||||
#
|
||||
def diff($before; $after):
|
||||
{
|
||||
added: $after | delpaths($before | keys | map([.])) | keys,
|
||||
removed: $before | delpaths($after | keys | map([.])) | keys,
|
||||
changed:
|
||||
$before
|
||||
| to_entries
|
||||
| map(
|
||||
$after."\(.key)" as $after2
|
||||
| select(
|
||||
# Filter out attributes that don't exist anymore
|
||||
($after2 != null)
|
||||
and
|
||||
# Filter out attributes that are the same as the new value
|
||||
(.value != $after2)
|
||||
)
|
||||
| .key
|
||||
)
|
||||
}
|
||||
;
|
||||
|
||||
($before[0] | expand_system) as $before
|
||||
| ($after[0] | expand_system) as $after
|
||||
| .attrdiff = diff($before; $after)
|
||||
| .rebuildsByKernel = (
|
||||
[
|
||||
(
|
||||
.attrdiff.changed[]
|
||||
| {
|
||||
key: .,
|
||||
value: diff($before."\(.)"; $after."\(.)").changed
|
||||
}
|
||||
)
|
||||
,
|
||||
(
|
||||
.attrdiff.added[]
|
||||
| {
|
||||
key: .,
|
||||
value: ($after."\(.)" | keys)
|
||||
}
|
||||
)
|
||||
]
|
||||
| from_entries
|
||||
| transpose
|
||||
)
|
||||
| .rebuildCountByKernel = (
|
||||
.rebuildsByKernel
|
||||
| with_entries(.value |= length)
|
||||
| pick(.linux, .darwin)
|
||||
| {
|
||||
linux: (.linux // 0),
|
||||
darwin: (.darwin // 0),
|
||||
}
|
||||
)
|
||||
| .labels = (
|
||||
.rebuildCountByKernel
|
||||
| to_entries
|
||||
| map(
|
||||
"10.rebuild-\(.key): " +
|
||||
if .value == 0 then
|
||||
"0"
|
||||
elif .value <= 10 then
|
||||
"1-10"
|
||||
elif .value <= 100 then
|
||||
"11-100"
|
||||
elif .value <= 500 then
|
||||
"101-500"
|
||||
elif .value <= 1000 then
|
||||
"501-1000"
|
||||
elif .value <= 2500 then
|
||||
"1001-2500"
|
||||
elif .value <= 5000 then
|
||||
"2501-5000"
|
||||
else
|
||||
"5001+"
|
||||
end
|
||||
)
|
||||
)
|
||||
53
ci/eval/compare/default.nix
Normal file
53
ci/eval/compare/default.nix
Normal file
@ -0,0 +1,53 @@
|
||||
{
|
||||
lib,
|
||||
jq,
|
||||
runCommand,
|
||||
writeText,
|
||||
supportedSystems,
|
||||
...
|
||||
}:
|
||||
{ beforeResultDir, afterResultDir }:
|
||||
let
|
||||
inherit (import ./utils.nix { inherit lib; })
|
||||
diff
|
||||
groupByKernel
|
||||
extractPackageNames
|
||||
getLabels
|
||||
uniqueStrings
|
||||
;
|
||||
|
||||
getAttrs = dir: builtins.fromJSON (builtins.readFile "${dir}/outpaths.json");
|
||||
beforeAttrs = getAttrs beforeResultDir;
|
||||
afterAttrs = getAttrs afterResultDir;
|
||||
|
||||
diffAttrs = diff beforeAttrs afterAttrs;
|
||||
|
||||
changed-paths =
|
||||
let
|
||||
rebuilds = uniqueStrings (diffAttrs.added ++ diffAttrs.changed);
|
||||
|
||||
rebuildsByKernel = groupByKernel rebuilds;
|
||||
rebuildCountByKernel = lib.mapAttrs (
|
||||
kernel: kernelRebuilds: lib.length kernelRebuilds
|
||||
) rebuildsByKernel;
|
||||
in
|
||||
writeText "changed-paths.json" (
|
||||
builtins.toJSON {
|
||||
attrdiff = lib.mapAttrs (_: v: extractPackageNames v) diffAttrs;
|
||||
inherit rebuildsByKernel rebuildCountByKernel;
|
||||
labels = getLabels rebuildCountByKernel;
|
||||
}
|
||||
);
|
||||
in
|
||||
runCommand "compare"
|
||||
{
|
||||
nativeBuildInputs = [ jq ];
|
||||
}
|
||||
''
|
||||
mkdir $out
|
||||
|
||||
cp ${changed-paths} $out/changed-paths.json
|
||||
|
||||
jq -r -f ${./generate-step-summary.jq} < ${changed-paths} > $out/step-summary.md
|
||||
# TODO: Compare eval stats
|
||||
''
|
||||
130
ci/eval/compare/utils.nix
Normal file
130
ci/eval/compare/utils.nix
Normal file
@ -0,0 +1,130 @@
|
||||
{ lib, ... }:
|
||||
rec {
|
||||
# Borrowed from https://github.com/NixOS/nixpkgs/pull/355616
|
||||
uniqueStrings = list: builtins.attrNames (builtins.groupBy lib.id list);
|
||||
|
||||
_processSystemPath =
|
||||
packageSystemPath:
|
||||
let
|
||||
# python312Packages.torch.aarch64-linux -> ["python312Packages" "torch" "aarch64-linux"]
|
||||
# splittedPath = lib.splitString "." attrName;
|
||||
splittedPath = lib.splitString "." packageSystemPath;
|
||||
|
||||
# ["python312Packages" "torch" "aarch64-linux"] -> ["python312Packages" "torch"]
|
||||
packagePath = lib.sublist 0 (lib.length splittedPath - 1) splittedPath;
|
||||
in
|
||||
{
|
||||
# "python312Packages.torch"
|
||||
name = lib.concatStringsSep "." packagePath;
|
||||
|
||||
# "aarch64-linux"
|
||||
system = lib.last splittedPath;
|
||||
};
|
||||
|
||||
# Turns
|
||||
# [
|
||||
# "hello.aarch64-linux"
|
||||
# "hello.x86_64-linux"
|
||||
# "hello.aarch64-darwin"
|
||||
# "hello.x86_64-darwin"
|
||||
# "bye.x86_64-darwin"
|
||||
# "bye.aarch64-darwin"
|
||||
# ]
|
||||
#
|
||||
# into
|
||||
#
|
||||
# [
|
||||
# "hello"
|
||||
# "bye"
|
||||
# ]
|
||||
extractPackageNames =
|
||||
packageSystemPaths:
|
||||
builtins.attrNames (
|
||||
builtins.removeAttrs (builtins.groupBy (
|
||||
packageSystemPath: (_processSystemPath packageSystemPath).name
|
||||
) packageSystemPaths) [ "" ]
|
||||
);
|
||||
|
||||
# Computes a diff between two attrs
|
||||
# {
|
||||
# added: [ <keys only in the second object> ],
|
||||
# removed: [ <keys only in the first object> ],
|
||||
# changed: [ <keys with different values between the two objects> ],
|
||||
# }
|
||||
#
|
||||
diff =
|
||||
let
|
||||
filterKeys = cond: attrs: lib.attrNames (lib.filterAttrs cond attrs);
|
||||
in
|
||||
old: new: {
|
||||
added = filterKeys (n: _: !(old ? ${n})) new;
|
||||
removed = filterKeys (n: _: !(new ? ${n})) old;
|
||||
changed = filterKeys (
|
||||
n: v:
|
||||
# Filter out attributes that don't exist anymore
|
||||
(new ? ${n})
|
||||
|
||||
# Filter out attributes that are the same as the new value
|
||||
&& (v != (new.${n}))
|
||||
) old;
|
||||
};
|
||||
|
||||
# Turns
|
||||
# [
|
||||
# "hello.aarch64-linux"
|
||||
# "hello.x86_64-linux"
|
||||
# "hello.aarch64-darwin"
|
||||
# "hello.x86_64-darwin"
|
||||
# "bye.x86_64-darwin"
|
||||
# "bye.aarch64-darwin"
|
||||
# ]
|
||||
#
|
||||
# into
|
||||
#
|
||||
# {
|
||||
# linux = [
|
||||
# "hello"
|
||||
# ];
|
||||
# darwin = [
|
||||
# "hello"
|
||||
# "bye"
|
||||
# ];
|
||||
# }
|
||||
groupByKernel =
|
||||
systemPaths:
|
||||
let
|
||||
systemPaths' = builtins.map _processSystemPath systemPaths;
|
||||
|
||||
filterKernel =
|
||||
kernel:
|
||||
builtins.attrNames (
|
||||
builtins.groupBy (systemPath: systemPath.name) (
|
||||
builtins.filter (systemPath: lib.hasSuffix kernel systemPath.system) systemPaths'
|
||||
)
|
||||
);
|
||||
in
|
||||
lib.genAttrs [ "linux" "darwin" ] filterKernel;
|
||||
|
||||
getLabels = lib.mapAttrs (
|
||||
kernel: rebuildCount:
|
||||
let
|
||||
number =
|
||||
if rebuildCount == 0 then
|
||||
"0"
|
||||
else if rebuildCount <= 10 then
|
||||
"1-10"
|
||||
else if rebuildCount <= 500 then
|
||||
"101-500"
|
||||
else if rebuildCount <= 1000 then
|
||||
"501-1000"
|
||||
else if rebuildCount <= 2500 then
|
||||
"1001-2500"
|
||||
else if rebuildCount <= 5000 then
|
||||
"2501-5000"
|
||||
else
|
||||
"5001+";
|
||||
|
||||
in
|
||||
"10.rebuild-${kernel}: ${number}"
|
||||
);
|
||||
}
|
||||
@ -2,6 +2,7 @@
|
||||
lib,
|
||||
runCommand,
|
||||
writeShellScript,
|
||||
writeText,
|
||||
linkFarm,
|
||||
time,
|
||||
procps,
|
||||
@ -246,24 +247,15 @@ let
|
||||
jq -s from_entries > $out/stats.json
|
||||
'';
|
||||
|
||||
compare =
|
||||
{ beforeResultDir, afterResultDir }:
|
||||
runCommand "compare"
|
||||
{
|
||||
nativeBuildInputs = [
|
||||
jq
|
||||
];
|
||||
}
|
||||
''
|
||||
mkdir $out
|
||||
jq -n -f ${./compare.jq} \
|
||||
--slurpfile before ${beforeResultDir}/outpaths.json \
|
||||
--slurpfile after ${afterResultDir}/outpaths.json \
|
||||
> $out/changed-paths.json
|
||||
|
||||
jq -r -f ${./generate-step-summary.jq} < $out/changed-paths.json > $out/step-summary.md
|
||||
# TODO: Compare eval stats
|
||||
'';
|
||||
compare = import ./compare {
|
||||
inherit
|
||||
lib
|
||||
jq
|
||||
runCommand
|
||||
writeText
|
||||
supportedSystems
|
||||
;
|
||||
};
|
||||
|
||||
full =
|
||||
{
|
||||
|
||||
Loading…
Reference in New Issue
Block a user