Merge pull request #297988 from a-n-n-a-l-e-e/bootstrap-script

maintainers/scripts/bootstrap-files: fix nar handling on linux and update README

maintainers/scripts/bootstrap-files/README.md

@@ -6,8 +6,9 @@ binaries (without the reliance on external inputs):
 - `bootstrap-tools`: an archive with the compiler toolchain and other
   helper tools enough to build the rest of the `nixpkgs`.
 - initial binaries needed to unpack `bootstrap-tools.*`. On `linux`
-  it's just `busybox`, on `darwin` it's `sh`, `bzip2`, `mkdir` and
-  `cpio`. These binaries can be executed directly from the store.
+  it's just `busybox`, on `darwin` it is unpack.nar.xz which contains
+  the binaries and script needed to unpack the tools. These binaries
+  can be executed directly from the store.
 
 These are called "bootstrap files".
 
@@ -74,12 +75,3 @@ There are two types of bootstrap files:
 The `.build` job contains `/on-server/` subdirectory with binaries to
 be uploaded to `tarballs.nixos.org`.
 The files are uploaded to `tarballs.nixos.org` by writers to `S3` store.
-
-## TODOs
-
-- `pkgs/stdenv/darwin` file layout is slightly different from
-  `pkgs/stdenv/linux`. Once `linux` seed update becomes a routine we can
-  bring `darwin` in sync if it's feasible.
-- `darwin` definition of `.build` `on-server/` directory layout differs
-  and should be updated.
-

maintainers/scripts/bootstrap-files/refresh-tarballs.bash

@@ -6,6 +6,8 @@
 #! nix-shell -p nix
 #! nix-shell -p jq
 
+set -o pipefail
+
 # How the refresher works:
 #
 # For a given list of <targets>:
@@ -15,6 +17,9 @@
 # 4. calculate hashes and craft the commit message with the details on
 #    how to upload the result to 'tarballs.nixos.org'
 
+scratch_dir=$(mktemp -d)
+trap 'rm -rf -- "${scratch_dir}"' EXIT
+
 usage() {
     cat >&2 <<EOF
 Usage:
@@ -101,15 +106,17 @@ is_cross() {
 }
 
 nar_sri_get() {
-    local ouput sri
-    output=$(nix-build  --expr \
-        'import <nix/fetchurl.nix> {
-           url = "'"$1"'";
-           unpack = true;
-         }' 2>&1 || true)
-    sri=$(echo "$output" | awk '/^\s+got:\s+/{ print $2 }')
-    [[ -z "$sri" ]] && die "$output"
-    echo "$sri"
+    local restore_path store_path
+    ((${#@} != 2)) && die "nar_sri_get /path/to/name.nar.xz name"
+    restore_path="${scratch_dir}/$2"
+    xz -d < "$1" | nix-store --restore "${restore_path}"
+    [[ $? -ne 0 ]] && die "Failed to unpack '$1'"
+
+    store_path=$(nix-store --add "${restore_path}")
+    [[ $? -ne 0 ]] && die "Failed to add '$restore_path' to store"
+    rm -rf -- "${restore_path}"
+
+    nix-hash --to-sri "$(nix-store --query --hash "${store_path}")"
 }
 
 # collect passed options
@@ -239,9 +246,12 @@ EOF
               executable_nix="executable = true;"
           fi
           unpack_nix=
-          if [[ $fname = *.nar.* ]]; then
+          name_nix=
+          if [[ $fname = *.nar.xz ]]; then
               unpack_nix="unpack = true;"
-              sri=$(nar_sri_get "file://$p")
+              name_nix="name = \"${fname%.nar.xz}\";"
+              sri=$(nar_sri_get "$p" "${fname%.nar.xz}")
+              [[ $? -ne 0 ]] && die "Failed to get hash of '$p'"
           else
               sha256=$(nix-prefetch-url $executable_arg --name "$fname" "file://$p")
               [[ $? -ne 0 ]] && die "Failed to get the hash for '$p'"
@@ -255,6 +265,7 @@ EOF
     url = "http://tarballs.nixos.org/${s3_prefix}/${nixpkgs_revision}/$fname";
     hash = "${sri}";$(
     [[ -n ${executable_nix} ]] && printf "\n    %s" "${executable_nix}"
+    [[ -n ${name_nix} ]]       && printf "\n    %s" "${name_nix}"
     [[ -n ${unpack_nix} ]]     && printf "\n    %s" "${unpack_nix}"
 )
   };