nixpkgs/maintainers/scripts/bootstrap-files/refresh-tarballs.bash

#!/usr/bin/env nix-shell
#! nix-shell --pure
#! nix-shell -i bash
#! nix-shell -p curl cacert
#! nix-shell -p git
#! nix-shell -p nix
#! nix-shell -p jq

set -o pipefail

# How the refresher works:
#
# For a given list of <targets>:
# 1. fetch latest successful '.build` job
# 2. fetch oldest evaluation that contained that '.build', extract nixpkgs commit
# 3. fetch all the `.build` artifacts from '$out/on-server/' directory
# 4. calculate hashes and craft the commit message with the details on
#    how to upload the result to 'tarballs.nixos.org'

scratch_dir=$(mktemp -d)
trap 'rm -rf -- "${scratch_dir}"' EXIT

usage() {
    cat >&2 <<EOF
Usage:
    $0 [ --commit ] --targets=<target>[,<target>,...]

    The tool must be ran from the root directory of 'nixpkgs' repository.

Synopsis:
    'refresh-tarballs.bash' script fetches latest bootstrapFiles built
    by hydra, registers them in 'nixpkgs' and provides commands to
    upload seed files to 'tarballs.nixos.org'.

    This is usually done in the following cases:

    1. Single target fix: current bootstrap files for a single target
       are problematic for some reason (target-specific bug). In this
       case we can refresh just that target as:

       \$ $0 --commit --targets=i686-unknown-linux-gnu

    2. Routine refresh: all bootstrap files should be refreshed to avoid
       debugging problems that only occur on very old binaries.

       \$ $0 --commit --all-targets

To get help on uploading refreshed binaries to 'tarballs.nixos.org'
please have a look at <maintainers/scripts/bootstrap-files/README.md>.
EOF
    exit 1
}

# log helpers

die() {
    echo "ERROR: $*" >&2
    exit 1
}

info() {
    echo "INFO: $*" >&2
}

[[ ${#@} -eq 0 ]] && usage

# known targets

NATIVE_TARGETS=(
    aarch64-unknown-linux-gnu
    aarch64-unknown-linux-musl
    i686-unknown-linux-gnu
    x86_64-unknown-linux-gnu
    x86_64-unknown-linux-musl
    aarch64-apple-darwin
    x86_64-apple-darwin
)

is_native() {
    local t target=$1
    for t in "${NATIVE_TARGETS[@]}"; do
        [[ $t == $target ]] && return 0
    done
    return 1
}

CROSS_TARGETS=(
    armv5tel-unknown-linux-gnueabi
    armv6l-unknown-linux-gnueabihf
    armv6l-unknown-linux-musleabihf
    armv7l-unknown-linux-gnueabihf
    mips64el-unknown-linux-gnuabi64
    mips64el-unknown-linux-gnuabin32
    mipsel-unknown-linux-gnu
    powerpc64-unknown-linux-gnuabielfv2
    powerpc64le-unknown-linux-gnu
    riscv64-unknown-linux-gnu
    s390x-unknown-linux-gnu
    x86_64-unknown-freebsd
)

is_cross() {
    local t target=$1
    for t in "${CROSS_TARGETS[@]}"; do
        [[ $t == $target ]] && return 0
    done
    return 1
}

nar_sri_get() {
    local restore_path store_path
    ((${#@} != 2)) && die "nar_sri_get /path/to/name.nar.xz name"
    restore_path="${scratch_dir}/$2"
    xz -d < "$1" | nix-store --restore "${restore_path}"
    [[ $? -ne 0 ]] && die "Failed to unpack '$1'"

    store_path=$(nix-store --add "${restore_path}")
    [[ $? -ne 0 ]] && die "Failed to add '$restore_path' to store"
    rm -rf -- "${restore_path}"

    nix-hash --to-sri "$(nix-store --query --hash "${store_path}")"
}

# collect passed options

targets=()
commit=no

for arg in "$@"; do
    case "$arg" in
        --all-targets)
            targets+=(
                ${CROSS_TARGETS[@]}
                ${NATIVE_TARGETS[@]}
            )
            ;;
        --targets=*)
            # Convert "--targets=a,b,c" to targets=(a b c) bash array.
            comma_targets=${arg#--targets=}
            targets+=(${comma_targets//,/ })
            ;;
        --commit)
            commit=yes
            ;;
        *)
            usage
            ;;
    esac
done

for target in "${targets[@]}"; do
    # Native and cross jobsets differ a bit. We'll have to pick the
    # one based on target name:
    if is_native $target; then
        jobset=nixpkgs/trunk
        job="stdenvBootstrapTools.${target}.build"
    elif is_cross $target; then
        jobset=nixpkgs/cross-trunk
        job="bootstrapTools.${target}.build"
    else
        die "'$target' is not present in either of 'NATIVE_TARGETS' or 'CROSS_TARGETS'. Please add one."
    fi

    # 'nixpkgs' prefix where we will write new tarball hashes
    case "$target" in
        *linux*) nixpkgs_prefix="pkgs/stdenv/linux" ;;
        *darwin*) nixpkgs_prefix="pkgs/stdenv/darwin" ;;
        *freebsd*) nixpkgs_prefix="pkgs/stdenv/freebsd" ;;
        *) die "don't know where to put '$target'" ;;
    esac

    # We enforce s3 prefix for all targets here. This slightly differs
    # from manual uploads targets where names were chosen inconsistently.
    s3_prefix="stdenv/$target"

    # resolve 'latest' build to the build 'id', construct the link.
    latest_build_uri="https://hydra.nixos.org/job/$jobset/$job/latest"
    latest_build="$target.latest-build"
    info "Fetching latest successful build from '${latest_build_uri}'"
    curl -s -H "Content-Type: application/json" -L "$latest_build_uri" > "$latest_build"
    [[ $? -ne 0 ]] && die "Failed to fetch latest successful build"
    latest_build_id=$(jq '.id' < "$latest_build")
    [[ $? -ne 0 ]] && die "Did not find 'id' in latest build"
    build_uri="https://hydra.nixos.org/build/${latest_build_id}"

    # We pick oldest jobset evaluation and extract the 'nicpkgs' commit.
    #
    # We use oldest instead of latest to make the result more stable
    # across unrelated 'nixpkgs' updates. Ideally two subsequent runs of
    # this refresher should produce the same output (provided there are
    # no bootstrapTools updates committed between the two runs).
    oldest_eval_id=$(jq '.jobsetevals|min' < "$latest_build")
    [[ $? -ne 0 ]] && die "Did not find 'jobsetevals' in latest build"
    eval_uri="https://hydra.nixos.org/eval/${oldest_eval_id}"
    eval_meta="$target.eval-meta"
    info "Fetching oldest eval details from '${eval_uri}' (can take a minute)"
    curl -s -H "Content-Type: application/json"  -L "${eval_uri}" > "$eval_meta"
    [[ $? -ne 0 ]] && die "Failed to fetch eval metadata"
    nixpkgs_revision=$(jq --raw-output ".jobsetevalinputs.nixpkgs.revision" < "$eval_meta")
    [[ $? -ne 0 ]] && die "Failed to fetch revision"

    # Extract the build paths out of the build metadata
    drvpath=$(jq --raw-output '.drvpath' < "${latest_build}")
    [[ $? -ne 0 ]] && die "Did not find 'drvpath' in latest build"
    outpath=$(jq --raw-output '.buildoutputs.out.path' < "${latest_build}")
    [[ $? -ne 0 ]] && die "Did not find 'buildoutputs' in latest build"
    build_timestamp=$(jq --raw-output '.timestamp' < "${latest_build}")
    [[ $? -ne 0 ]] && die "Did not find 'timestamp' in latest build"
    build_time=$(TZ=UTC LANG=C date --date="@${build_timestamp}" --rfc-email)
    [[ $? -ne 0 ]] && die "Failed to format timestamp"

    info "Fetching bootstrap tools to calculate hashes from '${outpath}'"
    nix-store --realize "$outpath"
    [[ $? -ne 0 ]] && die "Failed to fetch '${outpath}' from hydra"

    fnames=()

    target_file="${nixpkgs_prefix}/bootstrap-files/${target}.nix"
    info "Writing '${target_file}'"
    {
        # header
        cat <<EOF
# Autogenerated by maintainers/scripts/bootstrap-files/refresh-tarballs.bash as:
# $ ./refresh-tarballs.bash --targets=${target}
#
# Metadata:
# - nixpkgs revision: ${nixpkgs_revision}
# - hydra build: ${latest_build_uri}
# - resolved hydra build: ${build_uri}
# - instantiated derivation: ${drvpath}
# - output directory: ${outpath}
# - build time: ${build_time}
{
EOF
      for p in "${outpath}/on-server"/*; do
          fname=$(basename "$p")
          fnames+=("$fname")
          case "$fname" in
              bootstrap-tools.tar.xz) attr=bootstrapTools ;;
              busybox) attr=$fname ;;
              unpack.nar.xz) attr=unpack ;;
              *) die "Don't know how to map '$fname' to attribute name. Please update me."
          esac

          executable_arg=
          executable_nix=
          if [[ -x "$p" ]]; then
              executable_arg="--executable"
              executable_nix="executable = true;"
          fi
          unpack_nix=
          name_nix=
          if [[ $fname = *.nar.xz ]]; then
              unpack_nix="unpack = true;"
              name_nix="name = \"${fname%.nar.xz}\";"
              sri=$(nar_sri_get "$p" "${fname%.nar.xz}")
              [[ $? -ne 0 ]] && die "Failed to get hash of '$p'"
          else
              sha256=$(nix-prefetch-url $executable_arg --name "$fname" "file://$p")
              [[ $? -ne 0 ]] && die "Failed to get the hash for '$p'"
              sri=$(nix-hash --to-sri "sha256:$sha256")
              [[ $? -ne 0 ]] && die "Failed to convert '$sha256' hash to an SRI form"
          fi

          # individual file entries
          cat <<EOF
  $attr = import <nix/fetchurl.nix> {
    url = "http://tarballs.nixos.org/${s3_prefix}/${nixpkgs_revision}/$fname";
    hash = "${sri}";$(
    [[ -n ${executable_nix} ]] && printf "\n    %s" "${executable_nix}"
    [[ -n ${name_nix} ]]       && printf "\n    %s" "${name_nix}"
    [[ -n ${unpack_nix} ]]     && printf "\n    %s" "${unpack_nix}"
)
  };
EOF
      done
      # footer
      cat <<EOF
}
EOF
    } > "${target_file}"

        target_file_commit_msg=${target}.commit_message
        cat > "$target_file_commit_msg" <<EOF
${nixpkgs_prefix}: update ${target} bootstrap-files

sha256sum of files to be uploaded:

$(
echo "$ sha256sum ${outpath}/on-server/*"
sha256sum ${outpath}/on-server/*
)

Suggested commands to upload files to 'tarballs.nixos.org':

    $ nix-store --realize ${outpath}
    $ aws s3 cp --recursive --acl public-read ${outpath}/on-server/ s3://nixpkgs-tarballs/${s3_prefix}/${nixpkgs_revision}
    $ aws s3 cp --recursive s3://nixpkgs-tarballs/${s3_prefix}/${nixpkgs_revision} ./
    $ sha256sum ${fnames[*]}
    $ sha256sum ${outpath}/on-server/*
EOF

    cat "$target_file_commit_msg"
    if [[ $commit == yes ]]; then
        git commit "${target_file}" -F "$target_file_commit_msg"
    else
        info "DRY RUN: git commit ${target_file} -F $target_file_commit_msg"
    fi
    rm -- "$target_file_commit_msg"

    # delete temp files
    rm -- "$latest_build" "$eval_meta"
done