From b09a742323adeb4083adbc8cdc808c956fc00090 Mon Sep 17 00:00:00 2001 From: Guillaume Maudoux Date: Mon, 23 Oct 2023 23:14:41 +0200 Subject: [PATCH] bazel_7: Filter bzlmod dependencies by name predicate --- .../bazel/bazel_7/bazel-repository-cache.nix | 27 +++++++--------- .../build-managers/bazel/bazel_7/default.nix | 32 +++++++++---------- 2 files changed, 27 insertions(+), 32 deletions(-) diff --git a/pkgs/development/tools/build-managers/bazel/bazel_7/bazel-repository-cache.nix b/pkgs/development/tools/build-managers/bazel/bazel_7/bazel-repository-cache.nix index 07405a9a5e05..74aa49fae73f 100644 --- a/pkgs/development/tools/build-managers/bazel/bazel_7/bazel-repository-cache.nix +++ b/pkgs/development/tools/build-managers/bazel/bazel_7/bazel-repository-cache.nix @@ -4,9 +4,10 @@ , fetchurl # The path to the right MODULE.bazel.lock , lockfile - # The path to a json file containing the list of hashes we should prefetch -, requiredDeps ? null -, extraInputs ? [ ] + # A predicate used to select only some dependencies based on their name +, requiredDepNamePredicate ? _: true + # Extra deps in the form of derivations producing a "single file" output path +, extraDeps ? [ ] }: let modules = builtins.fromJSON (builtins.readFile lockfile); @@ -29,6 +30,9 @@ let # like foldl', force evaluation of intermediate results builtins.seq acc' children; + # remove the "--" prefix, abusing undocumented negative substring length + sanitize = builtins.substring 2 (-1); + extract_source = f: acc: value: # We take any "attributes" object that has a "sha256" field. Every value # under "attributes" is assumed to be an object, and all the "attributes" @@ -72,18 +76,8 @@ let && f attrs.name then accWithNewSource else acc; - # remove the "--" prefix, abusing undocumented negative substring length - sanitize = builtins.substring 2 (-1); - - inputs = foldlJSON (extract_source (n: builtins.trace (sanitize n) true)) { } modules; - - requiredHashes = builtins.fromJSON (builtins.readFile requiredDeps); - requiredAttrs = lib.genAttrs requiredHashes throw; - - requiredInputs = - if requiredDeps == null - then inputs - else builtins.intersectAttrs requiredAttrs (builtins.trace inputs inputs); + requiredSourcePredicate = n: requiredDepNamePredicate (sanitize n); + requiredDeps = foldlJSON (extract_source requiredSourcePredicate) { } modules; command = '' mkdir -p $out/content_addressable/sha256 @@ -92,13 +86,14 @@ let # TODO: Do not re-hash. Use nix-hash to convert hashes (drv: '' filename=$(basename "${lib.head drv.urls}") + echo Bundling $filename hash=$(${rnix-hashes}/bin/rnix-hashes --encoding BASE16 ${drv.sha256} | cut -f 2) mkdir -p content_addressable/sha256/$hash ln -sfn ${drv} content_addressable/sha256/$hash/file # Expect file name conflicts ln -sn ${drv} $filename || true '') - (builtins.attrValues requiredInputs ++ extraInputs) + (builtins.attrValues requiredDeps ++ extraDeps) ; repository_cache = runCommand "bazel-repository-cache" { } command; diff --git a/pkgs/development/tools/build-managers/bazel/bazel_7/default.nix b/pkgs/development/tools/build-managers/bazel/bazel_7/default.nix index 9b144544a42f..bc857986b44d 100644 --- a/pkgs/development/tools/build-managers/bazel/bazel_7/default.nix +++ b/pkgs/development/tools/build-managers/bazel/bazel_7/default.nix @@ -35,6 +35,7 @@ , gnupatch , file , installShellFiles +, lndir # updater , python3 , writeScript @@ -75,9 +76,10 @@ let distDir = repoCache; repoCache = callPackage ./bazel-repository-cache.nix { inherit lockfile; - # TODO: efficiently filter so that all tests find their dependencies - # without downloading 10 jdk versions - # requiredDeps = ./required-hashes.json; + # We use the release tarball that already has everything bundled, + # But we need one extra dep required by our nonprebuilt java toolchains. + requiredDepNamePredicate = name: + null != builtins.match "rules_java~.*~toolchains~remote_java_tools" name; }; defaultShellUtils = @@ -274,7 +276,6 @@ stdenv.mkDerivation rec { postPatch = let darwinPatches = '' - bazelLinkFlags () { eval set -- "$NIX_LDFLAGS" local flag @@ -367,30 +368,29 @@ stdenv.mkDerivation rec { -e 's!/bin/bash!${bash}/bin/bash!g' \ -e 's!shasum -a 256!sha256sum!g' - # Add compile options to command line. - # XXX: It would suit a bazelrc file better, but I found no way to pass it. - # It seems it is always ignored. - # Passing EXTRA_BAZEL_ARGS is tricky due to quoting. + # Augment bundled repository_cache with our extra paths + ${lndir}/bin/lndir ${repoCache}/content_addressable \ + $PWD/derived/repository_cache/content_addressable + # Add required flags to bazel command line. + # XXX: It would suit a bazelrc file better, but I found no way to pass it. + # It seems that bazel bootstrapping ignores it. + # Passing EXTRA_BAZEL_ARGS is tricky due to quoting. sedVerbose compile.sh \ -e "/bazel_build /a\ --verbose_failures \\\\" \ -e "/bazel_build /a\ --curses=no \\\\" \ -e "/bazel_build /a\ --features=-layering_check \\\\" \ -e "/bazel_build /a\ --experimental_strict_java_deps=off \\\\" \ -e "/bazel_build /a\ --strict_proto_deps=off \\\\" \ - -e "/bazel_build /a\ --action_env=NIX_CFLAGS_COMPILE=\"$NIX_CFLAGS_COMPILE\" \\\\" \ - -e "/bazel_build /a\ --action_env=NIX_LDFLAGS=\"$NIX_LDFLAGS\" \\\\" \ - -e "/bazel_build /a\ --host_action_env=NIX_CFLAGS_COMPILE=\"$NIX_CFLAGS_COMPILE\" \\\\" \ - -e "/bazel_build /a\ --host_action_env=NIX_LDFLAGS=\"$NIX_LDFLAGS\" \\\\" \ -e "/bazel_build /a\ --toolchain_resolution_debug='@bazel_tools//tools/jdk:(runtime_)?toolchain_type' \\\\" \ -e "/bazel_build /a\ --tool_java_runtime_version=local_jdk \\\\" \ -e "/bazel_build /a\ --java_runtime_version=local_jdk \\\\" \ -e "/bazel_build /a\ --extra_toolchains=@bazel_tools//tools/jdk:all \\\\" \ - -e "/bazel_build /a\ --distdir=${distDir} \\\\" \ - #-e "/bazel_build /a\ --action_env=NIX_BINTOOLS=\"$NIX_BINTOOLS\" \\\\" \ - #-e "/bazel_build /a\ --action_env=NIX_CC=\"$NIX_CC\" \\\\" \ - #-e "/bazel_build /a\ --action_env=nativeBuildInputs=\"$nativeBuildInputs\" \\\\" \ + #-e "/bazel_build /a\ --action_env=NIX_CFLAGS_COMPILE=\"$NIX_CFLAGS_COMPILE\" \\\\" \ + #-e "/bazel_build /a\ --action_env=NIX_LDFLAGS=\"$NIX_LDFLAGS\" \\\\" \ + #-e "/bazel_build /a\ --host_action_env=NIX_CFLAGS_COMPILE=\"$NIX_CFLAGS_COMPILE\" \\\\" \ + #-e "/bazel_build /a\ --host_action_env=NIX_LDFLAGS=\"$NIX_LDFLAGS\" \\\\" \ # Also build parser_deploy.jar with bootstrap bazel # TODO: Turn into a proper patch