flutter: Don't tarball the package cache

Co-authored-by: FlafyDev <flafyarazi@gmail.com>

The dependency fixed-output derivation now uses the recursive hash mode to avoid tarballing and copying all the files.

https://github.com/NixOS/nix/issues/6660 was ran into during the development of this change. Input references were found in the Git package cache before nukeReferences was used.

It turns out that the mirrors in the Git package cache do not actually need to be preserved, as the SDK does not use them during the build process. They are therefore deleted in the dependency derivation and re-created as blank repositories in the main derivation.

A description of the Git package cache layout can be found here: c890afa1d6/lib/src/source/git.dart (L339)

pkgs/applications/misc/yubioath-flutter/default.nix

@@ -21,7 +21,7 @@ flutter.mkFlutterApp rec {
 
   passthru.helper = python3.pkgs.callPackage ./helper.nix { inherit src version meta; };
 
-  vendorHash = "sha256-/WP6oN5avAILZd3SkC/jmsqNL+BxdQGUjNwsjLF9RB4=";
+  vendorHash = "sha256-OU433z6Cz+jNJAjIiPO6cxySyejQ4SpSvuUHJsdT8TA=";
 
   postPatch = ''
     substituteInPlace linux/CMakeLists.txt \

pkgs/build-support/flutter/default.nix

@@ -1,6 +1,5 @@
 { lib
 , stdenvNoCC
-, nukeReferences
 , llvmPackages_13
 , cacert
 , flutter
@@ -19,12 +18,11 @@ let
   self =
 (self: llvmPackages_13.stdenv.mkDerivation (args // {
   deps = stdenvNoCC.mkDerivation (lib.recursiveUpdate (getAttrsOrNull fetchAttrs args) {
-    name = "${self.name}-deps-flutter-v${flutter.unwrapped.version}.tar.gz";
+    name = "${self.name}-deps-flutter";
 
     nativeBuildInputs = [
       flutter
       git
-      nukeReferences
     ];
 
     # avoid pub phase
@@ -36,42 +34,33 @@ let
       TMP=$(mktemp -d)
       export HOME="$TMP"
 
+      # Configure the package cache
+      export PUB_CACHE="$out/cache/.pub-cache"
+      mkdir -p "$PUB_CACHE"
+
       flutter config --no-analytics &>/dev/null # mute first-run
       flutter config --enable-linux-desktop
       flutter packages get
       ${lib.optionalString (args ? flutterExtraFetchCommands) args.flutterExtraFetchCommands}
 
-      RES="$TMP"
-
       # so we can use lock, diff yaml
-      cp "pubspec.yaml" "$RES"
+      mkdir -p "$out/pubspec"
-      cp "pubspec.lock" "$RES"
+      cp "pubspec.yaml" "$out/pubspec"
-
+      cp "pubspec.lock" "$out/pubspec"
-      # replace paths with placeholders
-      find "$RES" -type f -exec sed -i \
-        -e s,$TMP,${placeholder_deps},g \
-        -e s,${flutter.unwrapped},${placeholder_flutter},g \
-        {} +
 
       # nuke nondeterminism
 
       # deterministic git repos
-      find "$RES" -iname .git -type d | while read -r repoGit; do
+      find "$PUB_CACHE" -iname .git -type d | while read -r repoGit; do
         make_deterministic_repo "$(dirname "$repoGit")"
       done
 
-      # Impure Pub files
+      # Impure package cache files
-      rm -rf "$RES"/.pub-cache/hosted/*/.cache # Not pinned by pubspec.lock
+      rm -rf "$PUB_CACHE"/hosted/*/.cache # Not pinned by pubspec.lock
-      rm -f "$RES"/.pub-cache/README.md # May change with different Dart versions
+      rm -f "$PUB_CACHE"/README.md # May change with different Dart versions
-
+      rm -rf "$PUB_CACHE"/_temp # https://github.com/dart-lang/pub/blob/c890afa1d65b340fa59308172029680c2f8b0fc6/lib/src/system_cache.dart#L131
-      # nuke refs
+      rm -rf "$PUB_CACHE"/log # https://github.com/dart-lang/pub/blob/c890afa1d65b340fa59308172029680c2f8b0fc6/lib/src/command.dart#L348
-      find "$RES" -type f -exec nuke-refs {} +
+      rm -rf "$PUB_CACHE"/git/cache/*/* # Recreate this on the other end. See: https://github.com/dart-lang/pub/blob/c890afa1d65b340fa59308172029680c2f8b0fc6/lib/src/source/git.dart#L531
-
-      # Build a reproducible tar, per instructions at https://reproducible-builds.org/docs/archives/
-      tar --owner=0 --group=0 --numeric-owner --format=gnu \
-          --sort=name --mtime="@$SOURCE_DATE_EPOCH" \
-          -czf "$out" -C "$RES" \
-          pubspec.yaml pubspec.lock .pub-cache
     '';
 
     GIT_SSL_CAINFO = "${cacert}/etc/ssl/certs/ca-bundle.crt";
@@ -84,51 +73,71 @@ let
     # unnecesarry
     dontFixup = true;
 
+    # Patching shebangs introduces input references to this fixed-output derivation.
+    # This triggers a bug in Nix, causing the output path to change unexpectedly.
+    # https://github.com/NixOS/nix/issues/6660
+    dontPatchShebangs = true;
+
     outputHashAlgo = if self ? vendorHash then null else "sha256";
-    # outputHashMode = "recursive";
+    outputHashMode = "recursive";
     outputHash = if self ? vendorHash then
       self.vendorHash
     else if self ? vendorSha256 then
       self.vendorSha256
     else
       lib.fakeSha256;
-
   });
 
-  nativeBuildInputs = [ flutter ] ++ lib.optionals (args ? nativeBuildInputs) args.nativeBuildInputs;
+  nativeBuildInputs = [
+    flutter
+    git
+  ] ++ lib.optionals (args ? nativeBuildInputs) args.nativeBuildInputs;
 
   buildInputs = lib.optionals (args ? buildInputs) args.buildInputs;
 
   configurePhase = ''
     runHook preConfigure
 
-    # for some reason fluffychat build breaks without this - seems file gets overriden by some tool
-    cp pubspec.yaml pubspec-backup
-
     TMP=$(mktemp -d)
     export HOME="$TMP"
 
     flutter config --no-analytics &>/dev/null # mute first-run
     flutter config --enable-linux-desktop
 
-    # extract deps
+    # Configure the package cache
-    tar xzf "$deps" -C "$HOME"
+    export PUB_CACHE="$TMP/.pub-cache"
+    mkdir -p "$PUB_CACHE"
 
-    # after extracting update paths to point to real paths
+    # Link the Git package cache.
-    find "$HOME" -type f -exec sed -i \
+    mkdir -p "$PUB_CACHE/git"
-      -e s,${placeholder_deps},"$HOME",g \
+    ln -s "$deps/cache/.pub-cache/git"/* "$PUB_CACHE/git"
-      -e s,${placeholder_flutter},${flutter},g \
+
-      {} +
+    # Recreate the internal Git cache subdirectory.
+    # See: https://github.com/dart-lang/pub/blob/c890afa1d65b340fa59308172029680c2f8b0fc6/lib/src/source/git.dart#L339)
+    # Blank repositories are created instead of attempting to match the cache mirrors to checkouts.
+    # This is not an issue, as pub does not need the mirrors in the Flutter build process.
+    rm "$PUB_CACHE/git/cache" && mkdir "$PUB_CACHE/git/cache"
+    for mirror in $(ls -A "$deps/cache/.pub-cache/git/cache"); do
+      git --git-dir="$PUB_CACHE/git/cache/$mirror" init --bare --quiet
+    done
+
+    # Link the remaining package cache directories.
+    # At this point, any subdirectories that must be writable must have been taken care of.
+    for file in $(comm -23 <(ls -A "$deps/cache/.pub-cache") <(ls -A "$PUB_CACHE")); do
+      ln -s "$deps/cache/.pub-cache/$file" "$PUB_CACHE/$file"
+    done
 
     # ensure we're using a lockfile for the right package version
     if [ -e pubspec.lock ]; then
       # FIXME: currently this is broken. in theory this should not break, but flutter has it's own way of doing things.
-      # diff -u pubspec.lock "$HOME/pubspec.lock"
+      # diff -u pubspec.lock "$deps/pubspec/pubspec.lock"
       true
     else
-      cp -v "$HOME/pubspec.lock" .
+      cp -v "$deps/pubspec/pubspec.lock" .
+      # Sometimes the pubspec.lock will get opened in write mode, even when offline.
+      chmod u+w pubspec.lock
     fi
-    diff -u pubspec.yaml "$HOME/pubspec.yaml"
+    diff -u pubspec.yaml "$deps/pubspec/pubspec.yaml"
 
     runHook postConfigure
   '';
@@ -136,8 +145,6 @@ let
   buildPhase = ''
     runHook preBuild
 
-    # for some reason fluffychat build breaks without this - seems file gets overriden by some tool
-    mv pubspec-backup pubspec.yaml
     mkdir -p build/flutter_assets/fonts
 
     flutter packages get --offline -v

pkgs/os-specific/linux/firmware/firmware-updater/default.nix

@@ -7,7 +7,7 @@ flutter2.mkFlutterApp {
   pname = "firmware-updater";
   version = "unstable";
 
-  vendorHash = "sha256-Pj6CU2W5juk4YE8oq+v7Z8CzuSwpyA3YqscoWGDaI4o=";
+  vendorHash = "sha256-kKfe+7obb2fihrca+mjCM2+51wNkbPLEPFLpXzK5Wvc=";
 
   src = fetchFromGitHub {
     owner = "canonical";