prefetch-npm-deps: download dev deps for git deps with install scripts

Git dependencies with install scripts are built isolated from the main
package, so their development dependencies are required.

To take advantage of this, #206477 is needed.

pkgs/build-support/node/build-npm-package/default.nix

@@ -12,6 +12,9 @@
   # The output hash of the dependencies for this project.
   # Can be calculated in advance with prefetch-npm-deps.
 , npmDepsHash ? ""
+  # Whether to force the usage of Git dependencies that have install scripts, but not a lockfile.
+  # Use with care.
+, forceGitDeps ? false
   # Whether to make the cache writable prior to installing dependencies.
   # Don't set this unless npm tries to write to the cache directory, as it can slow down the build.
 , makeCacheWritable ? false
@@ -32,7 +35,7 @@
 
 let
   npmDeps = fetchNpmDeps {
-    inherit src srcs sourceRoot prePatch patches postPatch;
+    inherit forceGitDeps src srcs sourceRoot prePatch patches postPatch;
     name = "${name}-npm-deps";
     hash = npmDepsHash;
   };

pkgs/build-support/node/fetch-npm-deps/default.nix

@@ -36,8 +36,8 @@
           '';
         };
 
-        makeTest = { name, src, hash }: testers.invalidateFetcherByDrvHash fetchNpmDeps {
-          inherit name hash;
+        makeTest = { name, src, hash, forceGitDeps ? false }: testers.invalidateFetcherByDrvHash fetchNpmDeps {
+          inherit name hash forceGitDeps;
 
           src = makeTestSrc { inherit name src; };
         };
@@ -108,6 +108,8 @@
           };
 
           hash = "sha256-+KA8/orSBJ4EhuSyQO8IKSxsN/FAsYU3lOzq+awuxNQ=";
+
+          forceGitDeps = true;
         };
       };
 
@@ -121,6 +123,7 @@
   fetchNpmDeps =
     { name ? "npm-deps"
     , hash ? ""
+    , forceGitDeps ? false
     , ...
     } @ args:
     let
@@ -131,6 +134,8 @@
           outputHash = "";
           outputHashAlgo = "sha256";
         };
+
+      forceGitDeps_ = lib.optionalAttrs forceGitDeps { FORCE_GIT_DEPS = true; };
     in
     stdenvNoCC.mkDerivation (args // {
       inherit name;
@@ -161,5 +166,5 @@
       dontInstall = true;
 
       outputHashMode = "recursive";
-    } // hash_);
+    } // hash_ // forceGitDeps_);
 }

pkgs/build-support/node/fetch-npm-deps/src/main.rs

@@ -97,7 +97,7 @@ fn main() -> anyhow::Result<()> {
         (out_tempdir.path(), true)
     };
 
-    let packages = parse::lockfile(&lock_content)?;
+    let packages = parse::lockfile(&lock_content, env::var("FORCE_GIT_DEPS").is_ok())?;
 
     let cache = Cache::new(out.join("_cacache"));
 

pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs

@@ -1,6 +1,7 @@
 use anyhow::{anyhow, bail, Context};
 use lock::UrlOrString;
 use rayon::prelude::*;
+use serde_json::{Map, Value};
 use std::{
     fs, io,
     process::{Command, Stdio},
@@ -10,17 +11,71 @@ use url::Url;
 
 mod lock;
 
-pub fn lockfile(lockfile: &str) -> anyhow::Result<Vec<Package>> {
-    let packages = lock::packages(lockfile).context("failed to extract packages from lockfile")?;
-
-    packages
+pub fn lockfile(content: &str, force_git_deps: bool) -> anyhow::Result<Vec<Package>> {
+    let mut packages = lock::packages(content)
+        .context("failed to extract packages from lockfile")?
         .into_par_iter()
         .map(|p| {
             let n = p.name.clone().unwrap();
 
             Package::from_lock(p).with_context(|| format!("failed to parse data for {n}"))
         })
-        .collect()
+        .collect::<anyhow::Result<Vec<_>>>()?;
+
+    let mut new = Vec::new();
+
+    for pkg in packages
+        .iter()
+        .filter(|p| matches!(p.specifics, Specifics::Git { .. }))
+    {
+        let dir = match &pkg.specifics {
+            Specifics::Git { workdir } => workdir,
+            Specifics::Registry { .. } => unimplemented!(),
+        };
+
+        let path = dir.path().join("package");
+
+        let lockfile_contents = fs::read_to_string(path.join("package-lock.json"));
+
+        let package_json_path = path.join("package.json");
+        let mut package_json: Map<String, Value> =
+            serde_json::from_str(&fs::read_to_string(package_json_path)?)?;
+
+        if let Some(scripts) = package_json
+            .get_mut("scripts")
+            .and_then(Value::as_object_mut)
+        {
+            // https://github.com/npm/pacote/blob/272edc1bac06991fc5f95d06342334bbacfbaa4b/lib/git.js#L166-L172
+            for typ in [
+                "postinstall",
+                "build",
+                "preinstall",
+                "install",
+                "prepack",
+                "prepare",
+            ] {
+                if scripts.contains_key(typ) && lockfile_contents.is_err() && !force_git_deps {
+                    bail!("Git dependency {} contains install scripts, but has no lockfile, which is something that will probably break. Open an issue if you can't feasibly patch this dependency out, and we'll come up with a workaround.\nIf you'd like to attempt to try to use this dependency anyways, set `forceGitDeps = true`.", pkg.name);
+                }
+            }
+        }
+
+        if let Ok(lockfile_contents) = lockfile_contents {
+            new.append(&mut lockfile(&lockfile_contents, force_git_deps)?);
+        }
+    }
+
+    packages.append(&mut new);
+
+    packages.par_sort_by(|x, y| {
+        x.url
+            .partial_cmp(&y.url)
+            .expect("resolved should be comparable")
+    });
+
+    packages.dedup_by(|x, y| x.url == y.url);
+
+    Ok(packages)
 }
 
 #[derive(Debug)]