From ec51a56dfc26f29b6ae3183b5e3132d529e01678 Mon Sep 17 00:00:00 2001 From: Lily Foster Date: Wed, 22 Nov 2023 15:18:16 -0500 Subject: [PATCH 1/3] prefetch-npm-deps: detect and error out when generating an empty cache --- .../node/fetch-npm-deps/src/cacache.rs | 7 +++++++ .../node/fetch-npm-deps/src/main.rs | 7 ++++++- .../node/fetch-npm-deps/src/parse/mod.rs | 18 ++++++++++++++++-- 3 files changed, 29 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/node/fetch-npm-deps/src/cacache.rs b/pkgs/build-support/node/fetch-npm-deps/src/cacache.rs index b7efedac59bd..75133b1b03ea 100644 --- a/pkgs/build-support/node/fetch-npm-deps/src/cacache.rs +++ b/pkgs/build-support/node/fetch-npm-deps/src/cacache.rs @@ -43,6 +43,13 @@ impl Cache { Cache(path) } + pub fn init(&self) -> anyhow::Result<()> { + fs::create_dir_all(self.0.join("content-v2"))?; + fs::create_dir_all(self.0.join("index-v5"))?; + + Ok(()) + } + pub fn put( &self, key: String, diff --git a/pkgs/build-support/node/fetch-npm-deps/src/main.rs b/pkgs/build-support/node/fetch-npm-deps/src/main.rs index 9d86bd8091a7..2b28e290ad51 100644 --- a/pkgs/build-support/node/fetch-npm-deps/src/main.rs +++ b/pkgs/build-support/node/fetch-npm-deps/src/main.rs @@ -234,9 +234,14 @@ fn main() -> anyhow::Result<()> { (out_tempdir.path(), true) }; - let packages = parse::lockfile(&lock_content, env::var("FORCE_GIT_DEPS").is_ok())?; + let packages = parse::lockfile( + &lock_content, + env::var("FORCE_GIT_DEPS").is_ok(), + env::var("FORCE_EMPTY_CACHE").is_ok(), + )?; let cache = Cache::new(out.join("_cacache")); + cache.init()?; packages.into_par_iter().try_for_each(|package| { eprintln!("{}", package.name); diff --git a/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs b/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs index 86e9120de02f..880a972e80e9 100644 --- a/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs +++ b/pkgs/build-support/node/fetch-npm-deps/src/parse/mod.rs @@ -14,7 +14,11 @@ use crate::util; pub mod lock; -pub fn lockfile(content: &str, force_git_deps: bool) -> anyhow::Result> { +pub fn lockfile( + content: &str, + force_git_deps: bool, + force_empty_cache: bool, +) -> anyhow::Result> { let mut packages = lock::packages(content) .context("failed to extract packages from lockfile")? .into_par_iter() @@ -25,6 +29,10 @@ pub fn lockfile(content: &str, force_git_deps: bool) -> anyhow::Result>>()?; + if packages.is_empty() && !force_empty_cache { + bail!("No cacheable dependencies were found. Please inspect the upstream `package-lock.json` file and ensure that remote dependencies have `resolved` URLs and `integrity` hashes. If the lockfile is missing this data, attempt to get upstream to fix it via a tool like . If generating an empty cache is intentional and you would like to do it anyways, set `forceEmptyCache = true`."); + } + let mut new = Vec::new(); for pkg in packages @@ -64,7 +72,13 @@ pub fn lockfile(content: &str, force_git_deps: bool) -> anyhow::Result Date: Wed, 22 Nov 2023 15:18:39 -0500 Subject: [PATCH 2/3] fetchNpmDeps: add forceEmptyCache option --- .../node/fetch-npm-deps/default.nix | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/pkgs/build-support/node/fetch-npm-deps/default.nix b/pkgs/build-support/node/fetch-npm-deps/default.nix index 67a4c337c0d2..cfa3366491f2 100644 --- a/pkgs/build-support/node/fetch-npm-deps/default.nix +++ b/pkgs/build-support/node/fetch-npm-deps/default.nix @@ -36,8 +36,8 @@ ''; }; - makeTest = { name, src, hash, forceGitDeps ? false }: testers.invalidateFetcherByDrvHash fetchNpmDeps { - inherit name hash forceGitDeps; + makeTest = { name, src, hash, forceGitDeps ? false, forceEmptyCache ? false }: testers.invalidateFetcherByDrvHash fetchNpmDeps { + inherit name hash forceGitDeps forceEmptyCache; src = makeTestSrc { inherit name src; }; }; @@ -98,6 +98,20 @@ hash = "sha256-VzQhArHoznYSXUT7l9HkJV4yoSOmoP8eYTLel1QwmB4="; }; + # This package has no resolved deps whatsoever, which will not actually work but does test the forceEmptyCache option. + emptyCache = makeTest { + name = "empty-cache"; + + src = fetchurl { + url = "https://raw.githubusercontent.com/bufbuild/protobuf-es/v1.2.1/package-lock.json"; + hash = "sha256-UdBUEb4YRHsbvyjymIyjemJEiaI9KQRirqt+SFSK0wA="; + }; + + hash = "sha256-Cdv40lQjRszzJtJydZt25uYfcJVeJGwH54A+agdH9wI="; + + forceEmptyCache = true; + }; + # This package contains both hosted Git shorthand, and a bundled dependency that happens to override an existing one. etherpadLite1818 = makeTest { name = "etherpad-lite-1.8.18"; @@ -124,6 +138,7 @@ { name ? "npm-deps" , hash ? "" , forceGitDeps ? false + , forceEmptyCache ? false , ... } @ args: let @@ -136,6 +151,7 @@ }; forceGitDeps_ = lib.optionalAttrs forceGitDeps { FORCE_GIT_DEPS = true; }; + forceEmptyCache_ = lib.optionalAttrs forceEmptyCache { FORCE_EMPTY_CACHE = true; }; in stdenvNoCC.mkDerivation (args // { inherit name; @@ -174,5 +190,5 @@ else "/no-cert-file.crt"; outputHashMode = "recursive"; - } // hash_ // forceGitDeps_); + } // hash_ // forceGitDeps_ // forceEmptyCache_); } From 8e3009d95c3334369a59ac9b02dab4393bb7c1c7 Mon Sep 17 00:00:00 2001 From: Lily Foster Date: Fri, 14 Jul 2023 08:03:54 -0400 Subject: [PATCH 3/3] buildNpmPackage: add forceEmptyCache option --- pkgs/build-support/node/build-npm-package/default.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkgs/build-support/node/build-npm-package/default.nix b/pkgs/build-support/node/build-npm-package/default.nix index eecea8b76758..42c6a9c065b2 100644 --- a/pkgs/build-support/node/build-npm-package/default.nix +++ b/pkgs/build-support/node/build-npm-package/default.nix @@ -21,6 +21,9 @@ # Whether to force the usage of Git dependencies that have install scripts, but not a lockfile. # Use with care. , forceGitDeps ? false + # Whether to force allow an empty dependency cache. + # This can be enabled if there are truly no remote dependencies, but generally an empty cache indicates something is wrong. +, forceEmptyCache ? false # Whether to make the cache writable prior to installing dependencies. # Don't set this unless npm tries to write to the cache directory, as it can slow down the build. , makeCacheWritable ? false @@ -42,7 +45,7 @@ , npmWorkspace ? null , nodejs ? topLevelArgs.nodejs , npmDeps ? fetchNpmDeps { - inherit forceGitDeps src srcs sourceRoot prePatch patches postPatch; + inherit forceGitDeps forceEmptyCache src srcs sourceRoot prePatch patches postPatch; name = "${name}-npm-deps"; hash = npmDepsHash; }