From 5c32e0ba7cf62bfcecff4b82e026de790bea394d Mon Sep 17 00:00:00 2001 From: Lily Foster Date: Tue, 27 Jun 2023 14:27:33 -0400 Subject: [PATCH] prefetch-yarn-deps: add --fixup-lockfile flag to fixup a yarn.lock (#214062) The flag iterates through the lockfile entries, rewrites `resolved` URLs to those that will be in the cache (like `fixup_yarn_lock` from yarn2nix), removes `integrity` for git deps whose hash won't match the reproducible repacking that the fetcher does, writes the amended lockfile, and exits. --- .../node/fetch-yarn-deps/common.js | 17 +++++ .../node/fetch-yarn-deps/default.nix | 5 +- .../node/fetch-yarn-deps/fixup.js | 74 +++++++++++++++++++ .../node/fetch-yarn-deps/index.js | 15 +--- 4 files changed, 95 insertions(+), 16 deletions(-) create mode 100644 pkgs/build-support/node/fetch-yarn-deps/common.js create mode 100755 pkgs/build-support/node/fetch-yarn-deps/fixup.js diff --git a/pkgs/build-support/node/fetch-yarn-deps/common.js b/pkgs/build-support/node/fetch-yarn-deps/common.js new file mode 100644 index 000000000000..8e0d1b0e470b --- /dev/null +++ b/pkgs/build-support/node/fetch-yarn-deps/common.js @@ -0,0 +1,17 @@ +const path = require('path') + +// This has to match the logic in pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/urlToName.js +// so that fixup_yarn_lock produces the same paths +const urlToName = url => { + const isCodeloadGitTarballUrl = url.startsWith('https://codeload.github.com/') && url.includes('/tar.gz/') + + if (url.startsWith('git+') || isCodeloadGitTarballUrl) { + return path.basename(url) + } else { + return url + .replace(/https:\/\/(.)*(.com)\//g, '') // prevents having long directory names + .replace(/[@/%:-]/g, '_') // replace @ and : and - and % characters with underscore + } +} + +module.exports = { urlToName }; diff --git a/pkgs/build-support/node/fetch-yarn-deps/default.nix b/pkgs/build-support/node/fetch-yarn-deps/default.nix index 4cf2507706ae..f3b5a6bd163b 100644 --- a/pkgs/build-support/node/fetch-yarn-deps/default.nix +++ b/pkgs/build-support/node/fetch-yarn-deps/default.nix @@ -21,8 +21,8 @@ in { mkdir libexec tar --strip-components=1 -xf ${yarnpkg-lockfile-tar} package/index.js mv index.js libexec/yarnpkg-lockfile.js - cp ${./index.js} libexec/index.js - patchShebangs libexec/index.js + cp ${./.}/*.js libexec/ + patchShebangs libexec runHook postBuild ''; @@ -34,6 +34,7 @@ in { cp -r libexec $out makeWrapper $out/libexec/index.js $out/bin/prefetch-yarn-deps \ --prefix PATH : ${lib.makeBinPath [ coreutils nix-prefetch-git nix ]} + makeWrapper $out/libexec/fixup.js $out/bin/fixup-yarn-lock runHook postInstall ''; diff --git a/pkgs/build-support/node/fetch-yarn-deps/fixup.js b/pkgs/build-support/node/fetch-yarn-deps/fixup.js new file mode 100755 index 000000000000..8b91e7efa63f --- /dev/null +++ b/pkgs/build-support/node/fetch-yarn-deps/fixup.js @@ -0,0 +1,74 @@ +#!/usr/bin/env node +'use strict' + +const fs = require('fs') +const process = require('process') +const lockfile = require('./yarnpkg-lockfile.js') +const { urlToName } = require('./common.js') + +const fixupYarnLock = async (lockContents, verbose) => { + const lockData = lockfile.parse(lockContents) + + const fixedData = Object.fromEntries( + Object.entries(lockData.object) + .map(([dep, pkg]) => { + const [ url, hash ] = pkg.resolved.split("#", 2) + + if (hash || url.startsWith("https://codeload.github.com")) { + if (verbose) console.log(`Removing integrity for git dependency ${dep}`) + delete pkg.integrity + } + + if (verbose) console.log(`Rewriting URL ${url} for dependency ${dep}`) + pkg.resolved = urlToName(url) + + return [dep, pkg] + }) + ) + + if (verbose) console.log('Done') + + return fixedData +} + +const showUsage = async () => { + process.stderr.write(` +syntax: fixup-yarn-lock [path to yarn.lock] [options] + +Options: + -h --help Show this help + -v --verbose Verbose output +`) + process.exit(1) +} + +const main = async () => { + const args = process.argv.slice(2) + let next, lockFile, verbose + while (next = args.shift()) { + if (next == '--verbose' || next == '-v') { + verbose = true + } else if (next == '--help' || next == '-h') { + showUsage() + } else if (!lockFile) { + lockFile = next + } else { + showUsage() + } + } + let lockContents + try { + lockContents = await fs.promises.readFile(lockFile || 'yarn.lock', 'utf-8') + } catch { + showUsage() + } + + const fixedData = await fixupYarnLock(lockContents, verbose) + await fs.promises.writeFile(lockFile || 'yarn.lock', lockfile.stringify(fixedData)) +} + +main() + .catch(e => { + console.error(e) + process.exit(1) + }) diff --git a/pkgs/build-support/node/fetch-yarn-deps/index.js b/pkgs/build-support/node/fetch-yarn-deps/index.js index b66e1220218d..9f95e1697030 100755 --- a/pkgs/build-support/node/fetch-yarn-deps/index.js +++ b/pkgs/build-support/node/fetch-yarn-deps/index.js @@ -10,6 +10,7 @@ const path = require('path') const lockfile = require('./yarnpkg-lockfile.js') const { promisify } = require('util') const url = require('url') +const { urlToName } = require('./common.js') const execFile = promisify(child_process.execFile) @@ -19,20 +20,6 @@ const exec = async (...args) => { return res } -// This has to match the logic in pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/urlToName.js -// so that fixup_yarn_lock produces the same paths -const urlToName = url => { - const isCodeloadGitTarballUrl = url.startsWith('https://codeload.github.com/') && url.includes('/tar.gz/') - - if (url.startsWith('git+') || isCodeloadGitTarballUrl) { - return path.basename(url) - } else { - return url - .replace(/https:\/\/(.)*(.com)\//g, '') // prevents having long directory names - .replace(/[@/%:-]/g, '_') // replace @ and : and - and % characters with underscore - } -} - const downloadFileHttps = (fileName, url, expectedHash, hashType = 'sha1') => { return new Promise((resolve, reject) => { https.get(url, (res) => {