prefetch-yarn-deps: add --fixup-lockfile flag to fixup a yarn.lock (#214062)

The flag iterates through the lockfile entries, rewrites `resolved` URLs
to those that will be in the cache (like `fixup_yarn_lock` from
yarn2nix), removes `integrity` for git deps whose hash won't match the
reproducible repacking that the fetcher does, writes the amended
lockfile, and exits.

pkgs/build-support/node/fetch-yarn-deps/common.js

@@ -0,0 +1,17 @@
+const path = require('path')
+
+// This has to match the logic in pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/urlToName.js
+// so that fixup_yarn_lock produces the same paths
+const urlToName = url => {
+	const isCodeloadGitTarballUrl = url.startsWith('https://codeload.github.com/') && url.includes('/tar.gz/')
+
+	if (url.startsWith('git+') || isCodeloadGitTarballUrl) {
+		return path.basename(url)
+	} else {
+		return url
+			.replace(/https:\/\/(.)*(.com)\//g, '') // prevents having long directory names
+			.replace(/[@/%:-]/g, '_') // replace @ and : and - and % characters with underscore
+	}
+}
+
+module.exports = { urlToName };

pkgs/build-support/node/fetch-yarn-deps/default.nix

@@ -21,8 +21,8 @@ in {
       mkdir libexec
       tar --strip-components=1 -xf ${yarnpkg-lockfile-tar} package/index.js
       mv index.js libexec/yarnpkg-lockfile.js
-      cp ${./index.js} libexec/index.js
-      patchShebangs libexec/index.js
+      cp ${./.}/*.js libexec/
+      patchShebangs libexec
 
       runHook postBuild
     '';
@@ -34,6 +34,7 @@ in {
       cp -r libexec $out
       makeWrapper $out/libexec/index.js $out/bin/prefetch-yarn-deps \
         --prefix PATH : ${lib.makeBinPath [ coreutils nix-prefetch-git nix ]}
+      makeWrapper $out/libexec/fixup.js $out/bin/fixup-yarn-lock
 
       runHook postInstall
     '';

pkgs/build-support/node/fetch-yarn-deps/fixup.js

@@ -0,0 +1,74 @@
+#!/usr/bin/env node
+'use strict'
+
+const fs = require('fs')
+const process = require('process')
+const lockfile = require('./yarnpkg-lockfile.js')
+const { urlToName } = require('./common.js')
+
+const fixupYarnLock = async (lockContents, verbose) => {
+	const lockData = lockfile.parse(lockContents)
+
+	const fixedData = Object.fromEntries(
+		Object.entries(lockData.object)
+		.map(([dep, pkg]) => {
+			const [ url, hash ] = pkg.resolved.split("#", 2)
+
+			if (hash || url.startsWith("https://codeload.github.com")) {
+				if (verbose) console.log(`Removing integrity for git dependency ${dep}`)
+				delete pkg.integrity
+			}
+
+			if (verbose) console.log(`Rewriting URL ${url} for dependency ${dep}`)
+			pkg.resolved = urlToName(url)
+
+			return [dep, pkg]
+		})
+	)
+
+	if (verbose) console.log('Done')
+
+	return fixedData
+}
+
+const showUsage = async () => {
+	process.stderr.write(`
+syntax: fixup-yarn-lock [path to yarn.lock] [options]
+
+Options:
+  -h --help         Show this help
+  -v --verbose      Verbose output
+`)
+	process.exit(1)
+}
+
+const main = async () => {
+	const args = process.argv.slice(2)
+	let next, lockFile, verbose
+	while (next = args.shift()) {
+		if (next == '--verbose' || next == '-v') {
+			verbose = true
+		} else if (next == '--help' || next == '-h') {
+			showUsage()
+		} else if (!lockFile) {
+			lockFile = next
+		} else {
+			showUsage()
+		}
+	}
+	let lockContents
+	try {
+		lockContents = await fs.promises.readFile(lockFile || 'yarn.lock', 'utf-8')
+	} catch {
+		showUsage()
+	}
+
+	const fixedData = await fixupYarnLock(lockContents, verbose)
+	await fs.promises.writeFile(lockFile || 'yarn.lock', lockfile.stringify(fixedData))
+}
+
+main()
+	.catch(e => {
+		console.error(e)
+		process.exit(1)
+	})

pkgs/build-support/node/fetch-yarn-deps/index.js

@@ -10,6 +10,7 @@ const path = require('path')
 const lockfile = require('./yarnpkg-lockfile.js')
 const { promisify } = require('util')
 const url = require('url')
+const { urlToName } = require('./common.js')
 
 const execFile = promisify(child_process.execFile)
 
@@ -19,20 +20,6 @@ const exec = async (...args) => {
 	return res
 }
 
-// This has to match the logic in pkgs/development/tools/yarn2nix-moretea/yarn2nix/lib/urlToName.js
-// so that fixup_yarn_lock produces the same paths
-const urlToName = url => {
-	const isCodeloadGitTarballUrl = url.startsWith('https://codeload.github.com/') && url.includes('/tar.gz/')
-
-	if (url.startsWith('git+') || isCodeloadGitTarballUrl) {
-		return path.basename(url)
-	} else {
-		return url
-			.replace(/https:\/\/(.)*(.com)\//g, '') // prevents having long directory names
-			.replace(/[@/%:-]/g, '_') // replace @ and : and - and % characters with underscore
-	}
-}
-
 const downloadFileHttps = (fileName, url, expectedHash, hashType = 'sha1') => {
 	return new Promise((resolve, reject) => {
 		https.get(url, (res) => {