diff --git a/nixos/tests/curl-impersonate.nix b/nixos/tests/curl-impersonate.nix index 97143951d4b0..13eb54b5b2d0 100644 --- a/nixos/tests/curl-impersonate.nix +++ b/nixos/tests/curl-impersonate.nix @@ -97,6 +97,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: let pyyaml pytest-asyncio dpkt + ts1-signatures ]}" # Prepare test root prefix diff --git a/pkgs/tools/networking/curl-impersonate/chrome/default.nix b/pkgs/tools/networking/curl-impersonate/chrome/default.nix new file mode 100644 index 000000000000..fd97c69b31c5 --- /dev/null +++ b/pkgs/tools/networking/curl-impersonate/chrome/default.nix @@ -0,0 +1,206 @@ +{ + lib, + stdenv, + fetchFromGitHub, + callPackage, + buildGoModule, + installShellFiles, + buildPackages, + zlib, + zstd, + sqlite, + cmake, + python3, + ninja, + perl, + autoconf, + automake, + libtool, + cctools, + cacert, + unzip, + go, + p11-kit, +}: +stdenv.mkDerivation rec { + pname = "curl-impersonate-chrome"; + version = "0.7.0"; + + outputs = [ + "out" + "dev" + ]; + + src = fetchFromGitHub { + owner = "yifeikong"; + repo = "curl-impersonate"; + rev = "v${version}"; + hash = "sha256-nxANiNgrbbp7F6k2y1HGGWGOUBRwc3tK8WcNIqEBLz4="; + }; + + patches = [ ./disable-building-docs.patch ]; + + # Disable blanket -Werror to fix build on `gcc-13` related to minor + # warnings on `boringssl`. + env.NIX_CFLAGS_COMPILE = "-Wno-error"; + + strictDeps = true; + + depsBuildBuild = lib.optionals (stdenv.buildPlatform != stdenv.hostPlatform) [ + buildPackages.stdenv.cc + ]; + + nativeBuildInputs = + lib.optionals stdenv.isDarwin [ + # Must come first so that it shadows the 'libtool' command but leaves 'libtoolize' + cctools + ] + ++ [ + installShellFiles + cmake + python3 + python3.pythonOnBuildForHost.pkgs.gyp + ninja + perl + autoconf + automake + libtool + unzip + go + ]; + + buildInputs = [ + zlib + zstd + sqlite + ]; + + configureFlags = [ + "--with-ca-bundle=${ + if stdenv.isDarwin then "/etc/ssl/cert.pem" else "/etc/ssl/certs/ca-certificates.crt" + }" + "--with-ca-path=${cacert}/etc/ssl/certs" + ]; + + buildFlags = [ "chrome-build" ]; + checkTarget = "chrome-checkbuild"; + installTargets = [ "chrome-install" ]; + + doCheck = true; + + dontUseCmakeConfigure = true; + dontUseNinjaBuild = true; + dontUseNinjaInstall = true; + dontUseNinjaCheck = true; + + postUnpack = + lib.concatStringsSep "\n" ( + lib.mapAttrsToList (name: dep: "ln -sT ${dep.outPath} source/${name}") ( + lib.filterAttrs (n: v: v ? outPath) passthru.deps + ) + ) + + '' + + curltar=$(realpath -s source/curl-*.tar.gz) + + pushd "$(mktemp -d)" + + tar -xf "$curltar" + + pushd curl-curl-*/ + patchShebangs scripts + popd + + rm "$curltar" + tar -czf "$curltar" . + + popd + ''; + + preConfigure = '' + export GOCACHE=$TMPDIR/go-cache + export GOPATH=$TMPDIR/go + export GOPROXY=file://${passthru.boringssl-go-modules} + export GOSUMDB=off + + # Need to get value of $out for this flag + configureFlagsArray+=("--with-libnssckbi=$out/lib") + ''; + + postInstall = + '' + # Remove vestigial *-config script + rm $out/bin/curl-impersonate-chrome-config + + # Patch all shebangs of installed scripts + patchShebangs $out/bin + + # Install headers + make -C curl-*/include install + '' + + lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' + # Build and install completions for each curl binary + + # Patch in correct binary name and alias it to all scripts + perl curl-*/scripts/completion.pl --curl $out/bin/curl-impersonate-chrome --shell zsh >$TMPDIR/curl-impersonate-chrome.zsh + substituteInPlace $TMPDIR/curl-impersonate-chrome.zsh \ + --replace-fail \ + '#compdef curl' \ + "#compdef curl-impersonate-chrome$(find $out/bin -name 'curl_*' -printf ' %f=curl-impersonate-chrome')" + + perl curl-*/scripts/completion.pl --curl $out/bin/curl-impersonate-chrome --shell fish >$TMPDIR/curl-impersonate-chrome.fish + substituteInPlace $TMPDIR/curl-impersonate-chrome.fish \ + --replace-fail \ + '--command curl' \ + "--command curl-impersonate-chrome$(find $out/bin -name 'curl_*' -printf ' --command %f')" + + # Install zsh and fish completions + installShellCompletion $TMPDIR/curl-impersonate-chrome.{zsh,fish} + ''; + + preFixup = + let + libext = stdenv.hostPlatform.extensions.sharedLibrary; + in + '' + # If libnssckbi.so is needed, link libnssckbi.so without needing nss in closure + if grep -F nssckbi $out/lib/libcurl-impersonate-*${libext} &>/dev/null; then + ln -s ${p11-kit}/lib/pkcs11/p11-kit-trust${libext} $out/lib/libnssckbi${libext} + ${lib.optionalString stdenv.hostPlatform.isElf '' + patchelf --add-needed libnssckbi${libext} $out/lib/libcurl-impersonate-*${libext} + ''} + fi + ''; + + disallowedReferences = [ go ]; + + passthru = { + deps = callPackage ./deps.nix { }; + + updateScript = ./update.sh; + + boringssl-go-modules = + (buildGoModule { + inherit (passthru.deps."boringssl.zip") name; + + src = passthru.deps."boringssl.zip"; + vendorHash = "sha256-oKlwh+Oup3lVgqgq42vY3iLg62VboF9N565yK2W0XxI="; + + nativeBuildInputs = [ unzip ]; + + proxyVendor = true; + }).goModules; + }; + + meta = { + description = "Special build of curl that can impersonate Chrome & Firefox"; + homepage = "https://github.com/yifeikong/curl-impersonate"; + license = with lib.licenses; [ + curl + mit + ]; + maintainers = with lib.maintainers; [ ggg ]; + platforms = lib.platforms.unix; + mainProgram = "curl-impersonate-chrome"; + }; +} diff --git a/pkgs/tools/networking/curl-impersonate/chrome/deps.nix b/pkgs/tools/networking/curl-impersonate/chrome/deps.nix new file mode 100644 index 000000000000..1bac96363b1d --- /dev/null +++ b/pkgs/tools/networking/curl-impersonate/chrome/deps.nix @@ -0,0 +1,24 @@ +# Generated by update.sh +{ fetchurl }: + +{ + "curl-8_7_1.tar.gz" = fetchurl { + url = "https://github.com/curl/curl/archive/curl-8_7_1.tar.gz"; + hash = "sha256-DkbIVvUXYCw0e7X+W3MXT47nmLyH8alyNclXYfdfzCg="; + }; + + "brotli-1.1.0.tar.gz" = fetchurl { + url = "https://github.com/google/brotli/archive/refs/tags/v1.1.0.tar.gz"; + hash = "sha256-5yCmyilCi4A/StFlNxdx9TmPq6OX7fZ3iDehhZnqE/8="; + }; + + "boringssl.zip" = fetchurl { + url = "https://github.com/google/boringssl/archive/d24a38200fef19150eef00cad35b138936c08767.zip"; + hash = "sha256-tzAAwL70VAyUEOZZ86ql+RgXsw4DZhkvW5l0d1eVVHU="; + }; + + "nghttp2-1.61.0.tar.bz2" = fetchurl { + url = "https://github.com/nghttp2/nghttp2/releases/download/v1.61.0/nghttp2-1.61.0.tar.bz2"; + hash = "sha256-Toz37DLUxaQwlmJC1yA10lXNlHCodm1h7tegGQ3VRP0="; + }; +} diff --git a/pkgs/tools/networking/curl-impersonate/chrome/disable-building-docs.patch b/pkgs/tools/networking/curl-impersonate/chrome/disable-building-docs.patch new file mode 100644 index 000000000000..ba7a8990ec1a --- /dev/null +++ b/pkgs/tools/networking/curl-impersonate/chrome/disable-building-docs.patch @@ -0,0 +1,20 @@ +From 5366ca35b3d20ef962ccf54399cc44f523d803be Mon Sep 17 00:00:00 2001 +From: GGG +Date: Mon, 5 Aug 2024 04:19:29 -0300 +Subject: [PATCH] Disable building docs +--- + Makefile.in | 1 + + 1 file changed, 1 insertion(+) + +diff --git a/Makefile.in b/Makefile.in +index 41d7324..b1f5ec6 100644 +--- a/Makefile.in ++++ b/Makefile.in +@@ -249,6 +249,7 @@ $(CURL_VERSION)/.chrome: $(chrome_libs) $(CURL_VERSION).tar.gz $(CURL_VERSION)/. + # (for cross compilation), then pass it on to curl. + { \ + config_flags="--prefix=@prefix@"; \ ++ config_flags="$$config_flags --disable-manual"; \ + config_flags="$$config_flags --with-nghttp2=$(nghttp2_install_dir)"; \ + config_flags="$$config_flags --with-brotli=$(brotli_install_dir)"; \ + config_flags="$$config_flags --with-openssl=$(boringssl_install_dir)"; \ diff --git a/pkgs/tools/networking/curl-impersonate/chrome/update.sh b/pkgs/tools/networking/curl-impersonate/chrome/update.sh new file mode 100755 index 000000000000..afda6e3fbf53 --- /dev/null +++ b/pkgs/tools/networking/curl-impersonate/chrome/update.sh @@ -0,0 +1,88 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p git nix jq coreutils gnugrep gnused curl common-updater-scripts +# shellcheck shell=bash +set -euo pipefail + +nixpkgs="$(git rev-parse --show-toplevel || (printf 'Could not find root of nixpkgs repo\nAre we running from within the nixpkgs git repo?\n' >&2; exit 1))" + +stripwhitespace() { + sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' +} + +narhash() { + nix --extra-experimental-features nix-command store prefetch-file --json "$1" | jq -r .hash +} + +nixeval() { + nix --extra-experimental-features nix-command eval --json --impure -f "$nixpkgs" "$1" | jq -r . +} + +vendorhash() { + (nix --extra-experimental-features nix-command build --no-link -f "$nixpkgs" --no-link "$1" 2>&1 >/dev/null | tail -n3 | grep -F got: | cut -d: -f2- | stripwhitespace) 2>/dev/null || true +} + +findpath() { + path="$(nix --extra-experimental-features nix-command eval --json --impure -f "$nixpkgs" "$1.meta.position" | jq -r . | cut -d: -f1)" + outpath="$(nix --extra-experimental-features nix-command eval --json --impure --expr "builtins.fetchGit \"$nixpkgs\"")" + + if [ -n "$outpath" ]; then + path="${path/$(echo "$outpath" | jq -r .)/$nixpkgs}" + fi + + echo "$path" +} + +getvar() { + echo "$2" | grep -F "$1" | sed -e 's/:=/:/g' | cut -d: -f2- | stripwhitespace +} + +attr="${UPDATE_NIX_ATTR_PATH:-curl-impersonate-chrome}" +version="$(curl -sSL "https://api.github.com/repos/yifeikong/curl-impersonate/releases/latest" | jq -r .tag_name | sed -e 's/^v//')" + +pkgpath="$(findpath "$attr")" + +updated="$(cd "$nixpkgs" && update-source-version "$attr" "$version" --file="$pkgpath" --print-changes | jq -r length)" + +if [ "$updated" -eq 0 ]; then + echo 'update.sh: Package version not updated, nothing to do.' + exit 0 +fi + +vars="$(curl -sSL "https://github.com/yifeikong/curl-impersonate/raw/v$version/Makefile.in" | grep '^ *[^ ]*_\(VERSION\|URL\|COMMIT\) *:=')" + +# TODO: Fix hash for curl. +cat >"$(dirname "$pkgpath")"/deps.nix <$TMPDIR/curl-impersonate-${name}.zsh - substituteInPlace $TMPDIR/curl-impersonate-${name}.zsh \ - --replace-fail \ - '#compdef curl' \ - "#compdef curl-impersonate-${name}$(find $out/bin -name 'curl_*' -printf ' %f=curl-impersonate-${name}')" - - perl curl-*/scripts/completion.pl --curl $out/bin/curl-impersonate-${name} --shell fish >$TMPDIR/curl-impersonate-${name}.fish - substituteInPlace $TMPDIR/curl-impersonate-${name}.fish \ - --replace-fail \ - '--command curl' \ - "--command curl-impersonate-${name}$(find $out/bin -name 'curl_*' -printf ' --command %f')" - - # Install zsh and fish completions - installShellCompletion $TMPDIR/curl-impersonate-${name}.{zsh,fish} - ''; - - preFixup = let - libext = stdenv.hostPlatform.extensions.sharedLibrary; - in '' - # If libnssckbi.so is needed, link libnssckbi.so without needing nss in closure - if grep -F nssckbi $out/lib/libcurl-impersonate-*${libext} &>/dev/null; then - ln -s ${p11-kit}/lib/pkcs11/p11-kit-trust${libext} $out/lib/libnssckbi${libext} - ${lib.optionalString stdenv.hostPlatform.isElf '' - patchelf --add-needed libnssckbi${libext} $out/lib/libcurl-impersonate-*${libext} - ''} - fi - ''; - - disallowedReferences = [ go ]; - - passthru = { - deps = callPackage ./deps.nix {}; - - boringssl-go-modules = (buildGoModule { - inherit (passthru.deps."boringssl.zip") name; - - src = passthru.deps."boringssl.zip"; - vendorHash = "sha256-SNUsBiKOGWmkRdTVABVrlbLAVMfu0Q9IgDe+kFC5vXs="; - - nativeBuildInputs = [ unzip ]; - - proxyVendor = true; - }).goModules; - }; - - meta = with lib; { - description = "Special build of curl that can impersonate Chrome & Firefox"; - homepage = "https://github.com/lwthiker/curl-impersonate"; - license = with licenses; [ curl mit ]; - maintainers = with maintainers; [ deliciouslytyped ]; - platforms = platforms.unix; - mainProgram = "curl-impersonate-${name}"; - }; - }; -in - symlinkJoin rec { pname = "curl-impersonate"; inherit (passthru.curl-impersonate-chrome) version meta; @@ -192,13 +15,11 @@ symlinkJoin rec { ]; passthru = { - curl-impersonate-ff = makeCurlImpersonate { name = "ff"; target = "firefox"; }; - curl-impersonate-chrome = makeCurlImpersonate { name = "chrome"; target = "chrome"; }; - - updateScript = ./update.sh; + curl-impersonate-ff = callPackage ./firefox {}; + curl-impersonate-chrome = callPackage ./chrome {}; inherit (passthru.curl-impersonate-chrome) src; - tests = { inherit (nixosTests) curl-impersonate; }; + tests = {inherit (nixosTests) curl-impersonate;}; }; } diff --git a/pkgs/tools/networking/curl-impersonate/curl-impersonate-0.6.1-fix-command-paths.patch b/pkgs/tools/networking/curl-impersonate/firefox/curl-impersonate-0.6.1-fix-command-paths.patch similarity index 100% rename from pkgs/tools/networking/curl-impersonate/curl-impersonate-0.6.1-fix-command-paths.patch rename to pkgs/tools/networking/curl-impersonate/firefox/curl-impersonate-0.6.1-fix-command-paths.patch diff --git a/pkgs/tools/networking/curl-impersonate/firefox/default.nix b/pkgs/tools/networking/curl-impersonate/firefox/default.nix new file mode 100644 index 000000000000..157aedab489a --- /dev/null +++ b/pkgs/tools/networking/curl-impersonate/firefox/default.nix @@ -0,0 +1,198 @@ +{ + lib, + stdenv, + fetchFromGitHub, + fetchpatch, + callPackage, + buildGoModule, + installShellFiles, + buildPackages, + zlib, + sqlite, + cmake, + python3, + ninja, + perl, + autoconf, + automake, + libtool, + cctools, + cacert, + unzip, + go, + p11-kit, +}: +stdenv.mkDerivation rec { + pname = "curl-impersonate-ff"; + version = "0.6.1"; + + outputs = [ + "out" + "dev" + ]; + + src = fetchFromGitHub { + owner = "lwthiker"; + repo = "curl-impersonate"; + rev = "v${version}"; + hash = "sha256-ExmEhjJC8FPzx08RuKOhRxKgJ4Dh+ElEl+OUHzRCzZc="; + }; + + patches = [ + # Fix shebangs and commands in the NSS build scripts + # (can't just patchShebangs or substituteInPlace since makefile unpacks it) + ./curl-impersonate-0.6.1-fix-command-paths.patch + + # SOCKS5 heap buffer overflow - https://curl.se/docs/CVE-2023-38545.html + (fetchpatch { + name = "curl-impersonate-patch-cve-2023-38545.patch"; + url = "https://github.com/lwthiker/curl-impersonate/commit/e7b90a0d9c61b6954aca27d346750240e8b6644e.diff"; + hash = "sha256-jFrz4Q+MJGfNmwwzHhThado4c9hTd/+b/bfRsr3FW5k="; + }) + ]; + + # Disable blanket -Werror to fix build on `gcc-13` related to minor + # warnings on `boringssl`. + env.NIX_CFLAGS_COMPILE = "-Wno-error"; + + strictDeps = true; + + depsBuildBuild = lib.optionals (stdenv.buildPlatform != stdenv.hostPlatform) [ + buildPackages.stdenv.cc + ]; + + nativeBuildInputs = + lib.optionals stdenv.isDarwin [ + # Must come first so that it shadows the 'libtool' command but leaves 'libtoolize' + cctools + ] + ++ [ + installShellFiles + cmake + python3 + python3.pythonOnBuildForHost.pkgs.gyp + ninja + perl + autoconf + automake + libtool + unzip + go + ]; + + buildInputs = [ + zlib + sqlite + ]; + + configureFlags = [ + "--with-ca-bundle=${ + if stdenv.isDarwin then "/etc/ssl/cert.pem" else "/etc/ssl/certs/ca-certificates.crt" + }" + "--with-ca-path=${cacert}/etc/ssl/certs" + ]; + + buildFlags = [ "firefox-build" ]; + checkTarget = "firefox-checkbuild"; + installTargets = [ "firefox-install" ]; + + doCheck = true; + + dontUseCmakeConfigure = true; + dontUseNinjaBuild = true; + dontUseNinjaInstall = true; + dontUseNinjaCheck = true; + + postUnpack = lib.concatStringsSep "\n" ( + lib.mapAttrsToList (name: dep: "ln -sT ${dep.outPath} source/${name}") ( + lib.filterAttrs (n: v: v ? outPath) passthru.deps + ) + ); + + preConfigure = '' + export GOCACHE=$TMPDIR/go-cache + export GOPATH=$TMPDIR/go + export GOPROXY=file://${passthru.boringssl-go-modules} + export GOSUMDB=off + + # Need to get value of $out for this flag + configureFlagsArray+=("--with-libnssckbi=$out/lib") + ''; + + postInstall = + '' + # Remove vestigial *-config script + rm $out/bin/curl-impersonate-ff-config + + # Patch all shebangs of installed scripts + patchShebangs $out/bin + + # Install headers + make -C curl-*/include install + '' + + lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' + # Build and install completions for each curl binary + + # Patch in correct binary name and alias it to all scripts + perl curl-*/scripts/completion.pl --curl $out/bin/curl-impersonate-ff --shell zsh >$TMPDIR/curl-impersonate-ff.zsh + substituteInPlace $TMPDIR/curl-impersonate-ff.zsh \ + --replace-fail \ + '#compdef curl' \ + "#compdef curl-impersonate-ff$(find $out/bin -name 'curl_*' -printf ' %f=curl-impersonate-ff')" + + perl curl-*/scripts/completion.pl --curl $out/bin/curl-impersonate-ff --shell fish >$TMPDIR/curl-impersonate-ff.fish + substituteInPlace $TMPDIR/curl-impersonate-ff.fish \ + --replace-fail \ + '--command curl' \ + "--command curl-impersonate-ff$(find $out/bin -name 'curl_*' -printf ' --command %f')" + + # Install zsh and fish completions + installShellCompletion $TMPDIR/curl-impersonate-ff.{zsh,fish} + ''; + + preFixup = + let + libext = stdenv.hostPlatform.extensions.sharedLibrary; + in + '' + # If libnssckbi.so is needed, link libnssckbi.so without needing nss in closure + if grep -F nssckbi $out/lib/libcurl-impersonate-*${libext} &>/dev/null; then + ln -s ${p11-kit}/lib/pkcs11/p11-kit-trust${libext} $out/lib/libnssckbi${libext} + ${lib.optionalString stdenv.hostPlatform.isElf '' + patchelf --add-needed libnssckbi${libext} $out/lib/libcurl-impersonate-*${libext} + ''} + fi + ''; + + disallowedReferences = [ go ]; + + passthru = { + deps = callPackage ./deps.nix { }; + + updateScript = ./update.sh; + + boringssl-go-modules = + (buildGoModule { + inherit (passthru.deps."boringssl.zip") name; + + src = passthru.deps."boringssl.zip"; + vendorHash = "sha256-SNUsBiKOGWmkRdTVABVrlbLAVMfu0Q9IgDe+kFC5vXs="; + + nativeBuildInputs = [ unzip ]; + + proxyVendor = true; + }).goModules; + }; + + meta = with lib; { + description = "Special build of curl that can impersonate Chrome & Firefox"; + homepage = "https://github.com/lwthiker/curl-impersonate"; + license = with licenses; [ + curl + mit + ]; + maintainers = with maintainers; [ deliciouslytyped ]; + platforms = platforms.unix; + mainProgram = "curl-impersonate-ff"; + }; +} diff --git a/pkgs/tools/networking/curl-impersonate/deps.nix b/pkgs/tools/networking/curl-impersonate/firefox/deps.nix similarity index 99% rename from pkgs/tools/networking/curl-impersonate/deps.nix rename to pkgs/tools/networking/curl-impersonate/firefox/deps.nix index 1b04659f0741..b8a70b935302 100644 --- a/pkgs/tools/networking/curl-impersonate/deps.nix +++ b/pkgs/tools/networking/curl-impersonate/firefox/deps.nix @@ -1,6 +1,5 @@ # Generated by update.sh { fetchurl }: - { "curl-8.1.1.tar.xz" = fetchurl { url = "https://curl.se/download/curl-8.1.1.tar.xz"; diff --git a/pkgs/tools/networking/curl-impersonate/update.sh b/pkgs/tools/networking/curl-impersonate/firefox/update.sh old mode 100755 new mode 100644 similarity index 99% rename from pkgs/tools/networking/curl-impersonate/update.sh rename to pkgs/tools/networking/curl-impersonate/firefox/update.sh index 3930c0768478..76dd6099d417 --- a/pkgs/tools/networking/curl-impersonate/update.sh +++ b/pkgs/tools/networking/curl-impersonate/firefox/update.sh @@ -1,5 +1,6 @@ #!/usr/bin/env nix-shell #!nix-shell -i bash -p git nix jq coreutils gnugrep gnused curl common-updater-scripts +# shellcheck shell=bash set -euo pipefail nixpkgs="$(git rev-parse --show-toplevel || (printf 'Could not find root of nixpkgs repo\nAre we running from within the nixpkgs git repo?\n' >&2; exit 1))"