Merge pull request #324894 from paveloom/flaresolverr

flaresolverr: init at 3.3.21 + nixos/flaresolverr: initial commit

nixos/doc/manual/release-notes/rl-2411.section.md

@@ -18,6 +18,8 @@
 
 ## New Services {#sec-release-24.11-new-services}
 
+- [FlareSolverr](https://github.com/FlareSolverr/FlareSolverr), proxy server to bypass Cloudflare protection. Available as [services.flaresolverr](#opt-services.flaresolverr.enable) service.
+
 - [Open-WebUI](https://github.com/open-webui/open-webui), a user-friendly WebUI
   for LLMs. Available as [services.open-webui](#opt-services.open-webui.enable)
   service.

nixos/modules/module-list.nix

@@ -732,6 +732,7 @@
   ./services/misc/etesync-dav.nix
   ./services/misc/evdevremapkeys.nix
   ./services/misc/felix.nix
+  ./services/misc/flaresolverr.nix
   ./services/misc/forgejo.nix
   ./services/misc/freeswitch.nix
   ./services/misc/fstrim.nix

nixos/modules/services/misc/flaresolverr.nix

@@ -0,0 +1,58 @@
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
+
+let
+  cfg = config.services.flaresolverr;
+in
+{
+  options = {
+    services.flaresolverr = {
+      enable = lib.mkEnableOption "FlareSolverr, a proxy server to bypass Cloudflare protection";
+
+      package = lib.mkPackageOption pkgs "flaresolverr" { };
+
+      openFirewall = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Open the port in the firewall for FlareSolverr.";
+      };
+
+      port = lib.mkOption {
+        type = lib.types.port;
+        default = 8191;
+        description = "The port on which FlareSolverr will listen for incoming HTTP traffic.";
+      };
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    systemd.services.flaresolverr = {
+      description = "FlareSolverr";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+
+      environment = {
+        HOME = "/run/flaresolverr";
+        PORT = toString cfg.port;
+      };
+
+      serviceConfig = {
+        SyslogIdentifier = "flaresolverr";
+        Restart = "always";
+        RestartSec = 5;
+        Type = "simple";
+        DynamicUser = true;
+        RuntimeDirectory = "flaresolverr";
+        WorkingDirectory = "/run/flaresolverr";
+        ExecStart = lib.getExe cfg.package;
+        TimeoutStopSec = 30;
+      };
+    };
+
+    networking.firewall = lib.mkIf cfg.openFirewall { allowedTCPPorts = [ cfg.port ]; };
+  };
+}

nixos/tests/all-tests.nix

@@ -332,6 +332,7 @@ in {
   firewall-nftables = handleTest ./firewall.nix { nftables = true; };
   fish = handleTest ./fish.nix {};
   flannel = handleTestOn ["x86_64-linux"] ./flannel.nix {};
+  flaresolverr = handleTest ./flaresolverr.nix {};
   flood = handleTest ./flood.nix {};
   floorp = handleTest ./firefox.nix { firefoxPackage = pkgs.floorp; };
   fluentd = handleTest ./fluentd.nix {};

nixos/tests/flaresolverr.nix

@@ -0,0 +1,22 @@
+import ./make-test-python.nix (
+  { lib, ... }:
+  {
+    name = "flaresolverr";
+    meta.maintainers = with lib.maintainers; [ paveloom ];
+
+    nodes.machine =
+      { pkgs, ... }:
+      {
+        services.flaresolverr = {
+          enable = true;
+          port = 8888;
+        };
+      };
+
+    testScript = ''
+      machine.wait_for_unit("flaresolverr.service")
+      machine.wait_for_open_port(8888)
+      machine.succeed("curl --fail http://localhost:8888/")
+    '';
+  }
+)

pkgs/by-name/fl/flaresolverr/package.nix

@@ -0,0 +1,82 @@
+{
+  lib,
+  stdenv,
+  fetchFromGitHub,
+
+  makeWrapper,
+
+  chromium,
+  python3,
+  undetected-chromedriver,
+  xorg,
+
+  nixosTests,
+}:
+
+let
+  python = python3.withPackages (
+    ps: with ps; [
+      bottle
+      func-timeout
+      prometheus-client
+      selenium
+      waitress
+      xvfbwrapper
+
+      # For `undetected_chromedriver`
+      looseversion
+      requests
+      websockets
+    ]
+  );
+in
+stdenv.mkDerivation (finalAttrs: {
+  pname = "flaresolverr";
+  version = "3.3.21";
+
+  src = fetchFromGitHub {
+    owner = "FlareSolverr";
+    repo = "FlareSolverr";
+    rev = "v${finalAttrs.version}";
+    hash = "sha256-M/snpYKZK3pgzlhYjRYEiAPlK9DUKYRiiu43KcrAy9g=";
+  };
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  postPatch = ''
+    substituteInPlace src/undetected_chromedriver/patcher.py \
+      --replace-fail \
+        "from distutils.version import LooseVersion" \
+        "from looseversion import LooseVersion"
+
+    substituteInPlace src/utils.py \
+      --replace-fail \
+        'CHROME_EXE_PATH = None' \
+        'CHROME_EXE_PATH = "${lib.getExe chromium}"' \
+      --replace-fail \
+        'PATCHED_DRIVER_PATH = None' \
+        'PATCHED_DRIVER_PATH = "${lib.getExe undetected-chromedriver}"'
+  '';
+
+  installPhase = ''
+    mkdir -p $out/{bin,share/${finalAttrs.pname}-${finalAttrs.version}}
+    cp -r * $out/share/${finalAttrs.pname}-${finalAttrs.version}/.
+
+    makeWrapper ${python}/bin/python $out/bin/flaresolverr \
+      --add-flags "$out/share/${finalAttrs.pname}-${finalAttrs.version}/src/flaresolverr.py" \
+      --prefix PATH : "${lib.makeBinPath [ xorg.xvfb ]}"
+  '';
+
+  passthru = {
+    tests.smoke-test = nixosTests.flaresolverr;
+  };
+
+  meta = with lib; {
+    description = "Proxy server to bypass Cloudflare protection";
+    homepage = "https://github.com/FlareSolverr/FlareSolverr";
+    license = licenses.mit;
+    mainProgram = "flaresolverr";
+    maintainers = with maintainers; [ paveloom ];
+    inherit (undetected-chromedriver.meta) platforms;
+  };
+})

pkgs/by-name/un/undetected-chromedriver/package.nix

@@ -0,0 +1,44 @@
+{
+  lib,
+  stdenv,
+
+  chromedriver,
+  python3,
+
+  testers,
+  undetected-chromedriver,
+}:
+
+stdenv.mkDerivation {
+  pname = "undetected-chromedriver";
+  inherit (chromedriver) version;
+
+  nativeBuildInputs = [ (python3.withPackages (ps: [ ps.undetected-chromedriver ])) ];
+
+  buildCommand = ''
+    export HOME=$(mktemp -d)
+
+    cp ${chromedriver}/bin/chromedriver .
+    chmod +w chromedriver
+
+    python <<EOF
+    import logging
+    from undetected_chromedriver.patcher import Patcher
+
+    logging.basicConfig(level=logging.DEBUG)
+
+    success = Patcher(executable_path="chromedriver").patch()
+    assert success, "Failed to patch ChromeDriver"
+    EOF
+
+    install -D -m 0555 chromedriver $out/bin/undetected-chromedriver
+  '';
+
+  passthru.tests.version = testers.testVersion { package = undetected-chromedriver; };
+
+  meta = chromedriver.meta // {
+    description = "Custom Selenium ChromeDriver that passes all bot mitigation systems";
+    mainProgram = "undetected-chromedriver";
+    maintainers = with lib.maintainers; [ paveloom ];
+  };
+}

pkgs/development/python-modules/undetected-chromedriver/default.nix

@@ -0,0 +1,54 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+
+  setuptools,
+
+  looseversion,
+  requests,
+  selenium,
+  websockets,
+}:
+
+buildPythonPackage {
+  pname = "undetected-chromedriver";
+  version = "3.5.5";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "ultrafunkamsterdam";
+    repo = "undetected-chromedriver";
+    # Upstream uses the summaries of commits for specifying versions
+    rev = "0aa5fbe252370b4cb2b95526add445392cad27ba";
+    hash = "sha256-Qe+GrsUPnhjJMDgjdUCloapjj0ggFlm/Dr42WLcmb1o=";
+  };
+
+  build-system = [ setuptools ];
+
+  dependencies = [
+    looseversion
+    requests
+    selenium
+    websockets
+  ];
+
+  # No tests
+  doCheck = false;
+
+  pythonImportsCheck = [ "undetected_chromedriver" ];
+
+  postPatch = ''
+    substituteInPlace undetected_chromedriver/patcher.py \
+      --replace-fail \
+        "from distutils.version import LooseVersion" \
+        "from looseversion import LooseVersion"
+  '';
+
+  meta = with lib; {
+    description = "Python library for the custom Selenium ChromeDriver that passes all bot mitigation systems";
+    homepage = "https://github.com/ultrafunkamsterdam/undetected-chromedriver";
+    license = licenses.gpl3Only;
+    maintainers = with lib.maintainers; [ paveloom ];
+  };
+}

pkgs/development/tools/selenium/chromedriver/binary.nix

@@ -0,0 +1,64 @@
+{ lib, stdenv, fetchurl, autoPatchelfHook
+, glib, nspr, nss, libxcb
+, testers, chromedriver
+}:
+
+let
+  upstream-info = (import ../../../../applications/networking/browsers/chromium/upstream-info.nix).stable.chromedriver;
+  allSpecs = {
+    x86_64-linux = {
+      system = "linux64";
+      hash = upstream-info.hash_linux;
+    };
+
+    x86_64-darwin = {
+      system = "mac-x64";
+      hash = upstream-info.hash_darwin;
+    };
+
+    aarch64-darwin = {
+      system = "mac-arm64";
+      hash = upstream-info.hash_darwin_aarch64;
+    };
+  };
+
+  spec = allSpecs.${stdenv.hostPlatform.system}
+    or (throw "missing chromedriver binary for ${stdenv.hostPlatform.system}");
+in stdenv.mkDerivation rec {
+  pname = "chromedriver";
+  version = upstream-info.version;
+
+  src = fetchurl {
+    url = "https://storage.googleapis.com/chrome-for-testing-public/${version}/${spec.system}/chromedriver-${spec.system}.zip";
+    hash = spec.hash;
+  };
+
+  nativeBuildInputs = [ autoPatchelfHook ];
+  buildInputs = lib.optionals (!stdenv.isDarwin) [
+    glib nspr nss libxcb
+  ];
+
+  installPhase = ''
+    install -m555 -D "chromedriver" $out/bin/chromedriver
+  '';
+
+  passthru.tests.version = testers.testVersion { package = chromedriver; };
+
+  meta = with lib; {
+    homepage = "https://chromedriver.chromium.org/";
+    description = "WebDriver server for running Selenium tests on Chrome";
+    longDescription = ''
+      WebDriver is an open source tool for automated testing of webapps across
+      many browsers. It provides capabilities for navigating to web pages, user
+      input, JavaScript execution, and more. ChromeDriver is a standalone
+      server that implements the W3C WebDriver standard.
+    '';
+    sourceProvenance = with sourceTypes; [ binaryNativeCode ];
+    license = licenses.bsd3;
+    maintainers = with maintainers; [ goibhniu primeos ];
+    # Note from primeos: By updating Chromium I also update Google Chrome and
+    # ChromeDriver.
+    platforms = attrNames allSpecs;
+    mainProgram = "chromedriver";
+  };
+}

pkgs/development/tools/selenium/chromedriver/default.nix

@@ -1,78 +1,4 @@
-{ lib, stdenv, fetchurl, unzip, makeWrapper
-, cairo, fontconfig, freetype, gdk-pixbuf, glib
-, glibc, gtk2, libX11, nspr, nss, pango
-, libxcb, libXi, libXrender, libXext, dbus
-, testers, chromedriver
-}:
-
-let
-  upstream-info = (import ../../../../applications/networking/browsers/chromium/upstream-info.nix).stable.chromedriver;
-  allSpecs = {
-    x86_64-linux = {
-      system = "linux64";
-      hash = upstream-info.hash_linux;
-    };
-
-    x86_64-darwin = {
-      system = "mac-x64";
-      hash = upstream-info.hash_darwin;
-    };
-
-    aarch64-darwin = {
-      system = "mac-arm64";
-      hash = upstream-info.hash_darwin_aarch64;
-    };
-  };
-
-  spec = allSpecs.${stdenv.hostPlatform.system}
-    or (throw "missing chromedriver binary for ${stdenv.hostPlatform.system}");
-
-  libs = lib.makeLibraryPath [
-    stdenv.cc.cc.lib
-    cairo fontconfig freetype
-    gdk-pixbuf glib gtk2
-    libX11 nspr nss pango libXrender
-    libxcb libXext libXi
-    dbus
-  ];
-
-in stdenv.mkDerivation rec {
-  pname = "chromedriver";
-  version = upstream-info.version;
-
-  src = fetchurl {
-    url = "https://storage.googleapis.com/chrome-for-testing-public/${version}/${spec.system}/chromedriver-${spec.system}.zip";
-    hash = spec.hash;
-  };
-
-  nativeBuildInputs = [ unzip makeWrapper ];
-
-  unpackPhase = "unzip $src";
-
-  installPhase = ''
-    install -m755 -D "chromedriver-${spec.system}/chromedriver" $out/bin/chromedriver
-  '' + lib.optionalString (!stdenv.isDarwin) ''
-    patchelf --set-interpreter ${glibc.out}/lib/ld-linux-x86-64.so.2 $out/bin/chromedriver
-    wrapProgram "$out/bin/chromedriver" --prefix LD_LIBRARY_PATH : "${libs}"
-  '';
-
-  passthru.tests.version = testers.testVersion { package = chromedriver; };
-
-  meta = with lib; {
-    homepage = "https://chromedriver.chromium.org/";
-    description = "WebDriver server for running Selenium tests on Chrome";
-    longDescription = ''
-      WebDriver is an open source tool for automated testing of webapps across
-      many browsers. It provides capabilities for navigating to web pages, user
-      input, JavaScript execution, and more. ChromeDriver is a standalone
-      server that implements the W3C WebDriver standard.
-    '';
-    sourceProvenance = with sourceTypes; [ binaryNativeCode ];
-    license = licenses.bsd3;
-    maintainers = with maintainers; [ goibhniu primeos ];
-    # Note from primeos: By updating Chromium I also update Google Chrome and
-    # ChromeDriver.
-    platforms = attrNames allSpecs;
-    mainProgram = "chromedriver";
-  };
-}
+{ lib, stdenv, chromium, callPackage }:
+if lib.meta.availableOn stdenv.hostPlatform chromium
+  then callPackage ./source.nix {}
+  else callPackage ./binary.nix {}

pkgs/development/tools/selenium/chromedriver/source.nix

@@ -0,0 +1,30 @@
+{ chromium, testers, chromedriver }:
+chromium.mkDerivation (_: {
+  name = "chromedriver";
+  packageName = "chromedriver";
+
+  # Build the unstripped target, because stripping in Chromium relies on a prebuilt strip binary
+  # that doesn't run on NixOS, and we will strip everything ourselves later anyway.
+  buildTargets = [ "chromedriver.unstripped" ];
+
+  installPhase = ''
+    install -Dm555 $buildPath/chromedriver.unstripped $out/bin/chromedriver
+  '';
+
+  # Kill existing postFixup that tries to patchelf things
+  postFixup = null;
+
+  passthru.tests.version = testers.testVersion { package = chromedriver; };
+
+  meta = chromium.meta // {
+    homepage = "https://chromedriver.chromium.org/";
+    description = "WebDriver server for running Selenium tests on Chrome";
+    longDescription = ''
+      WebDriver is an open source tool for automated testing of webapps across
+      many browsers. It provides capabilities for navigating to web pages, user
+      input, JavaScript execution, and more. ChromeDriver is a standalone
+      server that implements the W3C WebDriver standard.
+    '';
+    mainProgram = "chromedriver";
+  };
+})

pkgs/top-level/python-packages.nix

@@ -16738,6 +16738,8 @@ self: super: with self; {
 
   undefined = callPackage ../development/python-modules/undefined { };
 
+  undetected-chromedriver = callPackage ../development/python-modules/undetected-chromedriver { };
+
   unearth = callPackage ../development/python-modules/unearth { };
 
   unicodecsv = callPackage ../development/python-modules/unicodecsv { };