directoryListingUpdater: init

Update a package looking for available versions in an html page

pkgs/common-updater/directory-listing-updater.nix

@@ -0,0 +1,19 @@
+{ lib
+, genericUpdater
+, common-updater-scripts
+}:
+
+{ pname ? null
+, version ? null
+, attrPath ? null
+, ignoredVersions ? ""
+, rev-prefix ? ""
+, odd-unstable ? false
+, patchlevel-unstable ? false
+, url ? null
+}:
+
+genericUpdater {
+  inherit pname version attrPath ignoredVersions rev-prefix odd-unstable patchlevel-unstable;
+  versionLister = "${common-updater-scripts}/bin/list-directory-versions ${lib.optionalString (url != null) "--url=${lib.escapeShellArg url}"}";
+}

pkgs/common-updater/generic-updater.nix

@@ -62,7 +62,7 @@ let
       return 1
     }
 
-    tags=$($version_lister --pname=$pname --attr-path=$attr_path --file="${fileForGitCommands}") || exit 1
+    tags=$(sh -c "$version_lister --pname=$pname --attr-path=$attr_path --file=\"${fileForGitCommands}\"") || exit 1
 
     # print available tags
     for tag in $tags; do

pkgs/common-updater/scripts.nix

@@ -1,9 +1,28 @@
-{ lib, stdenv, makeWrapper, coreutils, gnused, gnugrep, diffutils, nix, git, jq }:
+{ lib
+, stdenv
+, makeWrapper
+, coreutils
+, diffutils
+, git
+, gnugrep
+, gnused
+, jq
+, nix
+, python3Packages
+}:
 
 stdenv.mkDerivation {
   name = "common-updater-scripts";
 
-  nativeBuildInputs = [ makeWrapper ];
+  nativeBuildInputs = [
+    makeWrapper
+    python3Packages.wrapPython
+  ];
+
+  pythonPath = [
+    python3Packages.beautifulsoup4
+    python3Packages.requests
+  ];
 
   dontUnpack = true;
 
@@ -11,8 +30,15 @@ stdenv.mkDerivation {
     mkdir -p $out/bin
     cp ${./scripts}/* $out/bin
 
+    # wrap non python scripts
     for f in $out/bin/*; do
-      wrapProgram $f --prefix PATH : ${lib.makeBinPath [ coreutils gnused gnugrep nix diffutils git jq ]}
+      if ! (head -n1 "$f" | grep -q '#!.*/env.*\(python\|pypy\)'); then
+        wrapProgram $f --prefix PATH : ${lib.makeBinPath [ coreutils diffutils git gnugrep gnused jq nix ]}
+      fi
     done
+
+    # wrap python scripts
+    makeWrapperArgs+=( --prefix PATH : "${lib.makeBinPath [ nix ]}" )
+    wrapPythonPrograms
   '';
 }

pkgs/common-updater/scripts/list-directory-versions

@@ -0,0 +1,65 @@
+#!/usr/bin/env python
+
+import argparse
+import requests
+import os
+import subprocess
+import json
+import re
+from bs4 import BeautifulSoup
+
+parser = argparse.ArgumentParser(
+    description="Get all available versions listed for a package in a site."
+)
+
+parser.add_argument(
+    "--pname",
+    default=os.environ.get("UPDATE_NIX_PNAME"),
+    required="UPDATE_NIX_PNAME" not in os.environ,
+    help="name of the package",
+)
+parser.add_argument(
+    "--attr-path",
+    default=os.environ.get("UPDATE_NIX_ATTR_PATH"),
+    help="attribute path of the package",
+)
+parser.add_argument("--url", help="url of the page that lists the package versions")
+parser.add_argument("--file", help="file name for writing debugging information")
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+
+    pname = args.pname
+
+    attr_path = args.attr_path or pname
+
+    url = args.url or json.loads(
+        subprocess.check_output(
+            [
+                "nix-instantiate",
+                "--json",
+                "--eval",
+                "-E",
+                f"with import ./. {{}}; dirOf (lib.head {attr_path}.src.urls)",
+            ],
+            text=True,
+        )
+    )
+
+    # print a debugging message
+    if args.file:
+        with open(args.file, "a") as f:
+            f.write(f"# Listing versions for {pname} from {url}\n")
+
+    page = requests.get(url)
+    soup = BeautifulSoup(page.content, "html.parser")
+    links = soup.find_all("a")
+    for link in links:
+        link_url = link.get("href", None)
+        if link_url is not None:
+            match = re.fullmatch(
+                rf"{args.pname}-([\d.]+?(-[\d\w.-]+?)?)(\.tar)?(\.[^.]*)", link_url
+            )
+            if match:
+                print(match.group(1))

pkgs/top-level/all-packages.nix

@@ -126,6 +126,8 @@ with pkgs;
 
   _experimental-update-script-combinators = callPackage ../common-updater/combinators.nix { };
 
+  directoryListingUpdater = callPackage ../common-updater/directory-listing-updater.nix { };
+
   gitUpdater = callPackage ../common-updater/git-updater.nix { };
 
   httpTwoLevelsUpdater = callPackage ../common-updater/http-two-levels-updater.nix { };