Merge pull request #254967 from nbraud/sha256-to-hash/top-level

doc/builders/fetchers.chapter.md

@@ -228,7 +228,7 @@ Otherwise, the builder will run, but fail with a message explaining to the user
 requireFile {
   name = "jdk-${version}_linux-x64_bin.tar.gz";
   url = "https://www.oracle.com/java/technologies/javase-jdk11-downloads.html";
-  sha256 = "94bd34f85ee38d3ef59e5289ec7450b9443b924c55625661fffe66b03f2c8de2";
+  hash = "sha256-lL00+F7jjT71nlKJ7HRQuUQ7kkxVYlZh//5msD8sjeI=";
 }
 ```
 results in this error message:

doc/languages-frameworks/dhall.section.md

@@ -91,7 +91,7 @@ buildDhallPackage {
 let
   nixpkgs = builtins.fetchTarball {
     url    = "https://github.com/NixOS/nixpkgs/archive/94b2848559b12a8ed1fe433084686b2a81123c99.tar.gz";
-    sha256 = "sha256-B4Q3c6IvTLg3Q92qYa8y+i4uTaphtFdjp+Ir3QQjdN0=";
+    hash = "sha256-B4Q3c6IvTLg3Q92qYa8y+i4uTaphtFdjp+Ir3QQjdN0=";
   };
 
   dhallOverlay = self: super: {

doc/languages-frameworks/rust.section.md

@@ -102,7 +102,7 @@ rustPlatform.buildRustPackage rec {
 
   src = fetchCrate {
     inherit pname version;
-    sha256 = "sha256-aDQA4A5mScX9or3Lyiv/5GyAehidnpKKE0grhbP1Ctc=";
+    hash = "sha256-aDQA4A5mScX9or3Lyiv/5GyAehidnpKKE0grhbP1Ctc=";
   };
 
   cargoHash = "sha256-tbrTbutUs5aPSV+yE0IBUZAAytgmZV7Eqxia7g+9zRs=";

doc/stdenv/stdenv.chapter.md

@@ -180,7 +180,7 @@ stdenv.mkDerivation rec {
 
   src = fetchurl {
     url = "https://github.com/Solo5/solo5/releases/download/v${version}/solo5-v${version}.tar.gz";
-    sha256 = "sha256-viwrS9lnaU8sTGuzK/+L/PlMM/xRRtgVuK5pixVeDEw=";
+    hash = "sha256-viwrS9lnaU8sTGuzK/+L/PlMM/xRRtgVuK5pixVeDEw=";
   };
 
   nativeBuildInputs = [ makeWrapper pkg-config ];

lib/customisation.nix

@@ -19,7 +19,7 @@ rec {
          name = "sed-4.2.2-pre";
          src = fetchurl {
            url = ftp://alpha.gnu.org/gnu/sed/sed-4.2.2-pre.tar.bz2;
-           sha256 = "11nq06d131y4wmf3drm0yk502d2xc6n5qy82cg88rb9nqd2lj41k";
+           hash = "sha256-MxBJRcM2rYzQYwJ5XKxhXTQByvSg5jZc5cSHEZoB2IY=";
          };
          patches = [];
        });

maintainers/scripts/sha256-to-SRI.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env nix-shell
+#! nix-shell -i "python3 -I" -p "python3.withPackages(p: with p; [ rich structlog ])"
+
+from contextlib import contextmanager
+from pathlib import Path
+from structlog.contextvars import bound_contextvars as log_context
+
+import re, structlog
+
+
+logger = structlog.getLogger("sha256-to-SRI")
+
+
+nix32alphabet = "0123456789abcdfghijklmnpqrsvwxyz"
+nix32inverted  = { c: i for i, c in enumerate(nix32alphabet) }
+
+def nix32decode(s: str) -> bytes:
+    # only support sha256 hashes for now
+    assert len(s) == 52
+    out = [ 0 for _ in range(32) ]
+    # TODO: Do better than a list of byte-sized ints
+
+    for n, c in enumerate(reversed(s)):
+        digit = nix32inverted[c]
+        i, j = divmod(5 * n, 8)
+        out[i] = out[i] | (digit << j) & 0xff
+        rem = digit >> (8 - j)
+        if rem == 0:
+            continue
+        elif i < 31:
+            out[i+1] = rem
+        else:
+            raise ValueError(f"Invalid nix32 hash: '{s}'")
+
+    return bytes(out)
+
+
+def toSRI(digest: bytes) -> str:
+    from base64 import b64encode
+    assert len(digest) == 32
+    return f"sha256-{b64encode(digest).decode()}"
+
+
+RE = {
+    'nix32': f"[{nix32alphabet}]" "{52}",
+    'hex':    "[0-9A-Fa-f]{64}",
+    'base64': "[A-Za-z0-9+/]{43}=",
+}
+RE['sha256'] = '|'.join(
+    f"{'(sha256-)?' if name == 'base64' else ''}"
+    f"(?P<{name}>{r})"
+    for name, r in RE.items()
+)
+
+def sha256toSRI(m: re.Match) -> str:
+    """Produce the equivalent SRI string for any match of RE['sha256']"""
+    if m['nix32'] is not None:
+        return toSRI(nix32decode(m['nix32']))
+    if m['hex'] is not None:
+        from binascii import unhexlify
+        return toSRI(unhexlify(m['hex']))
+    if m['base64'] is not None:
+        from base64 import b64decode
+        return toSRI(b64decode(m['base64']))
+
+    raise ValueError("Got a match where none of the groups captured")
+
+
+# Ohno I used evil, irregular backrefs instead of making 2 variants  ^^'
+_def_re = re.compile(
+    "sha256 = (?P<quote>[\"'])"
+    f"({RE['sha256']})"
+    "(?P=quote);"
+)
+
+def defToSRI(s: str) -> str:
+    def f(m: re.Match[str]) -> str:
+        try:
+            return f'hash = "{sha256toSRI(m)}";'
+
+        except ValueError as exn:
+            begin, end = m.span()
+            match = m.string[begin:end]
+
+            logger.error(
+                "Skipping",
+                exc_info = exn,
+            )
+            return match
+
+    return _def_re.sub(f, s)
+
+
+@contextmanager
+def atomicFileUpdate(target: Path):
+    '''Atomically replace the contents of a file.
+
+    Guarantees that no temporary files are left behind, and `target` is either
+    left untouched, or overwritten with new content if no exception was raised.
+
+    Yields a pair `(original, new)` of open files.
+    `original` is the pre-existing file at `target`, open for reading;
+    `new` is an empty, temporary file in the same filder, open for writing.
+
+    Upon exiting the context, the files are closed; if no exception was
+    raised, `new` (atomically) replaces the `target`, otherwise it is deleted.
+    '''
+    # That's mostly copied from noto-emoji.py, should DRY it out
+    from tempfile import mkstemp
+    fd, _p = mkstemp(
+        dir = target.parent,
+        prefix = target.name,
+    )
+    tmpPath = Path(_p)
+
+    try:
+        with target.open() as original:
+            with tmpPath.open('w') as new:
+                yield (original, new)
+
+        tmpPath.replace(target)
+
+    except Exception:
+        tmpPath.unlink(missing_ok = True)
+        raise
+
+
+def fileToSRI(p: Path):
+    with atomicFileUpdate(p) as (og, new):
+        for i, line in enumerate(og):
+            with log_context(line=i):
+                new.write(defToSRI(line))
+
+
+if __name__ == "__main__":
+    from sys import argv, stderr
+
+    for arg in argv[1:]:
+        p = Path(arg)
+        with log_context(path=str(p)):
+            try:
+                fileToSRI(p)
+            except Exception as exn:
+                logger.error(
+                    "Unhandled exception, skipping file!",
+                    exc_info = exn,
+                )
+            else:
+                logger.info("Finished processing file")

nixos/doc/manual/configuration/adding-custom-packages.section.md

@@ -44,7 +44,7 @@ environment.systemPackages =
       name = "hello-2.8";
       src = fetchurl {
         url = "mirror://gnu/hello/${name}.tar.gz";
-        sha256 = "0wqd8sjmxfskrflaxywc7gqw7sfawrfvdxd9skxawzfgyy0pzdz6";
+        hash = "sha256-5rd/gffPfa761Kn1tl3myunD8TuM+66oy1O7XqVGDXM=";
       };
     };
   in
@@ -67,7 +67,7 @@ stdenv.mkDerivation rec {
   name = "hello-2.8";
   src = fetchurl {
     url = "mirror://gnu/hello/${name}.tar.gz";
-    sha256 = "0wqd8sjmxfskrflaxywc7gqw7sfawrfvdxd9skxawzfgyy0pzdz6";
+    hash = "sha256-5rd/gffPfa761Kn1tl3myunD8TuM+66oy1O7XqVGDXM=";
   };
 }
 ```

pkgs/development/libraries/rocksdb/default.nix

@@ -24,7 +24,7 @@ stdenv.mkDerivation rec {
     owner = "facebook";
     repo = pname;
     rev = "v${version}";
-    sha256 = "sha256-mfIRQ8nkUbZ3Bugy3NAvOhcfzFY84J2kBUIUBcQ2/Qg=";
+    hash = "sha256-mfIRQ8nkUbZ3Bugy3NAvOhcfzFY84J2kBUIUBcQ2/Qg=";
   };
 
   nativeBuildInputs = [ cmake ninja ];

pkgs/top-level/all-packages.nix

@@ -4507,7 +4507,7 @@ with pkgs;
 
   bsh = fetchurl {
     url = "http://www.beanshell.org/bsh-2.0b5.jar";
-    sha256 = "0p2sxrpzd0vsk11zf3kb5h12yl1nq4yypb5mpjrm8ww0cfaijck2";
+    hash = "sha256-YjIZlWOAc1SzvLWs6z3BNlAvAixrDvdDmHqD9m/uWlw=";
   };
 
   btfs = callPackage ../os-specific/linux/btfs { };
@@ -14920,7 +14920,7 @@ with pkgs;
       src = fetchgit {
         url = "https://git.savannah.gnu.org/r/gnulib.git";
         rev = "0b38e1d69f03d3977d7ae7926c1efeb461a8a971";
-        sha256 = "06bj9y8wcfh35h653yk8j044k7h5g82d2j3z3ib69rg0gy1xagzp";
+        hash = "sha256-9z/Vg3/g5WRWHH9I0QR6BZ5JCJBo+lEMLAM6xpFPchk=";
       };
     };
   };
@@ -21710,7 +21710,7 @@ with pkgs;
       owner = "libgit2";
       repo = "libgit2";
       rev = "v${version}";
-      sha256 = "sha256-7atNkOBzX+nU1gtFQEaE+EF1L+eex+Ajhq2ocoJY920=";
+      hash = "sha256-7atNkOBzX+nU1gtFQEaE+EF1L+eex+Ajhq2ocoJY920=";
     };
     patches = [];
   };
@@ -24996,7 +24996,7 @@ with pkgs;
       owner = "facebook";
       repo = pname;
       rev = "v${version}";
-     sha256 = "sha256-U2ReSrJwjAXUdRmwixC0DQXht/h/6rV8SOf5e2NozIs=";
+     hash = "sha256-U2ReSrJwjAXUdRmwixC0DQXht/h/6rV8SOf5e2NozIs=";
    };
   };
 
@@ -25007,7 +25007,7 @@ with pkgs;
       owner = "facebook";
       repo = pname;
       rev = "v${version}";
-     sha256 = "sha256-SsDqhjdCdtIGNlsMj5kfiuS3zSGwcxi4KV71d95h7yk=";
+     hash = "sha256-SsDqhjdCdtIGNlsMj5kfiuS3zSGwcxi4KV71d95h7yk=";
    };
   };
 
@@ -28310,7 +28310,7 @@ with pkgs;
     version = "5.19.16";
     src = fetchurl {
       url = "mirror://kernel/linux/kernel/v5.x/linux-${version}.tar.xz";
-      sha256 = "13g0c6ljxk3sd0ja39ndih5vrzp2ssj78qxaf8nswn8hgrkazsx1";
+      hash = "sha256-oeuvZn4QWa4tcqpjdKTW4v68C4zNpqEkaHrMLqlh4I0=";
     };
   };
 
@@ -37320,7 +37320,7 @@ with pkgs;
       owner = "ElementsProject";
       repo = "elements";
       rev = "ea318a45094ab3d31dd017d7781a6f28f1ffaa33"; # simplicity branch latest
-      sha256 = "ooe+If3HWaJWpr2ux7DpiCTqB9Hv+aXjquEjplDjvhM=";
+      hash = "sha256-ooe+If3HWaJWpr2ux7DpiCTqB9Hv+aXjquEjplDjvhM=";
     };
   };
 
@@ -39799,7 +39799,7 @@ with pkgs;
         owner = "polyml";
         repo = "polyml";
         rev = "bafe319bc3a65bf63bd98a4721a6f4dd9e0eabd6";
-        sha256 = "1ygs09zzq8icq1gc8qf4sb24lxx7sbcyd5hw3vw67a3ryaki0qw2";
+        hash = "sha256-gmMQp/J5qGP4HhyW5tnSp3dKxNLEYcRewCwi/H8C+vk=";
       };
     };
 
@@ -41977,7 +41977,7 @@ with pkgs;
       version = "1.7.9";
       src = fetchurl {
         url = "http://www.openfst.org/twiki/pub/FST/FstDownload/openfst-${version}.tar.gz";
-        sha256 = "1pmx1yhn2gknj0an0zwqmzgwjaycapi896244np50a8y3nrsw6ck";
+        hash = "sha256-kxmusx0eKVCuJUSYhOJVzCvJ36+Yf2AVkHY+YaEPvd4=";
       };
     };
   };