kubo-migrator: rewrite

- Migrate to pkgs/by-name - Format with nixfmt-rfc-style - Make it possible to remove support for very old migrations in the future by increasing the `minRepoVersion` parameter - Rename kubo-migrator-all-fs-repo-migrations to kubo-fs-repo-migrations since it may no longer include all migrations - Add an alias for kubo-migrator-all-fs-repo-migrations to keep backwards compatibility - Update descriptions to differentiate between kubo-migrator and kubo-migrator-unwrapped and better describe the purpose of the migrator - Add a description to every individual migration - Add a description to kubo-fs-repo-migrations - Fetch the source code of the individual migrations from their specific Git tags, like upstream intends - Enable tests for some migrations - Check that the migrations don't crash on startup - Mark two broken migrations as broken. They are not compatible with the latest Go versions and upstream is not interested in fixing this - Change code to allow most updates to be done by only changing three lines (add new version and change git tag and hash) - Add a stub for any disabled or broken migration to prevent downloading unsigned binaries from the internet, see https://github.com/ipfs/fs-repo-migrations/issues/148#issuecomment-2351355627 and https://github.com/ipfs/fs-repo-migrations/issues/188 - Use `lib.getExe` instead of hardcoding the binary name in the kubo NixOS module - Use `substituteInPlace` with `--replace-fail` instead of `--replace`
2024-11-21 22:43:01 +00:00 · 2024-09-24 16:40:56 +02:00 · 2024-09-24 16:40:56 +02:00 · 9e2866d215
commit 9e2866d215
parent 5624e1334b
10 changed files with 319 additions and 147 deletions
--- a/nixos/modules/services/network-filesystems/kubo.nix
+++ b/nixos/modules/services/network-filesystems/kubo.nix
@ -339,7 +339,7 @@ in
          # After an unclean shutdown this file may exist which will cause the config command to attempt to talk to the daemon. This will hang forever if systemd is holding our sockets open.
          rm -vf "$IPFS_PATH/api"
      '' + lib.optionalString cfg.autoMigrate ''
-        ${pkgs.kubo-migrator}/bin/fs-repo-migrations -to '${cfg.package.repoVersion}' -y
+        '${lib.getExe pkgs.kubo-migrator}' -to '${cfg.package.repoVersion}' -y
      '' + ''
        fi
        ipfs --offline config show |
--- a/pkgs/applications/networking/kubo-migrator/all-migrations.nix
+++ b/pkgs/applications/networking/kubo-migrator/all-migrations.nix
@ -1,82 +0,0 @@
-{ lib
-, stdenv
-, symlinkJoin
-, buildGoModule
-, kubo-migrator-unwrapped
-}:
-
-# This package contains all the individual migrations in the bin directory.
-# This is used by fs-repo-migrations and could also be used by Kubo itself
-# when starting it like this: ipfs daemon --migrate
-
-let
-  fs-repo-common = pname: version: buildGoModule {
-    inherit pname version;
-    inherit (kubo-migrator-unwrapped) src;
-    sourceRoot = "${kubo-migrator-unwrapped.src.name}/${pname}";
-    vendorHash = null;
-    # Fix build on Go 1.17 and later: panic: qtls.ClientHelloInfo doesn't match
-    # See https://github.com/ipfs/fs-repo-migrations/pull/163
-    postPatch = lib.optionalString (lib.elem pname [ "fs-repo-10-to-11" "fs-repo-11-to-12" ]) ''
-      substituteInPlace 'vendor/github.com/marten-seemann/qtls-go1-15/common.go' \
-        --replace \
-          '"container/list"' \
-          '"container/list"
-          "context"' \
-        --replace \
-          'config *Config' \
-          'config *Config
-          ctx context.Context'
-    '';
-    doCheck = false;
-    meta = kubo-migrator-unwrapped.meta // {
-      mainProgram = pname;
-      description = "Individual migration for the filesystem repository of Kubo clients";
-    };
-  };
-
-  # Concatenation of the latest repo version and the version of that migration
-  version = "15.1.0.1";
-
-  fs-repo-14-to-15 = fs-repo-common "fs-repo-14-to-15" "1.0.1";
-  fs-repo-13-to-14 = fs-repo-common "fs-repo-13-to-14" "1.0.0";
-  fs-repo-12-to-13 = fs-repo-common "fs-repo-12-to-13" "1.0.0";
-  fs-repo-11-to-12 = fs-repo-common "fs-repo-11-to-12" "1.0.2";
-  fs-repo-10-to-11 = fs-repo-common "fs-repo-10-to-11" "1.0.1";
-  fs-repo-9-to-10  = fs-repo-common "fs-repo-9-to-10"  "1.0.1";
-  fs-repo-8-to-9   = fs-repo-common "fs-repo-8-to-9"   "1.0.1";
-  fs-repo-7-to-8   = fs-repo-common "fs-repo-7-to-8"   "1.0.1";
-  fs-repo-6-to-7   = fs-repo-common "fs-repo-6-to-7"   "1.0.1";
-  fs-repo-5-to-6   = fs-repo-common "fs-repo-5-to-6"   "1.0.1";
-  fs-repo-4-to-5   = fs-repo-common "fs-repo-4-to-5"   "1.0.1";
-  fs-repo-3-to-4   = fs-repo-common "fs-repo-3-to-4"   "1.0.1";
-  fs-repo-2-to-3   = fs-repo-common "fs-repo-2-to-3"   "1.0.1";
-  fs-repo-1-to-2   = fs-repo-common "fs-repo-1-to-2"   "1.0.1";
-  fs-repo-0-to-1   = fs-repo-common "fs-repo-0-to-1"   "1.0.1";
-
-  all-migrations = [
-    fs-repo-14-to-15
-    fs-repo-13-to-14
-    fs-repo-12-to-13
-    fs-repo-11-to-12
-    fs-repo-10-to-11
-    fs-repo-9-to-10
-    fs-repo-8-to-9
-    fs-repo-7-to-8
-  ] ++ lib.optional (!stdenv.hostPlatform.isDarwin) # I didn't manage to fix this on macOS:
-    fs-repo-6-to-7                     # gx/ipfs/QmSGRM5Udmy1jsFBr1Cawez7Lt7LZ3ZKA23GGVEsiEW6F3/eventfd/eventfd.go:27:32: undefined: syscall.SYS_EVENTFD2
-  ++ [
-    fs-repo-5-to-6
-    fs-repo-4-to-5
-    fs-repo-3-to-4
-    fs-repo-2-to-3
-    fs-repo-1-to-2
-    fs-repo-0-to-1
-  ];
-
-in
-
-symlinkJoin {
-  name = "kubo-migrator-all-fs-repo-migrations-${version}";
-  paths = all-migrations;
-}
--- a/pkgs/applications/networking/kubo-migrator/default.nix
+++ b/pkgs/applications/networking/kubo-migrator/default.nix
@ -1,23 +0,0 @@
-{ lib
-, buildEnv
-, makeWrapper
-, kubo-migrator-unwrapped
-, kubo-migrator-all-fs-repo-migrations
-}:
-
-buildEnv {
-  name = "kubo-migrator-${kubo-migrator-unwrapped.version}";
-
-  nativeBuildInputs = [ makeWrapper ];
-
-  paths = [ kubo-migrator-unwrapped ];
-
-  pathsToLink = [ "/bin" ];
-
-  postBuild = ''
-    wrapProgram "$out/bin/fs-repo-migrations" \
-      --prefix PATH ':' '${lib.makeBinPath [ kubo-migrator-all-fs-repo-migrations ]}'
-  '';
-
-  inherit (kubo-migrator-unwrapped) meta;
-}
--- a/pkgs/applications/networking/kubo-migrator/unwrapped.nix
+++ b/pkgs/applications/networking/kubo-migrator/unwrapped.nix
@ -1,35 +0,0 @@
-{ lib
-, buildGoModule
-, fetchFromGitHub
-}:
-
-buildGoModule rec {
-  pname = "kubo-migrator";
-  version = "2.0.2";
-
-  src = fetchFromGitHub {
-    owner = "ipfs";
-    repo = "fs-repo-migrations";
-    # Use the latest git tag here, since v2.0.2 does not
-    # contain the latest migration fs-repo-14-to-15/v1.0.1
-    # The fs-repo-migrations code itself is the same between
-    # the two versions but the migration code, which is built
-    # into separate binaries, is not.
-    rev = "fs-repo-14-to-15/v1.0.1";
-    hash = "sha256-oIGDZr0cv+TIl5glHr3U+eIqAlPAOWyFzgfQGGM+xNM=";
-  };
-
-  sourceRoot = "${src.name}/fs-repo-migrations";
-
-  vendorHash = "sha256-/DqkBBtR/nU8gk3TFqNKY5zQU6BFMc3N8Ti+38mi/jk=";
-
-  doCheck = false;
-
-  meta = with lib; {
-    description = "Migrations for the filesystem repository of Kubo clients";
-    homepage = "https://github.com/ipfs/fs-repo-migrations";
-    license = licenses.mit;
-    maintainers = with maintainers; [ Luflosi elitak ];
-    mainProgram = "fs-repo-migrations";
-  };
-}
--- a/pkgs/by-name/ku/kubo-fs-repo-migrations/package.nix
+++ b/pkgs/by-name/ku/kubo-fs-repo-migrations/package.nix
@ -0,0 +1,253 @@
+{
+  lib,
+  stdenv,
+  symlinkJoin,
+  buildGoModule,
+  fetchFromGitHub,
+  kubo-migrator-unwrapped,
+  writeShellApplication,
+  minRepoVersion ? 0, # The minimum supported Kubo repo version from which the migrations can start. Increasing this reduces the closure size
+  stubBrokenMigrations ? true, # This prevents the fs-repo-migrations program from downloading binaries off the internet without even checking any signatures
+}:
+
+let
+  mkMigration =
+    from: to: version: hash:
+    let
+      pname = "fs-repo-${toString from}-to-${toString to}";
+      src = fetchFromGitHub {
+        owner = "ipfs";
+        repo = "fs-repo-migrations";
+        rev = "${pname}/v${version}";
+        inherit hash;
+        sparseCheckout = [ pname ];
+      };
+    in
+    buildGoModule {
+      inherit pname version src;
+      sourceRoot = "${src.name}/${pname}";
+      vendorHash = null;
+
+      # Fix build on Go 1.17 and later: panic: qtls.ClientHelloInfo doesn't match
+      # See https://github.com/ipfs/fs-repo-migrations/pull/163
+      postPatch =
+        lib.optionalString
+          (lib.elem to [
+            11
+            12
+          ])
+          ''
+            substituteInPlace 'vendor/github.com/marten-seemann/qtls-go1-15/common.go' \
+              --replace-fail \
+                '"container/list"' \
+                '"container/list"
+                "context"' \
+              --replace-fail \
+                'config *Config' \
+                'config *Config
+                ctx context.Context'
+          '';
+
+      checkPhase = ''
+        runHook preCheck
+        ${
+          if to <= 11 then
+            "" # Migrations fs-repo-10-to-11 and earlier require too much effort to test, making it not worth it
+          else if to == 12 then
+            ''
+              cd migration
+              go test -mod=vendor
+            ''
+          else
+            ''
+              cd not-sharness
+              ./test.sh
+            ''
+        }
+        runHook postCheck
+      '';
+
+      # Check that it does not crash
+      doInstallCheck = true;
+      installCheckPhase = ''
+        runHook preInstallCheck
+        "$out/bin/${pname}" -help
+        runHook postInstallCheck
+      '';
+
+      meta = {
+        inherit (kubo-migrator-unwrapped.meta)
+          homepage
+          license
+          platforms
+          maintainers
+          ;
+        mainProgram = pname;
+        description = "Migrate the filesystem repository of Kubo from repo version ${toString from} to ${toString to}";
+
+        broken =
+          to == 7 && stdenv.hostPlatform.isDarwin # fs-repo-6-to-7 is broken on macOS: gx/ipfs/QmSGRM5Udmy1jsFBr1Cawez7Lt7LZ3ZKA23GGVEsiEW6F3/eventfd/eventfd.go:27:32: undefined: syscall.SYS_EVENTFD2
+          || (lib.elem to [
+            11 # fs-repo-10-to-11 fails (probably since Go 1.21) with: panic: qtls.ClientSessionState doesn't match
+            12 # fs-repo-11-to-12 fails (probably since Go 1.21) with: panic: qtls.ClientSessionState doesn't match
+          ]);
+      };
+    };
+
+  stubBecauseDisabled =
+    from: to: release:
+    let
+      pname = "fs-repo-${toString from}-to-${toString to}";
+    in
+    writeShellApplication {
+      name = pname;
+      text = ''
+        echo 'The kubo-fs-repo-migrations package was not buit with support for ${pname}.'
+        echo 'To enable support, set the minRepoVersion argument of this package to a lower value.'
+        echo 'The purpose of this stub is to prevent the fs-repo-migrations program from downloading unsigned binaries from the internet.'
+      '';
+    };
+
+  stubBecauseBroken =
+    pname:
+    writeShellApplication {
+      name = pname;
+      text = ''
+        echo '${pname} is broken with the latest Go version.'
+        echo 'The purpose of this stub is to prevent the fs-repo-migrations program from downloading unsigned binaries from the internet.'
+      '';
+    };
+
+  releases = [
+    {
+      from = 0;
+      to = 1;
+      release = "1.0.1";
+      hash = "sha256-2mKtr6ZXZdOOY+9GNaC85HKjOMsfeM91oxVuxHIWDO4=";
+    }
+    {
+      from = 1;
+      to = 2;
+      release = "1.0.1";
+      hash = "sha256-6/BewNcZc/fIBa8G1luNO2wqTdeHi8vL7ojJDjBfWYI=";
+    }
+    {
+      from = 2;
+      to = 3;
+      release = "1.0.1";
+      hash = "sha256-kESX/R25nb7G/uggwa7GB7I2IrdgeKe0chRzjr70Kuw=";
+    }
+    {
+      from = 3;
+      to = 4;
+      release = "1.0.1";
+      hash = "sha256-Mv3/7eUS8j7ZzbNR52baekDcXPwcaNpUfqkt0eRpP20=";
+    }
+    {
+      from = 4;
+      to = 5;
+      release = "1.0.1";
+      hash = "sha256-aEqXFhZGOBU5ql2RRqzwD5IXGySVGroaHxjrkpIGAeU=";
+    }
+    {
+      from = 5;
+      to = 6;
+      release = "1.0.1";
+      hash = "sha256-EhMe/3gIl3VjSh6KzBPGH4s6B3AWRnbJ+eHSc8GOHMw=";
+    }
+    {
+      from = 6;
+      to = 7;
+      release = "1.0.1";
+      hash = "sha256-+5kIPQZckloPujLS0QQT+ojIIndfCQaH6grftZdYQ88=";
+    }
+    {
+      from = 7;
+      to = 8;
+      release = "1.0.1";
+      hash = "sha256-82oSU7qhldPVTdbbol3xSnl8Ko7NUPvGpAnmFxvAceQ=";
+    }
+    {
+      from = 8;
+      to = 9;
+      release = "1.0.1";
+      hash = "sha256-9knC2CfiTUNJRlrOLRpKy70Hl9p9DQf6rfXnU2a0fig=";
+    }
+    {
+      from = 9;
+      to = 10;
+      release = "1.0.1";
+      hash = "sha256-732k76Kijs5izu404ES/YSnYfC9V88d9Qq5oHv5Qon0=";
+    }
+    {
+      from = 10;
+      to = 11;
+      release = "1.0.1";
+      hash = "sha256-WieBZpD8dpFDif7QxTGjRoZtNBbkI3KU4w4av7b+d4Q=";
+    }
+    {
+      from = 11;
+      to = 12;
+      release = "1.0.2";
+      hash = "sha256-x/4ps705Hnf+/875/tn3AsEHgaHHCc+cGXymXpRt0xA=";
+    }
+    {
+      from = 12;
+      to = 13;
+      release = "1.0.0";
+      hash = "sha256-HjtZ2izoZ+0BrhzXG/QJHcnwsxi0oY4Q3CHjTi29W9o=";
+    }
+    {
+      from = 13;
+      to = 14;
+      release = "1.0.0";
+      hash = "sha256-zvNq+AFp7HDHHZCJOh9OW/lalk3bXOl1Pi+rvJtjuSA=";
+    }
+    {
+      from = 14;
+      to = 15;
+      release = "1.0.1";
+      hash = "sha256-u7PM6kFCQUn07NGpeRYpBDEwc2pP+r5mf44LZU4DV5Y=";
+    }
+  ];
+
+  maxRepoVersion = builtins.length releases;
+
+  minRepoVersionValidated =
+    if minRepoVersion >= 0 then
+      minRepoVersion
+    else
+      throw "The minimum supported repo version is 0. Set `minRepoVersion` to a non-zero value.";
+
+  latestMigration = builtins.foldl' (x: y: if y.to == maxRepoVersion then y else x) {
+    release = throw "Could not get the latest Kubo migration";
+  } releases;
+  version = "${toString maxRepoVersion}.${latestMigration.release}";
+
+  mkMigrationOrStub =
+    x:
+    let
+      builder = if x.from >= minRepoVersionValidated then mkMigration else stubBecauseDisabled;
+    in
+    builder x.from x.to x.release x.hash;
+  migrations = builtins.map mkMigrationOrStub releases;
+
+  packageNotBroken = package: !package.meta.broken;
+  migrationsBrokenRemoved = builtins.filter packageNotBroken migrations;
+  migrationsBrokenStubbed = builtins.map (
+    x: if packageNotBroken x then x else (stubBecauseBroken x.pname)
+  ) migrations;
+in
+
+symlinkJoin {
+  name = "kubo-fs-repo-migrations-${version}";
+  paths = if stubBrokenMigrations then migrationsBrokenStubbed else migrationsBrokenRemoved;
+  meta = (builtins.removeAttrs kubo-migrator-unwrapped.meta [ "mainProgram" ]) // {
+    description = "Several individual migrations for migrating the filesystem repository of Kubo one version at a time";
+    longDescription = ''
+      This package contains all the individual migrations in the bin directory.
+      This is used by fs-repo-migrations and could also be used by Kubo itself
+      when starting it like this: ipfs daemon --migrate
+    '';
+  };
+}
--- a/pkgs/by-name/ku/kubo-migrator-unwrapped/package.nix
+++ b/pkgs/by-name/ku/kubo-migrator-unwrapped/package.nix
@ -0,0 +1,36 @@
+{
+  lib,
+  buildGoModule,
+  fetchFromGitHub,
+}:
+
+buildGoModule rec {
+  pname = "kubo-migrator";
+  version = "2.0.2-unstable-2024-08-02";
+
+  src = fetchFromGitHub {
+    owner = "ipfs";
+    repo = "fs-repo-migrations";
+    rev = "cbc31a03fb2f6aba80d577224c09472101427771";
+    hash = "sha256-wgWwDuL5Yv7dSYFrBiC4OS7SuTHh1D8RSabBnOTUiZ0=";
+    sparseCheckout = [ "fs-repo-migrations" ];
+  };
+
+  sourceRoot = "${src.name}/fs-repo-migrations";
+
+  vendorHash = "sha256-/DqkBBtR/nU8gk3TFqNKY5zQU6BFMc3N8Ti+38mi/jk=";
+
+  doCheck = false;
+
+  meta = {
+    description = "Run the appripriate migrations for migrating the filesystem repository of Kubo (migrations not included)";
+    homepage = "https://github.com/ipfs/fs-repo-migrations";
+    license = lib.licenses.mit;
+    platforms = lib.platforms.unix;
+    maintainers = with lib.maintainers; [
+      Luflosi
+      elitak
+    ];
+    mainProgram = "fs-repo-migrations";
+  };
+}
--- a/pkgs/by-name/ku/kubo-migrator/package.nix
+++ b/pkgs/by-name/ku/kubo-migrator/package.nix
@ -0,0 +1,26 @@
+{
+  lib,
+  buildEnv,
+  makeWrapper,
+  kubo-migrator-unwrapped,
+  kubo-fs-repo-migrations,
+}:
+
+buildEnv {
+  name = "kubo-migrator-${kubo-migrator-unwrapped.version}";
+
+  nativeBuildInputs = [ makeWrapper ];
+
+  paths = [ kubo-migrator-unwrapped ];
+
+  pathsToLink = [ "/bin" ];
+
+  postBuild = ''
+    wrapProgram "$out/bin/fs-repo-migrations" \
+      --prefix PATH ':' '${lib.makeBinPath [ kubo-fs-repo-migrations ]}'
+  '';
+
+  meta = kubo-migrator-unwrapped.meta // {
+    description = "Run the appripriate migrations for migrating the filesystem repository of Kubo";
+  };
+}
--- a/pkgs/by-name/ku/kubo/package.nix
+++ b/pkgs/by-name/ku/kubo/package.nix
@ -10,7 +10,7 @@ buildGoModule rec {
  version = "0.29.0"; # When updating, also check if the repo version changed and adjust repoVersion below
  rev = "v${version}";

-  passthru.repoVersion = "15"; # Also update kubo-migrator when changing the repo version
+  passthru.repoVersion = "15"; # Also update kubo-migrator-unwrapped and kubo-fs-repo-migrations when changing the repo version

  # Kubo makes changes to its source tarball that don't match the git source.
  src = fetchurl {
--- a/pkgs/top-level/aliases.nix
+++ b/pkgs/top-level/aliases.nix
@ -529,7 +529,7 @@ mapAliases {
  inotifyTools = inotify-tools;
  inter-ui = throw "'inter-ui' has been renamed to/replaced by 'inter'"; # Converted to throw 2024-10-17
  ipfs = kubo; # Added 2022-09-27
-  ipfs-migrator-all-fs-repo-migrations = kubo-migrator-all-fs-repo-migrations; # Added 2022-09-27
+  ipfs-migrator-all-fs-repo-migrations = kubo-fs-repo-migrations; # Added 2022-09-27
  ipfs-migrator-unwrapped = kubo-migrator-unwrapped; # Added 2022-09-27
  ipfs-migrator = kubo-migrator; # Added 2022-09-27
  iproute = throw "'iproute' has been renamed to/replaced by 'iproute2'"; # Converted to throw 2024-10-17
@ -586,6 +586,7 @@ mapAliases {
  krb5Full = krb5;
  krita-beta = throw "'krita-beta' has been renamed to/replaced by 'krita'"; # Converted to throw 2024-10-17
  kubei = kubeclarity; # Added 2023-05-20
+  kubo-migrator-all-fs-repo-migrations = kubo-fs-repo-migrations; # Added 2024-09-24

  ### L ###

--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@ -4086,10 +4086,6 @@ with pkgs;

  kdiff3 = libsForQt5.callPackage ../tools/text/kdiff3 { };

-  kubo-migrator-all-fs-repo-migrations = callPackage ../applications/networking/kubo-migrator/all-migrations.nix { };
-  kubo-migrator-unwrapped = callPackage ../applications/networking/kubo-migrator/unwrapped.nix { };
-  kubo-migrator = callPackage ../applications/networking/kubo-migrator { };
-
  kwalletcli = libsForQt5.callPackage ../tools/security/kwalletcli { };

  peruse = libsForQt5.callPackage ../tools/misc/peruse { };