diff --git a/nixos/modules/services/network-filesystems/kubo.nix b/nixos/modules/services/network-filesystems/kubo.nix index 35efd2ba0a57..3b0555b3a874 100644 --- a/nixos/modules/services/network-filesystems/kubo.nix +++ b/nixos/modules/services/network-filesystems/kubo.nix @@ -339,7 +339,7 @@ in # After an unclean shutdown this file may exist which will cause the config command to attempt to talk to the daemon. This will hang forever if systemd is holding our sockets open. rm -vf "$IPFS_PATH/api" '' + lib.optionalString cfg.autoMigrate '' - ${pkgs.kubo-migrator}/bin/fs-repo-migrations -to '${cfg.package.repoVersion}' -y + '${lib.getExe pkgs.kubo-migrator}' -to '${cfg.package.repoVersion}' -y '' + '' fi ipfs --offline config show | diff --git a/pkgs/applications/networking/kubo-migrator/all-migrations.nix b/pkgs/applications/networking/kubo-migrator/all-migrations.nix deleted file mode 100644 index d44fafb3be9e..000000000000 --- a/pkgs/applications/networking/kubo-migrator/all-migrations.nix +++ /dev/null @@ -1,82 +0,0 @@ -{ lib -, stdenv -, symlinkJoin -, buildGoModule -, kubo-migrator-unwrapped -}: - -# This package contains all the individual migrations in the bin directory. -# This is used by fs-repo-migrations and could also be used by Kubo itself -# when starting it like this: ipfs daemon --migrate - -let - fs-repo-common = pname: version: buildGoModule { - inherit pname version; - inherit (kubo-migrator-unwrapped) src; - sourceRoot = "${kubo-migrator-unwrapped.src.name}/${pname}"; - vendorHash = null; - # Fix build on Go 1.17 and later: panic: qtls.ClientHelloInfo doesn't match - # See https://github.com/ipfs/fs-repo-migrations/pull/163 - postPatch = lib.optionalString (lib.elem pname [ "fs-repo-10-to-11" "fs-repo-11-to-12" ]) '' - substituteInPlace 'vendor/github.com/marten-seemann/qtls-go1-15/common.go' \ - --replace \ - '"container/list"' \ - '"container/list" - "context"' \ - --replace \ - 'config *Config' \ - 'config *Config - ctx context.Context' - ''; - doCheck = false; - meta = kubo-migrator-unwrapped.meta // { - mainProgram = pname; - description = "Individual migration for the filesystem repository of Kubo clients"; - }; - }; - - # Concatenation of the latest repo version and the version of that migration - version = "15.1.0.1"; - - fs-repo-14-to-15 = fs-repo-common "fs-repo-14-to-15" "1.0.1"; - fs-repo-13-to-14 = fs-repo-common "fs-repo-13-to-14" "1.0.0"; - fs-repo-12-to-13 = fs-repo-common "fs-repo-12-to-13" "1.0.0"; - fs-repo-11-to-12 = fs-repo-common "fs-repo-11-to-12" "1.0.2"; - fs-repo-10-to-11 = fs-repo-common "fs-repo-10-to-11" "1.0.1"; - fs-repo-9-to-10 = fs-repo-common "fs-repo-9-to-10" "1.0.1"; - fs-repo-8-to-9 = fs-repo-common "fs-repo-8-to-9" "1.0.1"; - fs-repo-7-to-8 = fs-repo-common "fs-repo-7-to-8" "1.0.1"; - fs-repo-6-to-7 = fs-repo-common "fs-repo-6-to-7" "1.0.1"; - fs-repo-5-to-6 = fs-repo-common "fs-repo-5-to-6" "1.0.1"; - fs-repo-4-to-5 = fs-repo-common "fs-repo-4-to-5" "1.0.1"; - fs-repo-3-to-4 = fs-repo-common "fs-repo-3-to-4" "1.0.1"; - fs-repo-2-to-3 = fs-repo-common "fs-repo-2-to-3" "1.0.1"; - fs-repo-1-to-2 = fs-repo-common "fs-repo-1-to-2" "1.0.1"; - fs-repo-0-to-1 = fs-repo-common "fs-repo-0-to-1" "1.0.1"; - - all-migrations = [ - fs-repo-14-to-15 - fs-repo-13-to-14 - fs-repo-12-to-13 - fs-repo-11-to-12 - fs-repo-10-to-11 - fs-repo-9-to-10 - fs-repo-8-to-9 - fs-repo-7-to-8 - ] ++ lib.optional (!stdenv.hostPlatform.isDarwin) # I didn't manage to fix this on macOS: - fs-repo-6-to-7 # gx/ipfs/QmSGRM5Udmy1jsFBr1Cawez7Lt7LZ3ZKA23GGVEsiEW6F3/eventfd/eventfd.go:27:32: undefined: syscall.SYS_EVENTFD2 - ++ [ - fs-repo-5-to-6 - fs-repo-4-to-5 - fs-repo-3-to-4 - fs-repo-2-to-3 - fs-repo-1-to-2 - fs-repo-0-to-1 - ]; - -in - -symlinkJoin { - name = "kubo-migrator-all-fs-repo-migrations-${version}"; - paths = all-migrations; -} diff --git a/pkgs/applications/networking/kubo-migrator/default.nix b/pkgs/applications/networking/kubo-migrator/default.nix deleted file mode 100644 index e45cbf84d524..000000000000 --- a/pkgs/applications/networking/kubo-migrator/default.nix +++ /dev/null @@ -1,23 +0,0 @@ -{ lib -, buildEnv -, makeWrapper -, kubo-migrator-unwrapped -, kubo-migrator-all-fs-repo-migrations -}: - -buildEnv { - name = "kubo-migrator-${kubo-migrator-unwrapped.version}"; - - nativeBuildInputs = [ makeWrapper ]; - - paths = [ kubo-migrator-unwrapped ]; - - pathsToLink = [ "/bin" ]; - - postBuild = '' - wrapProgram "$out/bin/fs-repo-migrations" \ - --prefix PATH ':' '${lib.makeBinPath [ kubo-migrator-all-fs-repo-migrations ]}' - ''; - - inherit (kubo-migrator-unwrapped) meta; -} diff --git a/pkgs/applications/networking/kubo-migrator/unwrapped.nix b/pkgs/applications/networking/kubo-migrator/unwrapped.nix deleted file mode 100644 index 1e63c5852ca3..000000000000 --- a/pkgs/applications/networking/kubo-migrator/unwrapped.nix +++ /dev/null @@ -1,35 +0,0 @@ -{ lib -, buildGoModule -, fetchFromGitHub -}: - -buildGoModule rec { - pname = "kubo-migrator"; - version = "2.0.2"; - - src = fetchFromGitHub { - owner = "ipfs"; - repo = "fs-repo-migrations"; - # Use the latest git tag here, since v2.0.2 does not - # contain the latest migration fs-repo-14-to-15/v1.0.1 - # The fs-repo-migrations code itself is the same between - # the two versions but the migration code, which is built - # into separate binaries, is not. - rev = "fs-repo-14-to-15/v1.0.1"; - hash = "sha256-oIGDZr0cv+TIl5glHr3U+eIqAlPAOWyFzgfQGGM+xNM="; - }; - - sourceRoot = "${src.name}/fs-repo-migrations"; - - vendorHash = "sha256-/DqkBBtR/nU8gk3TFqNKY5zQU6BFMc3N8Ti+38mi/jk="; - - doCheck = false; - - meta = with lib; { - description = "Migrations for the filesystem repository of Kubo clients"; - homepage = "https://github.com/ipfs/fs-repo-migrations"; - license = licenses.mit; - maintainers = with maintainers; [ Luflosi elitak ]; - mainProgram = "fs-repo-migrations"; - }; -} diff --git a/pkgs/by-name/ku/kubo-fs-repo-migrations/package.nix b/pkgs/by-name/ku/kubo-fs-repo-migrations/package.nix new file mode 100644 index 000000000000..09c273e27d26 --- /dev/null +++ b/pkgs/by-name/ku/kubo-fs-repo-migrations/package.nix @@ -0,0 +1,253 @@ +{ + lib, + stdenv, + symlinkJoin, + buildGoModule, + fetchFromGitHub, + kubo-migrator-unwrapped, + writeShellApplication, + minRepoVersion ? 0, # The minimum supported Kubo repo version from which the migrations can start. Increasing this reduces the closure size + stubBrokenMigrations ? true, # This prevents the fs-repo-migrations program from downloading binaries off the internet without even checking any signatures +}: + +let + mkMigration = + from: to: version: hash: + let + pname = "fs-repo-${toString from}-to-${toString to}"; + src = fetchFromGitHub { + owner = "ipfs"; + repo = "fs-repo-migrations"; + rev = "${pname}/v${version}"; + inherit hash; + sparseCheckout = [ pname ]; + }; + in + buildGoModule { + inherit pname version src; + sourceRoot = "${src.name}/${pname}"; + vendorHash = null; + + # Fix build on Go 1.17 and later: panic: qtls.ClientHelloInfo doesn't match + # See https://github.com/ipfs/fs-repo-migrations/pull/163 + postPatch = + lib.optionalString + (lib.elem to [ + 11 + 12 + ]) + '' + substituteInPlace 'vendor/github.com/marten-seemann/qtls-go1-15/common.go' \ + --replace-fail \ + '"container/list"' \ + '"container/list" + "context"' \ + --replace-fail \ + 'config *Config' \ + 'config *Config + ctx context.Context' + ''; + + checkPhase = '' + runHook preCheck + ${ + if to <= 11 then + "" # Migrations fs-repo-10-to-11 and earlier require too much effort to test, making it not worth it + else if to == 12 then + '' + cd migration + go test -mod=vendor + '' + else + '' + cd not-sharness + ./test.sh + '' + } + runHook postCheck + ''; + + # Check that it does not crash + doInstallCheck = true; + installCheckPhase = '' + runHook preInstallCheck + "$out/bin/${pname}" -help + runHook postInstallCheck + ''; + + meta = { + inherit (kubo-migrator-unwrapped.meta) + homepage + license + platforms + maintainers + ; + mainProgram = pname; + description = "Migrate the filesystem repository of Kubo from repo version ${toString from} to ${toString to}"; + + broken = + to == 7 && stdenv.hostPlatform.isDarwin # fs-repo-6-to-7 is broken on macOS: gx/ipfs/QmSGRM5Udmy1jsFBr1Cawez7Lt7LZ3ZKA23GGVEsiEW6F3/eventfd/eventfd.go:27:32: undefined: syscall.SYS_EVENTFD2 + || (lib.elem to [ + 11 # fs-repo-10-to-11 fails (probably since Go 1.21) with: panic: qtls.ClientSessionState doesn't match + 12 # fs-repo-11-to-12 fails (probably since Go 1.21) with: panic: qtls.ClientSessionState doesn't match + ]); + }; + }; + + stubBecauseDisabled = + from: to: release: + let + pname = "fs-repo-${toString from}-to-${toString to}"; + in + writeShellApplication { + name = pname; + text = '' + echo 'The kubo-fs-repo-migrations package was not buit with support for ${pname}.' + echo 'To enable support, set the minRepoVersion argument of this package to a lower value.' + echo 'The purpose of this stub is to prevent the fs-repo-migrations program from downloading unsigned binaries from the internet.' + ''; + }; + + stubBecauseBroken = + pname: + writeShellApplication { + name = pname; + text = '' + echo '${pname} is broken with the latest Go version.' + echo 'The purpose of this stub is to prevent the fs-repo-migrations program from downloading unsigned binaries from the internet.' + ''; + }; + + releases = [ + { + from = 0; + to = 1; + release = "1.0.1"; + hash = "sha256-2mKtr6ZXZdOOY+9GNaC85HKjOMsfeM91oxVuxHIWDO4="; + } + { + from = 1; + to = 2; + release = "1.0.1"; + hash = "sha256-6/BewNcZc/fIBa8G1luNO2wqTdeHi8vL7ojJDjBfWYI="; + } + { + from = 2; + to = 3; + release = "1.0.1"; + hash = "sha256-kESX/R25nb7G/uggwa7GB7I2IrdgeKe0chRzjr70Kuw="; + } + { + from = 3; + to = 4; + release = "1.0.1"; + hash = "sha256-Mv3/7eUS8j7ZzbNR52baekDcXPwcaNpUfqkt0eRpP20="; + } + { + from = 4; + to = 5; + release = "1.0.1"; + hash = "sha256-aEqXFhZGOBU5ql2RRqzwD5IXGySVGroaHxjrkpIGAeU="; + } + { + from = 5; + to = 6; + release = "1.0.1"; + hash = "sha256-EhMe/3gIl3VjSh6KzBPGH4s6B3AWRnbJ+eHSc8GOHMw="; + } + { + from = 6; + to = 7; + release = "1.0.1"; + hash = "sha256-+5kIPQZckloPujLS0QQT+ojIIndfCQaH6grftZdYQ88="; + } + { + from = 7; + to = 8; + release = "1.0.1"; + hash = "sha256-82oSU7qhldPVTdbbol3xSnl8Ko7NUPvGpAnmFxvAceQ="; + } + { + from = 8; + to = 9; + release = "1.0.1"; + hash = "sha256-9knC2CfiTUNJRlrOLRpKy70Hl9p9DQf6rfXnU2a0fig="; + } + { + from = 9; + to = 10; + release = "1.0.1"; + hash = "sha256-732k76Kijs5izu404ES/YSnYfC9V88d9Qq5oHv5Qon0="; + } + { + from = 10; + to = 11; + release = "1.0.1"; + hash = "sha256-WieBZpD8dpFDif7QxTGjRoZtNBbkI3KU4w4av7b+d4Q="; + } + { + from = 11; + to = 12; + release = "1.0.2"; + hash = "sha256-x/4ps705Hnf+/875/tn3AsEHgaHHCc+cGXymXpRt0xA="; + } + { + from = 12; + to = 13; + release = "1.0.0"; + hash = "sha256-HjtZ2izoZ+0BrhzXG/QJHcnwsxi0oY4Q3CHjTi29W9o="; + } + { + from = 13; + to = 14; + release = "1.0.0"; + hash = "sha256-zvNq+AFp7HDHHZCJOh9OW/lalk3bXOl1Pi+rvJtjuSA="; + } + { + from = 14; + to = 15; + release = "1.0.1"; + hash = "sha256-u7PM6kFCQUn07NGpeRYpBDEwc2pP+r5mf44LZU4DV5Y="; + } + ]; + + maxRepoVersion = builtins.length releases; + + minRepoVersionValidated = + if minRepoVersion >= 0 then + minRepoVersion + else + throw "The minimum supported repo version is 0. Set `minRepoVersion` to a non-zero value."; + + latestMigration = builtins.foldl' (x: y: if y.to == maxRepoVersion then y else x) { + release = throw "Could not get the latest Kubo migration"; + } releases; + version = "${toString maxRepoVersion}.${latestMigration.release}"; + + mkMigrationOrStub = + x: + let + builder = if x.from >= minRepoVersionValidated then mkMigration else stubBecauseDisabled; + in + builder x.from x.to x.release x.hash; + migrations = builtins.map mkMigrationOrStub releases; + + packageNotBroken = package: !package.meta.broken; + migrationsBrokenRemoved = builtins.filter packageNotBroken migrations; + migrationsBrokenStubbed = builtins.map ( + x: if packageNotBroken x then x else (stubBecauseBroken x.pname) + ) migrations; +in + +symlinkJoin { + name = "kubo-fs-repo-migrations-${version}"; + paths = if stubBrokenMigrations then migrationsBrokenStubbed else migrationsBrokenRemoved; + meta = (builtins.removeAttrs kubo-migrator-unwrapped.meta [ "mainProgram" ]) // { + description = "Several individual migrations for migrating the filesystem repository of Kubo one version at a time"; + longDescription = '' + This package contains all the individual migrations in the bin directory. + This is used by fs-repo-migrations and could also be used by Kubo itself + when starting it like this: ipfs daemon --migrate + ''; + }; +} diff --git a/pkgs/by-name/ku/kubo-migrator-unwrapped/package.nix b/pkgs/by-name/ku/kubo-migrator-unwrapped/package.nix new file mode 100644 index 000000000000..6a47f45426c2 --- /dev/null +++ b/pkgs/by-name/ku/kubo-migrator-unwrapped/package.nix @@ -0,0 +1,36 @@ +{ + lib, + buildGoModule, + fetchFromGitHub, +}: + +buildGoModule rec { + pname = "kubo-migrator"; + version = "2.0.2-unstable-2024-08-02"; + + src = fetchFromGitHub { + owner = "ipfs"; + repo = "fs-repo-migrations"; + rev = "cbc31a03fb2f6aba80d577224c09472101427771"; + hash = "sha256-wgWwDuL5Yv7dSYFrBiC4OS7SuTHh1D8RSabBnOTUiZ0="; + sparseCheckout = [ "fs-repo-migrations" ]; + }; + + sourceRoot = "${src.name}/fs-repo-migrations"; + + vendorHash = "sha256-/DqkBBtR/nU8gk3TFqNKY5zQU6BFMc3N8Ti+38mi/jk="; + + doCheck = false; + + meta = { + description = "Run the appripriate migrations for migrating the filesystem repository of Kubo (migrations not included)"; + homepage = "https://github.com/ipfs/fs-repo-migrations"; + license = lib.licenses.mit; + platforms = lib.platforms.unix; + maintainers = with lib.maintainers; [ + Luflosi + elitak + ]; + mainProgram = "fs-repo-migrations"; + }; +} diff --git a/pkgs/by-name/ku/kubo-migrator/package.nix b/pkgs/by-name/ku/kubo-migrator/package.nix new file mode 100644 index 000000000000..ad681b9313c5 --- /dev/null +++ b/pkgs/by-name/ku/kubo-migrator/package.nix @@ -0,0 +1,26 @@ +{ + lib, + buildEnv, + makeWrapper, + kubo-migrator-unwrapped, + kubo-fs-repo-migrations, +}: + +buildEnv { + name = "kubo-migrator-${kubo-migrator-unwrapped.version}"; + + nativeBuildInputs = [ makeWrapper ]; + + paths = [ kubo-migrator-unwrapped ]; + + pathsToLink = [ "/bin" ]; + + postBuild = '' + wrapProgram "$out/bin/fs-repo-migrations" \ + --prefix PATH ':' '${lib.makeBinPath [ kubo-fs-repo-migrations ]}' + ''; + + meta = kubo-migrator-unwrapped.meta // { + description = "Run the appripriate migrations for migrating the filesystem repository of Kubo"; + }; +} diff --git a/pkgs/by-name/ku/kubo/package.nix b/pkgs/by-name/ku/kubo/package.nix index fcc690177da4..9cec847406b9 100644 --- a/pkgs/by-name/ku/kubo/package.nix +++ b/pkgs/by-name/ku/kubo/package.nix @@ -10,7 +10,7 @@ buildGoModule rec { version = "0.29.0"; # When updating, also check if the repo version changed and adjust repoVersion below rev = "v${version}"; - passthru.repoVersion = "15"; # Also update kubo-migrator when changing the repo version + passthru.repoVersion = "15"; # Also update kubo-migrator-unwrapped and kubo-fs-repo-migrations when changing the repo version # Kubo makes changes to its source tarball that don't match the git source. src = fetchurl { diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix index 5f6f3ad0eaa6..04ba2fdd74b8 100644 --- a/pkgs/top-level/aliases.nix +++ b/pkgs/top-level/aliases.nix @@ -529,7 +529,7 @@ mapAliases { inotifyTools = inotify-tools; inter-ui = throw "'inter-ui' has been renamed to/replaced by 'inter'"; # Converted to throw 2024-10-17 ipfs = kubo; # Added 2022-09-27 - ipfs-migrator-all-fs-repo-migrations = kubo-migrator-all-fs-repo-migrations; # Added 2022-09-27 + ipfs-migrator-all-fs-repo-migrations = kubo-fs-repo-migrations; # Added 2022-09-27 ipfs-migrator-unwrapped = kubo-migrator-unwrapped; # Added 2022-09-27 ipfs-migrator = kubo-migrator; # Added 2022-09-27 iproute = throw "'iproute' has been renamed to/replaced by 'iproute2'"; # Converted to throw 2024-10-17 @@ -586,6 +586,7 @@ mapAliases { krb5Full = krb5; krita-beta = throw "'krita-beta' has been renamed to/replaced by 'krita'"; # Converted to throw 2024-10-17 kubei = kubeclarity; # Added 2023-05-20 + kubo-migrator-all-fs-repo-migrations = kubo-fs-repo-migrations; # Added 2024-09-24 ### L ### diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 16b6aa095873..7ce033aa9779 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -4086,10 +4086,6 @@ with pkgs; kdiff3 = libsForQt5.callPackage ../tools/text/kdiff3 { }; - kubo-migrator-all-fs-repo-migrations = callPackage ../applications/networking/kubo-migrator/all-migrations.nix { }; - kubo-migrator-unwrapped = callPackage ../applications/networking/kubo-migrator/unwrapped.nix { }; - kubo-migrator = callPackage ../applications/networking/kubo-migrator { }; - kwalletcli = libsForQt5.callPackage ../tools/security/kwalletcli { }; peruse = libsForQt5.callPackage ../tools/misc/peruse { };