diff --git a/nixos/doc/manual/from_md/release-notes/rl-2305.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2305.section.xml index 83fa2b94641b..2d68e7f5426f 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2305.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2305.section.xml @@ -407,6 +407,21 @@ here. + + + Garage + version is based on + system.stateVersion, + existing installations will keep using version 0.7. New + installations will use version 0.8. In order to upgrade a + Garage cluster, please follow + upstream + instructions and force + services.garage.package + or upgrade accordingly + system.stateVersion. + + Resilio sync secret keys can now be provided using a secrets diff --git a/nixos/doc/manual/release-notes/rl-2305.section.md b/nixos/doc/manual/release-notes/rl-2305.section.md index 16c1e15052f3..d960ab03faae 100644 --- a/nixos/doc/manual/release-notes/rl-2305.section.md +++ b/nixos/doc/manual/release-notes/rl-2305.section.md @@ -111,6 +111,8 @@ In addition to numerous new and upgraded packages, this release has the followin - A new option `recommendedBrotliSettings` has been added to `services.nginx`. Learn more about compression in Brotli format [here](https://github.com/google/ngx_brotli/blob/master/README.md). +- [Garage](https://garagehq.deuxfleurs.fr/) version is based on [system.stateVersion](options.html#opt-system.stateVersion), existing installations will keep using version 0.7. New installations will use version 0.8. In order to upgrade a Garage cluster, please follow [upstream instructions](https://garagehq.deuxfleurs.fr/documentation/cookbook/upgrading/) and force [services.garage.package](options.html#opt-services.garage.package) or upgrade accordingly [system.stateVersion](options.html#opt-system.stateVersion). + - Resilio sync secret keys can now be provided using a secrets file at runtime, preventing these secrets from ending up in the Nix store. - The `firewall` and `nat` module now has a nftables based implementation. Enable `networking.nftables` to use it. diff --git a/nixos/modules/services/web-servers/garage-doc.xml b/nixos/modules/services/web-servers/garage-doc.xml new file mode 100644 index 000000000000..16f6fde94b5a --- /dev/null +++ b/nixos/modules/services/web-servers/garage-doc.xml @@ -0,0 +1,139 @@ + + Garage + + Garage + is an open-source, self-hostable S3 store, simpler than MinIO, for geodistributed stores. + The server setup can be automated using + services.garage. A + client configured to your local Garage instance is available in + the global environment as garage-manage. + + + The current default by NixOS is garage_0_8 which is also the latest + major version available. + +
+ General considerations on upgrades + + + Garage provides a cookbook documentation on how to upgrade: + https://garagehq.deuxfleurs.fr/documentation/cookbook/upgrading/ + + + + Garage has two types of upgrades: patch-level upgrades and minor/major version upgrades. + + In all cases, you should read the changelog and ideally test the upgrade on a staging cluster. + + Checking the health of your cluster can be achieved using garage-manage repair. + + + + + Until 1.0 is released, patch-level upgrades are considered as minor version upgrades. + Minor version upgrades are considered as major version upgrades. + i.e. 0.6 to 0.7 is a major version upgrade. + + + + + + Straightforward upgrades (patch-level upgrades) + + Upgrades must be performed one by one, i.e. for each node, stop it, upgrade it : change stateVersion or services.garage.package, restart it if it was not already by switching. + + + + + + + Multiple version upgrades + + Garage do not provide any guarantee on moving more than one major-version forward. + E.g., if you're on 0.7, you cannot upgrade to 0.9. + You need to upgrade to 0.8 first. + + As long as stateVersion is declared properly, + this is enforced automatically. The module will issue a warning to remind the user to upgrade to latest + Garage after that deploy. + + + + +
+ +
+ Advanced upgrades (minor/major version upgrades) + Here are some baseline instructions to handle advanced upgrades in Garage, when in doubt, please refer to upstream instructions. + + + Disable API and web access to Garage. + Perform garage-manage repair --all-nodes --yes tables and garage-manage repair --all-nodes --yes blocks. + Verify the resulting logs and check that data is synced properly between all nodes. + If you have time, do additional checks (scrub, block_refs, etc.). + Check if queues are empty by garage-manage stats or through monitoring tools. + Run systemctl stop garage to stop the actual Garage version. + Backup the metadata folder of ALL your nodes, e.g. for a metadata directory (the default one) in /var/lib/garage/meta, + you can run pushd /var/lib/garage; tar -acf meta-v0.7.tar.zst meta/; popd. + Run the offline migration: nix-shell -p garage_0_8 --run "garage offline-repair --yes", this can take some time depending on how many objects are stored in your cluster. + Bump Garage version in your NixOS configuration, either by changing stateVersion or bumping services.garage.package, this should restart Garage automatically. + Perform garage-manage repair --all-nodes --yes tables and garage-manage repair --all-nodes --yes blocks. + Wait for a full table sync to run. + + + + Your upgraded cluster should be in a working state, re-enable API and web access. + +
+ +
+ Maintainer information + + + As stated in the previous paragraph, we must provide a clean upgrade-path for Garage + since it cannot move more than one major version forward on a single upgrade. This chapter + adds some notes how Garage updates should be rolled out in the future. + + This is inspired from how Nextcloud does it. + + + + While patch-level updates are no problem and can be done directly in the + package-expression (and should be backported to supported stable branches after that), + major-releases should be added in a new attribute (e.g. Garage v0.8.0 + should be available in nixpkgs as pkgs.garage_0_8_0). + To provide simple upgrade paths it's generally useful to backport those as well to stable + branches. As long as the package-default isn't altered, this won't break existing setups. + After that, the versioning-warning in the garage-module should be + updated to make sure that the + package-option selects the latest version + on fresh setups. + + + + If major-releases will be abandoned by upstream, we should check first if those are needed + in NixOS for a safe upgrade-path before removing those. In that case we shold keep those + packages, but mark them as insecure in an expression like this (in + <nixpkgs/pkgs/tools/filesystem/garage/default.nix>): +/* ... */ +{ + garage_0_7_3 = generic { + version = "0.7.3"; + sha256 = "0000000000000000000000000000000000000000000000000000"; + eol = true; + }; +} + + + + Ideally we should make sure that it's possible to jump two NixOS versions forward: + i.e. the warnings and the logic in the module should guard a user to upgrade from a + Garage on e.g. 22.11 to a Garage on 23.11. + +
+ + diff --git a/nixos/modules/services/web-servers/garage.nix b/nixos/modules/services/web-servers/garage.nix index 76ab273483eb..d66bcd731508 100644 --- a/nixos/modules/services/web-servers/garage.nix +++ b/nixos/modules/services/web-servers/garage.nix @@ -8,7 +8,10 @@ let configFile = toml.generate "garage.toml" cfg.settings; in { - meta.maintainers = [ maintainers.raitobezarius ]; + meta = { + doc = ./garage-doc.xml; + maintainers = with pkgs.lib.maintainers; [ raitobezarius ]; + }; options.services.garage = { enable = mkEnableOption (lib.mdDoc "Garage Object Storage (S3 compatible)"); @@ -56,10 +59,12 @@ in }; package = mkOption { - default = pkgs.garage; - defaultText = literalExpression "pkgs.garage"; + # TODO: when 23.05 is released and if Garage 0.9 is the default, put a stateVersion check. + default = if versionAtLeast stateVersion "23.05" then pkgs.garage_0_8_0 + else pkgs.garage_0_7; + defaultText = literalExpression "pkgs.garage_0_7"; type = types.package; - description = lib.mdDoc "Garage package to use."; + description = lib.mdDoc "Garage package to use, if you are upgrading from a major version, please read NixOS and Garage release notes for upgrade instructions."; }; }; diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 661145afb74c..9fe1bd9e38f5 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -229,7 +229,7 @@ in { fsck = handleTest ./fsck.nix {}; ft2-clone = handleTest ./ft2-clone.nix {}; mimir = handleTest ./mimir.nix {}; - garage = handleTest ./garage.nix {}; + garage = handleTest ./garage {}; gerrit = handleTest ./gerrit.nix {}; geth = handleTest ./geth.nix {}; ghostunnel = handleTest ./ghostunnel.nix {}; diff --git a/nixos/tests/garage/basic.nix b/nixos/tests/garage/basic.nix new file mode 100644 index 000000000000..b6df1e72af98 --- /dev/null +++ b/nixos/tests/garage/basic.nix @@ -0,0 +1,98 @@ +args@{ mkNode, ... }: +(import ../make-test-python.nix ({ pkgs, ...} : { + name = "garage-basic"; + meta = { + maintainers = with pkgs.lib.maintainers; [ raitobezarius ]; + }; + + nodes = { + single_node = mkNode { replicationMode = "none"; }; + }; + + testScript = '' + from typing import List + from dataclasses import dataclass + import re + + start_all() + + cur_version_regex = re.compile('Current cluster layout version: (?P\d*)') + key_creation_regex = re.compile('Key name: (?P.*)\nKey ID: (?P.*)\nSecret key: (?P.*)') + + @dataclass + class S3Key: + key_name: str + key_id: str + secret_key: str + + @dataclass + class GarageNode: + node_id: str + host: str + + def get_node_fqn(machine: Machine) -> GarageNode: + node_id, host = machine.succeed("garage node id").split('@') + return GarageNode(node_id=node_id, host=host) + + def get_node_id(machine: Machine) -> str: + return get_node_fqn(machine).node_id + + def get_layout_version(machine: Machine) -> int: + version_data = machine.succeed("garage layout show") + m = cur_version_regex.search(version_data) + if m and m.group('ver') is not None: + return int(m.group('ver')) + 1 + else: + raise ValueError('Cannot find current layout version') + + def apply_garage_layout(machine: Machine, layouts: List[str]): + for layout in layouts: + machine.succeed(f"garage layout assign {layout}") + version = get_layout_version(machine) + machine.succeed(f"garage layout apply --version {version}") + + def create_api_key(machine: Machine, key_name: str) -> S3Key: + output = machine.succeed(f"garage key new --name {key_name}") + m = key_creation_regex.match(output) + if not m or not m.group('key_id') or not m.group('secret_key'): + raise ValueError('Cannot parse API key data') + return S3Key(key_name=key_name, key_id=m.group('key_id'), secret_key=m.group('secret_key')) + + def get_api_key(machine: Machine, key_pattern: str) -> S3Key: + output = machine.succeed(f"garage key info {key_pattern}") + m = key_creation_regex.match(output) + if not m or not m.group('key_name') or not m.group('key_id') or not m.group('secret_key'): + raise ValueError('Cannot parse API key data') + return S3Key(key_name=m.group('key_name'), key_id=m.group('key_id'), secret_key=m.group('secret_key')) + + def test_bucket_writes(node): + node.succeed("garage bucket create test-bucket") + s3_key = create_api_key(node, "test-api-key") + node.succeed("garage bucket allow --read --write test-bucket --key test-api-key") + other_s3_key = get_api_key(node, 'test-api-key') + assert other_s3_key.secret_key == other_s3_key.secret_key + node.succeed( + f"mc alias set test-garage http://[::1]:3900 {s3_key.key_id} {s3_key.secret_key} --api S3v4" + ) + node.succeed("echo test | mc pipe test-garage/test-bucket/test.txt") + assert node.succeed("mc cat test-garage/test-bucket/test.txt").strip() == "test" + + def test_bucket_over_http(node, bucket='test-bucket', url=None): + if url is None: + url = f"{bucket}.web.garage" + + node.succeed(f'garage bucket website --allow {bucket}') + node.succeed(f'echo hello world | mc pipe test-garage/{bucket}/index.html') + assert (node.succeed(f"curl -H 'Host: {url}' http://localhost:3902")).strip() == 'hello world' + + with subtest("Garage works as a single-node S3 storage"): + single_node.wait_for_unit("garage.service") + single_node.wait_for_open_port(3900) + # Now Garage is initialized. + single_node_id = get_node_id(single_node) + apply_garage_layout(single_node, [f'-z qemutest -c 1 "{single_node_id}"']) + # Now Garage is operational. + test_bucket_writes(single_node) + test_bucket_over_http(single_node) + ''; +})) args diff --git a/nixos/tests/garage/default.nix b/nixos/tests/garage/default.nix new file mode 100644 index 000000000000..5c9159276ace --- /dev/null +++ b/nixos/tests/garage/default.nix @@ -0,0 +1,54 @@ +{ system ? builtins.currentSystem +, config ? { } +, pkgs ? import ../../.. { inherit system config; } +}: +with pkgs.lib; + +let + mkNode = package: { replicationMode, publicV6Address ? "::1" }: { pkgs, ... }: { + networking.interfaces.eth1.ipv6.addresses = [{ + address = publicV6Address; + prefixLength = 64; + }]; + + networking.firewall.allowedTCPPorts = [ 3901 3902 ]; + + services.garage = { + enable = true; + inherit package; + settings = { + replication_mode = replicationMode; + + rpc_bind_addr = "[::]:3901"; + rpc_public_addr = "[${publicV6Address}]:3901"; + rpc_secret = "5c1915fa04d0b6739675c61bf5907eb0fe3d9c69850c83820f51b4d25d13868c"; + + s3_api = { + s3_region = "garage"; + api_bind_addr = "[::]:3900"; + root_domain = ".s3.garage"; + }; + + s3_web = { + bind_addr = "[::]:3902"; + root_domain = ".web.garage"; + index = "index.html"; + }; + }; + }; + environment.systemPackages = [ pkgs.minio-client ]; + + # Garage requires at least 1GiB of free disk space to run. + virtualisation.diskSize = 2 * 1024; + }; +in + foldl + (matrix: ver: matrix // { + "basic${toString ver}" = import ./basic.nix { inherit system pkgs; mkNode = mkNode pkgs."garage_${ver}"; }; + "with-3node-replication${toString ver}" = import ./with-3node-replication.nix { inherit system pkgs; mkNode = mkNode pkgs."garage_${ver}"; }; + }) + {} + [ + "0_7_3" + "0_8_0" + ] diff --git a/nixos/tests/garage.nix b/nixos/tests/garage/with-3node-replication.nix similarity index 74% rename from nixos/tests/garage.nix rename to nixos/tests/garage/with-3node-replication.nix index dc1f83e7f8f3..d372ad1aa000 100644 --- a/nixos/tests/garage.nix +++ b/nixos/tests/garage/with-3node-replication.nix @@ -1,50 +1,12 @@ -import ./make-test-python.nix ({ pkgs, ...} : -let - mkNode = { replicationMode, publicV6Address ? "::1" }: { pkgs, ... }: { - networking.interfaces.eth1.ipv6.addresses = [{ - address = publicV6Address; - prefixLength = 64; - }]; - - networking.firewall.allowedTCPPorts = [ 3901 3902 ]; - - services.garage = { - enable = true; - settings = { - replication_mode = replicationMode; - - rpc_bind_addr = "[::]:3901"; - rpc_public_addr = "[${publicV6Address}]:3901"; - rpc_secret = "5c1915fa04d0b6739675c61bf5907eb0fe3d9c69850c83820f51b4d25d13868c"; - - s3_api = { - s3_region = "garage"; - api_bind_addr = "[::]:3900"; - root_domain = ".s3.garage"; - }; - - s3_web = { - bind_addr = "[::]:3902"; - root_domain = ".web.garage"; - index = "index.html"; - }; - }; - }; - environment.systemPackages = [ pkgs.minio-client ]; - - # Garage requires at least 1GiB of free disk space to run. - virtualisation.diskSize = 2 * 1024; - }; - - -in { - name = "garage"; +args@{ mkNode, ... }: +(import ../make-test-python.nix ({ pkgs, ...} : +{ + name = "garage-3node-replication"; meta = { maintainers = with pkgs.lib.maintainers; [ raitobezarius ]; }; nodes = { - single_node = mkNode { replicationMode = "none"; }; node1 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::1"; }; node2 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::2"; }; node3 = mkNode { replicationMode = 3; publicV6Address = "fc00:1::3"; }; @@ -126,16 +88,6 @@ in { node.succeed(f'echo hello world | mc pipe test-garage/{bucket}/index.html') assert (node.succeed(f"curl -H 'Host: {url}' http://localhost:3902")).strip() == 'hello world' - with subtest("Garage works as a single-node S3 storage"): - single_node.wait_for_unit("garage.service") - single_node.wait_for_open_port(3900) - # Now Garage is initialized. - single_node_id = get_node_id(single_node) - apply_garage_layout(single_node, [f'-z qemutest -c 1 "{single_node_id}"']) - # Now Garage is operational. - test_bucket_writes(single_node) - test_bucket_over_http(single_node) - with subtest("Garage works as a multi-node S3 storage"): nodes = ('node1', 'node2', 'node3', 'node4') rev_machines = {m.name: m for m in machines} @@ -166,4 +118,4 @@ in { for node in nodes: test_bucket_over_http(get_machine(node)) ''; -}) +})) args diff --git a/pkgs/tools/filesystems/garage/default.nix b/pkgs/tools/filesystems/garage/default.nix index bdb04e36a633..63ac77057d4a 100644 --- a/pkgs/tools/filesystems/garage/default.nix +++ b/pkgs/tools/filesystems/garage/default.nix @@ -1,50 +1,90 @@ { lib, stdenv, rustPlatform, fetchFromGitea, openssl, pkg-config, protobuf -, testers, Security, garage }: +, testers, Security, garage, nixosTests }: +let + generic = { version, sha256, cargoSha256, eol ? false }: rustPlatform.buildRustPackage { + pname = "garage"; + inherit version; -rustPlatform.buildRustPackage rec { - pname = "garage"; - version = "0.7.3"; + src = fetchFromGitea { + domain = "git.deuxfleurs.fr"; + owner = "Deuxfleurs"; + repo = "garage"; + rev = "v${version}"; + inherit sha256; + }; - src = fetchFromGitea { - domain = "git.deuxfleurs.fr"; - owner = "Deuxfleurs"; - repo = "garage"; - rev = "v${version}"; - sha256 = "sha256-WDhe2L+NalMoIy2rhfmv8KCNDMkcqBC9ezEKKocihJg="; + inherit cargoSha256; + + nativeBuildInputs = [ protobuf pkg-config ]; + + buildInputs = [ + openssl + ] ++ lib.optional stdenv.isDarwin Security; + + OPENSSL_NO_VENDOR = true; + + # See https://git.deuxfleurs.fr/Deuxfleurs/garage/src/tag/v0.7.2/default.nix#L84-L98 + # on version changes for checking if changes are required here + buildFeatures = [ + "kubernetes-discovery" + ] ++ + (lib.optional (lib.versionAtLeast version "0.8") [ + "bundled-libs" + "sled" + "metrics" + "k2v" + "telemetry-otlp" + "lmdb" + "sqlite" + ]); + + # To make integration tests pass, we include the optional k2v feature here, + # but not in buildFeatures. See: + # https://garagehq.deuxfleurs.fr/documentation/reference-manual/k2v/ + checkFeatures = [ + "k2v" + "kubernetes-discovery" + ] ++ + (lib.optional (lib.versionAtLeast version "0.8") [ + "bundled-libs" + "sled" + "metrics" + "telemetry-otlp" + "lmdb" + "sqlite" + ]); + + passthru = nixosTests.garage; + + meta = { + description = "S3-compatible object store for small self-hosted geo-distributed deployments"; + homepage = "https://garagehq.deuxfleurs.fr"; + license = lib.licenses.agpl3Only; + maintainers = with lib.maintainers; [ nickcao _0x4A6F teutat3s raitobezarius ]; + knownVulnerabilities = (lib.optional eol "Garage version ${version} is EOL"); + }; }; +in + rec { + # Until Garage hits 1.0, 0.7.3 is equivalent to 7.3.0 for now, therefore + # we have to keep all the numbers in the version to handle major/minor/patch level. + # for <1.0. - cargoSha256 = "sha256-5m4c8/upBYN8nuysDhGKEnNVJjEGC+yLrraicrAQOfI="; + garage_0_7_3 = generic { + version = "0.7.3"; + sha256 = "sha256-WDhe2L+NalMoIy2rhfmv8KCNDMkcqBC9ezEKKocihJg="; + cargoSha256 = "sha256-5m4c8/upBYN8nuysDhGKEnNVJjEGC+yLrraicrAQOfI="; + }; - nativeBuildInputs = [ protobuf pkg-config ]; + garage_0_7 = garage_0_7_3; - buildInputs = [ - openssl - ] ++ lib.optional stdenv.isDarwin Security; + garage_0_8_0 = generic { + version = "0.8.0"; + sha256 = "sha256-c2RhHfg0+YV2E9Ckl1YSc+0nfzbHPIt0JgtT0DND9lA="; + cargoSha256 = "sha256-vITXckNOiJbMuQW6/8p7dsZThkjxg/zUy3AZBbn33no="; + }; - OPENSSL_NO_VENDOR = true; + garage_0_8 = garage_0_8_0; - # See https://git.deuxfleurs.fr/Deuxfleurs/garage/src/tag/v0.7.2/default.nix#L84-L98 - # on version changes for checking if changes are required here - buildFeatures = [ - "kubernetes-discovery" - ]; - - # To make integration tests pass, we include the optional k2v feature here, - # but not in buildFeatures. See: - # https://garagehq.deuxfleurs.fr/documentation/reference-manual/k2v/ - checkFeatures = [ - "k2v" - "kubernetes-discovery" - ]; - - passthru = { - tests.version = testers.testVersion { package = garage; }; - }; - - meta = { - description = "S3-compatible object store for small self-hosted geo-distributed deployments"; - homepage = "https://garagehq.deuxfleurs.fr"; - license = lib.licenses.agpl3Only; - maintainers = with lib.maintainers; [ nickcao _0x4A6F teutat3s ]; - }; -} + garage = garage_0_8; + } diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index c3c3409e7bf7..c89f1578c0b8 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -7355,9 +7355,12 @@ with pkgs; gaphor = python3Packages.callPackage ../tools/misc/gaphor { }; - garage = callPackage ../tools/filesystems/garage { + inherit (callPackage ../tools/filesystems/garage { inherit (darwin.apple_sdk.frameworks) Security; - }; + }) + garage + garage_0_7 garage_0_8 + garage_0_7_3 garage_0_8_0; garmin-plugin = callPackage ../applications/misc/garmin-plugin {};