Merge pull request #212087 from robryk/resticpaths

nixos/backups/restic: handle cases when both dynamicFileFrom and paths are set

nixos/modules/services/backup/restic.nix

@@ -113,12 +113,15 @@ in
         };
 
         paths = mkOption {
+          # This is nullable for legacy reasons only. We should consider making it a pure listOf
+          # after some time has passed since this comment was added.
           type = types.nullOr (types.listOf types.str);
-          default = null;
+          default = [ ];
           description = lib.mdDoc ''
-            Which paths to backup.  If null or an empty array, no
+            Which paths to backup, in addition to ones specified via
-            backup command will be run.  This can be used to create a
+            `dynamicFilesFrom`.  If null or an empty array and
-            prune-only job.
+            `dynamicFilesFrom` is also null, no backup command will be run.
+             This can be used to create a prune-only job.
           '';
           example = [
             "/var/lib/postgresql"
@@ -231,7 +234,7 @@ in
           description = lib.mdDoc ''
             A script that produces a list of files to back up.  The
             results of this command are given to the '--files-from'
-            option.
+            option. The result is merged with paths specified via `paths`.
           '';
           example = "find /home/matt/git -type d -name .git";
         };
@@ -310,10 +313,7 @@ in
             resticCmd = "${backup.package}/bin/restic${extraOptions}";
             excludeFlags = optional (backup.exclude != []) "--exclude-file=${pkgs.writeText "exclude-patterns" (concatStringsSep "\n" backup.exclude)}";
             filesFromTmpFile = "/run/restic-backups-${name}/includes";
-            backupPaths =
+            doBackup = (backup.dynamicFilesFrom != null) || (backup.paths != null && backup.paths != []);
-              if (backup.dynamicFilesFrom == null)
-              then optionalString (backup.paths != null) (concatStringsSep " " backup.paths)
-              else "--files-from ${filesFromTmpFile}";
             pruneCmd = optionals (builtins.length backup.pruneOpts > 0) [
               (resticCmd + " forget --prune " + (concatStringsSep " " backup.pruneOpts))
               (resticCmd + " check " + (concatStringsSep " " backup.checkOpts))
@@ -348,7 +348,7 @@ in
             after = [ "network-online.target" ];
             serviceConfig = {
               Type = "oneshot";
-              ExecStart = (optionals (backupPaths != "") [ "${resticCmd} backup ${concatStringsSep " " (backup.extraBackupArgs ++ excludeFlags)} ${backupPaths}" ])
+              ExecStart = (optionals doBackup [ "${resticCmd} backup ${concatStringsSep " " (backup.extraBackupArgs ++ excludeFlags)} --files-from=${filesFromTmpFile}" ])
                 ++ pruneCmd;
               User = backup.user;
               RuntimeDirectory = "restic-backups-${name}";
@@ -366,8 +366,11 @@ in
               ${optionalString (backup.initialize) ''
                 ${resticCmd} snapshots || ${resticCmd} init
               ''}
+              ${optionalString (backup.paths != null && backup.paths != []) ''
+                cat ${pkgs.writeText "staticPaths" (concatStringsSep "\n" backup.paths)} >> ${filesFromTmpFile}
+              ''}
               ${optionalString (backup.dynamicFilesFrom != null) ''
-                ${pkgs.writeScript "dynamicFilesFromScript" backup.dynamicFilesFrom} > ${filesFromTmpFile}
+                ${pkgs.writeScript "dynamicFilesFromScript" backup.dynamicFilesFrom} >> ${filesFromTmpFile}
               ''}
             '';
           } // optionalAttrs (backup.dynamicFilesFrom != null || backup.backupCleanupCommand != null) {

nixos/tests/restic.nix

@@ -21,7 +21,10 @@ import ./make-test-python.nix (
       unpackPhase = "true";
       installPhase = ''
         mkdir $out
-        touch $out/some_file
+        echo some_file > $out/some_file
+        echo some_other_file > $out/some_other_file
+        mkdir $out/a_dir
+        echo a_file > $out/a_dir/a_file
       '';
     };
 
@@ -53,9 +56,13 @@ import ./make-test-python.nix (
               initialize = true;
             };
             remote-from-file-backup = {
-              inherit passwordFile paths exclude pruneOpts;
+              inherit passwordFile exclude pruneOpts;
               initialize = true;
               repositoryFile = pkgs.writeText "repositoryFile" remoteFromFileRepository;
+              paths = [ "/opt/a_dir" ];
+              dynamicFilesFrom = ''
+                find /opt -mindepth 1 -maxdepth 1 ! -name a_dir # all files in /opt except for a_dir
+              '';
             };
             rclonebackup = {
               inherit passwordFile paths exclude pruneOpts;
@@ -123,13 +130,18 @@ import ./make-test-python.nix (
           "systemctl start restic-backups-remote-from-file-backup.service",
           'restic-remote-from-file-backup snapshots --json | ${pkgs.jq}/bin/jq "length | . == 1"',
 
+          # test that restoring that snapshot produces the same directory
+          "mkdir /tmp/restore-2",
+          "${pkgs.restic}/bin/restic -r ${remoteRepository} -p ${passwordFile} restore latest -t /tmp/restore-2",
+          "diff -ru ${testDir} /tmp/restore-2/opt",
+
           # test that rclonebackup produces a snapshot
           "systemctl start restic-backups-rclonebackup.service",
           'restic-rclonebackup snapshots --json | ${pkgs.jq}/bin/jq "length | . == 1"',
 
           # test that custompackage runs both `restic backup` and `restic check` with reasonable commandlines
           "systemctl start restic-backups-custompackage.service",
-          "grep 'backup.* /opt' /root/fake-restic.log",
+          "grep 'backup' /root/fake-restic.log",
           "grep 'check.* --some-check-option' /root/fake-restic.log",
 
           # test that we can create four snapshots in remotebackup and rclonebackup