Merge pull request #10922 from hercules-ci/functional-tests-on-nixos

Run the functional tests in a NixOS environment

.github/workflows/ci.yml

@@ -31,6 +31,23 @@ jobs:
         name: '${{ env.CACHIX_NAME }}'
         signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}'
         authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}'
+    - if: matrix.os == 'ubuntu-latest'
+      run: |
+        free -h
+        swapon --show
+        swap=$(swapon --show --noheadings | head -n 1 | awk '{print $1}')
+        echo "Found swap: $swap"
+        sudo swapoff $swap
+        # resize it (fallocate)
+        sudo fallocate -l 10G $swap
+        sudo mkswap $swap
+        sudo swapon $swap
+        free -h
+        (
+          while sleep 60; do
+            free -h
+          done
+        ) &
     - run: nix --experimental-features 'nix-command flakes' flake check -L
 
   # Steps to test CI automation in your own fork.
@@ -175,4 +192,4 @@ jobs:
       - uses: actions/checkout@v4
       - uses: DeterminateSystems/nix-installer-action@main
       - uses: DeterminateSystems/magic-nix-cache-action@main
-      - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes
+      - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.functional_user

doc/manual/src/contributing/testing.md

@@ -114,6 +114,8 @@ On other platforms they wouldn't be run at all.
 The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`.
 Each test is a bash script.
 
+Functional tests are run during `installCheck` in the `nix` package build, as well as separately from the build, in VM tests.
+
 ### Running the whole test suite
 
 The whole test suite can be run with:
@@ -252,13 +254,30 @@ Regressions are caught, and improvements always show up in code review.
 
 To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`.
 
+### Running functional tests on NixOS
+
+We run the functional tests not just in the build, but also in VM tests.
+This helps us ensure that Nix works correctly on NixOS, and environments that have similar characteristics that are hard to reproduce in a build environment.
+
+The recommended way to run these tests during development is:
+
+```shell
+nix build .#hydraJobs.tests.functional_user.quickBuild
+```
+
+The `quickBuild` attribute configures the test to use a `nix` package that's built without integration tests, so that you can iterate on the tests without performing recompilations due to the changed sources for `installCheck`.
+
+Generally, this build is sufficient, but in nightly or CI we also test the attributes `functional_root` and `functional_trusted`, in which the test suite is run with different levels of authorization.
+
 ## Integration tests
 
 The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute.
 These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup.
 Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on <https://hydra.nixos.org/jobset/nix/master>).
 
-You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`
+You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`.
+
+If you are testing a build of `nix` that you haven't compiled yet, you may iterate faster by appending the `quickBuild` attribute: `nix build .#hydraJobs.tests.{testName}.quickBuild`.
 
 ## Installer tests
 

flake.nix

@@ -227,6 +227,12 @@
               ;
           };
 
+          nix_noTests = final.nix.override {
+            doCheck = false;
+            doInstallCheck = false;
+            installUnitTests = false;
+          };
+
           # See https://github.com/NixOS/nixpkgs/pull/214409
           # Remove when fixed in this flake's nixpkgs
           pre-commit =

maintainers/hydra.nix

@@ -58,13 +58,7 @@ in
     self.packages.${system}.nix.override { enableGC = false; }
   );
 
-  buildNoTests = forAllSystems (system:
+  buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nix_noTests);
-    self.packages.${system}.nix.override {
-      doCheck = false;
-      doInstallCheck = false;
-      installUnitTests = false;
-    }
-  );
 
   # Toggles some settings for better coverage. Windows needs these
   # library combinations, and Debian build Nix with GNU readline too.

tests/functional/add.sh

@@ -31,7 +31,7 @@ test "$hash1" = "sha256:$hash2"
 
 #### New style commands
 
-clearStore
+clearStoreIfPossible
 
 (
     path1=$(nix store add ./dummy)

tests/functional/binary-cache-build-remote.sh

@@ -2,7 +2,9 @@
 
 source common.sh
 
-clearStore
+TODO_NixOS
+
+clearStoreIfPossible
 clearCacheCache
 
 # Fails without remote builders

tests/functional/binary-cache.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 needLocalStore "'--no-require-sigs' can’t be used with the daemon"
 
 # We can produce drvs directly into the binary cache

tests/functional/brotli.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 clearStore
 clearCache
 

tests/functional/build-delete.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 
 # https://github.com/NixOS/nix/issues/6572
 issue_6572_independent_outputs() {

tests/functional/build-dry.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 ###################################################
 # Check that --dry-run isn't confused with read-only mode
 # https://github.com/NixOS/nix/issues/1795

tests/functional/build-remote-input-addressed.sh

@@ -23,7 +23,7 @@ EOF
     chmod +x "$TEST_ROOT/post-build-hook.sh"
     rm -f "$TEST_ROOT/post-hook-counter"
 
-    echo "post-build-hook = $TEST_ROOT/post-build-hook.sh" >> "$NIX_CONF_DIR/nix.conf"
+    echo "post-build-hook = $TEST_ROOT/post-build-hook.sh" >> "$test_nix_conf"
 }
 
 registerBuildHook

tests/functional/build-remote-trustless-should-fail-0.sh

@@ -4,6 +4,7 @@ source common.sh
 
 enableFeatures "daemon-trust-override"
 
+TODO_NixOS
 restartDaemon
 
 requireSandboxSupport

tests/functional/build-remote-trustless-should-pass-2.sh

@@ -4,6 +4,8 @@ source common.sh
 
 enableFeatures "daemon-trust-override"
 
+TODO_NixOS
+
 restartDaemon
 
 # Remote doesn't trust us

tests/functional/build-remote-trustless-should-pass-3.sh

@@ -4,6 +4,7 @@ source common.sh
 
 enableFeatures "daemon-trust-override"
 
+TODO_NixOS
 restartDaemon
 
 # Remote doesn't trusts us, but this is fine because we are only

tests/functional/build.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 
 # Make sure that 'nix build' returns all outputs by default.
 nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status '

tests/functional/ca/common.sh

@@ -2,4 +2,6 @@ source ../common.sh
 
 enableFeatures "ca-derivations"
 
+TODO_NixOS
+
 restartDaemon

tests/functional/case-hack.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 clearStore
 
 rm -rf "$TEST_ROOT/case"

tests/functional/check-refs.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 clearStore
 
 RESULT=$TEST_ROOT/result
@@ -43,13 +45,18 @@ nix-build -o "$RESULT" check-refs.nix -A test7
 # test10 should succeed (no disallowed references).
 nix-build -o "$RESULT" check-refs.nix -A test10
 
-if isDaemonNewer 2.12pre20230103; then
+if ! isTestOnNixOS; then
-    if ! isDaemonNewer 2.16.0; then
+    # If we have full control over our store, we can test some more things.
-        enableFeatures discard-references
+
-        restartDaemon
+    if isDaemonNewer 2.12pre20230103; then
+        if ! isDaemonNewer 2.16.0; then
+            enableFeatures discard-references
+            restartDaemon
+        fi
+
+        # test11 should succeed.
+        test11=$(nix-build -o "$RESULT" check-refs.nix -A test11)
+        [[ -z $(nix-store -q --references "$test11") ]]
     fi
 
-    # test11 should succeed.
-    test11=$(nix-build -o "$RESULT" check-refs.nix -A test11)
-    [[ -z $(nix-store -q --references "$test11") ]]
 fi

tests/functional/check-reqs.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 
 RESULT=$TEST_ROOT/result
 

tests/functional/check.sh

@@ -15,6 +15,8 @@ checkBuildTempDirRemoved ()
 # written to build temp directories to verify created by this instance
 checkBuildId=$(date +%s%N)
 
+TODO_NixOS
+
 clearStore
 
 nix-build dependencies.nix --no-out-link
@@ -76,6 +78,8 @@ grep 'may not be deterministic' $TEST_ROOT/log
 [ "$status" = "104" ]
 if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi
 
+TODO_NixOS
+
 clearStore
 
 path=$(nix-build check.nix -A fetchurl --no-out-link)

tests/functional/chroot-store.sh

@@ -39,6 +39,8 @@ EOF
 
     cp simple.nix shell.nix simple.builder.sh config.nix "$flakeDir/"
 
+    TODO_NixOS
+
     outPath=$(nix build --print-out-paths --no-link --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store "$TEST_ROOT/x" path:"$flakeDir")
 
     [[ $outPath =~ ^/nix2/store/.*-simple$ ]]

tests/functional/common/init.sh

@@ -1,5 +1,31 @@
 # shellcheck shell=bash
 
+# for shellcheck
+: "${test_nix_conf_dir?}" "${test_nix_conf?}"
+
+if isTestOnNixOS; then
+
+  mkdir -p "$test_nix_conf_dir" "$TEST_HOME"
+
+  export NIX_USER_CONF_FILES="$test_nix_conf_dir/nix.conf"
+  mkdir -p "$test_nix_conf_dir" "$TEST_HOME"
+  ! test -e "$test_nix_conf"
+  cat > "$test_nix_conf_dir/nix.conf" <<EOF
+# TODO: this is not needed for all tests and prevents stable commands from be tested in isolation
+experimental-features = nix-command flakes
+flake-registry = $TEST_ROOT/registry.json
+show-trace = true
+EOF
+
+  # When we're doing everything in the same store, we need to bring
+  # dependencies into context.
+  sed -i "$(dirname "${BASH_SOURCE[0]}")"/../config.nix \
+    -e 's^\(shell\) = "/nix/store/\([^/]*\)/\(.*\)";^\1 = builtins.appendContext "/nix/store/\2" { "/nix/store/\2".path = true; } + "/\3";^' \
+    -e 's^\(path\) = "/nix/store/\([^/]*\)/\(.*\)";^\1 = builtins.appendContext "/nix/store/\2" { "/nix/store/\2".path = true; } + "/\3";^' \
+    ;
+
+else
+
 test -n "$TEST_ROOT"
 # We would delete any daemon socket, so let's stop the daemon first.
 killDaemon
@@ -41,3 +67,5 @@ EOF
 nix-store --init
 # Sanity check
 test -e "$NIX_STATE_DIR"/db/db.sqlite
+
+fi # !isTestOnNixOS

tests/functional/common/subst-vars.sh.in

@@ -9,7 +9,6 @@ export coreutils=@coreutils@
 #lsof=@lsof@
 
 export dot=@dot@
-export SHELL="@bash@"
 export PAGER=cat
 export busybox="@sandbox_shell@"
 
@@ -18,4 +17,9 @@ export system=@system@
 
 export BUILD_SHARED_LIBS=@BUILD_SHARED_LIBS@
 
+if ! isTestOnNixOS; then
+  export SHELL="@bash@"
+  export PATH=@bindir@:$PATH
+fi
+
 fi

tests/functional/common/vars-and-functions.sh

@@ -6,6 +6,15 @@ if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then
 
 COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1
 
+isTestOnNixOS() {
+  [[ "${isTestOnNixOS:-}" == 1 ]]
+}
+
+die() {
+  echo "unexpected fatal error: $*" >&2
+  exit 1
+}
+
 set +x
 
 commonDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")"
@@ -17,27 +26,35 @@ source "$commonDir/subst-vars.sh"
 source "$commonDir/paths.sh"
 source "$commonDir/test-root.sh"
 
-export NIX_STORE_DIR
+test_nix_conf_dir=$TEST_ROOT/etc
-if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then
+test_nix_conf=$test_nix_conf_dir/nix.conf
-    # Maybe the build directory is symlinked.
+
-    export NIX_IGNORE_SYMLINK_STORE=1
-    NIX_STORE_DIR=$TEST_ROOT/store
-fi
-export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
-export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
-export NIX_STATE_DIR=$TEST_ROOT/var/nix
-export NIX_CONF_DIR=$TEST_ROOT/etc
-export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket
-unset NIX_USER_CONF_FILES
-export _NIX_TEST_SHARED=$TEST_ROOT/shared
-if [[ -n $NIX_STORE ]]; then
-    export _NIX_TEST_NO_SANDBOX=1
-fi
-export _NIX_IN_TEST=$TEST_ROOT/shared
-export _NIX_TEST_NO_LSOF=1
-export NIX_REMOTE=${NIX_REMOTE_-}
-unset NIX_PATH
 export TEST_HOME=$TEST_ROOT/test-home
+
+if ! isTestOnNixOS; then
+  export NIX_STORE_DIR
+  if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then
+      # Maybe the build directory is symlinked.
+      export NIX_IGNORE_SYMLINK_STORE=1
+      NIX_STORE_DIR=$TEST_ROOT/store
+  fi
+  export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
+  export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
+  export NIX_STATE_DIR=$TEST_ROOT/var/nix
+  export NIX_CONF_DIR=$test_nix_conf_dir
+  export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket
+  unset NIX_USER_CONF_FILES
+  export _NIX_TEST_SHARED=$TEST_ROOT/shared
+  if [[ -n $NIX_STORE ]]; then
+      export _NIX_TEST_NO_SANDBOX=1
+  fi
+  export _NIX_IN_TEST=$TEST_ROOT/shared
+  export _NIX_TEST_NO_LSOF=1
+  export NIX_REMOTE=${NIX_REMOTE_-}
+
+fi # ! isTestOnNixOS
+
+unset NIX_PATH
 export HOME=$TEST_HOME
 unset XDG_STATE_HOME
 unset XDG_DATA_HOME
@@ -59,7 +76,25 @@ clearProfiles() {
     rm -rf "$profiles"
 }
 
+# Clear the store, but do not fail if we're in an environment where we can't.
+# This allows the test to run in a NixOS test environment, where we use the system store.
+# See doc/manual/src/contributing/testing.md / Running functional tests on NixOS.
+clearStoreIfPossible() {
+    if isTestOnNixOS; then
+        echo "clearStoreIfPossible: Not clearing store, because we're on NixOS. Moving on."
+    else
+        doClearStore
+    fi
+}
+
 clearStore() {
+    if isTestOnNixOS; then
+      die "clearStore: not supported when testing on NixOS. If not essential, call clearStoreIfPossible. If really needed, add conditionals; e.g. if ! isTestOnNixOS; then ..."
+    fi
+    doClearStore
+}
+
+doClearStore() {
     echo "clearing store..."
     chmod -R +w "$NIX_STORE_DIR"
     rm -rf "$NIX_STORE_DIR"
@@ -78,6 +113,10 @@ clearCacheCache() {
 }
 
 startDaemon() {
+    if isTestOnNixOS; then
+      die "startDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..."
+    fi
+
     # Don’t start the daemon twice, as this would just make it loop indefinitely
     if [[ "${_NIX_TEST_DAEMON_PID-}" != '' ]]; then
         return
@@ -104,6 +143,10 @@ startDaemon() {
 }
 
 killDaemon() {
+    if isTestOnNixOS; then
+      die "killDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..."
+    fi
+
     # Don’t fail trying to stop a non-existant daemon twice
     if [[ "${_NIX_TEST_DAEMON_PID-}" == '' ]]; then
         return
@@ -124,6 +167,10 @@ killDaemon() {
 }
 
 restartDaemon() {
+    if isTestOnNixOS; then
+      die "restartDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..."
+    fi
+
     [[ -z "${_NIX_TEST_DAEMON_PID:-}" ]] && return 0
 
     killDaemon
@@ -146,6 +193,12 @@ skipTest () {
     exit 99
 }
 
+TODO_NixOS() {
+    if isTestOnNixOS; then
+        skipTest "This test has not been adapted for NixOS yet"
+    fi
+}
+
 requireDaemonNewerThan () {
     isDaemonNewer "$1" || skipTest "Daemon is too old"
 }
@@ -163,7 +216,7 @@ requireGit() {
 }
 
 fail() {
-    echo "$1" >&2
+    echo "test failed: $*" >&2
     exit 1
 }
 
@@ -228,7 +281,7 @@ buggyNeedLocalStore() {
 
 enableFeatures() {
     local features="$1"
-    sed -i 's/experimental-features .*/& '"$features"'/' "$NIX_CONF_DIR"/nix.conf
+    sed -i 's/experimental-features .*/& '"$features"'/' "$test_nix_conf_dir"/nix.conf
 }
 
 set -x

tests/functional/compression-levels.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 clearCache
 
 outPath=$(nix-build dependencies.nix --no-out-link)

tests/functional/config.sh

@@ -28,6 +28,8 @@ nix registry remove userhome-with-xdg
 # Assert the .config folder hasn't been created.
 [ ! -e "$HOME/.config" ]
 
+TODO_NixOS # Very specific test setup not compatible with the NixOS test environment?
+
 # Test that files are loaded from XDG by default
 export XDG_CONFIG_HOME=$TEST_ROOT/confighome
 export XDG_CONFIG_DIRS=$TEST_ROOT/dir1:$TEST_ROOT/dir2

tests/functional/db-migration.sh

@@ -10,6 +10,8 @@ if [[ -z "${NIX_DAEMON_PACKAGE-}" ]]; then
     skipTest "not using the Nix daemon"
 fi
 
+TODO_NixOS
+
 killDaemon
 
 # Fill the db using the older Nix

tests/functional/debugger.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 
 # regression #9932
 echo ":env" | expect 1 nix eval --debugger --expr '(_: throw "oh snap") 42'

tests/functional/dependencies.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 
 drvPath=$(nix-instantiate dependencies.nix)
 
@@ -65,6 +65,8 @@ drvPath2=$(nix-instantiate dependencies.nix --argstr hashInvalidator yay)
 # now --valid-derivers returns both
 test "$(nix-store -q --valid-derivers "$outPath" | sort)" = "$(sort <<< "$drvPath"$'\n'"$drvPath2")"
 
+TODO_NixOS # The following --delete fails, because it seems to be still alive. This might be caused by a different test using the same path. We should try make the derivations unique, e.g. naming after tests, and adding a timestamp that's constant for that test script run.
+
 # check that nix-store --valid-derivers only returns existing drv
 nix-store --delete "$drvPath"
 test "$(nix-store -q --valid-derivers "$outPath")" = "$drvPath2"

tests/functional/dump-db.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 needLocalStore "--dump-db requires a local store"
 
 clearStore

tests/functional/dyn-drv/common.sh

@@ -5,4 +5,6 @@ requireDaemonNewerThan "2.16.0pre20230419"
 
 enableFeatures "ca-derivations dynamic-derivations"
 
+TODO_NixOS
+
 restartDaemon

tests/functional/eval-store.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 # Using `--eval-store` with the daemon will eventually copy everything
 # to the build store, invalidating most of the tests here
 needLocalStore "“--eval-store” doesn't achieve much with the daemon"

tests/functional/eval.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 
 testStdinHeredoc=$(nix eval -f - <<EOF
 {

tests/functional/export-graph.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 clearStore
 clearProfiles
 

tests/functional/export.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 clearStore
 
 outPath=$(nix-build dependencies.nix --no-out-link)

tests/functional/fetchClosure.sh

@@ -4,6 +4,8 @@ source common.sh
 
 enableFeatures "fetch-closure"
 
+TODO_NixOS
+
 clearStore
 clearCacheCache
 

tests/functional/fetchGit.sh

@@ -4,7 +4,7 @@ source common.sh
 
 requireGit
 
-clearStore
+clearStoreIfPossible
 
 # Intentionally not in a canonical form
 # See https://github.com/NixOS/nix/issues/6195

tests/functional/fetchGitRefs.sh

@@ -4,7 +4,7 @@ source common.sh
 
 requireGit
 
-clearStore
+clearStoreIfPossible
 
 repo="$TEST_ROOT/git"
 

tests/functional/fetchGitSubmodules.sh

@@ -6,7 +6,7 @@ set -u
 
 requireGit
 
-clearStore
+clearStoreIfPossible
 
 rootRepo=$TEST_ROOT/gitSubmodulesRoot
 subRepo=$TEST_ROOT/gitSubmodulesSub

tests/functional/fetchGitVerification.sh

@@ -7,7 +7,7 @@ requireGit
 
 enableFeatures "verified-fetches"
 
-clearStore
+clearStoreIfPossible
 
 repo="$TEST_ROOT/git"
 

tests/functional/fetchMercurial.sh

@@ -4,6 +4,8 @@ source common.sh
 
 [[ $(type -p hg) ]] || skipTest "Mercurial not installed"
 
+TODO_NixOS
+
 clearStore
 
 # Intentionally not in a canonical form

tests/functional/fetchTree-file.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 clearStore
 
 cd "$TEST_ROOT"

tests/functional/fetchurl.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 clearStore
 
 # Test fetching a flat file.

tests/functional/fixed.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 clearStore
 
 path=$(nix-store -q $(nix-instantiate fixed.nix -A good.0))

tests/functional/flakes/config.sh

@@ -28,6 +28,7 @@ EOF
 # Without --accept-flake-config, the post hook should not run.
 nix build < /dev/null
 (! [[ -f post-hook-ran ]])
+TODO_NixOS
 clearStore
 
 nix build --accept-flake-config

tests/functional/flakes/develop.sh

@@ -2,6 +2,8 @@
 
 source ../common.sh
 
+TODO_NixOS
+
 clearStore
 rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local
 

tests/functional/flakes/flake-in-submodule.sh

@@ -15,6 +15,8 @@ source common.sh
 
 requireGit
 
+TODO_NixOS
+
 clearStore
 
 # Submodules can't be fetched locally by default.

tests/functional/flakes/flakes.sh

@@ -2,6 +2,8 @@
 
 source ./common.sh
 
+TODO_NixOS
+
 requireGit
 
 clearStore

tests/functional/flakes/run.sh

@@ -2,6 +2,8 @@
 
 source ../common.sh
 
+TODO_NixOS
+
 clearStore
 rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local
 cp ../shell-hello.nix ../config.nix $TEST_HOME

tests/functional/flakes/search-root.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 
 writeSimpleFlake "$TEST_HOME"
 cd "$TEST_HOME"

tests/functional/fmt.sh

@@ -2,7 +2,9 @@
 
 source common.sh
 
-clearStore
+TODO_NixOS # Provide a `shell` variable. Try not to `export` it, perhaps.
+
+clearStoreIfPossible
 rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local
 
 cp ./simple.nix ./simple.builder.sh ./fmt.simple.sh ./config.nix $TEST_HOME
@@ -31,5 +33,3 @@ EOF
 nix fmt ./file ./folder | grep 'Formatting: ./file ./folder'
 nix flake check
 nix flake show | grep -P "package 'formatter'"
-
-clearStore

tests/functional/gc-auto.sh

@@ -4,6 +4,8 @@ source common.sh
 
 needLocalStore "“min-free” and “max-free” are daemon options"
 
+TODO_NixOS
+
 clearStore
 
 garbage1=$(nix store add-path --name garbage1 ./nar-access.sh)

tests/functional/gc-concurrent.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 clearStore
 
 lockFifo1=$TEST_ROOT/test1.fifo

tests/functional/gc-non-blocking.sh

@@ -4,6 +4,8 @@
 # parallel with it.
 source common.sh
 
+TODO_NixOS
+
 needLocalStore "the GC test needs a synchronisation point"
 
 clearStore

tests/functional/gc-runtime.sh

@@ -11,6 +11,8 @@ esac
 
 set -m # enable job control, needed for kill
 
+TODO_NixOS
+
 profiles="$NIX_STATE_DIR"/profiles
 rm -rf "$profiles"
 

tests/functional/gc.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 clearStore
 
 drvPath=$(nix-instantiate dependencies.nix)

tests/functional/git-hashing/common.sh

@@ -1,5 +1,7 @@
 source ../common.sh
 
+TODO_NixOS # Need to enable git hashing feature and make sure test is ok for store we don't clear
+
 clearStore
 clearCache
 

tests/functional/help.sh

@@ -2,8 +2,6 @@
 
 source common.sh
 
-clearStore
-
 # test help output
 
 nix-build --help

tests/functional/import-derivation.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 
 if nix-instantiate --readonly-mode ./import-derivation.nix; then
     echo "read-only evaluation of an imported derivation unexpectedly failed"

tests/functional/impure-derivations.sh

@@ -4,10 +4,12 @@ source common.sh
 
 requireDaemonNewerThan "2.8pre20220311"
 
+TODO_NixOS
+
 enableFeatures "ca-derivations impure-derivations"
 restartDaemon
 
-clearStore
+clearStoreIfPossible
 
 # Basic test of impure derivations: building one a second time should not use the previous result.
 printf 0 > $TEST_ROOT/counter

tests/functional/impure-env.sh

@@ -5,6 +5,8 @@ source common.sh
 # Needs the config option 'impure-env' to work
 requireDaemonNewerThan "2.19.0"
 
+TODO_NixOS
+
 enableFeatures "configurable-impure-env"
 restartDaemon
 
@@ -20,13 +22,13 @@ startDaemon
 
 varTest env_name value --impure-env env_name=value
 
-echo 'impure-env = set_in_config=config_value' >> "$NIX_CONF_DIR/nix.conf"
+echo 'impure-env = set_in_config=config_value' >> "$test_nix_conf"
 set_in_config=daemon_value restartDaemon
 
 varTest set_in_config config_value
 varTest set_in_config client_value --impure-env set_in_config=client_value
 
-sed -i -e '/^trusted-users =/d' "$NIX_CONF_DIR/nix.conf"
+sed -i -e '/^trusted-users =/d' "$test_nix_conf"
 
 env_name=daemon_value restartDaemon
 

tests/functional/linux-sandbox.sh

@@ -4,6 +4,8 @@ source common.sh
 
 needLocalStore "the sandbox only runs on the builder side, so it makes no sense to test it with the daemon"
 
+TODO_NixOS
+
 clearStore
 
 requireSandboxSupport

tests/functional/local-overlay-store/bad-uris.sh

@@ -15,6 +15,8 @@ declare -a storesBad=(
     "$storeBadRoot" "$storeBadLower" "$storeBadUpper"
 )
 
+TODO_NixOS
+
 for i in "${storesBad[@]}"; do
     echo $i
     unshare --mount --map-root-user bash <<EOF

tests/functional/local-overlay-store/common.sh

@@ -1,5 +1,7 @@
 source ../common/vars-and-functions.sh
 
+TODO_NixOS
+
 # The new Linux mount interface does not seem to support remounting
 # OverlayFS mount points.
 #
@@ -31,7 +33,7 @@ requireEnvironment () {
 }
 
 addConfig () {
-    echo "$1" >> "$NIX_CONF_DIR/nix.conf"
+    echo "$1" >> "$test_nix_conf"
 }
 
 setupConfig () {

tests/functional/logging.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 clearStore
 
 path=$(nix-build dependencies.nix --no-out-link)

tests/functional/multiple-outputs.sh

@@ -2,7 +2,9 @@
 
 source common.sh
 
-clearStore
+TODO_NixOS
+
+clearStoreIfPossible
 
 rm -f $TEST_ROOT/result*
 

tests/functional/nar-access.sh

@@ -27,6 +27,8 @@ diff -u baz.cat-nar $storePath/foo/baz
 nix store cat $storePath/foo/baz > baz.cat-nar
 diff -u baz.cat-nar $storePath/foo/baz
 
+TODO_NixOS
+
 # Check that 'nix store cat' fails on invalid store paths.
 invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo"
 cp -r $storePath $invalidPath

tests/functional/nested-sandboxing.sh

@@ -4,6 +4,8 @@ source common.sh
 # This test is run by `tests/functional/nested-sandboxing/runner.nix` in an extra layer of sandboxing.
 [[ -d /nix/store ]] || skipTest "running this test without Nix's deps being drawn from /nix/store is not yet supported"
 
+TODO_NixOS
+
 requireSandboxSupport
 
 source ./nested-sandboxing/command.sh

tests/functional/nix-build.sh

@@ -2,7 +2,9 @@
 
 source common.sh
 
-clearStore
+TODO_NixOS
+
+clearStoreIfPossible
 
 outPath=$(nix-build dependencies.nix -o $TEST_ROOT/result)
 test "$(cat $TEST_ROOT/result/foobar)" = FOOBAR

tests/functional/nix-collect-garbage-d.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 clearStore
 
 ## Test `nix-collect-garbage -d`

tests/functional/nix-copy-ssh-common.sh

@@ -2,6 +2,8 @@ proto=$1
 shift
 (( $# == 0 ))
 
+TODO_NixOS
+
 clearStore
 clearCache
 

tests/functional/nix-copy-ssh-ng.sh

@@ -4,6 +4,8 @@ source common.sh
 
 source nix-copy-ssh-common.sh "ssh-ng"
 
+TODO_NixOS
+
 clearStore
 clearRemoteStore
 

tests/functional/nix-profile.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 clearStore
 clearProfiles
 

tests/functional/nix-shell.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 
 if [[ -n ${CONTENT_ADDRESSED:-} ]]; then
     shellDotNix="$PWD/ca-shell.nix"

tests/functional/optimise-store.sh

@@ -2,11 +2,14 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 
 outPath1=$(echo 'with import ./config.nix; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store)
 outPath2=$(echo 'with import ./config.nix; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store)
 
+TODO_NixOS # ignoring the client-specified setting 'auto-optimise-store', because it is a restricted setting and you are not a trusted user
+  # TODO: only continue when trusted user or root
+
 inode1="$(stat --format=%i $outPath1/foo)"
 inode2="$(stat --format=%i $outPath2/foo)"
 if [ "$inode1" != "$inode2" ]; then

tests/functional/output-normalization.sh

@@ -3,6 +3,7 @@
 source common.sh
 
 testNormalization () {
+    TODO_NixOS
     clearStore
     outPath=$(nix-build ./simple.nix --no-out-link)
     test "$(stat -c %Y $outPath)" -eq 1

tests/functional/parallel.sh

@@ -4,6 +4,8 @@ source common.sh
 # First, test that -jN performs builds in parallel.
 echo "testing nix-build -j..."
 
+TODO_NixOS
+
 clearStore
 
 rm -f $_NIX_TEST_SHARED.cur $_NIX_TEST_SHARED.max

tests/functional/pass-as-file.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 
 outPath=$(nix-build --no-out-link -E "
 with import ./config.nix;

tests/functional/placeholders.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 
 nix-build --no-out-link -E '
   with import ./config.nix;

tests/functional/post-hook.sh

@@ -2,12 +2,14 @@
 
 source common.sh
 
+TODO_NixOS
+
 clearStore
 
 rm -f $TEST_ROOT/result
 
 export REMOTE_STORE=file:$TEST_ROOT/remote_store
-echo 'require-sigs = false' >> $NIX_CONF_DIR/nix.conf
+echo 'require-sigs = false' >> $test_nix_conf
 
 restartDaemon
 

tests/functional/pure-eval.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 
 nix eval --expr 'assert 1 + 2 == 3; true'
 

tests/functional/read-only-store.sh

@@ -6,6 +6,8 @@ enableFeatures "read-only-local-store"
 
 needLocalStore "cannot open store read-only when daemon has already opened it writeable"
 
+TODO_NixOS
+
 clearStore
 
 happy () {

tests/functional/readfile-context.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS # NixOS doesn't provide $NIX_STATE_DIR (and shouldn't)
+
 clearStore
 
 outPath=$(nix-build --no-out-link readfile-context.nix)

tests/functional/recursive.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS # can't enable a sandbox feature easily
+
 enableFeatures 'recursive-nix'
 restartDaemon
 

tests/functional/referrers.sh

@@ -4,6 +4,8 @@ source common.sh
 
 needLocalStore "uses some low-level store manipulations that aren’t available through the daemon"
 
+TODO_NixOS
+
 clearStore
 
 max=500

tests/functional/remote-store.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 clearStore
 
 # Ensure "fake ssh" remote store works just as legacy fake ssh would.

tests/functional/repair.sh

@@ -4,6 +4,8 @@ source common.sh
 
 needLocalStore "--repair needs a local store"
 
+TODO_NixOS
+
 clearStore
 
 path=$(nix-build dependencies.nix -o $TEST_ROOT/result)

tests/functional/repl.sh

@@ -22,6 +22,8 @@ replUndefinedVariable="
 import $testDir/undefined-variable.nix
 "
 
+TODO_NixOS
+
 testRepl () {
     local nixArgs
     nixArgs=("$@")

tests/functional/restricted.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 
 nix-instantiate --restrict-eval --eval -E '1 + 2'
 (! nix-instantiate --eval --restrict-eval ./restricted.nix)

tests/functional/search.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 clearCache
 
 (( $(nix search -f search.nix '' hello | wc -l) > 0 ))

tests/functional/secure-drv-outputs.sh

@@ -6,6 +6,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 clearStore
 
 startDaemon

tests/functional/selfref-gc.sh

@@ -4,7 +4,7 @@ source common.sh
 
 requireDaemonNewerThan "2.6.0pre20211215"
 
-clearStore
+clearStoreIfPossible
 
 nix-build --no-out-link -E '
   with import ./config.nix;

tests/functional/shell.sh

@@ -2,6 +2,8 @@
 
 source common.sh
 
+TODO_NixOS
+
 clearStore
 clearCache
 

tests/functional/signing.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 clearCache
 
 nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1
@@ -16,6 +16,7 @@ outPath=$(nix-build dependencies.nix --no-out-link --secret-key-files "$TEST_ROO
 # Verify that the path got signed.
 info=$(nix path-info --json $outPath)
 echo $info | jq -e '.[] | .ultimate == true'
+TODO_NixOS # looks like an actual bug? Following line fails on NixOS:
 echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))'
 echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))'
 

tests/functional/simple.sh

@@ -17,6 +17,8 @@ echo "output path is $outPath"
 text=$(cat "$outPath/hello")
 if test "$text" != "Hello World!"; then exit 1; fi
 
+TODO_NixOS
+
 # Directed delete: $outPath is not reachable from a root, so it should
 # be deleteable.
 nix-store --delete $outPath

tests/functional/store-info.sh

@@ -16,4 +16,6 @@ fi
 expect 127 NIX_REMOTE=unix:$PWD/store nix store info || \
     fail "nix store info on a non-existent store should fail"
 
+TODO_NixOS
+
 [[ "$(echo "$STORE_INFO_JSON" | jq -r ".url")" == "${NIX_REMOTE:-local}" ]]

tests/functional/structured-attrs.sh

@@ -6,7 +6,7 @@ source common.sh
 # tests for the older versions
 requireDaemonNewerThan "2.4pre20210712"
 
-clearStore
+clearStoreIfPossible
 
 rm -f $TEST_ROOT/result
 
@@ -21,6 +21,8 @@ env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \
 
 nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"'
 
+TODO_NixOS # following line fails.
+
 # `nix develop` is a slightly special way of dealing with environment vars, it parses
 # these from a shell-file exported from a derivation. This is to test especially `outputs`
 # (which is an associative array in thsi case) being fine.

tests/functional/suggestions.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 
 cd "$TEST_HOME"
 

tests/functional/supplementary-groups.sh

@@ -7,6 +7,8 @@ requireSandboxSupport
 if ! command -p -v unshare; then skipTest "Need unshare"; fi
 needLocalStore "The test uses --store always so we would just be bypassing the daemon"
 
+TODO_NixOS
+
 unshare --mount --map-root-user bash <<EOF
   source common.sh
 

tests/functional/tarball.sh

@@ -2,7 +2,7 @@
 
 source common.sh
 
-clearStore
+clearStoreIfPossible
 
 rm -rf $TEST_HOME