diff --git a/.editorconfig b/.editorconfig index 86360e658..e1c8bae39 100644 --- a/.editorconfig +++ b/.editorconfig @@ -4,20 +4,20 @@ # Top-most EditorConfig file root = true -# Unix-style newlines with a newline ending every file, utf-8 charset +# Unix-style newlines with a newline ending every file, UTF-8 charset [*] end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true charset = utf-8 -# Match nix files, set indent to spaces with width of two +# Match Nix files, set indent to spaces with width of two [*.nix] indent_style = space indent_size = 2 -# Match c++/shell/perl, set indent to spaces with width of four -[*.{hpp,cc,hh,sh,pl,xs}] +# Match C++/C/shell/Perl, set indent to spaces with width of four +[*.{hpp,cc,hh,c,h,sh,pl,xs}] indent_style = space indent_size = 4 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1aa3b776e..103ce4ff4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,6 +31,23 @@ jobs: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + - if: matrix.os == 'ubuntu-latest' + run: | + free -h + swapon --show + swap=$(swapon --show --noheadings | head -n 1 | awk '{print $1}') + echo "Found swap: $swap" + sudo swapoff $swap + # resize it (fallocate) + sudo fallocate -l 10G $swap + sudo mkswap $swap + sudo swapon $swap + free -h + ( + while sleep 60; do + free -h + done + ) & - run: nix --experimental-features 'nix-command flakes' flake check -L # Steps to test CI automation in your own fork. @@ -175,4 +192,16 @@ jobs: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main - uses: DeterminateSystems/magic-nix-cache-action@main - - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes + - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.functional_user + + meson_build: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main + - run: nix build -L .#hydraJobs.build.{nix-fetchers,nix-store,nix-util}.$(nix-instantiate --eval --expr builtins.currentSystem | sed -e 's/"//g') diff --git a/dep-patches/boehmgc-traceable_allocator-public.diff b/dep-patches/boehmgc-traceable_allocator-public.diff deleted file mode 100644 index 903c707a6..000000000 --- a/dep-patches/boehmgc-traceable_allocator-public.diff +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/include/gc_allocator.h b/include/gc_allocator.h -index 597c7f13..587286be 100644 ---- a/include/gc_allocator.h -+++ b/include/gc_allocator.h -@@ -312,6 +312,7 @@ public: - - template<> - class traceable_allocator { -+public: - typedef size_t size_type; - typedef ptrdiff_t difference_type; - typedef void* pointer; diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 633932e55..f7b479106 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -1,7 +1,7 @@ // redirect rules for URL fragments (client-side) to prevent link rot. // this must be done on the client side, as web servers do not see the fragment part of the URL. // it will only work with JavaScript enabled in the browser, but this is the best we can do here. -// see ./_redirects for path redirects (client-side) +// see src/_redirects for path redirects (server-side) // redirects are declared as follows: // each entry has as its key a path matching the requested URL path, relative to the mdBook document root. diff --git a/doc/manual/src/_redirects b/doc/manual/src/_redirects index a04a36f1e..7f8edca8e 100644 --- a/doc/manual/src/_redirects +++ b/doc/manual/src/_redirects @@ -1,5 +1,5 @@ # redirect rules for paths (server-side) to prevent link rot. -# see ./redirects.js for redirects based on URL fragments (client-side) +# see ../redirects.js for redirects based on URL fragments (client-side) # # concrete user story this supports: # - user finds URL to the manual for Nix x.y diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index 88b3c5cd9..717deabd7 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -114,6 +114,8 @@ On other platforms they wouldn't be run at all. The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. Each test is a bash script. +Functional tests are run during `installCheck` in the `nix` package build, as well as separately from the build, in VM tests. + ### Running the whole test suite The whole test suite can be run with: @@ -252,13 +254,30 @@ Regressions are caught, and improvements always show up in code review. To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. +### Running functional tests on NixOS + +We run the functional tests not just in the build, but also in VM tests. +This helps us ensure that Nix works correctly on NixOS, and environments that have similar characteristics that are hard to reproduce in a build environment. + +The recommended way to run these tests during development is: + +```shell +nix build .#hydraJobs.tests.functional_user.quickBuild +``` + +The `quickBuild` attribute configures the test to use a `nix` package that's built without integration tests, so that you can iterate on the tests without performing recompilations due to the changed sources for `installCheck`. + +Generally, this build is sufficient, but in nightly or CI we also test the attributes `functional_root` and `functional_trusted`, in which the test suite is run with different levels of authorization. + ## Integration tests The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). -You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`. + +If you are testing a build of `nix` that you haven't compiled yet, you may iterate faster by appending the `quickBuild` attribute: `nix build .#hydraJobs.tests.{testName}.quickBuild`. ## Installer tests diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 70ec06f57..e916c7360 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -302,6 +302,12 @@ Derivations can declare some infrequently used optional attributes. (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. + > **Warning** + > + > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize. + will have no effect. + - [`outputChecks`]{#adv-attr-outputChecks}\ When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks` attribute allows defining checks per-output. diff --git a/flake.nix b/flake.nix index 4e7363cd6..582a946c4 100644 --- a/flake.nix +++ b/flake.nix @@ -58,6 +58,18 @@ "stdenv" ]; + /** + `flatMapAttrs attrs f` applies `f` to each attribute in `attrs` and + merges the results into a single attribute set. + + This can be nested to form a build matrix where all the attributes + generated by the innermost `f` are returned as is. + (Provided that the names are unique.) + + See https://nixos.org/manual/nixpkgs/stable/index.html#function-library-lib.attrsets.concatMapAttrs + */ + flatMapAttrs = attrs: f: lib.concatMapAttrs f attrs; + forAllSystems = lib.genAttrs systems; forAllCrossSystems = lib.genAttrs crossSystems; @@ -117,114 +129,25 @@ { nixStable = prev.nix; - default-busybox-sandbox-shell = final.busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y + # A new scope, so that we can use `callPackage` to inject our own interdependencies + # without "polluting" the top level "`pkgs`" attrset. + # This also has the benefit of providing us with a distinct set of packages + # we can iterate over. + nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix); - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }; - - libgit2-nix = final.libgit2.overrideAttrs (attrs: { - src = libgit2; - version = libgit2.lastModifiedDate; - cmakeFlags = attrs.cmakeFlags or [] - ++ [ "-DUSE_SSH=exec" ]; + # The dependencies are in their own scope, so that they don't have to be + # in Nixpkgs top level `pkgs` or `nixComponents`. + nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { + inherit inputs stdenv versionSuffix; + pkgs = final; }); - boehmgc-nix = final.boehmgc.override { - enableLargeConfig = true; - }; + nix = final.nixComponents.nix; - libseccomp-nix = final.libseccomp.overrideAttrs (_: rec { - version = "2.5.5"; - src = final.fetchurl { - url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; - hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; - }; - }); - - nix-util = final.callPackage ./src/libutil/package.nix { - inherit - fileset - stdenv - officialRelease - versionSuffix - ; - }; - - nix-store = final.callPackage ./src/libstore/package.nix { - inherit - fileset - stdenv - officialRelease - versionSuffix - ; - libseccomp = final.libseccomp-nix; - busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; - }; - - nix-fetchers = final.callPackage ./src/libfetchers/package.nix { - inherit - fileset - stdenv - officialRelease - versionSuffix - ; - }; - - nix = - final.callPackage ./package.nix { - inherit - fileset - stdenv - officialRelease - versionSuffix - ; - boehmgc = final.boehmgc-nix; - libgit2 = final.libgit2-nix; - libseccomp = final.libseccomp-nix; - busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; - }; - - nix-perl-bindings = final.callPackage ./src/perl/package.nix { - inherit - fileset - stdenv - versionSuffix - ; - }; - - nix-internal-api-docs = final.callPackage ./src/internal-api-docs/package.nix { - inherit - fileset - stdenv - versionSuffix - ; - }; - - nix-external-api-docs = final.callPackage ./src/external-api-docs/package.nix { - inherit - fileset - stdenv - versionSuffix - ; + nix_noTests = final.nix.override { + doCheck = false; + doInstallCheck = false; + installUnitTests = false; }; # See https://github.com/NixOS/nixpkgs/pull/214409 @@ -241,7 +164,7 @@ # 'nix-perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); - hydraJobs = import ./maintainers/hydra.nix { + hydraJobs = import ./packaging/hydra.nix { inherit inputs binaryTarball @@ -275,39 +198,61 @@ # the old build system is gone and we are back to one build # system, we should reenable this. #perlBindings = self.hydraJobs.perlBindings.${system}; - } // devFlake.checks.${system} or {} + } + # Add "passthru" tests + // flatMapAttrs ({ + "" = nixpkgsFor.${system}.native; + } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) { + # TODO: enable static builds for darwin, blocked on: + # https://github.com/NixOS/nixpkgs/issues/320448 + "static-" = nixpkgsFor.${system}.static; + }) + (nixpkgsPrefix: nixpkgs: + flatMapAttrs nixpkgs.nixComponents + (pkgName: pkg: + flatMapAttrs pkg.tests or {} + (testName: test: { + "${nixpkgsPrefix}${pkgName}-${testName}" = test; + }) + ) + ) + // devFlake.checks.${system} or {} ); - packages = forAllSystems (system: { - inherit (nixpkgsFor.${system}.native) - changelog-d; - default = self.packages.${system}.nix; - nix-internal-api-docs = nixpkgsFor.${system}.native.nix-internal-api-docs; - nix-external-api-docs = nixpkgsFor.${system}.native.nix-external-api-docs; - } // lib.concatMapAttrs - # We need to flatten recursive attribute sets of derivations to pass `flake check`. - (pkgName: {}: { - "${pkgName}" = nixpkgsFor.${system}.native.${pkgName}; - "${pkgName}-static" = nixpkgsFor.${system}.static.${pkgName}; - } // lib.concatMapAttrs - (crossSystem: {}: { - "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.${pkgName}; - }) - (lib.genAttrs crossSystems (_: { })) - // lib.concatMapAttrs - (stdenvName: {}: { - "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".${pkgName}; - }) - (lib.genAttrs stdenvs (_: { }))) - { - "nix" = { }; - # Temporarily disabled because GitHub Actions OOM issues. Once - # the old build system is gone and we are back to one build - # system, we should reenable these. - #"nix-util" = { }; - #"nix-store" = { }; - #"nix-fetchers" = { }; + packages = forAllSystems (system: + { # Here we put attributes that map 1:1 into packages., ie + # for which we don't apply the full build matrix such as cross or static. + inherit (nixpkgsFor.${system}.native) + changelog-d; + default = self.packages.${system}.nix; + nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; + nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; } + # We need to flatten recursive attribute sets of derivations to pass `flake check`. + // flatMapAttrs + { # Components we'll iterate over in the upcoming lambda + "nix" = { }; + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable these. + #"nix-util" = { }; + #"nix-store" = { }; + #"nix-fetchers" = { }; + } + (pkgName: {}: { + # These attributes go right into `packages.`. + "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; + } + // flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; + }) + // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName}; + }) + ) // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let @@ -367,19 +312,19 @@ }; mesonFlags = - map (transformFlag "libutil") pkgs.nix-util.mesonFlags - ++ map (transformFlag "libstore") pkgs.nix-store.mesonFlags - ++ map (transformFlag "libfetchers") pkgs.nix-fetchers.mesonFlags - ++ lib.optionals havePerl (map (transformFlag "perl") pkgs.nix-perl-bindings.mesonFlags) + map (transformFlag "libutil") pkgs.nixComponents.nix-util.mesonFlags + ++ map (transformFlag "libstore") pkgs.nixComponents.nix-store.mesonFlags + ++ map (transformFlag "libfetchers") pkgs.nixComponents.nix-fetchers.mesonFlags + ++ lib.optionals havePerl (map (transformFlag "perl") pkgs.nixComponents.nix-perl-bindings.mesonFlags) ; nativeBuildInputs = attrs.nativeBuildInputs or [] - ++ pkgs.nix-util.nativeBuildInputs - ++ pkgs.nix-store.nativeBuildInputs - ++ pkgs.nix-fetchers.nativeBuildInputs - ++ lib.optionals havePerl pkgs.nix-perl-bindings.nativeBuildInputs - ++ pkgs.nix-internal-api-docs.nativeBuildInputs - ++ pkgs.nix-external-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-util.nativeBuildInputs + ++ pkgs.nixComponents.nix-store.nativeBuildInputs + ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs + ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs + ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs ++ [ modular.pre-commit.settings.package (pkgs.writeScriptBin "pre-commit-hooks-install" diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index b1d21e8fe..5e4291fcb 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -447,7 +447,6 @@ ''^tests/unit/libfetchers/public-key\.cc'' ''^tests/unit/libstore-support/tests/derived-path\.cc'' ''^tests/unit/libstore-support/tests/derived-path\.hh'' - ''^tests/unit/libstore-support/tests/libstore\.hh'' ''^tests/unit/libstore-support/tests/nix_api_store\.hh'' ''^tests/unit/libstore-support/tests/outputs-spec\.cc'' ''^tests/unit/libstore-support/tests/outputs-spec\.hh'' @@ -522,6 +521,7 @@ ''^tests/functional/ca/repl\.sh$'' ''^tests/functional/ca/selfref-gc\.sh$'' ''^tests/functional/ca/why-depends\.sh$'' + ''^tests/functional/characterisation-test-infra\.sh$'' ''^tests/functional/check\.sh$'' ''^tests/functional/common/vars-and-functions\.sh$'' ''^tests/functional/completions\.sh$'' @@ -579,9 +579,7 @@ ''^tests/functional/impure-env\.sh$'' ''^tests/functional/impure-eval\.sh$'' ''^tests/functional/install-darwin\.sh$'' - ''^tests/functional/lang-test-infra\.sh$'' ''^tests/functional/lang\.sh$'' - ''^tests/functional/lang/framework\.sh$'' ''^tests/functional/legacy-ssh-store\.sh$'' ''^tests/functional/linux-sandbox\.sh$'' ''^tests/functional/local-overlay-store/add-lower-inner\.sh$'' diff --git a/meson.build b/meson.build index e67fd4a7a..085ac0865 100644 --- a/meson.build +++ b/meson.build @@ -12,3 +12,10 @@ subproject('libfetchers') subproject('perl') subproject('internal-api-docs') subproject('external-api-docs') + +# C wrappers +subproject('libutil-c') + +# Testing +subproject('libutil-test-support') +subproject('libutil-test') diff --git a/package.nix b/package.nix index f414b9a73..9a6fc272a 100644 --- a/package.nix +++ b/package.nix @@ -1,12 +1,10 @@ { lib -, fetchurl , stdenv , releaseTools , autoconf-archive , autoreconfHook , aws-sdk-cpp , boehmgc -, buildPackages , nlohmann_json , bison , boost @@ -15,7 +13,6 @@ , curl , editline , readline -, fileset , flex , git , gtest @@ -50,7 +47,6 @@ , pname ? "nix" , versionSuffix ? "" -, officialRelease ? false # Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true @@ -113,6 +109,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; # selected attributes with defaults, will be used to define some diff --git a/packaging/components.nix b/packaging/components.nix new file mode 100644 index 000000000..b5e47969e --- /dev/null +++ b/packaging/components.nix @@ -0,0 +1,28 @@ +scope: +let + inherit (scope) callPackage; +in + +# This becomes the pkgs.nixComponents attribute set +{ + nix = callPackage ../package.nix { }; + + nix-util = callPackage ../src/libutil/package.nix { }; + + nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { }; + + nix-util-test = callPackage ../tests/unit/libutil/package.nix { }; + + nix-util-c = callPackage ../src/libutil-c/package.nix { }; + + nix-store = callPackage ../src/libstore/package.nix { }; + + nix-fetchers = callPackage ../src/libfetchers/package.nix { }; + + nix-perl-bindings = callPackage ../src/perl/package.nix { }; + + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { }; + + nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { }; + +} diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix new file mode 100644 index 000000000..88273df22 --- /dev/null +++ b/packaging/dependencies.nix @@ -0,0 +1,58 @@ +# These overrides are applied to the dependencies of the Nix components. + +{ + # Flake inputs; used for sources + inputs, + + # The raw Nixpkgs, not affected by this scope + pkgs, + + stdenv, + versionSuffix, +}: +scope: { + inherit stdenv versionSuffix; + + libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { + version = "2.5.5"; + src = pkgs.fetchurl { + url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; + hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; + }; + }); + + boehmgc = pkgs.boehmgc.override { + enableLargeConfig = true; + }; + + libgit2 = pkgs.libgit2.overrideAttrs (attrs: { + src = inputs.libgit2; + version = inputs.libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; + }); + + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); +} diff --git a/maintainers/hydra.nix b/packaging/hydra.nix similarity index 88% rename from maintainers/hydra.nix rename to packaging/hydra.nix index abd44efef..5715abd8e 100644 --- a/maintainers/hydra.nix +++ b/packaging/hydra.nix @@ -9,7 +9,6 @@ }: let inherit (inputs) nixpkgs nixpkgs-regression; - inherit (lib) fileset; installScriptFor = tarballs: nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix { @@ -25,17 +24,21 @@ let lib.versionAtLeast client.version "2.4pre20211005") "-${client.version}-against-${daemon.version}"; - inherit fileset; - test-client = client; test-daemon = daemon; doBuild = false; }; + # Technically we could just return `pkgs.nixComponents`, but for Hydra it's + # convention to transpose it, and to transpose it efficiently, we need to + # enumerate them manually, so that we don't evaluate unnecessary package sets. forAllPackages = lib.genAttrs [ "nix" "nix-util" + "nix-util-c" + "nix-util-test-support" + "nix-util-test" "nix-store" "nix-fetchers" ]; @@ -43,28 +46,22 @@ in { # Binary package for various platforms. build = forAllPackages (pkgName: - forAllSystems (system: nixpkgsFor.${system}.native.${pkgName})); + forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName})); shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); buildStatic = forAllPackages (pkgName: - lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.${pkgName})); + lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})); buildCross = forAllPackages (pkgName: forAllCrossSystems (crossSystem: - lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.${pkgName}))); + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}))); buildNoGc = forAllSystems (system: self.packages.${system}.nix.override { enableGC = false; } ); - buildNoTests = forAllSystems (system: - self.packages.${system}.nix.override { - doCheck = false; - doInstallCheck = false; - installUnitTests = false; - } - ); + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nix_noTests); # Toggles some settings for better coverage. Windows needs these # library combinations, and Debian build Nix with GNU readline too. @@ -76,7 +73,7 @@ in ); # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix-perl-bindings); + perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings); # Binary tarball for various platforms, containing a Nix store # with the closure of 'nix' package, and the second half of @@ -125,10 +122,10 @@ in }; # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.nix-internal-api-docs; + internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs; # API docs for Nix's C bindings. - external-api-docs = nixpkgsFor.x86_64-linux.native.nix-external-api-docs; + external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs; # System tests. tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // { diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix index aa5cd49eb..352698360 100644 --- a/src/external-api-docs/package.nix +++ b/src/external-api-docs/package.nix @@ -1,7 +1,5 @@ { lib , stdenv -, releaseTools -, fileset , meson , ninja @@ -12,6 +10,10 @@ , versionSuffix ? "" }: +let + inherit (lib) fileset; +in + stdenv.mkDerivation (finalAttrs: { pname = "nix-external-api-docs"; version = lib.fileContents ./.version + versionSuffix; diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix index b5f1b0da1..fa54d55f3 100644 --- a/src/internal-api-docs/package.nix +++ b/src/internal-api-docs/package.nix @@ -1,7 +1,5 @@ { lib , stdenv -, releaseTools -, fileset , meson , ninja @@ -12,6 +10,10 @@ , versionSuffix ? "" }: +let + inherit (lib) fileset; +in + stdenv.mkDerivation (finalAttrs: { pname = "nix-internal-api-docs"; version = lib.fileContents ./.version + versionSuffix; diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 543250da3..74d146c66 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -127,12 +127,12 @@ ref EvalCommand::getEvalState() if (!evalState) { evalState = #if HAVE_BOEHMGC - std::allocate_shared(traceable_allocator(), - lookupPath, getEvalStore(), getStore()) + std::allocate_shared( + traceable_allocator(), #else std::make_shared( - lookupPath, getEvalStore(), getStore()) #endif + lookupPath, getEvalStore(), evalSettings, getStore()) ; evalState->repair = repair; diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index cd0f19257..77dba546d 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,6 +1,7 @@ #include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" +#include "config-global.hh" #include "filetransfer.hh" #include "eval.hh" #include "fetchers.hh" @@ -13,6 +14,12 @@ namespace nix { +EvalSettings evalSettings { + settings.readOnlyMode +}; + +static GlobalConfig::Register rEvalSettings(&evalSettings); + MixEvalArgs::MixEvalArgs() { addFlag({ diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 75cb19334..189abf0ed 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -12,9 +12,15 @@ namespace nix { class Store; class EvalState; +struct EvalSettings; class Bindings; struct SourcePath; +/** + * @todo Get rid of global setttings variables + */ +extern EvalSettings evalSettings; + struct MixEvalArgs : virtual Args, virtual MixRepair { static constexpr auto category = "Common evaluation options"; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 1e80d14e1..53dd94c7a 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -8,6 +8,7 @@ #include "ansicolor.hh" #include "shared.hh" +#include "config-global.hh" #include "eval.hh" #include "eval-cache.hh" #include "eval-inline.hh" diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index bdf7a1e63..13e0f5f3a 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -7,6 +7,7 @@ #include "eval.hh" #include "globals.hh" #include "util.hh" +#include "eval-settings.hh" #include "nix_api_expr.h" #include "nix_api_expr_internal.h" @@ -106,7 +107,21 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c for (size_t i = 0; lookupPath_c[i] != nullptr; i++) lookupPath.push_back(lookupPath_c[i]); - return new EvalState{nix::EvalState(nix::LookupPath::parse(lookupPath), store->ptr)}; + void * p = ::operator new( + sizeof(EvalState), + static_cast(alignof(EvalState))); + auto * p2 = static_cast(p); + new (p) EvalState { + .settings = nix::EvalSettings{ + nix::settings.readOnlyMode, + }, + .state = nix::EvalState( + nix::LookupPath::parse(lookupPath), + store->ptr, + p2->settings), + }; + loadConfFile(p2->settings); + return p2; } NIXC_CATCH_ERRS_NULL } diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index 5a875ef39..d4ccffd29 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -2,11 +2,13 @@ #define NIX_API_EXPR_INTERNAL_H #include "eval.hh" +#include "eval-settings.hh" #include "attr-set.hh" #include "nix_api_value.h" struct EvalState { + nix::EvalSettings settings; nix::EvalState state; }; diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 85b1677ea..11a62b0fd 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -1,4 +1,5 @@ #include "users.hh" +#include "config-global.hh" #include "globals.hh" #include "profiles.hh" #include "eval.hh" @@ -44,7 +45,8 @@ static Strings parseNixPath(const std::string & s) return res; } -EvalSettings::EvalSettings() +EvalSettings::EvalSettings(bool & readOnlyMode) + : readOnlyMode{readOnlyMode} { auto var = getEnv("NIX_PATH"); if (var) nixPath = parseNixPath(*var); @@ -54,7 +56,7 @@ EvalSettings::EvalSettings() builtinsAbortOnWarn = true; } -Strings EvalSettings::getDefaultNixPath() +Strings EvalSettings::getDefaultNixPath() const { Strings res; auto add = [&](const Path & p, const std::string & s = std::string()) { @@ -67,7 +69,7 @@ Strings EvalSettings::getDefaultNixPath() } }; - if (!evalSettings.restrictEval && !evalSettings.pureEval) { + if (!restrictEval && !pureEval) { add(getNixDefExpr() + "/channels"); add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); add(rootChannelsDir()); @@ -93,16 +95,12 @@ std::string EvalSettings::resolvePseudoUrl(std::string_view url) return std::string(url); } -const std::string & EvalSettings::getCurrentSystem() +const std::string & EvalSettings::getCurrentSystem() const { const auto & evalSystem = currentSystem.get(); return evalSystem != "" ? evalSystem : settings.thisSystem.get(); } -EvalSettings evalSettings; - -static GlobalConfig::Register rEvalSettings(&evalSettings); - Path getNixDefExpr() { return settings.useXDGBaseDirectories diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index d01de980f..2689a8465 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -7,9 +7,11 @@ namespace nix { struct EvalSettings : Config { - EvalSettings(); + EvalSettings(bool & readOnlyMode); - static Strings getDefaultNixPath(); + bool & readOnlyMode; + + Strings getDefaultNixPath() const; static bool isPseudoUrl(std::string_view s); @@ -74,7 +76,7 @@ struct EvalSettings : Config * Implements the `eval-system` vs `system` defaulting logic * described for `eval-system`. */ - const std::string & getCurrentSystem(); + const std::string & getCurrentSystem() const; Setting restrictEval{ this, false, "restrict-eval", @@ -193,8 +195,6 @@ struct EvalSettings : Config )"}; }; -extern EvalSettings evalSettings; - /** * Conventionally part of the default nix path in impure mode. */ diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 301b81a62..717ccc803 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -9,7 +9,6 @@ #include "store-api.hh" #include "derivations.hh" #include "downstream-placeholder.hh" -#include "globals.hh" #include "eval-inline.hh" #include "filetransfer.hh" #include "function-trace.hh" @@ -219,8 +218,10 @@ static constexpr size_t BASE_ENV_SIZE = 128; EvalState::EvalState( const LookupPath & _lookupPath, ref store, + const EvalSettings & settings, std::shared_ptr buildStore) - : sWith(symbols.create("")) + : settings{settings} + , sWith(symbols.create("")) , sOutPath(symbols.create("outPath")) , sDrvPath(symbols.create("drvPath")) , sType(symbols.create("type")) @@ -240,6 +241,12 @@ EvalState::EvalState( , sRight(symbols.create("right")) , sWrong(symbols.create("wrong")) , sStructuredAttrs(symbols.create("__structuredAttrs")) + , sAllowedReferences(symbols.create("allowedReferences")) + , sAllowedRequisites(symbols.create("allowedRequisites")) + , sDisallowedReferences(symbols.create("disallowedReferences")) + , sDisallowedRequisites(symbols.create("disallowedRequisites")) + , sMaxSize(symbols.create("maxSize")) + , sMaxClosureSize(symbols.create("maxClosureSize")) , sBuilder(symbols.create("builder")) , sArgs(symbols.create("args")) , sContentAddressed(symbols.create("__contentAddressed")) @@ -270,10 +277,10 @@ EvalState::EvalState( , repair(NoRepair) , emptyBindings(0) , rootFS( - evalSettings.restrictEval || evalSettings.pureEval + settings.restrictEval || settings.pureEval ? ref(AllowListSourceAccessor::create(getFSSourceAccessor(), {}, - [](const CanonPath & path) -> RestrictedPathError { - auto modeInformation = evalSettings.pureEval + [&settings](const CanonPath & path) -> RestrictedPathError { + auto modeInformation = settings.pureEval ? "in pure evaluation mode (use '--impure' to override)" : "in restricted mode"; throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); @@ -324,10 +331,10 @@ EvalState::EvalState( vStringUnknown.mkString("unknown"); /* Initialise the Nix expression search path. */ - if (!evalSettings.pureEval) { + if (!settings.pureEval) { for (auto & i : _lookupPath.elements) lookupPath.elements.emplace_back(LookupPath::Elem {i}); - for (auto & i : evalSettings.nixPath.get()) + for (auto & i : settings.nixPath.get()) lookupPath.elements.emplace_back(LookupPath::Elem::parse(i)); } @@ -405,9 +412,9 @@ bool isAllowedURI(std::string_view uri, const Strings & allowedUris) void EvalState::checkURI(const std::string & uri) { - if (!evalSettings.restrictEval) return; + if (!settings.restrictEval) return; - if (isAllowedURI(uri, evalSettings.allowedUris.get())) return; + if (isAllowedURI(uri, settings.allowedUris.get())) return; /* If the URI is a path, then check it against allowedPaths as well. */ @@ -452,7 +459,7 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info) constantInfos.push_back({name2, info}); - if (!(evalSettings.pureEval && info.impureOnly)) { + if (!(settings.pureEval && info.impureOnly)) { /* Check the type, if possible. We might know the type of a thunk in advance, so be allowed @@ -1407,11 +1414,11 @@ public: void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos) { - if (callDepth > evalSettings.maxCallDepth) + if (callDepth > settings.maxCallDepth) error("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow(); CallDepth _level(callDepth); - auto trace = evalSettings.traceFunctionCalls + auto trace = settings.traceFunctionCalls ? std::make_unique(positions[pos]) : nullptr; @@ -2297,7 +2304,7 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat path.resolveSymlinks(), settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, path.baseName(), - FileIngestionMethod::Recursive, + ContentAddressMethod::Raw::NixArchive, nullptr, repair); allowPath(dstPath); @@ -2739,7 +2746,7 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ return {corepkgsFS, CanonPath(path.substr(3))}; error( - evalSettings.pureEval + settings.pureEval ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", path @@ -2819,7 +2826,7 @@ Expr * EvalState::parse( const SourcePath & basePath, std::shared_ptr & staticEnv) { - auto result = parseExprFromBuf(text, length, origin, basePath, symbols, positions, rootFS, exprSymbols); + auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, rootFS, exprSymbols); result->bindVars(*this, staticEnv); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0e5e0433e..b84bc9907 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -30,6 +30,7 @@ namespace nix { constexpr size_t maxPrimOpArity = 8; class Store; +struct EvalSettings; class EvalState; class StorePath; struct SingleDerivedPath; @@ -39,7 +40,6 @@ namespace eval_cache { class EvalCache; } - /** * Function that implements a primop. */ @@ -162,13 +162,17 @@ struct DebugTrace { class EvalState : public std::enable_shared_from_this { public: + const EvalSettings & settings; SymbolTable symbols; PosTable positions; const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, - sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, + sRight, sWrong, sStructuredAttrs, + sAllowedReferences, sAllowedRequisites, sDisallowedReferences, sDisallowedRequisites, + sMaxSize, sMaxClosureSize, + sBuilder, sArgs, sContentAddressed, sImpure, sOutputHash, sOutputHashAlgo, sOutputHashMode, sRecurseForDerivations, @@ -349,6 +353,7 @@ public: EvalState( const LookupPath & _lookupPath, ref store, + const EvalSettings & settings, std::shared_ptr buildStore = nullptr); ~EvalState(); diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index e0c5d4512..b348f6d44 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,5 +1,5 @@ #include "users.hh" -#include "globals.hh" +#include "config-global.hh" #include "fetch-settings.hh" #include "flake.hh" diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 3af9ef14e..67b19bd57 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -803,7 +803,7 @@ static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, V { std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); auto flakeRef = parseFlakeRef(flakeRefS, {}, true); - if (evalSettings.pureEval && !flakeRef.input.isLocked()) + if (state.settings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, state.positions[pos]); callFlake(state, @@ -811,8 +811,8 @@ static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, V LockFlags { .updateLockFile = false, .writeLockFile = false, - .useRegistries = !evalSettings.pureEval && fetchSettings.useRegistries, - .allowUnlocked = !evalSettings.pureEval, + .useRegistries = !state.settings.pureEval && fetchSettings.useRegistries, + .allowUnlocked = !state.settings.pureEval, }), v); } diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 5a928e9aa..cff6282fa 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -46,6 +46,7 @@ struct ParserState PosTable::Origin origin; const ref rootFS; const Expr::AstSymbols & s; + const EvalSettings & settings; void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 00300449f..709a4532a 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -25,7 +25,6 @@ #include "nixexpr.hh" #include "eval.hh" #include "eval-settings.hh" -#include "globals.hh" #include "parser-state.hh" #define YYLTYPE ::nix::ParserLocation @@ -40,6 +39,7 @@ Expr * parseExprFromBuf( Pos::Origin origin, const SourcePath & basePath, SymbolTable & symbols, + const EvalSettings & settings, PosTable & positions, const ref rootFS, const Expr::AstSymbols & astSymbols); @@ -294,7 +294,7 @@ path_start $$ = new ExprPath(ref(state->rootFS), std::move(path)); } | HPATH { - if (evalSettings.pureEval) { + if (state->settings.pureEval) { throw Error( "the path '%s' can not be resolved in pure mode", std::string_view($1.p, $1.l) @@ -429,6 +429,7 @@ Expr * parseExprFromBuf( Pos::Origin origin, const SourcePath & basePath, SymbolTable & symbols, + const EvalSettings & settings, PosTable & positions, const ref rootFS, const Expr::AstSymbols & astSymbols) @@ -441,6 +442,7 @@ Expr * parseExprFromBuf( .origin = positions.addOrigin(origin, length), .rootFS = rootFS, .s = astSymbols, + .settings = settings, }; yylex_init(&scanner); diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 9177b0a2f..212441019 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -5,7 +5,6 @@ #include "eval.hh" #include "eval-settings.hh" #include "gc-small-vector.hh" -#include "globals.hh" #include "json-to-value.hh" #include "names.hh" #include "path-references.hh" @@ -78,7 +77,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS if (drvs.empty()) return {}; - if (isIFD && !evalSettings.enableImportFromDerivation) + if (isIFD && !settings.enableImportFromDerivation) error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", drvs.begin()->to_string(*store) @@ -901,7 +900,7 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va MaintainCount trylevel(state.trylevel); ReplExitStatus (* savedDebugRepl)(ref es, const ValMap & extraEnv) = nullptr; - if (state.debugRepl && evalSettings.ignoreExceptionsDuringTry) + if (state.debugRepl && state.settings.ignoreExceptionsDuringTry) { /* to prevent starting the repl from exceptions withing a tryEval, null it. */ savedDebugRepl = state.debugRepl; @@ -950,7 +949,7 @@ static RegisterPrimOp primop_tryEval({ static void prim_getEnv(EvalState & state, const PosIdx pos, Value * * args, Value & v) { std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getEnv")); - v.mkString(evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or("")); + v.mkString(state.settings.restrictEval || state.settings.pureEval ? "" : getEnv(name).value_or("")); } static RegisterPrimOp primop_getEnv({ @@ -1017,7 +1016,7 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu printError("trace: %1%", args[0]->string_view()); else printError("trace: %1%", ValuePrinter(state, *args[0])); - if (evalSettings.builtinsTraceDebugger) { + if (state.settings.builtinsTraceDebugger) { state.runDebugRepl(nullptr); } state.forceValue(*args[1], pos); @@ -1056,11 +1055,11 @@ static void prim_warn(EvalState & state, const PosIdx pos, Value * * args, Value logWarning(info); } - if (evalSettings.builtinsAbortOnWarn) { + if (state.settings.builtinsAbortOnWarn) { // Not an EvalError or subclass, which would cause the error to be stored in the eval cache. state.error("aborting to reveal stack trace of warning, as abort-on-warn is set").setIsFromExpr().debugThrow(); } - if (evalSettings.builtinsTraceDebugger || evalSettings.builtinsDebuggerOnWarn) { + if (state.settings.builtinsTraceDebugger || state.settings.builtinsDebuggerOnWarn) { state.runDebugRepl(nullptr); } state.forceValue(*args[1], pos); @@ -1163,12 +1162,34 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * } } +/** + * Early validation for the derivation name, for better error message. + * It is checked again when constructing store paths. + * + * @todo Check that the `.drv` suffix also fits. + */ +static void checkDerivationName(EvalState & state, std::string_view drvName) +{ + try { + checkName(drvName); + } catch (BadStorePathName & e) { + // "Please pass a different name": Users may not be aware that they can + // pass a different one, in functions like `fetchurl` where the name + // is optional. + // Note that Nixpkgs generally won't trigger this, because `mkDerivation` + // sanitizes the name. + state.error("invalid derivation name: %s. Please pass a different '%s'.", Uncolored(e.message()), "name").debugThrow(); + } +} + static void derivationStrictInternal( EvalState & state, const std::string & drvName, const Bindings * attrs, Value & v) { + checkDerivationName(state, drvName); + /* Check whether attributes should be passed as a JSON file. */ using nlohmann::json; std::optional jsonObject; @@ -1209,7 +1230,7 @@ static void derivationStrictInternal( auto handleHashMode = [&](const std::string_view s) { if (s == "recursive") { // back compat, new name is "nar" - ingestionMethod = FileIngestionMethod::Recursive; + ingestionMethod = ContentAddressMethod::Raw::NixArchive; } else try { ingestionMethod = ContentAddressMethod::parse(s); } catch (UsageError &) { @@ -1217,9 +1238,9 @@ static void derivationStrictInternal( "invalid value '%s' for 'outputHashMode' attribute", s ).atPos(v).debugThrow(); } - if (ingestionMethod == TextIngestionMethod {}) + if (ingestionMethod == ContentAddressMethod::Raw::Text) experimentalFeatureSettings.require(Xp::DynamicDerivations); - if (ingestionMethod == FileIngestionMethod::Git) + if (ingestionMethod == ContentAddressMethod::Raw::Git) experimentalFeatureSettings.require(Xp::GitHashing); }; @@ -1308,6 +1329,20 @@ static void derivationStrictInternal( handleOutputs(ss); } + if (i->name == state.sAllowedReferences) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks..allowedReferences' instead", drvName); + if (i->name == state.sAllowedRequisites) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks..allowedRequisites' instead", drvName); + if (i->name == state.sDisallowedReferences) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks..disallowedReferences' instead", drvName); + if (i->name == state.sDisallowedRequisites) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks..disallowedRequisites' instead", drvName); + if (i->name == state.sMaxSize) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks..maxSize' instead", drvName); + if (i->name == state.sMaxClosureSize) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks..maxClosureSize' instead", drvName); + + } else { auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned(); drv.env.emplace(key, s); @@ -1377,7 +1412,7 @@ static void derivationStrictInternal( /* Check whether the derivation name is valid. */ if (isDerivation(drvName) && - !(ingestionMethod == ContentAddressMethod { TextIngestionMethod { } } && + !(ingestionMethod == ContentAddressMethod::Raw::Text && outputs.size() == 1 && *(outputs.begin()) == "out")) { @@ -1399,7 +1434,7 @@ static void derivationStrictInternal( auto h = newHashAllowEmpty(*outputHash, outputHashAlgo); - auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); + auto method = ingestionMethod.value_or(ContentAddressMethod::Raw::Flat); DerivationOutput::CAFixed dof { .ca = ContentAddress { @@ -1418,7 +1453,7 @@ static void derivationStrictInternal( .atPos(v).debugThrow(); auto ha = outputHashAlgo.value_or(HashAlgorithm::SHA256); - auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); + auto method = ingestionMethod.value_or(ContentAddressMethod::Raw::NixArchive); for (auto & i : outputs) { drv.env[i] = hashPlaceholder(i); @@ -1564,7 +1599,7 @@ static RegisterPrimOp primop_toPath({ corner cases. */ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - if (evalSettings.pureEval) + if (state.settings.pureEval) state.error( "'%s' is not allowed in pure evaluation mode", "builtins.storePath" @@ -2194,7 +2229,7 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val }) : ({ StringSource s { contents }; - state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair); + state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, state.repair); }); /* Note: we don't need to add `context' to the context of the @@ -2377,7 +2412,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); - addPath(state, pos, path.baseName(), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); + addPath(state, pos, path.baseName(), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ @@ -2440,7 +2475,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value std::optional path; std::string name; Value * filterFun = nullptr; - ContentAddressMethod method = FileIngestionMethod::Recursive; + auto method = ContentAddressMethod::Raw::NixArchive; std::optional expectedHash; NixStringContext context; @@ -2456,8 +2491,8 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value state.forceFunction(*(filterFun = attr.value), attr.pos, "while evaluating the `filter` parameter passed to builtins.path"); else if (n == "recursive") method = state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") - ? FileIngestionMethod::Recursive - : FileIngestionMethod::Flat; + ? ContentAddressMethod::Raw::NixArchive + : ContentAddressMethod::Raw::Flat; else if (n == "sha256") expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256); else @@ -4548,7 +4583,7 @@ void EvalState::createBaseEnv() )", }); - if (!evalSettings.pureEval) { + if (!settings.pureEval) { v.mkInt(time(0)); } addConstant("__currentTime", v, { @@ -4575,8 +4610,8 @@ void EvalState::createBaseEnv() .impureOnly = true, }); - if (!evalSettings.pureEval) - v.mkString(evalSettings.getCurrentSystem()); + if (!settings.pureEval) + v.mkString(settings.getCurrentSystem()); addConstant("__currentSystem", v, { .type = nString, .doc = R"( @@ -4656,7 +4691,7 @@ void EvalState::createBaseEnv() #ifndef _WIN32 // TODO implement on Windows // Miscellaneous - if (evalSettings.enableNativeCode) { + if (settings.enableNativeCode) { addPrimOp({ .name = "__importNative", .arity = 2, @@ -4679,7 +4714,7 @@ void EvalState::createBaseEnv() error if `--trace-verbose` is enabled. Then return *e2*. This function is useful for debugging. )", - .fun = evalSettings.traceVerbose ? prim_trace : prim_second, + .fun = settings.traceVerbose ? prim_trace : prim_second, }); /* Add a value containing the current Nix expression search path. */ diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index d9ba6aa97..7b5f4193a 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -53,7 +53,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a // whitelist. Ah well. state.checkURI(url); - if (evalSettings.pureEval && !rev) + if (state.settings.pureEval && !rev) throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); fetchers::Attrs attrs; diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index a99a71577..fa6b1c4b6 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -171,10 +171,10 @@ static void fetchTree( } } - if (!evalSettings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) + if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) input = lookupInRegistries(state.store, input).first; - if (evalSettings.pureEval && !input.isLocked()) { + if (state.settings.pureEval && !input.isLocked()) { auto fetcher = "fetchTree"; if (params.isFetchGit) fetcher = "fetchGit"; @@ -431,7 +431,10 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v state.forceValue(*args[0], pos); - if (args[0]->type() == nAttrs) { + bool isArgAttrs = args[0]->type() == nAttrs; + bool nameAttrPassed = false; + + if (isArgAttrs) { for (auto & attr : *args[0]->attrs()) { std::string_view n(state.symbols[attr.name]); @@ -439,8 +442,10 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch"); else if (n == "sha256") expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), HashAlgorithm::SHA256); - else if (n == "name") + else if (n == "name") { + nameAttrPassed = true; name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch"); + } else state.error("unsupported argument '%s' to '%s'", n, who) .atPos(pos).debugThrow(); @@ -453,14 +458,27 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v url = state.forceStringNoCtx(*args[0], pos, "while evaluating the url we should fetch"); if (who == "fetchTarball") - url = evalSettings.resolvePseudoUrl(*url); + url = state.settings.resolvePseudoUrl(*url); state.checkURI(*url); if (name == "") name = baseNameOf(*url); - if (evalSettings.pureEval && !expectedHash) + try { + checkName(name); + } catch (BadStorePathName & e) { + auto resolution = + nameAttrPassed ? HintFmt("Please change the value for the 'name' attribute passed to '%s', so that it can create a valid store path.", who) : + isArgAttrs ? HintFmt("Please add a valid 'name' attribute to the argument for '%s', so that it can create a valid store path.", who) : + HintFmt("Please pass an attribute set with 'url' and 'name' attributes to '%s', so that it can create a valid store path.", who); + + state.error( + std::string("invalid store path name when fetching URL '%s': %s. %s"), *url, Uncolored(e.message()), Uncolored(resolution.str())) + .atPos(pos).debugThrow(); + } + + if (state.settings.pureEval && !expectedHash) state.error("in pure evaluation mode, '%s' requires a 'sha256' argument", who).atPos(pos).debugThrow(); // early exit if pinned and already in the store @@ -468,7 +486,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { - .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, + .method = unpack ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat, .hash = *expectedHash, .references = {} }); diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc index e7d5244dc..21c42567c 100644 --- a/src/libfetchers/fetch-settings.cc +++ b/src/libfetchers/fetch-settings.cc @@ -1,4 +1,5 @@ #include "fetch-settings.hh" +#include "config-global.hh" namespace nix { diff --git a/src/libfetchers/fetch-to-store.hh b/src/libfetchers/fetch-to-store.hh index 81af1e240..c762629f3 100644 --- a/src/libfetchers/fetch-to-store.hh +++ b/src/libfetchers/fetch-to-store.hh @@ -18,7 +18,7 @@ StorePath fetchToStore( const SourcePath & path, FetchMode mode, std::string_view name = "source", - ContentAddressMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, PathFilter * filter = nullptr, RepairFlag repair = NoRepair); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 73923907c..170a8910c 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -305,7 +305,7 @@ StorePath Input::computeStorePath(Store & store) const if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); return store.makeFixedOutputPath(getName(), FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = *narHash, .references = {}, }); diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index ce80932f6..184c1383e 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -41,21 +41,6 @@ bool isCacheFileWithinTtl(time_t now, const struct stat & st) return st.st_mtime + settings.tarballTtl > now; } -bool touchCacheFile(const Path & path, time_t touch_time) -{ -#ifndef _WIN32 // TODO implement - struct timeval times[2]; - times[0].tv_sec = touch_time; - times[0].tv_usec = 0; - times[1].tv_sec = touch_time; - times[1].tv_usec = 0; - - return lutimes(path.c_str(), times) == 0; -#else - return false; -#endif -} - Path getCachePath(std::string_view key, bool shallow) { return getCacheDir() @@ -594,8 +579,11 @@ struct GitInputScheme : InputScheme warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.url); } - if (!touchCacheFile(localRefFile, now)) - warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno)); + try { + setWriteTime(localRefFile, now, now); + } catch (Error & e) { + warn("could not update mtime for file '%s': %s", localRefFile, e.msg()); + } if (!originalRef && !storeCachedHead(repoInfo.url, ref)) warn("could not update cached head '%s' for '%s'", ref, repoInfo.url); } diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 267e8607f..ddb41e63f 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -433,7 +433,7 @@ struct GitLabInputScheme : GitArchiveInputScheme store->toRealPath( downloadFile(store, url, "source", headers).storePath))); - if (json.is_array() && json.size() == 1 && json[0]["id"] != nullptr) { + if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) { return RefInfo { .rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1) }; diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 7bdf1e937..198795caa 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -213,7 +213,7 @@ struct MercurialInputScheme : InputScheme auto storePath = store->addToStore( input.getName(), {getFSSourceAccessor(), CanonPath(actualPath)}, - FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {}, + ContentAddressMethod::Raw::NixArchive, HashAlgorithm::SHA256, {}, filter); return storePath; diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index 800256030..a5583d14c 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -1,7 +1,6 @@ { lib , stdenv , releaseTools -, fileset , meson , ninja @@ -16,17 +15,16 @@ # Configuration Options , versionSuffix ? "" -, officialRelease ? false # Check test coverage of Nix. Probably want to use with with at least # one of `doCheck` or `doInstallCheck` enabled. , withCoverageChecks ? false -# Avoid setting things that would interfere with a functioning devShell -, forDevShell ? false }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; mkDerivation = diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 5b49aaabc..a94845ab8 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,5 +1,6 @@ #include "common-args.hh" #include "args/root.hh" +#include "config-global.hh" #include "globals.hh" #include "logging.hh" #include "loggers.hh" diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 95a8d5a7a..e8c8892b3 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -322,7 +322,7 @@ StorePath BinaryCacheStore::addToStoreFromDump( if (static_cast(dumpMethod) == hashMethod.getFileIngestionMethod()) caHash = hashString(HashAlgorithm::SHA256, dump2.s); switch (dumpMethod) { - case FileSerialisationMethod::Recursive: + case FileSerialisationMethod::NixArchive: // The dump is already NAR in this case, just use it. nar = dump2.s; break; @@ -339,7 +339,7 @@ StorePath BinaryCacheStore::addToStoreFromDump( } else { // Otherwise, we have to do th same hashing as NAR so our single // hash will suffice for both purposes. - if (dumpMethod != FileSerialisationMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) + if (dumpMethod != FileSerialisationMethod::NixArchive || hashAlgo != HashAlgorithm::SHA256) unsupported("addToStoreFromDump"); } StringSource narDump { nar }; diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 146a060f3..64b8495e1 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -3,6 +3,7 @@ # include "hook-instance.hh" #endif #include "processes.hh" +#include "config-global.hh" #include "worker.hh" #include "builtins.hh" #include "builtins/buildenv.hh" diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index b53dc771a..8a5d6de72 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -19,7 +19,6 @@ Worker::Worker(Store & store, Store & evalStore) , store(store) , evalStore(evalStore) { - /* Debugging: prevent recursive workers. */ nrLocalBuilds = 0; nrSubstitutions = 0; lastWokenUp = steady_time_point::min(); @@ -530,7 +529,7 @@ bool Worker::pathContentsGood(const StorePath & path) else { auto current = hashPath( {store.getFSAccessor(), CanonPath(store.printStorePath(path))}, - FileIngestionMethod::Recursive, info->narHash.algo).first; + FileIngestionMethod::NixArchive, info->narHash.algo).first; Hash nullHash(HashAlgorithm::SHA256); res = info->narHash == nullHash || info->narHash == current; } diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh index 7d67030d7..33a7bf015 100644 --- a/src/libstore/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -59,7 +59,7 @@ struct HookInstance; #endif /** - * The worker class. + * Coordinates one or more realisations and their interdependencies. */ class Worker { diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 4ed4f2de5..e1cdfece6 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -8,98 +8,136 @@ std::string_view makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { case FileIngestionMethod::Flat: + // Not prefixed for back compat return ""; - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: return "r:"; case FileIngestionMethod::Git: experimentalFeatureSettings.require(Xp::GitHashing); return "git:"; default: - throw Error("impossible, caught both cases"); + assert(false); } } std::string_view ContentAddressMethod::render() const { - return std::visit(overloaded { - [](TextIngestionMethod) -> std::string_view { return "text"; }, - [](FileIngestionMethod m2) { - /* Not prefixed for back compat with things that couldn't produce text before. */ - return renderFileIngestionMethod(m2); - }, - }, raw); + switch (raw) { + case ContentAddressMethod::Raw::Text: + return "text"; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return renderFileIngestionMethod(getFileIngestionMethod()); + default: + assert(false); + } +} + +/** + * **Not surjective** + * + * This is not exposed because `FileIngestionMethod::Flat` maps to + * `ContentAddressMethod::Raw::Flat` and + * `ContentAddressMethod::Raw::Text` alike. We can thus only safely use + * this when the latter is ruled out (e.g. because it is already + * handled). + */ +static ContentAddressMethod fileIngestionMethodToContentAddressMethod(FileIngestionMethod m) +{ + switch (m) { + case FileIngestionMethod::Flat: + return ContentAddressMethod::Raw::Flat; + case FileIngestionMethod::NixArchive: + return ContentAddressMethod::Raw::NixArchive; + case FileIngestionMethod::Git: + return ContentAddressMethod::Raw::Git; + default: + assert(false); + } } ContentAddressMethod ContentAddressMethod::parse(std::string_view m) { if (m == "text") - return TextIngestionMethod {}; + return ContentAddressMethod::Raw::Text; else - return parseFileIngestionMethod(m); + return fileIngestionMethodToContentAddressMethod( + parseFileIngestionMethod(m)); } std::string_view ContentAddressMethod::renderPrefix() const { - return std::visit(overloaded { - [](TextIngestionMethod) -> std::string_view { return "text:"; }, - [](FileIngestionMethod m2) { - /* Not prefixed for back compat with things that couldn't produce text before. */ - return makeFileIngestionPrefix(m2); - }, - }, raw); + switch (raw) { + case ContentAddressMethod::Raw::Text: + return "text:"; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return makeFileIngestionPrefix(getFileIngestionMethod()); + default: + assert(false); + } } ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) { if (splitPrefix(m, "r:")) { - return FileIngestionMethod::Recursive; + return ContentAddressMethod::Raw::NixArchive; } else if (splitPrefix(m, "git:")) { experimentalFeatureSettings.require(Xp::GitHashing); - return FileIngestionMethod::Git; + return ContentAddressMethod::Raw::Git; } else if (splitPrefix(m, "text:")) { - return TextIngestionMethod {}; + return ContentAddressMethod::Raw::Text; + } + return ContentAddressMethod::Raw::Flat; +} + +/** + * This is slightly more mindful of forward compat in that it uses `fixed:` + * rather than just doing a raw empty prefix or `r:`, which doesn't "save room" + * for future changes very well. + */ +static std::string renderPrefixModern(const ContentAddressMethod & ca) +{ + switch (ca.raw) { + case ContentAddressMethod::Raw::Text: + return "text:"; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return "fixed:" + makeFileIngestionPrefix(ca.getFileIngestionMethod()); + default: + assert(false); } - return FileIngestionMethod::Flat; } std::string ContentAddressMethod::renderWithAlgo(HashAlgorithm ha) const { - return std::visit(overloaded { - [&](const TextIngestionMethod & th) { - return std::string{"text:"} + printHashAlgo(ha); - }, - [&](const FileIngestionMethod & fim) { - return "fixed:" + makeFileIngestionPrefix(fim) + printHashAlgo(ha); - } - }, raw); + return renderPrefixModern(*this) + printHashAlgo(ha); } FileIngestionMethod ContentAddressMethod::getFileIngestionMethod() const { - return std::visit(overloaded { - [&](const TextIngestionMethod & th) { - return FileIngestionMethod::Flat; - }, - [&](const FileIngestionMethod & fim) { - return fim; - } - }, raw); + switch (raw) { + case ContentAddressMethod::Raw::Flat: + return FileIngestionMethod::Flat; + case ContentAddressMethod::Raw::NixArchive: + return FileIngestionMethod::NixArchive; + case ContentAddressMethod::Raw::Git: + return FileIngestionMethod::Git; + case ContentAddressMethod::Raw::Text: + return FileIngestionMethod::Flat; + default: + assert(false); + } } std::string ContentAddress::render() const { - return std::visit(overloaded { - [](const TextIngestionMethod &) -> std::string { - return "text:"; - }, - [](const FileIngestionMethod & method) { - return "fixed:" - + makeFileIngestionPrefix(method); - }, - }, method.raw) - + this->hash.to_string(HashFormat::Nix32, true); + return renderPrefixModern(method) + this->hash.to_string(HashFormat::Nix32, true); } /** @@ -130,17 +168,17 @@ static std::pair parseContentAddressMethodP // No parsing of the ingestion method, "text" only support flat. HashAlgorithm hashAlgo = parseHashAlgorithm_(); return { - TextIngestionMethod {}, + ContentAddressMethod::Raw::Text, std::move(hashAlgo), }; } else if (prefix == "fixed") { // Parse method - auto method = FileIngestionMethod::Flat; + auto method = ContentAddressMethod::Raw::Flat; if (splitPrefix(rest, "r:")) - method = FileIngestionMethod::Recursive; + method = ContentAddressMethod::Raw::NixArchive; else if (splitPrefix(rest, "git:")) { experimentalFeatureSettings.require(Xp::GitHashing); - method = FileIngestionMethod::Git; + method = ContentAddressMethod::Raw::Git; } HashAlgorithm hashAlgo = parseHashAlgorithm_(); return { @@ -201,57 +239,58 @@ size_t StoreReferences::size() const ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept { - return std::visit(overloaded { - [&](const TextIngestionMethod &) -> ContentAddressWithReferences { - return TextInfo { - .hash = ca.hash, - .references = {}, - }; - }, - [&](const FileIngestionMethod & method) -> ContentAddressWithReferences { - return FixedOutputInfo { - .method = method, - .hash = ca.hash, - .references = {}, - }; - }, - }, ca.method.raw); + switch (ca.method.raw) { + case ContentAddressMethod::Raw::Text: + return TextInfo { + .hash = ca.hash, + .references = {}, + }; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return FixedOutputInfo { + .method = ca.method.getFileIngestionMethod(), + .hash = ca.hash, + .references = {}, + }; + default: + assert(false); + } } ContentAddressWithReferences ContentAddressWithReferences::fromParts( ContentAddressMethod method, Hash hash, StoreReferences refs) { - return std::visit(overloaded { - [&](TextIngestionMethod _) -> ContentAddressWithReferences { - if (refs.self) - throw Error("self-reference not allowed with text hashing"); - return ContentAddressWithReferences { - TextInfo { - .hash = std::move(hash), - .references = std::move(refs.others), - } - }; - }, - [&](FileIngestionMethod m2) -> ContentAddressWithReferences { - return ContentAddressWithReferences { - FixedOutputInfo { - .method = m2, - .hash = std::move(hash), - .references = std::move(refs), - } - }; - }, - }, method.raw); + switch (method.raw) { + case ContentAddressMethod::Raw::Text: + if (refs.self) + throw Error("self-reference not allowed with text hashing"); + return TextInfo { + .hash = std::move(hash), + .references = std::move(refs.others), + }; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return FixedOutputInfo { + .method = method.getFileIngestionMethod(), + .hash = std::move(hash), + .references = std::move(refs), + }; + default: + assert(false); + } } ContentAddressMethod ContentAddressWithReferences::getMethod() const { return std::visit(overloaded { [](const TextInfo & th) -> ContentAddressMethod { - return TextIngestionMethod {}; + return ContentAddressMethod::Raw::Text; }, [](const FixedOutputInfo & fsh) -> ContentAddressMethod { - return fsh.method; + return fileIngestionMethodToContentAddressMethod( + fsh.method); }, }, raw); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 5925f8e01..6cc3b7cd9 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -5,7 +5,6 @@ #include "hash.hh" #include "path.hh" #include "file-content-address.hh" -#include "comparator.hh" #include "variant-wrapper.hh" namespace nix { @@ -14,24 +13,6 @@ namespace nix { * Content addressing method */ -/* We only have one way to hash text with references, so this is a single-value - type, mainly useful with std::variant. -*/ - -/** - * The single way we can serialize "text" file system objects. - * - * Somewhat obscure, used by \ref Derivation derivations and - * `builtins.toFile` currently. - * - * TextIngestionMethod is identical to FileIngestionMethod::Fixed except that - * the former may not have self-references and is tagged `text:${algo}:${hash}` - * rather than `fixed:${algo}:${hash}`. The contents of the store path are - * ingested and hashed identically, aside from the slightly different tag and - * restriction on self-references. - */ -struct TextIngestionMethod : std::monostate { }; - /** * Compute the prefix to the hash algorithm which indicates how the * files were ingested. @@ -48,14 +29,51 @@ std::string_view makeFileIngestionPrefix(FileIngestionMethod m); */ struct ContentAddressMethod { - typedef std::variant< - TextIngestionMethod, - FileIngestionMethod - > Raw; + enum struct Raw { + /** + * Calculate a store path using the `FileIngestionMethod::Flat` + * hash of the file system objects, and references. + * + * See `store-object/content-address.md#method-flat` in the + * manual. + */ + Flat, + + /** + * Calculate a store path using the + * `FileIngestionMethod::NixArchive` hash of the file system + * objects, and references. + * + * See `store-object/content-address.md#method-flat` in the + * manual. + */ + NixArchive, + + /** + * Calculate a store path using the `FileIngestionMethod::Git` + * hash of the file system objects, and references. + * + * Part of `ExperimentalFeature::GitHashing`. + * + * See `store-object/content-address.md#method-git` in the + * manual. + */ + Git, + + /** + * Calculate a store path using the `FileIngestionMethod::Flat` + * hash of the file system objects, and references, but in a + * different way than `ContentAddressMethod::Raw::Flat`. + * + * See `store-object/content-address.md#method-text` in the + * manual. + */ + Text, + }; Raw raw; - GENERATE_CMP(ContentAddressMethod, me->raw); + auto operator <=>(const ContentAddressMethod &) const = default; MAKE_WRAPPER_CONSTRUCTOR(ContentAddressMethod); @@ -141,7 +159,7 @@ struct ContentAddress */ Hash hash; - GENERATE_CMP(ContentAddress, me->method, me->hash); + auto operator <=>(const ContentAddress &) const = default; /** * Compute the content-addressability assertion @@ -200,7 +218,7 @@ struct StoreReferences */ size_t size() const; - GENERATE_CMP(StoreReferences, me->self, me->others); + auto operator <=>(const StoreReferences &) const = default; }; // This matches the additional info that we need for makeTextPath @@ -217,7 +235,7 @@ struct TextInfo */ StorePathSet references; - GENERATE_CMP(TextInfo, me->hash, me->references); + auto operator <=>(const TextInfo &) const = default; }; struct FixedOutputInfo @@ -237,7 +255,7 @@ struct FixedOutputInfo */ StoreReferences references; - GENERATE_CMP(FixedOutputInfo, me->hash, me->references); + auto operator <=>(const FixedOutputInfo &) const = default; }; /** @@ -254,7 +272,7 @@ struct ContentAddressWithReferences Raw raw; - GENERATE_CMP(ContentAddressWithReferences, me->raw); + auto operator <=>(const ContentAddressWithReferences &) const = default; MAKE_WRAPPER_CONSTRUCTOR(ContentAddressWithReferences); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index fe60cb918..40163a621 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -415,12 +415,12 @@ static void performOp(TunnelLogger * logger, ref store, case FileIngestionMethod::Flat: dumpMethod = FileSerialisationMethod::Flat; break; - case FileIngestionMethod::Recursive: - dumpMethod = FileSerialisationMethod::Recursive; + case FileIngestionMethod::NixArchive: + dumpMethod = FileSerialisationMethod::NixArchive; break; case FileIngestionMethod::Git: // Use NAR; Git is not a serialization method - dumpMethod = FileSerialisationMethod::Recursive; + dumpMethod = FileSerialisationMethod::NixArchive; break; default: assert(false); @@ -435,19 +435,21 @@ static void performOp(TunnelLogger * logger, ref store, } else { HashAlgorithm hashAlgo; std::string baseName; - FileIngestionMethod method; + ContentAddressMethod method; { bool fixed; uint8_t recursive; std::string hashAlgoRaw; from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; - if (recursive > (uint8_t) FileIngestionMethod::Recursive) + if (recursive > true) throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); - method = FileIngestionMethod { recursive }; + method = recursive + ? ContentAddressMethod::Raw::NixArchive + : ContentAddressMethod::Raw::Flat; /* Compatibility hack. */ if (!fixed) { hashAlgoRaw = "sha256"; - method = FileIngestionMethod::Recursive; + method = ContentAddressMethod::Raw::NixArchive; } hashAlgo = parseHashAlgo(hashAlgoRaw); } @@ -468,7 +470,7 @@ static void performOp(TunnelLogger * logger, ref store, }); logger->startWork(); auto path = store->addToStoreFromDump( - *dumpSource, baseName, FileSerialisationMethod::Recursive, method, hashAlgo); + *dumpSource, baseName, FileSerialisationMethod::NixArchive, method, hashAlgo); logger->stopWork(); to << store->printStorePath(path); @@ -500,7 +502,7 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto path = ({ StringSource source { s }; - store->addToStoreFromDump(source, suffix, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair); + store->addToStoreFromDump(source, suffix, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, NoRepair); }); logger->stopWork(); to << store->printStorePath(path); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 869880112..6dfcc408c 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -150,7 +150,7 @@ StorePath writeDerivation(Store & store, }) : ({ StringSource s { contents }; - store.addToStoreFromDump(s, suffix, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair); + store.addToStoreFromDump(s, suffix, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references, repair); }); } @@ -274,7 +274,7 @@ static DerivationOutput parseDerivationOutput( { if (hashAlgoStr != "") { ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgoStr); - if (method == TextIngestionMethod {}) + if (method == ContentAddressMethod::Raw::Text) xpSettings.require(Xp::DynamicDerivations); const auto hashAlgo = parseHashAlgo(hashAlgoStr); if (hashS == "impure") { @@ -1249,7 +1249,7 @@ DerivationOutput DerivationOutput::fromJSON( auto methodAlgo = [&]() -> std::pair { auto & method_ = getString(valueAt(json, "method")); ContentAddressMethod method = ContentAddressMethod::parse(method_); - if (method == TextIngestionMethod {}) + if (method == ContentAddressMethod::Raw::Text) xpSettings.require(Xp::DynamicDerivations); auto & hashAlgo_ = getString(valueAt(json, "hashAlgo")); diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 0d5d03091..17ebaace6 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -64,8 +64,8 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store virtual StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, - ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index a54ebdcf3..cbbb0fe7a 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,5 +1,6 @@ #include "filetransfer.hh" #include "globals.hh" +#include "config-global.hh" #include "store-api.hh" #include "s3.hh" #include "compression.hh" diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 88f899dbf..7e1d7ea6d 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,4 +1,5 @@ #include "globals.hh" +#include "config-global.hh" #include "current-process.hh" #include "archive.hh" #include "args.hh" @@ -123,12 +124,12 @@ Settings::Settings() }; } -void loadConfFile() +void loadConfFile(AbstractConfig & config) { auto applyConfigFile = [&](const Path & path) { try { std::string contents = readFile(path); - globalConfig.applyConfig(contents, path); + config.applyConfig(contents, path); } catch (SystemError &) { } }; @@ -136,7 +137,7 @@ void loadConfFile() /* We only want to send overrides to the daemon, i.e. stuff from ~/.nix/nix.conf or the command line. */ - globalConfig.resetOverridden(); + config.resetOverridden(); auto files = settings.nixUserConfFiles; for (auto file = files.rbegin(); file != files.rend(); file++) { @@ -145,7 +146,7 @@ void loadConfFile() auto nixConfEnv = getEnv("NIX_CONFIG"); if (nixConfEnv.has_value()) { - globalConfig.applyConfig(nixConfEnv.value(), "NIX_CONFIG"); + config.applyConfig(nixConfEnv.value(), "NIX_CONFIG"); } } @@ -437,7 +438,7 @@ void initLibStore(bool loadConfig) { initLibUtil(); if (loadConfig) - loadConfFile(); + loadConfFile(globalConfig); preloadNSS(); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 843e77bcf..439e9f4fc 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1284,7 +1284,13 @@ extern Settings settings; */ void initPlugins(); -void loadConfFile(); +/** + * Load the configuration (from `nix.conf`, `NIX_CONFIG`, etc.) into the + * given configuration object. + * + * Usually called with `globalConfig`. + */ +void loadConfFile(AbstractConfig & config); // Used by the Settings constructor std::vector getUserConfigFiles(); diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index b683ed580..db49188ec 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -76,8 +76,8 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor virtual StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, - ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 676a035fa..2b4e01eb3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1155,7 +1155,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, auto fim = specified.method.getFileIngestionMethod(); switch (fim) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: { HashModuloSink caSink { specified.hash.algo, @@ -1253,7 +1253,7 @@ StorePath LocalStore::addToStoreFromDump( std::filesystem::path tempDir; AutoCloseFD tempDirFd; - bool methodsMatch = ContentAddressMethod(FileIngestionMethod(dumpMethod)) == hashMethod; + bool methodsMatch = static_cast(dumpMethod) == hashMethod.getFileIngestionMethod(); /* If the methods don't match, our streaming hash of the dump is the wrong sort, and we need to rehash. */ @@ -1314,7 +1314,7 @@ StorePath LocalStore::addToStoreFromDump( auto fim = hashMethod.getFileIngestionMethod(); switch (fim) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: restorePath(realPath, dumpSource, (FileSerialisationMethod) fim); break; case FileIngestionMethod::Git: @@ -1330,7 +1330,7 @@ StorePath LocalStore::addToStoreFromDump( /* For computing the nar hash. In recursive SHA-256 mode, this is the same as the store hash, so no need to do it again. */ auto narHash = std::pair { dumpHash, size }; - if (dumpMethod != FileSerialisationMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) { + if (dumpMethod != FileSerialisationMethod::NixArchive || hashAlgo != HashAlgorithm::SHA256) { HashSink narSink { HashAlgorithm::SHA256 }; dumpPath(realPath, narSink); narHash = narSink.finish(); @@ -1423,7 +1423,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) PosixSourceAccessor accessor; std::string hash = hashPath( PosixSourceAccessor::createAtRoot(link.path()), - FileIngestionMethod::Recursive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false); + FileIngestionMethod::NixArchive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false); if (hash != name.string()) { printError("link '%s' was modified! expected hash '%s', got '%s'", link.path(), name, hash); diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 170fe67b9..a3130d7cc 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -52,7 +52,7 @@ std::map makeContentAddressed( dstStore, path.name(), FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = narModuloHash, .references = std::move(refs), }, diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index e9b6d2d50..9d903f218 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -151,7 +151,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, Hash hash = ({ hashPath( {make_ref(), CanonPath(path)}, - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first; + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256).first; }); debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true)); @@ -165,7 +165,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, || (repair && hash != ({ hashPath( PosixSourceAccessor::createAtRoot(linkPath), - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first; + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256).first; }))) { // XXX: Consider overwriting linkPath with our valid version. diff --git a/src/libstore/package.nix b/src/libstore/package.nix index e54dfe597..e118f3cd2 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -1,7 +1,6 @@ { lib , stdenv , releaseTools -, fileset , meson , ninja @@ -13,7 +12,6 @@ , aws-sdk-cpp , libseccomp , nlohmann_json -, man , sqlite , busybox-sandbox-shell ? null @@ -21,7 +19,6 @@ # Configuration Options , versionSuffix ? "" -, officialRelease ? false # Check test coverage of Nix. Probably want to use with at least # one of `doCheck` or `doInstallCheck` enabled. @@ -32,6 +29,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; mkDerivation = diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ddd7f50d9..5c27182b7 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -48,15 +48,21 @@ std::optional ValidPathInfo::contentAddressWithRef if (! ca) return std::nullopt; - return std::visit(overloaded { - [&](const TextIngestionMethod &) -> ContentAddressWithReferences { + switch (ca->method.raw) { + case ContentAddressMethod::Raw::Text: + { assert(references.count(path) == 0); return TextInfo { .hash = ca->hash, .references = references, }; - }, - [&](const FileIngestionMethod & m2) -> ContentAddressWithReferences { + } + + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + default: + { auto refs = references; bool hasSelfReference = false; if (refs.count(path)) { @@ -64,15 +70,15 @@ std::optional ValidPathInfo::contentAddressWithRef refs.erase(path); } return FixedOutputInfo { - .method = m2, + .method = ca->method.getFileIngestionMethod(), .hash = ca->hash, .references = { .others = std::move(refs), .self = hasSelfReference, }, }; - }, - }, ca->method.raw); + } + } } bool ValidPathInfo::isContentAddressed(const Store & store) const @@ -127,22 +133,18 @@ ValidPathInfo::ValidPathInfo( : UnkeyedValidPathInfo(narHash) , path(store.makeFixedOutputPathFromCA(name, ca)) { + this->ca = ContentAddress { + .method = ca.getMethod(), + .hash = ca.getHash(), + }; std::visit(overloaded { [this](TextInfo && ti) { this->references = std::move(ti.references); - this->ca = ContentAddress { - .method = TextIngestionMethod {}, - .hash = std::move(ti.hash), - }; }, [this](FixedOutputInfo && foi) { this->references = std::move(foi.references.others); if (foi.references.self) this->references.insert(path); - this->ca = ContentAddress { - .method = std::move(foi.method), - .hash = std::move(foi.hash), - }; }, }, std::move(ca).raw); } diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 8d9726722..3e9d05477 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -2,25 +2,24 @@ namespace nix { -static void checkName(std::string_view path, std::string_view name) +void checkName(std::string_view name) { if (name.empty()) - throw BadStorePath("store path '%s' has an empty name", path); + throw BadStorePathName("name must not be empty"); if (name.size() > StorePath::MaxPathLen) - throw BadStorePath("store path '%s' has a name longer than %d characters", - path, StorePath::MaxPathLen); + throw BadStorePathName("name '%s' must be no longer than %d characters", name, StorePath::MaxPathLen); // See nameRegexStr for the definition if (name[0] == '.') { // check against "." and "..", followed by end or dash if (name.size() == 1) - throw BadStorePath("store path '%s' has invalid name '%s'", path, name); + throw BadStorePathName("name '%s' is not valid", name); if (name[1] == '-') - throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, "."); + throw BadStorePathName("name '%s' is not valid: first dash-separated component must not be '%s'", name, "."); if (name[1] == '.') { if (name.size() == 2) - throw BadStorePath("store path '%s' has invalid name '%s'", path, name); + throw BadStorePathName("name '%s' is not valid", name); if (name[2] == '-') - throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, ".."); + throw BadStorePathName("name '%s' is not valid: first dash-separated component must not be '%s'", name, ".."); } } for (auto c : name) @@ -28,7 +27,16 @@ static void checkName(std::string_view path, std::string_view name) || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '+' || c == '-' || c == '.' || c == '_' || c == '?' || c == '=')) - throw BadStorePath("store path '%s' contains illegal character '%s'", path, c); + throw BadStorePathName("name '%s' contains illegal character '%s'", name, c); +} + +static void checkPathName(std::string_view path, std::string_view name) +{ + try { + checkName(name); + } catch (BadStorePathName & e) { + throw BadStorePath("path '%s' is not a valid store path: %s", path, Uncolored(e.message())); + } } StorePath::StorePath(std::string_view _baseName) @@ -40,13 +48,13 @@ StorePath::StorePath(std::string_view _baseName) if (c == 'e' || c == 'o' || c == 'u' || c == 't' || !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z'))) throw BadStorePath("store path '%s' contains illegal base-32 character '%s'", baseName, c); - checkName(baseName, name()); + checkPathName(baseName, name()); } StorePath::StorePath(const Hash & hash, std::string_view _name) : baseName((hash.to_string(HashFormat::Nix32, false) + "-").append(std::string(_name))) { - checkName(baseName, name()); + checkPathName(baseName, name()); } bool StorePath::isDerivation() const noexcept diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 4abbfcd7c..2380dc6a2 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -9,6 +9,13 @@ namespace nix { struct Hash; +/** + * Check whether a name is a valid store path name. + * + * @throws BadStorePathName if the name is invalid. The message is of the format "name %s is not valid, for this specific reason". + */ +void checkName(std::string_view name); + /** * \ref StorePath "Store path" is the fundamental reference type of Nix. * A store paths refers to a Store object. @@ -31,8 +38,10 @@ public: StorePath() = delete; + /** @throws BadStorePath */ StorePath(std::string_view baseName); + /** @throws BadStorePath */ StorePath(const Hash & hash, std::string_view name); std::string_view to_string() const noexcept diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc index 8cb13d810..46a78cc86 100644 --- a/src/libstore/posix-fs-canonicalise.cc +++ b/src/libstore/posix-fs-canonicalise.cc @@ -33,19 +33,9 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct #ifndef _WIN32 // TODO implement if (st.st_mtime != mtimeStore) { - struct timeval times[2]; - times[0].tv_sec = st.st_atime; - times[0].tv_usec = 0; - times[1].tv_sec = mtimeStore; - times[1].tv_usec = 0; -#if HAVE_LUTIMES - if (lutimes(path.c_str(), times) == -1) - if (errno != ENOSYS || - (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) -#else - if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) -#endif - throw SysError("changing modification time of '%1%'", path); + struct stat st2 = st; + st2.st_mtime = mtimeStore, + setWriteTime(path, st2); } #endif } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index d6efc14f9..d749ccd0a 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -392,8 +392,9 @@ ref RemoteStore::addCAToStore( else { if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); - std::visit(overloaded { - [&](const TextIngestionMethod & thm) -> void { + switch (caMethod.raw) { + case ContentAddressMethod::Raw::Text: + { if (hashAlgo != HashAlgorithm::SHA256) throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", name, printHashAlgo(hashAlgo)); @@ -401,13 +402,19 @@ ref RemoteStore::addCAToStore( conn->to << WorkerProto::Op::AddTextToStore << name << s; WorkerProto::write(*this, *conn, references); conn.processStderr(); - }, - [&](const FileIngestionMethod & fim) -> void { + break; + } + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + default: + { + auto fim = caMethod.getFileIngestionMethod(); conn->to << WorkerProto::Op::AddToStore << name - << ((hashAlgo == HashAlgorithm::SHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (fim == FileIngestionMethod::Recursive ? 1 : 0) + << ((hashAlgo == HashAlgorithm::SHA256 && fim == FileIngestionMethod::NixArchive) ? 0 : 1) /* backwards compatibility hack */ + << (fim == FileIngestionMethod::NixArchive ? 1 : 0) << printHashAlgo(hashAlgo); try { @@ -415,7 +422,7 @@ ref RemoteStore::addCAToStore( connections->incCapacity(); { Finally cleanup([&]() { connections->decCapacity(); }); - if (fim == FileIngestionMethod::Recursive) { + if (fim == FileIngestionMethod::NixArchive) { dump.drainInto(conn->to); } else { std::string contents = dump.drain(); @@ -432,9 +439,9 @@ ref RemoteStore::addCAToStore( } catch (EndOfFile & e) { } throw; } - + break; } - }, caMethod.raw); + } auto path = parseStorePath(readString(conn->from)); // Release our connection to prevent a deadlock in queryPathInfo(). conn_.reset(); @@ -457,12 +464,12 @@ StorePath RemoteStore::addToStoreFromDump( case FileIngestionMethod::Flat: fsm = FileSerialisationMethod::Flat; break; - case FileIngestionMethod::Recursive: - fsm = FileSerialisationMethod::Recursive; + case FileIngestionMethod::NixArchive: + fsm = FileSerialisationMethod::NixArchive; break; case FileIngestionMethod::Git: // Use NAR; Git is not a serialization method - fsm = FileSerialisationMethod::Recursive; + fsm = FileSerialisationMethod::NixArchive; break; default: assert(false); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index d630adc08..4e1896268 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -87,8 +87,8 @@ public: StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, - ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index c67ccd7d4..05c4e1c5e 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -19,6 +19,7 @@ #include "signals.hh" #include "users.hh" +#include #include using json = nlohmann::json; @@ -121,7 +122,7 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed if (info.method == FileIngestionMethod::Git && info.hash.algo != HashAlgorithm::SHA1) throw Error("Git file ingestion must use SHA-1 hash"); - if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) { + if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::NixArchive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { if (!info.references.empty()) { @@ -199,12 +200,12 @@ StorePath Store::addToStore( case FileIngestionMethod::Flat: fsm = FileSerialisationMethod::Flat; break; - case FileIngestionMethod::Recursive: - fsm = FileSerialisationMethod::Recursive; + case FileIngestionMethod::NixArchive: + fsm = FileSerialisationMethod::NixArchive; break; case FileIngestionMethod::Git: // Use NAR; Git is not a serialization method - fsm = FileSerialisationMethod::Recursive; + fsm = FileSerialisationMethod::NixArchive; break; } auto source = sinkToSource([&](Sink & sink) { @@ -355,7 +356,7 @@ ValidPathInfo Store::addToStoreSlow( RegularFileSink fileSink { caHashSink }; TeeSink unusualHashTee { narHashSink, caHashSink }; - auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != HashAlgorithm::SHA256 + auto & narSink = method == ContentAddressMethod::Raw::NixArchive && hashAlgo != HashAlgorithm::SHA256 ? static_cast(unusualHashTee) : narHashSink; @@ -383,9 +384,9 @@ ValidPathInfo Store::addToStoreSlow( finish. */ auto [narHash, narSize] = narHashSink.finish(); - auto hash = method == FileIngestionMethod::Recursive && hashAlgo == HashAlgorithm::SHA256 + auto hash = method == ContentAddressMethod::Raw::NixArchive && hashAlgo == HashAlgorithm::SHA256 ? narHash - : method == FileIngestionMethod::Git + : method == ContentAddressMethod::Raw::Git ? git::dumpHash(hashAlgo, srcPath).hash : caHashSink.finish().first; @@ -1303,7 +1304,7 @@ ref openStore(StoreReference && storeURI) if (!pathExists(chrootStore)) { try { createDirs(chrootStore); - } catch (Error & e) { + } catch (SystemError & e) { return std::make_shared(params); } warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 15712458c..a5effb4c1 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -441,7 +441,7 @@ public: virtual StorePath addToStore( std::string_view name, const SourcePath & path, - ContentAddressMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), PathFilter & filter = defaultPathFilter, @@ -455,7 +455,7 @@ public: ValidPathInfo addToStoreSlow( std::string_view name, const SourcePath & path, - ContentAddressMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), std::optional expectedCAHash = {}); @@ -470,7 +470,7 @@ public: * * @param dumpMethod What serialisation format is `dump`, i.e. how * to deserialize it. Must either match hashMethod or be - * `FileSerialisationMethod::Recursive`. + * `FileSerialisationMethod::NixArchive`. * * @param hashMethod How content addressing? Need not match be the * same as `dumpMethod`. @@ -480,8 +480,8 @@ public: virtual StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, - ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = ContentAddressMethod::Raw::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) = 0; diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index 643f8854d..64c0dd8b7 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -16,6 +16,7 @@ namespace nix { struct SourcePath; MakeError(BadStorePath, Error); +MakeError(BadStorePathName, BadStorePath); struct StoreDirConfig : public Config { @@ -97,7 +98,7 @@ struct StoreDirConfig : public Config std::pair computeStorePath( std::string_view name, const SourcePath & path, - ContentAddressMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = {}, PathFilter & filter = defaultPathFilter) const; diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index 5d045ec3d..dfc208798 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -1,4 +1,5 @@ #include "globals.hh" +#include "config-global.hh" #include "hook-instance.hh" #include "file-system.hh" #include "child.hh" diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 34b20925a..a20ed5300 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -2499,7 +2499,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() auto fim = outputHash.method.getFileIngestionMethod(); switch (fim) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: { HashModuloSink caSink { outputHash.hashAlgo, oldHashPart }; auto fim = outputHash.method.getFileIngestionMethod(); @@ -2541,7 +2541,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() { HashResult narHashAndSize = hashPath( {getFSSourceAccessor(), CanonPath(actualPath)}, - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256); + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); newInfo0.narHash = narHashAndSize.first; newInfo0.narSize = narHashAndSize.second; } @@ -2564,7 +2564,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() rewriteOutput(outputRewrites); HashResult narHashAndSize = hashPath( {getFSSourceAccessor(), CanonPath(actualPath)}, - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256); + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; auto refs = rewriteRefs(); @@ -2914,6 +2914,24 @@ void LocalDerivationGoal::checkOutputs(const std::mapgetStructuredAttrs()) { + if (get(*structuredAttrs, "allowedReferences")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "allowedRequisites")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "disallowedRequisites")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "disallowedReferences")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "maxSize")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "maxClosureSize")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead"); + } if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { if (auto output = get(*outputChecks, outputName)) { Checks checks; diff --git a/src/libutil-c/.version b/src/libutil-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libutil-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build new file mode 100644 index 000000000..a2b020818 --- /dev/null +++ b/src/libutil-c/meson.build @@ -0,0 +1,117 @@ +project('nix-util-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +# See note in ../nix-util/meson.build +deps_private = [ ] + +# See note in ../nix-util/meson.build +deps_public = [ ] + +# See note in ../nix-util/meson.build +deps_other = [ ] + +configdata = configuration_data() + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.h', + # '-include', 'config-store.h', + '-Wno-deprecated-declarations', + '-Wimplicit-fallthrough', + '-Werror=switch', + '-Werror=switch-enum', + '-Wdeprecated-copy', + '-Wignored-qualifiers', + # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked + # at ~1% overhead in `nix search`. + # + # FIXME: remove when we get meson 1.4.0 which will default this to on for us: + # https://mesonbuild.com/Release-notes-for-1-4-0.html#ndebug-setting-now-controls-c-stdlib-assertions + '-D_GLIBCXX_ASSERTIONS=1', + language : 'cpp', +) + +sources = files( + 'nix_api_util.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'nix_api_util.h', + 'nix_api_util_internal.h', +) + +if host_machine.system() == 'cygwin' or host_machine.system() == 'windows' + # Windows DLLs are stricter about symbol visibility than Unix shared + # objects --- see https://gcc.gnu.org/wiki/Visibility for details. + # This is a temporary sledgehammer to export everything like on Unix, + # and not detail with this yet. + # + # TODO do not do this, and instead do fine-grained export annotations. + linker_export_flags = ['-Wl,--export-all-symbols'] +else + linker_export_flags = [] +endif + +nix_util = dependency('nix-util') +if nix_util.type_name() == 'internal' + # subproject sadly no good for pkg-config module + deps_other += nix_util +else + deps_public += nix_util +endif + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-util.h', +) + +this_library = library( + 'nixutilc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +import('pkgconfig').generate( + this_library, + filebase : meson.project_name(), + name : 'Nix', + description : 'Nix Package Manager', + subdirs : ['nix'], + extra_cflags : ['-std=c++2a'], + requires : deps_public, + requires_private : deps_private, + libraries_private : libraries_private, +) + +meson.override_dependency(meson.project_name(), declare_dependency( + include_directories : include_dirs, + link_with : this_library, + compile_args : ['-std=c++2a'], + dependencies : [], +)) diff --git a/src/libutil-c/meson.options b/src/libutil-c/meson.options new file mode 100644 index 000000000..04422feaf --- /dev/null +++ b/src/libutil-c/meson.options @@ -0,0 +1 @@ +# vim: filetype=meson diff --git a/src/libutil-c/nix-util-c.pc.in b/src/libutil-c/nix-util-c.pc.in new file mode 100644 index 000000000..0ccae3f8a --- /dev/null +++ b/src/libutil-c/nix-util-c.pc.in @@ -0,0 +1,9 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix libutil C API +Description: Common functions for the Nix C API, such as error handling +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lnixutil +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libutil-c/nix_api_util.cc b/src/libutil-c/nix_api_util.cc index 0a9b49345..4f65a4c12 100644 --- a/src/libutil-c/nix_api_util.cc +++ b/src/libutil-c/nix_api_util.cc @@ -1,5 +1,5 @@ #include "nix_api_util.h" -#include "config.hh" +#include "config-global.hh" #include "error.hh" #include "nix_api_util_internal.h" #include "util.hh" diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix new file mode 100644 index 000000000..05a26c17e --- /dev/null +++ b/src/libutil-c/package.nix @@ -0,0 +1,97 @@ +{ lib +, stdenv +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util + +# Configuration Options + +, versionSuffix ? "" + +# Check test coverage of Nix. Probably want to use with at least +# one of `doCheck` or `doInstallCheck` enabled. +, withCoverageChecks ? false +}: + +let + inherit (lib) fileset; + + version = lib.fileContents ./.version + versionSuffix; + + mkDerivation = + if withCoverageChecks + then + # TODO support `finalAttrs` args function in + # `releaseTools.coverageAnalysis`. + argsFun: + releaseTools.coverageAnalysis (let args = argsFun args; in args) + else stdenv.mkDerivation; +in + +mkDerivation (finalAttrs: { + pname = "nix-util-c"; + inherit version; + + src = fileset.toSource { + root = ./.; + fileset = fileset.unions [ + ./meson.build + ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + }; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-util + ] + ; + + propagatedBuildInputs = [ + nix-util + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix + '' + echo ${version} > .version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 + strictDeps = !withCoverageChecks; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +} // lib.optionalAttrs withCoverageChecks { + lcovFilter = [ "*/boost/*" "*-tab.*" ]; + + hardeningDisable = [ "fortify" ]; +}) diff --git a/src/libutil-test b/src/libutil-test new file mode 120000 index 000000000..62c86f54b --- /dev/null +++ b/src/libutil-test @@ -0,0 +1 @@ +../tests/unit/libutil/ \ No newline at end of file diff --git a/src/libutil-test-support b/src/libutil-test-support new file mode 120000 index 000000000..f7da46d4c --- /dev/null +++ b/src/libutil-test-support @@ -0,0 +1 @@ +../tests/unit/libutil-support/ \ No newline at end of file diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index d20936de4..22be392d4 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -6,7 +6,7 @@ #include // for strcasecmp #include "archive.hh" -#include "config.hh" +#include "config-global.hh" #include "posix-source-accessor.hh" #include "source-path.hh" #include "file-system.hh" diff --git a/src/libutil/config-global.cc b/src/libutil/config-global.cc new file mode 100644 index 000000000..4c08898a4 --- /dev/null +++ b/src/libutil/config-global.cc @@ -0,0 +1,67 @@ +#include "config-global.hh" + +namespace nix { + +bool GlobalConfig::set(const std::string & name, const std::string & value) +{ + for (auto & config : *configRegistrations) + if (config->set(name, value)) + return true; + + unknownSettings.emplace(name, value); + + return false; +} + +void GlobalConfig::getSettings(std::map & res, bool overriddenOnly) +{ + for (auto & config : *configRegistrations) + config->getSettings(res, overriddenOnly); +} + +void GlobalConfig::resetOverridden() +{ + for (auto & config : *configRegistrations) + config->resetOverridden(); +} + +nlohmann::json GlobalConfig::toJSON() +{ + auto res = nlohmann::json::object(); + for (const auto & config : *configRegistrations) + res.update(config->toJSON()); + return res; +} + +std::string GlobalConfig::toKeyValue() +{ + std::string res; + std::map settings; + globalConfig.getSettings(settings); + for (const auto & s : settings) + res += fmt("%s = %s\n", s.first, s.second.value); + return res; +} + +void GlobalConfig::convertToArgs(Args & args, const std::string & category) +{ + for (auto & config : *configRegistrations) + config->convertToArgs(args, category); +} + +GlobalConfig globalConfig; + +GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; + +GlobalConfig::Register::Register(Config * config) +{ + if (!configRegistrations) + configRegistrations = new ConfigRegistrations; + configRegistrations->emplace_back(config); +} + +ExperimentalFeatureSettings experimentalFeatureSettings; + +static GlobalConfig::Register rSettings(&experimentalFeatureSettings); + +} diff --git a/src/libutil/config-global.hh b/src/libutil/config-global.hh new file mode 100644 index 000000000..2caf51524 --- /dev/null +++ b/src/libutil/config-global.hh @@ -0,0 +1,33 @@ +#pragma once +///@file + +#include "config.hh" + +namespace nix { + +struct GlobalConfig : public AbstractConfig +{ + typedef std::vector ConfigRegistrations; + static ConfigRegistrations * configRegistrations; + + bool set(const std::string & name, const std::string & value) override; + + void getSettings(std::map & res, bool overriddenOnly = false) override; + + void resetOverridden() override; + + nlohmann::json toJSON() override; + + std::string toKeyValue() override; + + void convertToArgs(Args & args, const std::string & category) override; + + struct Register + { + Register(Config * config); + }; +}; + +extern GlobalConfig globalConfig; + +} diff --git a/src/libutil/config.cc b/src/libutil/config.cc index efde8591b..192a4ecb9 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -443,67 +443,6 @@ void OptionalPathSetting::operator =(const std::optional & v) this->assign(v); } -bool GlobalConfig::set(const std::string & name, const std::string & value) -{ - for (auto & config : *configRegistrations) - if (config->set(name, value)) return true; - - unknownSettings.emplace(name, value); - - return false; -} - -void GlobalConfig::getSettings(std::map & res, bool overriddenOnly) -{ - for (auto & config : *configRegistrations) - config->getSettings(res, overriddenOnly); -} - -void GlobalConfig::resetOverridden() -{ - for (auto & config : *configRegistrations) - config->resetOverridden(); -} - -nlohmann::json GlobalConfig::toJSON() -{ - auto res = nlohmann::json::object(); - for (const auto & config : *configRegistrations) - res.update(config->toJSON()); - return res; -} - -std::string GlobalConfig::toKeyValue() -{ - std::string res; - std::map settings; - globalConfig.getSettings(settings); - for (const auto & s : settings) - res += fmt("%s = %s\n", s.first, s.second.value); - return res; -} - -void GlobalConfig::convertToArgs(Args & args, const std::string & category) -{ - for (auto & config : *configRegistrations) - config->convertToArgs(args, category); -} - -GlobalConfig globalConfig; - -GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; - -GlobalConfig::Register::Register(Config * config) -{ - if (!configRegistrations) - configRegistrations = new ConfigRegistrations; - configRegistrations->emplace_back(config); -} - -ExperimentalFeatureSettings experimentalFeatureSettings; - -static GlobalConfig::Register rSettings(&experimentalFeatureSettings); - bool ExperimentalFeatureSettings::isEnabled(const ExperimentalFeature & feature) const { auto & f = experimentalFeatures.get(); diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 07322b60d..1952ba1b8 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -375,31 +375,6 @@ public: void operator =(const std::optional & v); }; -struct GlobalConfig : public AbstractConfig -{ - typedef std::vector ConfigRegistrations; - static ConfigRegistrations * configRegistrations; - - bool set(const std::string & name, const std::string & value) override; - - void getSettings(std::map & res, bool overriddenOnly = false) override; - - void resetOverridden() override; - - nlohmann::json toJSON() override; - - std::string toKeyValue() override; - - void convertToArgs(Args & args, const std::string & category) override; - - struct Register - { - Register(Config * config); - }; -}; - -extern GlobalConfig globalConfig; - struct ExperimentalFeatureSettings : Config { diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 269000016..4b08a045e 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -155,6 +155,7 @@ public: : err(e) { } + /** The error message without "error: " prefixed to it. */ std::string message() { return err.msg.str(); } diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc index 8b1e3117a..438dac7da 100644 --- a/src/libutil/file-content-address.cc +++ b/src/libutil/file-content-address.cc @@ -10,7 +10,7 @@ static std::optional parseFileSerialisationMethodOpt(st if (input == "flat") { return FileSerialisationMethod::Flat; } else if (input == "nar") { - return FileSerialisationMethod::Recursive; + return FileSerialisationMethod::NixArchive; } else { return std::nullopt; } @@ -45,7 +45,7 @@ std::string_view renderFileSerialisationMethod(FileSerialisationMethod method) switch (method) { case FileSerialisationMethod::Flat: return "flat"; - case FileSerialisationMethod::Recursive: + case FileSerialisationMethod::NixArchive: return "nar"; default: assert(false); @@ -57,7 +57,7 @@ std::string_view renderFileIngestionMethod(FileIngestionMethod method) { switch (method) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: return renderFileSerialisationMethod( static_cast(method)); case FileIngestionMethod::Git: @@ -78,7 +78,7 @@ void dumpPath( case FileSerialisationMethod::Flat: path.readFile(sink); break; - case FileSerialisationMethod::Recursive: + case FileSerialisationMethod::NixArchive: path.dumpPath(sink, filter); break; } @@ -94,7 +94,7 @@ void restorePath( case FileSerialisationMethod::Flat: writeFile(path, source); break; - case FileSerialisationMethod::Recursive: + case FileSerialisationMethod::NixArchive: restorePath(path, source); break; } @@ -119,7 +119,7 @@ std::pair> hashPath( { switch (method) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: { + case FileIngestionMethod::NixArchive: { auto res = hashPath(path, (FileSerialisationMethod) method, ht, filter); return {res.first, {res.second}}; } diff --git a/src/libutil/file-content-address.hh b/src/libutil/file-content-address.hh index e216ee4a7..4c7218f19 100644 --- a/src/libutil/file-content-address.hh +++ b/src/libutil/file-content-address.hh @@ -35,14 +35,14 @@ enum struct FileSerialisationMethod : uint8_t { * See `file-system-object/content-address.md#serial-nix-archive` in * the manual. */ - Recursive, + NixArchive, }; /** * Parse a `FileSerialisationMethod` by name. Choice of: * * - `flat`: `FileSerialisationMethod::Flat` - * - `nar`: `FileSerialisationMethod::Recursive` + * - `nar`: `FileSerialisationMethod::NixArchive` * * Opposite of `renderFileSerialisationMethod`. */ @@ -107,16 +107,18 @@ enum struct FileIngestionMethod : uint8_t { Flat, /** - * Hash `FileSerialisationMethod::Recursive` serialisation. + * Hash `FileSerialisationMethod::NixArchive` serialisation. * * See `file-system-object/content-address.md#serial-flat` in the * manual. */ - Recursive, + NixArchive, /** * Git hashing. * + * Part of `ExperimentalFeature::GitHashing`. + * * See `file-system-object/content-address.md#serial-git` in the * manual. */ @@ -127,7 +129,7 @@ enum struct FileIngestionMethod : uint8_t { * Parse a `FileIngestionMethod` by name. Choice of: * * - `flat`: `FileIngestionMethod::Flat` - * - `nar`: `FileIngestionMethod::Recursive` + * - `nar`: `FileIngestionMethod::NixArchive` * - `git`: `FileIngestionMethod::Git` * * Opposite of `renderFileIngestionMethod`. diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 7b20e078b..f75851bbd 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -418,30 +418,13 @@ void createDir(const Path & path, mode_t mode) throw SysError("creating directory '%1%'", path); } -Paths createDirs(const Path & path) +void createDirs(const Path & path) { - Paths created; - if (path == "/") return created; - - struct stat st; - if (STAT(path.c_str(), &st) == -1) { - created = createDirs(dirOf(path)); - if (mkdir(path.c_str() -#ifndef _WIN32 // TODO abstract mkdir perms for Windows - , 0777 -#endif - ) == -1 && errno != EEXIST) - throw SysError("creating directory '%1%'", path); - st = STAT(path); - created.push_back(path); + try { + fs::create_directories(path); + } catch (fs::filesystem_error & e) { + throw SysError("creating directory '%1%'", path); } - - if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) - throw SysError("statting symlink '%1%'", path); - - if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); - - return created; } @@ -579,29 +562,69 @@ void replaceSymlink(const Path & target, const Path & link) } } -#ifndef _WIN32 -static void setWriteTime(const fs::path & p, const struct stat & st) +void setWriteTime( + const std::filesystem::path & path, + time_t accessedTime, + time_t modificationTime, + std::optional optIsSymlink) { - struct timeval times[2]; - times[0] = { - .tv_sec = st.st_atime, - .tv_usec = 0, +#ifndef _WIN32 + struct timeval times[2] = { + { + .tv_sec = accessedTime, + .tv_usec = 0, + }, + { + .tv_sec = modificationTime, + .tv_usec = 0, + }, }; - times[1] = { - .tv_sec = st.st_mtime, - .tv_usec = 0, - }; - if (lutimes(p.c_str(), times) != 0) - throw SysError("changing modification time of '%s'", p); -} #endif + auto nonSymlink = [&]{ + bool isSymlink = optIsSymlink + ? *optIsSymlink + : fs::is_symlink(path); + + if (!isSymlink) { +#ifdef _WIN32 + // FIXME use `fs::last_write_time`. + // + // Would be nice to use std::filesystem unconditionally, but + // doesn't support access time just modification time. + // + // System clock vs File clock issues also make that annoying. + warn("Changing file times is not yet implemented on Windows, path is '%s'", path); +#else + if (utimes(path.c_str(), times) == -1) { + + throw SysError("changing modification time of '%s' (not a symlink)", path); + } +#endif + } else { + throw Error("Cannot modification time of symlink '%s'", path); + } + }; + +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) { + if (errno == ENOSYS) + nonSymlink(); + else + throw SysError("changing modification time of '%s'", path); + } +#else + nonSymlink(); +#endif +} + +void setWriteTime(const fs::path & path, const struct stat & st) +{ + setWriteTime(path, st.st_atime, st.st_mtime, S_ISLNK(st.st_mode)); +} + void copyFile(const fs::path & from, const fs::path & to, bool andDelete) { -#ifndef _WIN32 - // TODO: Rewrite the `is_*` to use `symlink_status()` - auto statOfFrom = lstat(from.c_str()); -#endif auto fromStatus = fs::symlink_status(from); // Mark the directory as writable so that we can delete its children @@ -621,9 +644,7 @@ void copyFile(const fs::path & from, const fs::path & to, bool andDelete) throw Error("file '%s' has an unsupported type", from); } -#ifndef _WIN32 - setWriteTime(to, statOfFrom); -#endif + setWriteTime(to, lstat(from.string().c_str())); if (andDelete) { if (!fs::is_symlink(fromStatus)) fs::permissions(from, fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index e7bf087eb..4b215162d 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -148,11 +148,10 @@ void deletePath(const std::filesystem::path & path); void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed); /** - * Create a directory and all its parents, if necessary. Returns the - * list of created directories, in order of creation. + * Create a directory and all its parents, if necessary. */ -Paths createDirs(const Path & path); -inline Paths createDirs(PathView path) +void createDirs(const Path & path); +inline void createDirs(PathView path) { return createDirs(Path(path)); } @@ -162,6 +161,30 @@ inline Paths createDirs(PathView path) */ void createDir(const Path & path, mode_t mode = 0755); +/** + * Set the access and modification times of the given path, not + * following symlinks. + * + * @param accessTime Specified in seconds. + * + * @param modificationTime Specified in seconds. + * + * @param isSymlink Whether the file in question is a symlink. Used for + * fallback code where we don't have `lutimes` or similar. if + * `std::optional` is passed, the information will be recomputed if it + * is needed. Race conditions are possible so be careful! + */ +void setWriteTime( + const std::filesystem::path & path, + time_t accessedTime, + time_t modificationTime, + std::optional isSymlink = std::nullopt); + +/** + * Convenience wrapper that takes all arguments from the `struct stat`. + */ +void setWriteTime(const std::filesystem::path & path, const struct stat & st); + /** * Create a symlink. */ diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index ef44a8409..850b7162d 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -111,6 +111,8 @@ std::ostream & operator<<(std::ostream & out, const Magenta & y) /** * Values wrapped in this class are printed without coloring. * + * Specifically, the color is reset to normal before printing the value. + * * By default, arguments to `HintFmt` are printed in magenta (see `Magenta`). */ template diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 91070ea89..a6a743737 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -1,7 +1,7 @@ #include #include "error.hh" -#include "config.hh" +#include "config-global.hh" #include "fs-sink.hh" #if _WIN32 diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 5fa01f0d9..55751b4cf 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -3,7 +3,7 @@ #include "environment-variables.hh" #include "terminal.hh" #include "util.hh" -#include "config.hh" +#include "config-global.hh" #include "source-path.hh" #include "position.hh" diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 2259d4e22..9d0b28539 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -161,6 +161,7 @@ sources = files( 'compression.cc', 'compute-levels.cc', 'config.cc', + 'config-global.cc', 'current-process.cc', 'english.cc', 'environment-variables.cc', @@ -211,6 +212,7 @@ headers = [config_h] + files( 'comparator.hh', 'compression.hh', 'compute-levels.hh', + 'config-global.hh', 'config-impl.hh', 'config.hh', 'current-process.hh', diff --git a/src/libutil/package.nix b/src/libutil/package.nix index b36e3879c..892951cdf 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -1,7 +1,6 @@ { lib , stdenv , releaseTools -, fileset , meson , ninja @@ -18,7 +17,6 @@ # Configuration Options , versionSuffix ? "" -, officialRelease ? false # Check test coverage of Nix. Probably want to use with at least # one of `doCheck` or `doInstallCheck` enabled. @@ -26,6 +24,8 @@ }: let + inherit (lib) fileset; + version = lib.fileContents ./.version + versionSuffix; mkDerivation = diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh index 168fcaa55..bbbe7dcab 100644 --- a/src/libutil/processes.hh +++ b/src/libutil/processes.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "error.hh" +#include "file-descriptor.hh" #include "logging.hh" #include "ansicolor.hh" @@ -23,26 +24,36 @@ namespace nix { struct Sink; struct Source; -#ifndef _WIN32 class Pid { +#ifndef _WIN32 pid_t pid = -1; bool separatePG = false; int killSignal = SIGKILL; +#else + AutoCloseFD pid = INVALID_DESCRIPTOR; +#endif public: Pid(); +#ifndef _WIN32 Pid(pid_t pid); - ~Pid(); void operator =(pid_t pid); operator pid_t(); +#else + Pid(AutoCloseFD pid); + void operator =(AutoCloseFD pid); +#endif + ~Pid(); int kill(); int wait(); + // TODO: Implement for Windows +#ifndef _WIN32 void setSeparatePG(bool separatePG); void setKillSignal(int signal); pid_t release(); -}; #endif +}; #ifndef _WIN32 diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index 44a32f6a1..9cd714f84 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -1,9 +1,15 @@ #include "current-process.hh" #include "environment-variables.hh" +#include "error.hh" +#include "file-descriptor.hh" +#include "file-path.hh" #include "signals.hh" #include "processes.hh" #include "finally.hh" #include "serialise.hh" +#include "file-system.hh" +#include "util.hh" +#include "windows-error.hh" #include #include @@ -16,25 +22,347 @@ #include #include +#define WIN32_LEAN_AND_MEAN +#include + namespace nix { -std::string runProgram(Path program, bool lookupPath, const Strings & args, - const std::optional & input, bool isInteractive) +using namespace nix::windows; + +Pid::Pid() {} + +Pid::Pid(AutoCloseFD pid) + : pid(std::move(pid)) { - throw UnimplementedError("Cannot shell out to git on Windows yet"); } +Pid::~Pid() +{ + if (pid.get() != INVALID_DESCRIPTOR) + kill(); +} +void Pid::operator=(AutoCloseFD pid) +{ + if (this->pid.get() != INVALID_DESCRIPTOR && this->pid.get() != pid.get()) + kill(); + this->pid = std::move(pid); +} + +// TODO: Implement (not needed for process spawning yet) +int Pid::kill() +{ + assert(pid.get() != INVALID_DESCRIPTOR); + + debug("killing process %1%", pid.get()); + + throw UnimplementedError("Pid::kill unimplemented"); +} + +int Pid::wait() +{ + // https://github.com/nix-windows/nix/blob/windows-meson/src/libutil/util.cc#L1938 + assert(pid.get() != INVALID_DESCRIPTOR); + DWORD status = WaitForSingleObject(pid.get(), INFINITE); + if (status != WAIT_OBJECT_0) { + debug("WaitForSingleObject returned %1%", status); + } + + DWORD exitCode = 0; + if (GetExitCodeProcess(pid.get(), &exitCode) == FALSE) { + debug("GetExitCodeProcess failed on pid %1%", pid.get()); + } + + pid.close(); + return exitCode; +} + +// TODO: Merge this with Unix's runProgram since it's identical logic. +std::string runProgram( + Path program, bool lookupPath, const Strings & args, const std::optional & input, bool isInteractive) +{ + auto res = runProgram(RunOptions{ + .program = program, .lookupPath = lookupPath, .args = args, .input = input, .isInteractive = isInteractive}); + + if (!statusOk(res.first)) + throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); + + return res.second; +} + +std::optional getProgramInterpreter(const Path & program) +{ + // These extensions are automatically handled by Windows and don't require an interpreter. + static constexpr const char * exts[] = {".exe", ".cmd", ".bat"}; + for (const auto ext : exts) { + if (hasSuffix(program, ext)) { + return {}; + } + } + // TODO: Open file and read the shebang + throw UnimplementedError("getProgramInterpreter unimplemented"); +} + +// TODO: Not sure if this is needed in the unix version but it might be useful as a member func +void setFDInheritable(AutoCloseFD & fd, bool inherit) +{ + if (fd.get() != INVALID_DESCRIPTOR) { + if (!SetHandleInformation(fd.get(), HANDLE_FLAG_INHERIT, inherit ? HANDLE_FLAG_INHERIT : 0)) { + throw WinError("Couldn't disable inheriting of handle"); + } + } +} + +AutoCloseFD nullFD() +{ + // Create null handle to discard reads / writes + // https://stackoverflow.com/a/25609668 + // https://github.com/nix-windows/nix/blob/windows-meson/src/libutil/util.cc#L2228 + AutoCloseFD nul = CreateFileW( + L"NUL", + GENERIC_READ | GENERIC_WRITE, + // We don't care who reads / writes / deletes this file since it's NUL anyways + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + 0, + NULL); + if (!nul.get()) { + throw WinError("Couldn't open NUL device"); + } + // Let this handle be inheritable by child processes + setFDInheritable(nul, true); + return nul; +} + +// Adapted from +// https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ +std::string windowsEscape(const std::string & str, bool cmd) +{ + // TODO: This doesn't handle cmd.exe escaping. + if (cmd) { + throw UnimplementedError("cmd.exe escaping is not implemented"); + } + + if (str.find_first_of(" \t\n\v\"") == str.npos && !str.empty()) { + // No need to escape this one, the nonempty contents don't have a special character + return str; + } + std::string buffer; + // Add the opening quote + buffer += '"'; + for (auto iter = str.begin();; ++iter) { + size_t backslashes = 0; + while (iter != str.end() && *iter == '\\') { + ++iter; + ++backslashes; + } + + // We only escape backslashes if: + // - They come immediately before the closing quote + // - They come immediately before a quote in the middle of the string + // Both of these cases break the escaping if not handled. Otherwise backslashes are fine as-is + if (iter == str.end()) { + // Need to escape each backslash + buffer.append(backslashes * 2, '\\'); + // Exit since we've reached the end of the string + break; + } else if (*iter == '"') { + // Need to escape each backslash and the intermediate quote character + buffer.append(backslashes * 2, '\\'); + buffer += "\\\""; + } else { + // Don't escape the backslashes since they won't break the delimiter + buffer.append(backslashes, '\\'); + buffer += *iter; + } + } + // Add the closing quote + return buffer + '"'; +} + +Pid spawnProcess(const Path & realProgram, const RunOptions & options, Pipe & out, Pipe & in) +{ + // Setup pipes. + if (options.standardOut) { + // Don't inherit the read end of the output pipe + setFDInheritable(out.readSide, false); + } else { + out.writeSide = nullFD(); + } + if (options.standardIn) { + // Don't inherit the write end of the input pipe + setFDInheritable(in.writeSide, false); + } else { + in.readSide = nullFD(); + } + + STARTUPINFOW startInfo = {0}; + startInfo.cb = sizeof(startInfo); + startInfo.dwFlags = STARTF_USESTDHANDLES; + startInfo.hStdInput = in.readSide.get(); + startInfo.hStdOutput = out.writeSide.get(); + startInfo.hStdError = out.writeSide.get(); + + std::string envline; + // Retain the current processes' environment variables. + for (const auto & envVar : getEnv()) { + envline += (envVar.first + '=' + envVar.second + '\0'); + } + // Also add new ones specified in options. + if (options.environment) { + for (const auto & envVar : *options.environment) { + envline += (envVar.first + '=' + envVar.second + '\0'); + } + } + + std::string cmdline = windowsEscape(realProgram, false); + for (const auto & arg : options.args) { + // TODO: This isn't the right way to escape windows command + // See https://learn.microsoft.com/en-us/windows/win32/api/shellapi/nf-shellapi-commandlinetoargvw + cmdline += ' ' + windowsEscape(arg, false); + } + + PROCESS_INFORMATION procInfo = {0}; + if (CreateProcessW( + // EXE path is provided in the cmdline + NULL, + string_to_os_string(cmdline).data(), + NULL, + NULL, + TRUE, + CREATE_UNICODE_ENVIRONMENT | CREATE_SUSPENDED, + string_to_os_string(envline).data(), + options.chdir.has_value() ? string_to_os_string(*options.chdir).data() : NULL, + &startInfo, + &procInfo) + == 0) { + throw WinError("CreateProcessW failed (%1%)", cmdline); + } + + // Convert these to use RAII + AutoCloseFD process = procInfo.hProcess; + AutoCloseFD thread = procInfo.hThread; + + // Add current process and child to job object so child terminates when parent terminates + // TODO: This spawns one job per child process. We can probably keep this as a global, and + // add children a single job so we don't use so many jobs at once. + Descriptor job = CreateJobObjectW(NULL, NULL); + if (job == NULL) { + TerminateProcess(procInfo.hProcess, 0); + throw WinError("Couldn't create job object for child process"); + } + if (AssignProcessToJobObject(job, procInfo.hProcess) == FALSE) { + TerminateProcess(procInfo.hProcess, 0); + throw WinError("Couldn't assign child process to job object"); + } + if (ResumeThread(procInfo.hThread) == (DWORD) -1) { + TerminateProcess(procInfo.hProcess, 0); + throw WinError("Couldn't resume child process thread"); + } + + return process; +} + +// TODO: Merge this with Unix's runProgram since it's identical logic. // Output = error code + "standard out" output stream std::pair runProgram(RunOptions && options) { - throw UnimplementedError("Cannot shell out to git on Windows yet"); -} + StringSink sink; + options.standardOut = &sink; + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(sink.s)}; +} void runProgram2(const RunOptions & options) { - throw UnimplementedError("Cannot shell out to git on Windows yet"); + checkInterrupt(); + + assert(!(options.standardIn && options.input)); + + std::unique_ptr source_; + Source * source = options.standardIn; + + if (options.input) { + source_ = std::make_unique(*options.input); + source = source_.get(); + } + + /* Create a pipe. */ + Pipe out, in; + // TODO: I copied this from unix but this is handled again in spawnProcess, so might be weird to split it up like + // this + if (options.standardOut) + out.create(); + if (source) + in.create(); + + Path realProgram = options.program; + // TODO: Implement shebang / program interpreter lookup on Windows + auto interpreter = getProgramInterpreter(realProgram); + + std::optional>> resumeLoggerDefer; + if (options.isInteractive) { + logger->pause(); + resumeLoggerDefer.emplace([]() { logger->resume(); }); + } + + Pid pid = spawnProcess(interpreter.has_value() ? *interpreter : realProgram, options, out, in); + + // TODO: This is identical to unix, deduplicate? + out.writeSide.close(); + + std::thread writerThread; + + std::promise promise; + + Finally doJoin([&] { + if (writerThread.joinable()) + writerThread.join(); + }); + + if (source) { + in.readSide.close(); + writerThread = std::thread([&] { + try { + std::vector buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + writeFull(in.writeSide.get(), {buf.data(), n}); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide.close(); + }); + } + + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); + + /* Wait for the child to finish. */ + int status = pid.wait(); + + /* Wait for the writer thread to finish. */ + if (source) + promise.get_future().get(); + + if (status) + throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); } std::string statusToString(int status) @@ -45,10 +373,8 @@ std::string statusToString(int status) return "succeeded"; } - bool statusOk(int status) { return status == 0; } - } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 77df08edd..57630c8c3 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -259,7 +259,7 @@ static void main_nix_build(int argc, char * * argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.lookupPath, evalStore, store); + auto state = std::make_unique(myArgs.lookupPath, evalStore, evalSettings, store); state->repair = myArgs.repair; if (myArgs.repair) buildMode = bmRepair; diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index b5e13cc23..c72378cd5 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1525,7 +1525,7 @@ static int main_nix_env(int argc, char * * argv) auto store = openStore(); - globals.state = std::shared_ptr(new EvalState(myArgs.lookupPath, store)); + globals.state = std::shared_ptr(new EvalState(myArgs.lookupPath, store, evalSettings)); globals.state->repair = myArgs.repair; globals.instSource.nixExprPath = std::make_shared( diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 5246b03e4..a24dd11d6 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -115,7 +115,7 @@ bool createUserEnv(EvalState & state, PackageInfos & elems, std::string str2 = str.str(); StringSource source { str2 }; state.store->addToStoreFromDump( - source, "env-manifest.nix", FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, references); + source, "env-manifest.nix", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references); }); /* Get the environment builder expression. */ diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 35664374c..a4bfeebbb 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -157,7 +157,7 @@ static int main_nix_instantiate(int argc, char * * argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.lookupPath, evalStore, store); + auto state = std::make_unique(myArgs.lookupPath, evalStore, evalSettings, store); state->repair = myArgs.repair; Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 6d028e0a7..d0840a02e 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -194,10 +194,10 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - auto method = FileIngestionMethod::Flat; + ContentAddressMethod method = ContentAddressMethod::Raw::Flat; for (auto & i : opFlags) - if (i == "--recursive") method = FileIngestionMethod::Recursive; + if (i == "--recursive") method = ContentAddressMethod::Raw::NixArchive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.empty()) @@ -223,7 +223,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") method = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::NixArchive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) @@ -563,7 +563,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) if (!hashGiven) { HashResult hash = hashPath( {store->getFSAccessor(false), CanonPath { store->printStorePath(info->path) }}, - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256); + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); info->narHash = hash.first; info->narSize = hash.second; } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index af6743375..5c08f7616 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -12,7 +12,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; - ContentAddressMethod caMethod = FileIngestionMethod::Recursive; + ContentAddressMethod caMethod = ContentAddressMethod::Raw::NixArchive; HashAlgorithm hashAlgo = HashAlgorithm::SHA256; CmdAddToStore() @@ -68,7 +68,7 @@ struct CmdAddFile : CmdAddToStore { CmdAddFile() { - caMethod = FileIngestionMethod::Flat; + caMethod = ContentAddressMethod::Raw::Flat; } std::string description() override diff --git a/src/nix/config.cc b/src/nix/config.cc index 52706afcf..07f975a00 100644 --- a/src/nix/config.cc +++ b/src/nix/config.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "config-global.hh" #include diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 27287a1a8..6bd3dc9ef 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -1,3 +1,4 @@ +#include "config-global.hh" #include "eval.hh" #include "installable-flake.hh" #include "command-installable-value.hh" @@ -238,7 +239,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore auto getEnvShPath = ({ StringSource source { getEnvSh }; evalStore->addToStoreFromDump( - source, "get-env.sh", FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, {}); + source, "get-env.sh", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, {}); }); drv.args = {store->printStorePath(getEnvShPath)}; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index f57b224d2..62266fda1 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -68,7 +68,7 @@ struct CmdHashBase : Command switch (mode) { case FileIngestionMethod::Flat: return "print cryptographic hash of a regular file"; - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: return "print cryptographic hash of the NAR serialisation of a path"; case FileIngestionMethod::Git: return "print cryptographic hash of the Git serialisation of a path"; @@ -91,7 +91,7 @@ struct CmdHashBase : Command Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++ switch (mode) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: { auto hashSink = makeSink(); dumpPath(path2, *hashSink, (FileSerialisationMethod) mode); @@ -126,7 +126,7 @@ struct CmdHashBase : Command struct CmdHashPath : CmdHashBase { CmdHashPath() - : CmdHashBase(FileIngestionMethod::Recursive) + : CmdHashBase(FileIngestionMethod::NixArchive) { addFlag(flag::hashAlgo("algo", &hashAlgo)); addFlag(flag::fileIngestionMethod(&mode)); @@ -311,7 +311,7 @@ static int compatNixHash(int argc, char * * argv) }); if (op == opHash) { - CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); + CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::NixArchive); if (!hashAlgo.has_value()) hashAlgo = HashAlgorithm::MD5; cmd.hashAlgo = hashAlgo.value(); cmd.hashFormat = hashFormat; diff --git a/src/nix/main.cc b/src/nix/main.cc index 81d368d6a..e95558781 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -242,7 +242,7 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) evalSettings.restrictEval = false; evalSettings.pureEval = false; - EvalState state({}, openStore("dummy://")); + EvalState state({}, openStore("dummy://"), evalSettings); auto vGenerateManpage = state.allocValue(); state.eval(state.parseExprFromString( @@ -418,7 +418,7 @@ void mainWrapped(int argc, char * * argv) Xp::FetchTree, }; evalSettings.pureEval = false; - EvalState state({}, openStore("dummy://")); + EvalState state({}, openStore("dummy://"), evalSettings); auto res = nlohmann::json::object(); res["builtins"] = ({ auto builtinsJson = nlohmann::json::object(); diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 3ce52acc5..5e890f3c8 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -57,7 +57,9 @@ std::tuple prefetchFile( bool unpack, bool executable) { - auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + ContentAddressMethod method = unpack || executable + ? ContentAddressMethod::Raw::NixArchive + : ContentAddressMethod::Raw::Flat; /* Figure out a name in the Nix store. */ if (!name) { @@ -73,11 +75,10 @@ std::tuple prefetchFile( the store. */ if (expectedHash) { hashAlgo = expectedHash->algo; - storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { - .method = ingestionMethod, - .hash = *expectedHash, - .references = {}, - }); + storePath = store->makeFixedOutputPathFromCA(*name, ContentAddressWithReferences::fromParts( + method, + *expectedHash, + {})); if (store->isValidPath(*storePath)) hash = expectedHash; else @@ -128,7 +129,7 @@ std::tuple prefetchFile( auto info = store->addToStoreSlow( *name, PosixSourceAccessor::createAtRoot(tmpFile), - ingestionMethod, hashAlgo, {}, expectedHash); + method, hashAlgo, {}, expectedHash); storePath = info.path; assert(info.ca); hash = info.ca->hash; @@ -193,7 +194,7 @@ static int main_nix_prefetch_url(int argc, char * * argv) startProgressBar(); auto store = openStore(); - auto state = std::make_unique(myArgs.lookupPath, store); + auto state = std::make_unique(myArgs.lookupPath, store, evalSettings); Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index a5a40e4f6..c89b8c9bd 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -258,7 +258,7 @@ struct ProfileManifest *store, "profile", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = narHash, .references = { .others = std::move(references), diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index f1fc51682..41ea1f5a4 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -10,6 +10,7 @@ #include "serialise.hh" #include "archive.hh" #include "globals.hh" +#include "config-global.hh" #include "derivations.hh" #include "finally.hh" #include "legacy.hh" diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 17d1edb97..29297253b 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -147,7 +147,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto req = FileTransferRequest((std::string&) settings.upgradeNixStorePathUrl); auto res = getFileTransfer()->download(req); - auto state = std::make_unique(LookupPath{}, store); + auto state = std::make_unique(LookupPath{}, store, evalSettings); auto v = state->allocValue(); state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); Bindings & bindings(*state->allocBindings(0)); diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index 15eb5c4f8..acce25f3a 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -259,7 +259,7 @@ hashPath(char * algo, int base32, char * path) try { Hash h = hashPath( PosixSourceAccessor::createAtRoot(path), - FileIngestionMethod::Recursive, parseHashAlgo(algo)).first; + FileIngestionMethod::NixArchive, parseHashAlgo(algo)).first; auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { @@ -335,7 +335,7 @@ SV * StoreWrapper::addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { - auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto method = recursive ? ContentAddressMethod::Raw::NixArchive : ContentAddressMethod::Raw::Flat; auto path = THIS->store->addToStore( std::string(baseNameOf(srcPath)), PosixSourceAccessor::createAtRoot(srcPath), @@ -351,7 +351,7 @@ StoreWrapper::makeFixedOutputPath(int recursive, char * algo, char * hash, char PPCODE: try { auto h = Hash::parseAny(hash, parseHashAlgo(algo)); - auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto method = recursive ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat; auto path = THIS->store->makeFixedOutputPath(name, FixedOutputInfo { .method = method, .hash = h, diff --git a/src/perl/package.nix b/src/perl/package.nix index a31b1b66c..85f1547b7 100644 --- a/src/perl/package.nix +++ b/src/perl/package.nix @@ -1,5 +1,4 @@ { lib -, fileset , stdenv , perl , perlPackages @@ -16,6 +15,10 @@ , versionSuffix ? "" }: +let + inherit (lib) fileset; +in + perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: { pname = "nix-perl"; version = lib.fileContents ./.version + versionSuffix; diff --git a/tests/functional/add.sh b/tests/functional/add.sh index a6cf88e1a..3b37ee7d4 100755 --- a/tests/functional/add.sh +++ b/tests/functional/add.sh @@ -31,7 +31,7 @@ test "$hash1" = "sha256:$hash2" #### New style commands -clearStore +clearStoreIfPossible ( path1=$(nix store add ./dummy) diff --git a/tests/functional/binary-cache-build-remote.sh b/tests/functional/binary-cache-build-remote.sh index 4edda85b6..5046d0064 100755 --- a/tests/functional/binary-cache-build-remote.sh +++ b/tests/functional/binary-cache-build-remote.sh @@ -2,7 +2,9 @@ source common.sh -clearStore +TODO_NixOS + +clearStoreIfPossible clearCacheCache # Fails without remote builders diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 5ef6d89d4..6a177b657 100755 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + needLocalStore "'--no-require-sigs' can’t be used with the daemon" # We can produce drvs directly into the binary cache diff --git a/tests/functional/brotli.sh b/tests/functional/brotli.sh index 672e771c2..327eab4a5 100755 --- a/tests/functional/brotli.sh +++ b/tests/functional/brotli.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore clearCache diff --git a/tests/functional/build-delete.sh b/tests/functional/build-delete.sh index 59cf95bd2..18841509d 100755 --- a/tests/functional/build-delete.sh +++ b/tests/functional/build-delete.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible # https://github.com/NixOS/nix/issues/6572 issue_6572_independent_outputs() { diff --git a/tests/functional/build-dry.sh b/tests/functional/build-dry.sh index dca5888a6..cff0f9a49 100755 --- a/tests/functional/build-dry.sh +++ b/tests/functional/build-dry.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + ################################################### # Check that --dry-run isn't confused with read-only mode # https://github.com/NixOS/nix/issues/1795 diff --git a/tests/functional/build-remote-input-addressed.sh b/tests/functional/build-remote-input-addressed.sh index 986692dbc..11199a408 100755 --- a/tests/functional/build-remote-input-addressed.sh +++ b/tests/functional/build-remote-input-addressed.sh @@ -23,7 +23,7 @@ EOF chmod +x "$TEST_ROOT/post-build-hook.sh" rm -f "$TEST_ROOT/post-hook-counter" - echo "post-build-hook = $TEST_ROOT/post-build-hook.sh" >> "$NIX_CONF_DIR/nix.conf" + echo "post-build-hook = $TEST_ROOT/post-build-hook.sh" >> "$test_nix_conf" } registerBuildHook diff --git a/tests/functional/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh index 269f7f112..4eccb73e0 100755 --- a/tests/functional/build-remote-trustless-should-fail-0.sh +++ b/tests/functional/build-remote-trustless-should-fail-0.sh @@ -4,6 +4,7 @@ source common.sh enableFeatures "daemon-trust-override" +TODO_NixOS restartDaemon requireSandboxSupport diff --git a/tests/functional/build-remote-trustless-should-pass-2.sh b/tests/functional/build-remote-trustless-should-pass-2.sh index ba5d1ff7a..34ce7fbe4 100755 --- a/tests/functional/build-remote-trustless-should-pass-2.sh +++ b/tests/functional/build-remote-trustless-should-pass-2.sh @@ -4,6 +4,8 @@ source common.sh enableFeatures "daemon-trust-override" +TODO_NixOS + restartDaemon # Remote doesn't trust us diff --git a/tests/functional/build-remote-trustless-should-pass-3.sh b/tests/functional/build-remote-trustless-should-pass-3.sh index 187b89948..d01d79191 100755 --- a/tests/functional/build-remote-trustless-should-pass-3.sh +++ b/tests/functional/build-remote-trustless-should-pass-3.sh @@ -4,6 +4,7 @@ source common.sh enableFeatures "daemon-trust-override" +TODO_NixOS restartDaemon # Remote doesn't trusts us, but this is fine because we are only diff --git a/tests/functional/build.sh b/tests/functional/build.sh index a14e6d672..9de953d8c 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible # Make sure that 'nix build' returns all outputs by default. nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status ' diff --git a/tests/functional/ca/common.sh b/tests/functional/ca/common.sh index b104b5a78..48f1ac46b 100644 --- a/tests/functional/ca/common.sh +++ b/tests/functional/ca/common.sh @@ -2,4 +2,6 @@ source ../common.sh enableFeatures "ca-derivations" +TODO_NixOS + restartDaemon diff --git a/tests/functional/case-hack.sh b/tests/functional/case-hack.sh index 48a2ab13f..feddc6583 100755 --- a/tests/functional/case-hack.sh +++ b/tests/functional/case-hack.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore rm -rf "$TEST_ROOT/case" diff --git a/tests/functional/lang-test-infra.sh b/tests/functional/characterisation-test-infra.sh similarity index 98% rename from tests/functional/lang-test-infra.sh rename to tests/functional/characterisation-test-infra.sh index f32ccef05..279454550 100755 --- a/tests/functional/lang-test-infra.sh +++ b/tests/functional/characterisation-test-infra.sh @@ -3,7 +3,7 @@ # Test the function for lang.sh source common.sh -source lang/framework.sh +source characterisation/framework.sh # We are testing this, so don't want outside world to affect us. unset _NIX_TEST_ACCEPT diff --git a/tests/functional/lang/empty.exp b/tests/functional/characterisation/empty similarity index 100% rename from tests/functional/lang/empty.exp rename to tests/functional/characterisation/empty diff --git a/tests/functional/characterisation/framework.sh b/tests/functional/characterisation/framework.sh new file mode 100644 index 000000000..913fdd967 --- /dev/null +++ b/tests/functional/characterisation/framework.sh @@ -0,0 +1,77 @@ +# shellcheck shell=bash + +# Golden test support +# +# Test that the output of the given test matches what is expected. If +# `_NIX_TEST_ACCEPT` is non-empty also update the expected output so +# that next time the test succeeds. +function diffAndAcceptInner() { + local -r testName=$1 + local -r got="$2" + local -r expected="$3" + + # Absence of expected file indicates empty output expected. + if test -e "$expected"; then + local -r expectedOrEmpty="$expected" + else + local -r expectedOrEmpty=characterisation/empty + fi + + # Diff so we get a nice message + if ! diff --color=always --unified "$expectedOrEmpty" "$got"; then + echo "FAIL: evaluation result of $testName not as expected" + # shellcheck disable=SC2034 + badDiff=1 + fi + + # Update expected if `_NIX_TEST_ACCEPT` is non-empty. + if test -n "${_NIX_TEST_ACCEPT-}"; then + cp "$got" "$expected" + # Delete empty expected files to avoid bloating the repo with + # empty files. + if ! test -s "$expected"; then + rm "$expected" + fi + fi +} + +function characterisationTestExit() { + # Make sure shellcheck knows all these will be defined by the caller + : "${badDiff?} ${badExitCode?}" + + if test -n "${_NIX_TEST_ACCEPT-}"; then + if (( "$badDiff" )); then + set +x + echo 'Output did mot match, but accepted output as the persisted expected output.' + echo 'That means the next time the tests are run, they should pass.' + set -x + else + set +x + echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,' + echo 'indicating the unexpected output should be accepted as the expected output going forward,' + echo 'but no tests had unexpected output so there was no expected output to update.' + set -x + fi + if (( "$badExitCode" )); then + exit "$badExitCode" + else + skipTest "regenerating golden masters" + fi + else + if (( "$badDiff" )); then + set +x + echo '' + echo 'You can rerun this test with:' + echo '' + echo " _NIX_TEST_ACCEPT=1 make tests/functional/${TEST_NAME}.test" + echo '' + echo 'to regenerate the files containing the expected output,' + echo 'and then view the git diff to decide whether a change is' + echo 'good/intentional or bad/unintentional.' + echo 'If the diff contains arbitrary or impure information,' + echo 'please improve the normalization that the test applies to the output.' + set -x + fi + exit $(( "$badExitCode" + "$badDiff" )) + fi +} diff --git a/tests/functional/check-refs.sh b/tests/functional/check-refs.sh index 6534e55c6..5c3ac915e 100755 --- a/tests/functional/check-refs.sh +++ b/tests/functional/check-refs.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore RESULT=$TEST_ROOT/result @@ -43,13 +45,18 @@ nix-build -o "$RESULT" check-refs.nix -A test7 # test10 should succeed (no disallowed references). nix-build -o "$RESULT" check-refs.nix -A test10 -if isDaemonNewer 2.12pre20230103; then - if ! isDaemonNewer 2.16.0; then - enableFeatures discard-references - restartDaemon +if ! isTestOnNixOS; then + # If we have full control over our store, we can test some more things. + + if isDaemonNewer 2.12pre20230103; then + if ! isDaemonNewer 2.16.0; then + enableFeatures discard-references + restartDaemon + fi + + # test11 should succeed. + test11=$(nix-build -o "$RESULT" check-refs.nix -A test11) + [[ -z $(nix-store -q --references "$test11") ]] fi - # test11 should succeed. - test11=$(nix-build -o "$RESULT" check-refs.nix -A test11) - [[ -z $(nix-store -q --references "$test11") ]] fi diff --git a/tests/functional/check-reqs.sh b/tests/functional/check-reqs.sh index 4d795391e..34eb133db 100755 --- a/tests/functional/check-reqs.sh +++ b/tests/functional/check-reqs.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible RESULT=$TEST_ROOT/result diff --git a/tests/functional/check.sh b/tests/functional/check.sh index 7b70d6aef..9b15dccb6 100755 --- a/tests/functional/check.sh +++ b/tests/functional/check.sh @@ -15,6 +15,8 @@ checkBuildTempDirRemoved () # written to build temp directories to verify created by this instance checkBuildId=$(date +%s%N) +TODO_NixOS + clearStore nix-build dependencies.nix --no-out-link @@ -79,6 +81,8 @@ grep 'may not be deterministic' $TEST_ROOT/log [ "$status" = "104" ] if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi +TODO_NixOS + clearStore path=$(nix-build check.nix -A fetchurl --no-out-link) diff --git a/tests/functional/chroot-store.sh b/tests/functional/chroot-store.sh index 741907fca..03803a2b9 100755 --- a/tests/functional/chroot-store.sh +++ b/tests/functional/chroot-store.sh @@ -39,6 +39,8 @@ EOF cp simple.nix shell.nix simple.builder.sh config.nix "$flakeDir/" + TODO_NixOS + outPath=$(nix build --print-out-paths --no-link --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store "$TEST_ROOT/x" path:"$flakeDir") [[ $outPath =~ ^/nix2/store/.*-simple$ ]] diff --git a/tests/functional/common/init.sh b/tests/functional/common/init.sh index 4f2a393af..d33ad5d57 100755 --- a/tests/functional/common/init.sh +++ b/tests/functional/common/init.sh @@ -1,5 +1,31 @@ # shellcheck shell=bash +# for shellcheck +: "${test_nix_conf_dir?}" "${test_nix_conf?}" + +if isTestOnNixOS; then + + mkdir -p "$test_nix_conf_dir" "$TEST_HOME" + + export NIX_USER_CONF_FILES="$test_nix_conf_dir/nix.conf" + mkdir -p "$test_nix_conf_dir" "$TEST_HOME" + ! test -e "$test_nix_conf" + cat > "$test_nix_conf_dir/nix.conf" <&2 + exit 1 +} + set +x commonDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")" source "$commonDir/subst-vars.sh" # Make sure shellcheck knows all these will be defined by the above generated snippet -: "${PATH?} ${coreutils?} ${dot?} ${SHELL?} ${PAGER?} ${busybox?} ${version?} ${system?} ${BUILD_SHARED_LIBS?}" +: "${bindir?} ${coreutils?} ${dot?} ${SHELL?} ${PAGER?} ${busybox?} ${version?} ${system?} ${BUILD_SHARED_LIBS?}" + +source "$commonDir/paths.sh" +source "$commonDir/test-root.sh" + +test_nix_conf_dir=$TEST_ROOT/etc +test_nix_conf=$test_nix_conf_dir/nix.conf -export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default/tests\/functional//} -export NIX_STORE_DIR -if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then - # Maybe the build directory is symlinked. - export NIX_IGNORE_SYMLINK_STORE=1 - NIX_STORE_DIR=$TEST_ROOT/store -fi -export NIX_LOCALSTATE_DIR=$TEST_ROOT/var -export NIX_LOG_DIR=$TEST_ROOT/var/log/nix -export NIX_STATE_DIR=$TEST_ROOT/var/nix -export NIX_CONF_DIR=$TEST_ROOT/etc -export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket -unset NIX_USER_CONF_FILES -export _NIX_TEST_SHARED=$TEST_ROOT/shared -if [[ -n $NIX_STORE ]]; then - export _NIX_TEST_NO_SANDBOX=1 -fi -export _NIX_IN_TEST=$TEST_ROOT/shared -export _NIX_TEST_NO_LSOF=1 -export NIX_REMOTE=${NIX_REMOTE_-} -unset NIX_PATH export TEST_HOME=$TEST_ROOT/test-home + +if ! isTestOnNixOS; then + export NIX_STORE_DIR + if ! NIX_STORE_DIR=$(readlink -f $TEST_ROOT/store 2> /dev/null); then + # Maybe the build directory is symlinked. + export NIX_IGNORE_SYMLINK_STORE=1 + NIX_STORE_DIR=$TEST_ROOT/store + fi + export NIX_LOCALSTATE_DIR=$TEST_ROOT/var + export NIX_LOG_DIR=$TEST_ROOT/var/log/nix + export NIX_STATE_DIR=$TEST_ROOT/var/nix + export NIX_CONF_DIR=$test_nix_conf_dir + export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket + unset NIX_USER_CONF_FILES + export _NIX_TEST_SHARED=$TEST_ROOT/shared + if [[ -n $NIX_STORE ]]; then + export _NIX_TEST_NO_SANDBOX=1 + fi + export _NIX_IN_TEST=$TEST_ROOT/shared + export _NIX_TEST_NO_LSOF=1 + export NIX_REMOTE=${NIX_REMOTE_-} + +fi # ! isTestOnNixOS + +unset NIX_PATH export HOME=$TEST_HOME unset XDG_STATE_HOME unset XDG_DATA_HOME @@ -43,14 +62,6 @@ unset XDG_CONFIG_HOME unset XDG_CONFIG_DIRS unset XDG_CACHE_HOME -if [[ -n "${NIX_CLIENT_PACKAGE:-}" ]]; then - export PATH="$NIX_CLIENT_PACKAGE/bin":$PATH -fi -DAEMON_PATH="$PATH" -if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then - DAEMON_PATH="${NIX_DAEMON_PACKAGE}/bin:$DAEMON_PATH" -fi - export IMPURE_VAR1=foo export IMPURE_VAR2=bar @@ -65,7 +76,25 @@ clearProfiles() { rm -rf "$profiles" } +# Clear the store, but do not fail if we're in an environment where we can't. +# This allows the test to run in a NixOS test environment, where we use the system store. +# See doc/manual/src/contributing/testing.md / Running functional tests on NixOS. +clearStoreIfPossible() { + if isTestOnNixOS; then + echo "clearStoreIfPossible: Not clearing store, because we're on NixOS. Moving on." + else + doClearStore + fi +} + clearStore() { + if isTestOnNixOS; then + die "clearStore: not supported when testing on NixOS. If not essential, call clearStoreIfPossible. If really needed, add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + doClearStore +} + +doClearStore() { echo "clearing store..." chmod -R +w "$NIX_STORE_DIR" rm -rf "$NIX_STORE_DIR" @@ -84,6 +113,10 @@ clearCacheCache() { } startDaemon() { + if isTestOnNixOS; then + die "startDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + # Don’t start the daemon twice, as this would just make it loop indefinitely if [[ "${_NIX_TEST_DAEMON_PID-}" != '' ]]; then return @@ -110,6 +143,10 @@ startDaemon() { } killDaemon() { + if isTestOnNixOS; then + die "killDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + # Don’t fail trying to stop a non-existant daemon twice if [[ "${_NIX_TEST_DAEMON_PID-}" == '' ]]; then return @@ -130,6 +167,10 @@ killDaemon() { } restartDaemon() { + if isTestOnNixOS; then + die "restartDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + [[ -z "${_NIX_TEST_DAEMON_PID:-}" ]] && return 0 killDaemon @@ -152,6 +193,12 @@ skipTest () { exit 99 } +TODO_NixOS() { + if isTestOnNixOS; then + skipTest "This test has not been adapted for NixOS yet" + fi +} + requireDaemonNewerThan () { isDaemonNewer "$1" || skipTest "Daemon is too old" } @@ -169,7 +216,7 @@ requireGit() { } fail() { - echo "$1" >&2 + echo "test failed: $*" >&2 exit 1 } @@ -234,7 +281,7 @@ buggyNeedLocalStore() { enableFeatures() { local features="$1" - sed -i 's/experimental-features .*/& '"$features"'/' "$NIX_CONF_DIR"/nix.conf + sed -i 's/experimental-features .*/& '"$features"'/' "$test_nix_conf_dir"/nix.conf } set -x diff --git a/tests/functional/compression-levels.sh b/tests/functional/compression-levels.sh index 6a2111f10..399265f9c 100755 --- a/tests/functional/compression-levels.sh +++ b/tests/functional/compression-levels.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible clearCache outPath=$(nix-build dependencies.nix --no-out-link) diff --git a/tests/functional/config.sh b/tests/functional/config.sh index 1811b755c..50858eaa4 100755 --- a/tests/functional/config.sh +++ b/tests/functional/config.sh @@ -28,6 +28,8 @@ nix registry remove userhome-with-xdg # Assert the .config folder hasn't been created. [ ! -e "$HOME/.config" ] +TODO_NixOS # Very specific test setup not compatible with the NixOS test environment? + # Test that files are loaded from XDG by default export XDG_CONFIG_HOME=$TEST_ROOT/confighome export XDG_CONFIG_DIRS=$TEST_ROOT/dir1:$TEST_ROOT/dir2 diff --git a/tests/functional/db-migration.sh b/tests/functional/db-migration.sh index a6a5c7744..6feabb90d 100755 --- a/tests/functional/db-migration.sh +++ b/tests/functional/db-migration.sh @@ -10,6 +10,8 @@ if [[ -z "${NIX_DAEMON_PACKAGE-}" ]]; then skipTest "not using the Nix daemon" fi +TODO_NixOS + killDaemon # Fill the db using the older Nix diff --git a/tests/functional/debugger.sh b/tests/functional/debugger.sh index 47e644bb7..b96b7e5d3 100755 --- a/tests/functional/debugger.sh +++ b/tests/functional/debugger.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible # regression #9932 echo ":env" | expect 1 nix eval --debugger --expr '(_: throw "oh snap") 42' diff --git a/tests/functional/dependencies.sh b/tests/functional/dependencies.sh index 1b266935d..972bc5a9b 100755 --- a/tests/functional/dependencies.sh +++ b/tests/functional/dependencies.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible drvPath=$(nix-instantiate dependencies.nix) @@ -65,6 +65,8 @@ drvPath2=$(nix-instantiate dependencies.nix --argstr hashInvalidator yay) # now --valid-derivers returns both test "$(nix-store -q --valid-derivers "$outPath" | sort)" = "$(sort <<< "$drvPath"$'\n'"$drvPath2")" +TODO_NixOS # The following --delete fails, because it seems to be still alive. This might be caused by a different test using the same path. We should try make the derivations unique, e.g. naming after tests, and adding a timestamp that's constant for that test script run. + # check that nix-store --valid-derivers only returns existing drv nix-store --delete "$drvPath" test "$(nix-store -q --valid-derivers "$outPath")" = "$drvPath2" diff --git a/tests/functional/derivation-advanced-attributes.sh b/tests/functional/derivation-advanced-attributes.sh new file mode 100755 index 000000000..6c0c76b4c --- /dev/null +++ b/tests/functional/derivation-advanced-attributes.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +source common/test-root.sh +source common/paths.sh + +set -o pipefail + +source characterisation/framework.sh + +badDiff=0 +badExitCode=0 + +store="$TEST_ROOT/store" + +for nixFile in derivation/*.nix; do + drvPath=$(nix-instantiate --store "$store" --pure-eval --expr "$(< "$nixFile")") + testName=$(basename "$nixFile" .nix) + got="${store}${drvPath}" + expected="derivation/$testName.drv" + diffAndAcceptInner "$testName" "$got" "$expected" +done + +characterisationTestExit diff --git a/tests/functional/derivation/advanced-attributes-defaults.drv b/tests/functional/derivation/advanced-attributes-defaults.drv new file mode 100644 index 000000000..391c6ab80 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-defaults.drv @@ -0,0 +1 @@ +Derive([("out","/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults","","")],[],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("builder","/bin/bash"),("name","advanced-attributes-defaults"),("out","/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-defaults.nix b/tests/functional/derivation/advanced-attributes-defaults.nix new file mode 100644 index 000000000..51a8d0e7e --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-defaults.nix @@ -0,0 +1,6 @@ +derivation { + name = "advanced-attributes-defaults"; + system = "my-system"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; +} diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv new file mode 100644 index 000000000..9dd402057 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv @@ -0,0 +1 @@ +Derive([("dev","/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev","",""),("out","/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults","","")],[],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}"),("dev","/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev"),("out","/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix new file mode 100644 index 000000000..0c13a7691 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix @@ -0,0 +1,8 @@ +derivation { + name = "advanced-attributes-structured-attrs-defaults"; + system = "my-system"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; + outputs = [ "out" "dev" ]; + __structuredAttrs = true; +} diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs.drv b/tests/functional/derivation/advanced-attributes-structured-attrs.drv new file mode 100644 index 000000000..e47a41ad5 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +Derive([("bin","/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin","",""),("dev","/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev","",""),("out","/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin"),("dev","/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev"),("out","/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs.nix b/tests/functional/derivation/advanced-attributes-structured-attrs.nix new file mode 100644 index 000000000..0044b65fd --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs.nix @@ -0,0 +1,45 @@ +let + system = "my-system"; + foo = derivation { + inherit system; + name = "foo"; + builder = "/bin/bash"; + args = ["-c" "echo foo > $out"]; + }; + bar = derivation { + inherit system; + name = "bar"; + builder = "/bin/bash"; + args = ["-c" "echo bar > $out"]; + }; +in +derivation { + inherit system; + name = "advanced-attributes-structured-attrs"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; + __sandboxProfile = "sandcastle"; + __noChroot = true; + __impureHostDeps = ["/usr/bin/ditto"]; + impureEnvVars = ["UNICORN"]; + __darwinAllowLocalNetworking = true; + outputs = [ "out" "bin" "dev" ]; + __structuredAttrs = true; + outputChecks = { + out = { + allowedReferences = [foo]; + allowedRequisites = [foo]; + }; + bin = { + disallowedReferences = [bar]; + disallowedRequisites = [bar]; + }; + dev = { + maxSize = 789; + maxClosureSize = 5909; + }; + }; + requiredSystemFeatures = ["rainbow" "uid-range"]; + preferLocalBuild = true; + allowSubstitutes = false; +} diff --git a/tests/functional/derivation/advanced-attributes.drv b/tests/functional/derivation/advanced-attributes.drv new file mode 100644 index 000000000..ec3112ab2 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes.drv @@ -0,0 +1 @@ +Derive([("out","/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("allowedRequisites","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("builder","/bin/bash"),("disallowedReferences","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("disallowedRequisites","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes.nix b/tests/functional/derivation/advanced-attributes.nix new file mode 100644 index 000000000..ff680c567 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes.nix @@ -0,0 +1,33 @@ +let + system = "my-system"; + foo = derivation { + inherit system; + name = "foo"; + builder = "/bin/bash"; + args = ["-c" "echo foo > $out"]; + }; + bar = derivation { + inherit system; + name = "bar"; + builder = "/bin/bash"; + args = ["-c" "echo bar > $out"]; + }; +in +derivation { + inherit system; + name = "advanced-attributes"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; + __sandboxProfile = "sandcastle"; + __noChroot = true; + __impureHostDeps = ["/usr/bin/ditto"]; + impureEnvVars = ["UNICORN"]; + __darwinAllowLocalNetworking = true; + allowedReferences = [foo]; + allowedRequisites = [foo]; + disallowedReferences = [bar]; + disallowedRequisites = [bar]; + requiredSystemFeatures = ["rainbow" "uid-range"]; + preferLocalBuild = true; + allowSubstitutes = false; +} diff --git a/tests/functional/dump-db.sh b/tests/functional/dump-db.sh index 2d0460275..14181b4b6 100755 --- a/tests/functional/dump-db.sh +++ b/tests/functional/dump-db.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + needLocalStore "--dump-db requires a local store" clearStore diff --git a/tests/functional/dyn-drv/common.sh b/tests/functional/dyn-drv/common.sh index c786f6925..0d95881b6 100644 --- a/tests/functional/dyn-drv/common.sh +++ b/tests/functional/dyn-drv/common.sh @@ -5,4 +5,6 @@ requireDaemonNewerThan "2.16.0pre20230419" enableFeatures "ca-derivations dynamic-derivations" +TODO_NixOS + restartDaemon diff --git a/tests/functional/eval-store.sh b/tests/functional/eval-store.sh index 0ab608acc..202e7b004 100755 --- a/tests/functional/eval-store.sh +++ b/tests/functional/eval-store.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + # Using `--eval-store` with the daemon will eventually copy everything # to the build store, invalidating most of the tests here needLocalStore "“--eval-store” doesn't achieve much with the daemon" diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh index acd6e2915..27cdce478 100755 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible testStdinHeredoc=$(nix eval -f - < $TEST_ROOT/counter diff --git a/tests/functional/impure-env.sh b/tests/functional/impure-env.sh index 3c7df169e..ca32c1030 100755 --- a/tests/functional/impure-env.sh +++ b/tests/functional/impure-env.sh @@ -5,6 +5,8 @@ source common.sh # Needs the config option 'impure-env' to work requireDaemonNewerThan "2.19.0" +TODO_NixOS + enableFeatures "configurable-impure-env" restartDaemon @@ -20,13 +22,13 @@ startDaemon varTest env_name value --impure-env env_name=value -echo 'impure-env = set_in_config=config_value' >> "$NIX_CONF_DIR/nix.conf" +echo 'impure-env = set_in_config=config_value' >> "$test_nix_conf" set_in_config=daemon_value restartDaemon varTest set_in_config config_value varTest set_in_config client_value --impure-env set_in_config=client_value -sed -i -e '/^trusted-users =/d' "$NIX_CONF_DIR/nix.conf" +sed -i -e '/^trusted-users =/d' "$test_nix_conf" env_name=daemon_value restartDaemon diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index 569e7082e..8cb8e98fb 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -4,7 +4,7 @@ source common.sh set -o pipefail -source lang/framework.sh +source characterisation/framework.sh # specialize function a bit function diffAndAccept() { @@ -138,32 +138,4 @@ for i in lang/eval-okay-*.nix; do fi done -if test -n "${_NIX_TEST_ACCEPT-}"; then - if (( "$badDiff" )); then - echo 'Output did mot match, but accepted output as the persisted expected output.' - echo 'That means the next time the tests are run, they should pass.' - else - echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,' - echo 'indicating the unexpected output should be accepted as the expected output going forward,' - echo 'but no tests had unexpected output so there was no expected output to update.' - fi - if (( "$badExitCode" )); then - exit "$badExitCode" - else - skipTest "regenerating golden masters" - fi -else - if (( "$badDiff" )); then - echo '' - echo 'You can rerun this test with:' - echo '' - echo ' _NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test' - echo '' - echo 'to regenerate the files containing the expected output,' - echo 'and then view the git diff to decide whether a change is' - echo 'good/intentional or bad/unintentional.' - echo 'If the diff contains arbitrary or impure information,' - echo 'please improve the normalization that the test applies to the output.' - fi - exit $(( "$badExitCode" + "$badDiff" )) -fi +characterisationTestExit diff --git a/tests/functional/lang/eval-fail-derivation-name.err.exp b/tests/functional/lang/eval-fail-derivation-name.err.exp new file mode 100644 index 000000000..eb2206df1 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivation-name.err.exp @@ -0,0 +1,26 @@ +error: + … while evaluating the attribute 'outPath' + at :19:9: + 18| value = commonAttrs // { + 19| outPath = builtins.getAttr outputName strict; + | ^ + 20| drvPath = strict.drvPath; + + … while calling the 'getAttr' builtin + at :19:19: + 18| value = commonAttrs // { + 19| outPath = builtins.getAttr outputName strict; + | ^ + 20| drvPath = strict.drvPath; + + … while calling the 'derivationStrict' builtin + at :9:12: + 8| + 9| strict = derivationStrict drvAttrs; + | ^ + 10| + + … while evaluating derivation '~jiggle~' + whose name attribute is located at /pwd/lang/eval-fail-derivation-name.nix:2:3 + + error: invalid derivation name: name '~jiggle~' contains illegal character '~'. Please pass a different 'name'. diff --git a/tests/functional/lang/eval-fail-derivation-name.nix b/tests/functional/lang/eval-fail-derivation-name.nix new file mode 100644 index 000000000..e779ad6ff --- /dev/null +++ b/tests/functional/lang/eval-fail-derivation-name.nix @@ -0,0 +1,5 @@ +derivation { + name = "~jiggle~"; + system = "some-system"; + builder = "/dontcare"; +} diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.err.exp b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.err.exp new file mode 100644 index 000000000..30f8b6a35 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchurl' builtin + at /pwd/lang/eval-fail-fetchurl-baseName-attrs-name.nix:1:1: + 1| builtins.fetchurl { url = "https://example.com/foo.tar.gz"; name = "~wobble~"; } + | ^ + 2| + + error: invalid store path name when fetching URL 'https://example.com/foo.tar.gz': name '~wobble~' contains illegal character '~'. Please change the value for the 'name' attribute passed to 'fetchurl', so that it can create a valid store path. diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.nix b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.nix new file mode 100644 index 000000000..583805539 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.nix @@ -0,0 +1 @@ +builtins.fetchurl { url = "https://example.com/foo.tar.gz"; name = "~wobble~"; } diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.err.exp b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.err.exp new file mode 100644 index 000000000..cef532e94 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchurl' builtin + at /pwd/lang/eval-fail-fetchurl-baseName-attrs.nix:1:1: + 1| builtins.fetchurl { url = "https://example.com/~wiggle~"; } + | ^ + 2| + + error: invalid store path name when fetching URL 'https://example.com/~wiggle~': name '~wiggle~' contains illegal character '~'. Please add a valid 'name' attribute to the argument for 'fetchurl', so that it can create a valid store path. diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.nix b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.nix new file mode 100644 index 000000000..068120edb --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.nix @@ -0,0 +1 @@ +builtins.fetchurl { url = "https://example.com/~wiggle~"; } diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName.err.exp b/tests/functional/lang/eval-fail-fetchurl-baseName.err.exp new file mode 100644 index 000000000..0950e8e70 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchurl' builtin + at /pwd/lang/eval-fail-fetchurl-baseName.nix:1:1: + 1| builtins.fetchurl "https://example.com/~wiggle~" + | ^ + 2| + + error: invalid store path name when fetching URL 'https://example.com/~wiggle~': name '~wiggle~' contains illegal character '~'. Please pass an attribute set with 'url' and 'name' attributes to 'fetchurl', so that it can create a valid store path. diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName.nix b/tests/functional/lang/eval-fail-fetchurl-baseName.nix new file mode 100644 index 000000000..965093843 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName.nix @@ -0,0 +1 @@ +builtins.fetchurl "https://example.com/~wiggle~" diff --git a/tests/functional/lang/eval-okay-derivation-legacy.err.exp b/tests/functional/lang/eval-okay-derivation-legacy.err.exp new file mode 100644 index 000000000..94f0854dd --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.err.exp @@ -0,0 +1,6 @@ +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks..allowedReferences' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks..allowedRequisites' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks..disallowedReferences' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks..disallowedRequisites' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks..maxClosureSize' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks..maxSize' instead diff --git a/tests/functional/lang/eval-okay-derivation-legacy.exp b/tests/functional/lang/eval-okay-derivation-legacy.exp new file mode 100644 index 000000000..4f374a1aa --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.exp @@ -0,0 +1 @@ +"/nix/store/mzgwvrjjir216ra58mwwizi8wj6y9ddr-eval-okay-derivation-legacy" diff --git a/tests/functional/lang/eval-okay-derivation-legacy.nix b/tests/functional/lang/eval-okay-derivation-legacy.nix new file mode 100644 index 000000000..b529cdf90 --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.nix @@ -0,0 +1,12 @@ +(builtins.derivationStrict { + name = "eval-okay-derivation-legacy"; + system = "x86_64-linux"; + builder = "/dontcare"; + __structuredAttrs = true; + allowedReferences = [ ]; + disallowedReferences = [ ]; + allowedRequisites = [ ]; + disallowedRequisites = [ ]; + maxSize = 1234; + maxClosureSize = 12345; +}).out diff --git a/tests/functional/lang/framework.sh b/tests/functional/lang/framework.sh deleted file mode 100644 index 9b886e983..000000000 --- a/tests/functional/lang/framework.sh +++ /dev/null @@ -1,33 +0,0 @@ -# Golden test support -# -# Test that the output of the given test matches what is expected. If -# `_NIX_TEST_ACCEPT` is non-empty also update the expected output so -# that next time the test succeeds. -function diffAndAcceptInner() { - local -r testName=$1 - local -r got="$2" - local -r expected="$3" - - # Absence of expected file indicates empty output expected. - if test -e "$expected"; then - local -r expectedOrEmpty="$expected" - else - local -r expectedOrEmpty=lang/empty.exp - fi - - # Diff so we get a nice message - if ! diff --color=always --unified "$expectedOrEmpty" "$got"; then - echo "FAIL: evaluation result of $testName not as expected" - badDiff=1 - fi - - # Update expected if `_NIX_TEST_ACCEPT` is non-empty. - if test -n "${_NIX_TEST_ACCEPT-}"; then - cp "$got" "$expected" - # Delete empty expected files to avoid bloating the repo with - # empty files. - if ! test -s "$expected"; then - rm "$expected" - fi - fi -} diff --git a/tests/functional/linux-sandbox.sh b/tests/functional/linux-sandbox.sh index e71224f5e..653a3873f 100755 --- a/tests/functional/linux-sandbox.sh +++ b/tests/functional/linux-sandbox.sh @@ -4,6 +4,8 @@ source common.sh needLocalStore "the sandbox only runs on the builder side, so it makes no sense to test it with the daemon" +TODO_NixOS + clearStore requireSandboxSupport diff --git a/tests/functional/local-overlay-store/bad-uris.sh b/tests/functional/local-overlay-store/bad-uris.sh index 42a6d47f7..b7930e32e 100644 --- a/tests/functional/local-overlay-store/bad-uris.sh +++ b/tests/functional/local-overlay-store/bad-uris.sh @@ -15,6 +15,8 @@ declare -a storesBad=( "$storeBadRoot" "$storeBadLower" "$storeBadUpper" ) +TODO_NixOS + for i in "${storesBad[@]}"; do echo $i unshare --mount --map-root-user bash <> "$NIX_CONF_DIR/nix.conf" + echo "$1" >> "$test_nix_conf" } setupConfig () { diff --git a/tests/functional/local.mk b/tests/functional/local.mk index b379eeefe..49ee31284 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -23,7 +23,7 @@ nix_tests = \ remote-store.sh \ legacy-ssh-store.sh \ lang.sh \ - lang-test-infra.sh \ + characterisation-test-infra.sh \ experimental-features.sh \ fetchMercurial.sh \ gc-auto.sh \ @@ -106,6 +106,7 @@ nix_tests = \ eval-store.sh \ why-depends.sh \ derivation-json.sh \ + derivation-advanced-attributes.sh \ import-derivation.sh \ nix_path.sh \ case-hack.sh \ diff --git a/tests/functional/logging.sh b/tests/functional/logging.sh index 63752f6db..bd80a9163 100755 --- a/tests/functional/logging.sh +++ b/tests/functional/logging.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore path=$(nix-build dependencies.nix --no-out-link) diff --git a/tests/functional/multiple-outputs.sh b/tests/functional/multiple-outputs.sh index 31ce2a3a4..35a78d152 100755 --- a/tests/functional/multiple-outputs.sh +++ b/tests/functional/multiple-outputs.sh @@ -2,7 +2,9 @@ source common.sh -clearStore +TODO_NixOS + +clearStoreIfPossible rm -f $TEST_ROOT/result* diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh index 8839fd043..b254081cf 100755 --- a/tests/functional/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -27,6 +27,8 @@ diff -u baz.cat-nar $storePath/foo/baz nix store cat $storePath/foo/baz > baz.cat-nar diff -u baz.cat-nar $storePath/foo/baz +TODO_NixOS + # Check that 'nix store cat' fails on invalid store paths. invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo" cp -r $storePath $invalidPath diff --git a/tests/functional/nested-sandboxing.sh b/tests/functional/nested-sandboxing.sh index 44c3bb2bc..ae0256de2 100755 --- a/tests/functional/nested-sandboxing.sh +++ b/tests/functional/nested-sandboxing.sh @@ -4,6 +4,8 @@ source common.sh # This test is run by `tests/functional/nested-sandboxing/runner.nix` in an extra layer of sandboxing. [[ -d /nix/store ]] || skipTest "running this test without Nix's deps being drawn from /nix/store is not yet supported" +TODO_NixOS + requireSandboxSupport source ./nested-sandboxing/command.sh diff --git a/tests/functional/nix-build.sh b/tests/functional/nix-build.sh index 45ff314c7..091e429e0 100755 --- a/tests/functional/nix-build.sh +++ b/tests/functional/nix-build.sh @@ -2,7 +2,9 @@ source common.sh -clearStore +TODO_NixOS + +clearStoreIfPossible outPath=$(nix-build dependencies.nix -o $TEST_ROOT/result) test "$(cat $TEST_ROOT/result/foobar)" = FOOBAR diff --git a/tests/functional/nix-collect-garbage-d.sh b/tests/functional/nix-collect-garbage-d.sh index 07aaf61e9..119efe629 100755 --- a/tests/functional/nix-collect-garbage-d.sh +++ b/tests/functional/nix-collect-garbage-d.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore ## Test `nix-collect-garbage -d` diff --git a/tests/functional/nix-copy-ssh-common.sh b/tests/functional/nix-copy-ssh-common.sh index cc8314ff7..5eea9612d 100644 --- a/tests/functional/nix-copy-ssh-common.sh +++ b/tests/functional/nix-copy-ssh-common.sh @@ -2,6 +2,8 @@ proto=$1 shift (( $# == 0 )) +TODO_NixOS + clearStore clearCache diff --git a/tests/functional/nix-copy-ssh-ng.sh b/tests/functional/nix-copy-ssh-ng.sh index 1fd735b9d..41958c2c3 100755 --- a/tests/functional/nix-copy-ssh-ng.sh +++ b/tests/functional/nix-copy-ssh-ng.sh @@ -4,6 +4,8 @@ source common.sh source nix-copy-ssh-common.sh "ssh-ng" +TODO_NixOS + clearStore clearRemoteStore diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index 3e5846cf2..e2f19b99e 100755 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore clearProfiles diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh index c38107e64..2c94705de 100755 --- a/tests/functional/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible if [[ -n ${CONTENT_ADDRESSED:-} ]]; then shellDotNix="$PWD/ca-shell.nix" diff --git a/tests/functional/optimise-store.sh b/tests/functional/optimise-store.sh index 70ce954f9..0bedafc43 100755 --- a/tests/functional/optimise-store.sh +++ b/tests/functional/optimise-store.sh @@ -2,11 +2,14 @@ source common.sh -clearStore +clearStoreIfPossible outPath1=$(echo 'with import ./config.nix; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) outPath2=$(echo 'with import ./config.nix; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) +TODO_NixOS # ignoring the client-specified setting 'auto-optimise-store', because it is a restricted setting and you are not a trusted user + # TODO: only continue when trusted user or root + inode1="$(stat --format=%i $outPath1/foo)" inode2="$(stat --format=%i $outPath2/foo)" if [ "$inode1" != "$inode2" ]; then diff --git a/tests/functional/output-normalization.sh b/tests/functional/output-normalization.sh index 2b319201a..c55f1b1d1 100755 --- a/tests/functional/output-normalization.sh +++ b/tests/functional/output-normalization.sh @@ -3,6 +3,7 @@ source common.sh testNormalization () { + TODO_NixOS clearStore outPath=$(nix-build ./simple.nix --no-out-link) test "$(stat -c %Y $outPath)" -eq 1 diff --git a/tests/functional/parallel.sh b/tests/functional/parallel.sh index 3b7bbe5a2..7e420688d 100644 --- a/tests/functional/parallel.sh +++ b/tests/functional/parallel.sh @@ -4,6 +4,8 @@ source common.sh # First, test that -jN performs builds in parallel. echo "testing nix-build -j..." +TODO_NixOS + clearStore rm -f $_NIX_TEST_SHARED.cur $_NIX_TEST_SHARED.max diff --git a/tests/functional/pass-as-file.sh b/tests/functional/pass-as-file.sh index 21d9ffc6d..6487bfffd 100755 --- a/tests/functional/pass-as-file.sh +++ b/tests/functional/pass-as-file.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible outPath=$(nix-build --no-out-link -E " with import ./config.nix; diff --git a/tests/functional/placeholders.sh b/tests/functional/placeholders.sh index f2b8bf6bf..33ec0c2b7 100755 --- a/tests/functional/placeholders.sh +++ b/tests/functional/placeholders.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible nix-build --no-out-link -E ' with import ./config.nix; diff --git a/tests/functional/plugins/plugintest.cc b/tests/functional/plugins/plugintest.cc index e02fd68d5..7433ad190 100644 --- a/tests/functional/plugins/plugintest.cc +++ b/tests/functional/plugins/plugintest.cc @@ -1,4 +1,4 @@ -#include "config.hh" +#include "config-global.hh" #include "primops.hh" using namespace nix; diff --git a/tests/functional/post-hook.sh b/tests/functional/post-hook.sh index c0b1ab3aa..94a6d0d69 100755 --- a/tests/functional/post-hook.sh +++ b/tests/functional/post-hook.sh @@ -2,12 +2,14 @@ source common.sh +TODO_NixOS + clearStore rm -f $TEST_ROOT/result export REMOTE_STORE=file:$TEST_ROOT/remote_store -echo 'require-sigs = false' >> $NIX_CONF_DIR/nix.conf +echo 'require-sigs = false' >> $test_nix_conf restartDaemon diff --git a/tests/functional/pure-eval.sh b/tests/functional/pure-eval.sh index 6d8aa35ec..250381099 100755 --- a/tests/functional/pure-eval.sh +++ b/tests/functional/pure-eval.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible nix eval --expr 'assert 1 + 2 == 3; true' diff --git a/tests/functional/read-only-store.sh b/tests/functional/read-only-store.sh index ecc57642d..f6b6eaf32 100755 --- a/tests/functional/read-only-store.sh +++ b/tests/functional/read-only-store.sh @@ -6,6 +6,8 @@ enableFeatures "read-only-local-store" needLocalStore "cannot open store read-only when daemon has already opened it writeable" +TODO_NixOS + clearStore happy () { diff --git a/tests/functional/readfile-context.sh b/tests/functional/readfile-context.sh index d0644471d..cb9ef6234 100755 --- a/tests/functional/readfile-context.sh +++ b/tests/functional/readfile-context.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS # NixOS doesn't provide $NIX_STATE_DIR (and shouldn't) + clearStore outPath=$(nix-build --no-out-link readfile-context.nix) diff --git a/tests/functional/recursive.sh b/tests/functional/recursive.sh index a9966aabd..640fb92d2 100755 --- a/tests/functional/recursive.sh +++ b/tests/functional/recursive.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS # can't enable a sandbox feature easily + enableFeatures 'recursive-nix' restartDaemon diff --git a/tests/functional/referrers.sh b/tests/functional/referrers.sh index 0fda97378..411cdb7c1 100755 --- a/tests/functional/referrers.sh +++ b/tests/functional/referrers.sh @@ -4,6 +4,8 @@ source common.sh needLocalStore "uses some low-level store manipulations that aren’t available through the daemon" +TODO_NixOS + clearStore max=500 diff --git a/tests/functional/remote-store.sh b/tests/functional/remote-store.sh index 171a5d391..841b6b27a 100755 --- a/tests/functional/remote-store.sh +++ b/tests/functional/remote-store.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore # Ensure "fake ssh" remote store works just as legacy fake ssh would. diff --git a/tests/functional/repair.sh b/tests/functional/repair.sh index 552e04280..1f6004b2c 100755 --- a/tests/functional/repair.sh +++ b/tests/functional/repair.sh @@ -4,6 +4,8 @@ source common.sh needLocalStore "--repair needs a local store" +TODO_NixOS + clearStore path=$(nix-build dependencies.nix -o $TEST_ROOT/result) diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index fca982807..86cd6f458 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -22,6 +22,8 @@ replUndefinedVariable=" import $testDir/undefined-variable.nix " +TODO_NixOS + testRepl () { local nixArgs nixArgs=("$@") diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index ab4cad5cf..915d973b0 100755 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible nix-instantiate --restrict-eval --eval -E '1 + 2' (! nix-instantiate --eval --restrict-eval ./restricted.nix) diff --git a/tests/functional/search.sh b/tests/functional/search.sh index ce17411d2..3fadecd02 100755 --- a/tests/functional/search.sh +++ b/tests/functional/search.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible clearCache (( $(nix search -f search.nix '' hello | wc -l) > 0 )) diff --git a/tests/functional/secure-drv-outputs.sh b/tests/functional/secure-drv-outputs.sh index 7d81db58b..5cc4af435 100755 --- a/tests/functional/secure-drv-outputs.sh +++ b/tests/functional/secure-drv-outputs.sh @@ -6,6 +6,8 @@ source common.sh +TODO_NixOS + clearStore startDaemon diff --git a/tests/functional/selfref-gc.sh b/tests/functional/selfref-gc.sh index 37ce33089..518aea66b 100755 --- a/tests/functional/selfref-gc.sh +++ b/tests/functional/selfref-gc.sh @@ -4,7 +4,7 @@ source common.sh requireDaemonNewerThan "2.6.0pre20211215" -clearStore +clearStoreIfPossible nix-build --no-out-link -E ' with import ./config.nix; diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh index 1760eefff..b4c03f547 100755 --- a/tests/functional/shell.sh +++ b/tests/functional/shell.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore clearCache diff --git a/tests/functional/signing.sh b/tests/functional/signing.sh index cf84ab377..890d1446f 100755 --- a/tests/functional/signing.sh +++ b/tests/functional/signing.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible clearCache nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 @@ -16,6 +16,7 @@ outPath=$(nix-build dependencies.nix --no-out-link --secret-key-files "$TEST_ROO # Verify that the path got signed. info=$(nix path-info --json $outPath) echo $info | jq -e '.[] | .ultimate == true' +TODO_NixOS # looks like an actual bug? Following line fails on NixOS: echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' echo $info | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' diff --git a/tests/functional/simple.sh b/tests/functional/simple.sh index 4e7d37f59..86acca0c2 100755 --- a/tests/functional/simple.sh +++ b/tests/functional/simple.sh @@ -17,6 +17,8 @@ echo "output path is $outPath" text=$(cat "$outPath/hello") if test "$text" != "Hello World!"; then exit 1; fi +TODO_NixOS + # Directed delete: $outPath is not reachable from a root, so it should # be deleteable. nix-store --delete $outPath diff --git a/tests/functional/store-info.sh b/tests/functional/store-info.sh index 2398f5beb..f37889fbb 100755 --- a/tests/functional/store-info.sh +++ b/tests/functional/store-info.sh @@ -16,4 +16,6 @@ fi expect 127 NIX_REMOTE=unix:$PWD/store nix store info || \ fail "nix store info on a non-existent store should fail" +TODO_NixOS + [[ "$(echo "$STORE_INFO_JSON" | jq -r ".url")" == "${NIX_REMOTE:-local}" ]] diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index ba7f5967e..ec1282668 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -6,7 +6,7 @@ source common.sh # tests for the older versions requireDaemonNewerThan "2.4pre20210712" -clearStore +clearStoreIfPossible rm -f $TEST_ROOT/result @@ -21,6 +21,8 @@ env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' +TODO_NixOS # following line fails. + # `nix develop` is a slightly special way of dealing with environment vars, it parses # these from a shell-file exported from a derivation. This is to test especially `outputs` # (which is an associative array in thsi case) being fine. diff --git a/tests/functional/suggestions.sh b/tests/functional/suggestions.sh index 6ec1cd322..8db6f7b97 100755 --- a/tests/functional/suggestions.sh +++ b/tests/functional/suggestions.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible cd "$TEST_HOME" diff --git a/tests/functional/supplementary-groups.sh b/tests/functional/supplementary-groups.sh index 9d474219f..5d329efc9 100755 --- a/tests/functional/supplementary-groups.sh +++ b/tests/functional/supplementary-groups.sh @@ -7,6 +7,8 @@ requireSandboxSupport if ! command -p -v unshare; then skipTest "Need unshare"; fi needLocalStore "The test uses --store always so we would just be bypassing the daemon" +TODO_NixOS + unshare --mount --map-root-user bash <&1) && status= if [ $status -ne 101 ]; then echo "error: 'nix-store' exited with '$status'; should have exited 101" + + # FIXME: https://github.com/NixOS/nix/issues/4813 + skipTest "Do not block CI until fixed" + exit 1 fi diff --git a/tests/functional/user-envs-migration.sh b/tests/functional/user-envs-migration.sh index 992586b95..0f33074e1 100755 --- a/tests/functional/user-envs-migration.sh +++ b/tests/functional/user-envs-migration.sh @@ -13,6 +13,8 @@ fi killDaemon unset NIX_REMOTE +TODO_NixOS + clearStore clearProfiles rm -rf ~/.nix-profile diff --git a/tests/functional/user-envs-test-case.sh b/tests/functional/user-envs-test-case.sh index f4a90a675..117c6c7a4 100644 --- a/tests/functional/user-envs-test-case.sh +++ b/tests/functional/user-envs-test-case.sh @@ -11,6 +11,8 @@ outPath10=$(nix-env -f ./user-envs.nix -qa --out-path --no-name '*' | grep foo-1 drvPath10=$(nix-env -f ./user-envs.nix -qa --drv-path --no-name '*' | grep foo-1.0) [ -n "$outPath10" -a -n "$drvPath10" ] +TODO_NixOS + # Query with json nix-env -f ./user-envs.nix -qa --json | jq -e '.[] | select(.name == "bar-0.1") | [ .outputName == "out", diff --git a/tests/functional/why-depends.sh b/tests/functional/why-depends.sh index 69b365069..ce53546d8 100755 --- a/tests/functional/why-depends.sh +++ b/tests/functional/why-depends.sh @@ -2,7 +2,7 @@ source common.sh -clearStore +clearStoreIfPossible cp ./dependencies.nix ./dependencies.builder0.sh ./config.nix $TEST_HOME diff --git a/tests/functional/zstd.sh b/tests/functional/zstd.sh index 3bf9d5601..90fe58539 100755 --- a/tests/functional/zstd.sh +++ b/tests/functional/zstd.sh @@ -2,6 +2,8 @@ source common.sh +TODO_NixOS + clearStore clearCache diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 2cf7a64c6..c0c7b42fd 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -7,7 +7,13 @@ let # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests runNixOSTestFor = system: test: (nixos-lib.runTest { - imports = [ test ]; + imports = [ + test + + # Add the quickBuild attribute to the check packages + ./quick-build.nix + ]; + hostPkgs = nixpkgsFor.${system}.native; defaults = { nixpkgs.pkgs = nixpkgsFor.${system}.native; @@ -133,5 +139,11 @@ in gzip-content-encoding = runNixOSTestFor "x86_64-linux" ./gzip-content-encoding.nix; + functional_user = runNixOSTestFor "x86_64-linux" ./functional/as-user.nix; + + functional_trusted = runNixOSTestFor "x86_64-linux" ./functional/as-trusted-user.nix; + + functional_root = runNixOSTestFor "x86_64-linux" ./functional/as-root.nix; + user-sandboxing = runNixOSTestFor "x86_64-linux" ./user-sandboxing; } diff --git a/tests/nixos/functional/as-root.nix b/tests/nixos/functional/as-root.nix new file mode 100644 index 000000000..96be3d593 --- /dev/null +++ b/tests/nixos/functional/as-root.nix @@ -0,0 +1,12 @@ +{ + name = "functional-tests-on-nixos_root"; + + imports = [ ./common.nix ]; + + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + run-test-suite >&2 + """) + ''; +} diff --git a/tests/nixos/functional/as-trusted-user.nix b/tests/nixos/functional/as-trusted-user.nix new file mode 100644 index 000000000..d6f825697 --- /dev/null +++ b/tests/nixos/functional/as-trusted-user.nix @@ -0,0 +1,18 @@ +{ + name = "functional-tests-on-nixos_trusted-user"; + + imports = [ ./common.nix ]; + + nodes.machine = { + users.users.alice = { isNormalUser = true; }; + nix.settings.trusted-users = [ "alice" ]; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + export TEST_TRUSTED_USER=1 + su --login --command "run-test-suite" alice >&2 + """) + ''; +} \ No newline at end of file diff --git a/tests/nixos/functional/as-user.nix b/tests/nixos/functional/as-user.nix new file mode 100644 index 000000000..1443f6e6c --- /dev/null +++ b/tests/nixos/functional/as-user.nix @@ -0,0 +1,16 @@ +{ + name = "functional-tests-on-nixos_user"; + + imports = [ ./common.nix ]; + + nodes.machine = { + users.users.alice = { isNormalUser = true; }; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + su --login --command "run-test-suite" alice >&2 + """) + ''; +} diff --git a/tests/nixos/functional/common.nix b/tests/nixos/functional/common.nix new file mode 100644 index 000000000..51fd76884 --- /dev/null +++ b/tests/nixos/functional/common.nix @@ -0,0 +1,71 @@ +{ lib, ... }: + +let + # FIXME (roberth) reference issue + inputDerivation = pkg: (pkg.overrideAttrs (o: { + disallowedReferences = [ ]; + })).inputDerivation; + +in +{ + # We rarely change the script in a way that benefits from type checking, so + # we skip it to save time. + skipTypeCheck = true; + + nodes.machine = { config, pkgs, ... }: { + + virtualisation.writableStore = true; + system.extraDependencies = [ + (inputDerivation config.nix.package) + ]; + + nix.settings.substituters = lib.mkForce []; + + environment.systemPackages = let + run-test-suite = pkgs.writeShellApplication { + name = "run-test-suite"; + runtimeInputs = [ pkgs.gnumake pkgs.jq pkgs.git ]; + text = '' + set -x + cat /proc/sys/fs/file-max + ulimit -Hn + ulimit -Sn + cd ~ + cp -r ${pkgs.nix.overrideAttrs (o: { + name = "nix-configured-source"; + outputs = [ "out" ]; + separateDebugInfo = false; + disallowedReferences = [ ]; + buildPhase = ":"; + checkPhase = ":"; + installPhase = '' + cp -r . $out + ''; + installCheckPhase = ":"; + fixupPhase = ":"; + doInstallCheck = true; + })} nix + chmod -R +w nix + cd nix + + # Tests we don't need + echo >tests/functional/plugins/local.mk + sed -i tests/functional/local.mk \ + -e 's!nix_tests += plugins\.sh!!' \ + -e 's!nix_tests += test-libstoreconsumer\.sh!!' \ + ; + + export isTestOnNixOS=1 + export version=${config.nix.package.version} + export NIX_REMOTE_=daemon + export NIX_REMOTE=daemon + export NIX_STORE=${builtins.storeDir} + make -j1 installcheck --keep-going + ''; + }; + in [ + run-test-suite + pkgs.git + ]; + }; +} diff --git a/tests/nixos/quick-build.nix b/tests/nixos/quick-build.nix new file mode 100644 index 000000000..37169b4e2 --- /dev/null +++ b/tests/nixos/quick-build.nix @@ -0,0 +1,47 @@ +test@{ lib, extendModules, ... }: +let + inherit (lib) mkOption types; +in +{ + options = { + quickBuild = mkOption { + description = '' + Whether to perform a "quick" build of the Nix package to test. + + When iterating on the functional tests, it's recommended to "set" this + to `true`, so that changes to the functional tests don't require any + recompilation of the package. + You can do so by building the `.quickBuild` attribute on the check package, + e.g: + ```console + nix build .#hydraJobs.functional_user.quickBuild + ``` + + We don't enable this by default to avoid the mostly unnecessary work of + performing an additional build of the package in cases where we build + the package normally anyway, such as in our pre-merge CI. + ''; + type = types.bool; + default = false; + }; + }; + + config = { + passthru.quickBuild = + let withQuickBuild = extendModules { modules = [{ quickBuild = true; }]; }; + in withQuickBuild.config.test; + + defaults = { pkgs, ... }: { + config = lib.mkIf test.config.quickBuild { + nix.package = pkgs.nix_noTests; + + system.forbiddenDependenciesRegexes = [ + # This would indicate that the quickBuild feature is broken. + # It could happen if NixOS has a dependency on pkgs.nix instead of + # config.nix.package somewhere. + (builtins.unsafeDiscardStringContext pkgs.nix.outPath) + ]; + }; + }; + }; +} \ No newline at end of file diff --git a/tests/unit/libexpr-support/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh index 1a4313990..eacbf0d5c 100644 --- a/tests/unit/libexpr-support/tests/libexpr.hh +++ b/tests/unit/libexpr-support/tests/libexpr.hh @@ -19,14 +19,14 @@ namespace nix { static void SetUpTestSuite() { LibStoreTest::SetUpTestSuite(); initGC(); - evalSettings.nixPath = {}; } protected: LibExprTest() : LibStoreTest() - , state({}, store) + , state({}, store, evalSettings, nullptr) { + evalSettings.nixPath = {}; } Value eval(std::string input, bool forceValue = true) { Value v; @@ -42,6 +42,8 @@ namespace nix { return state.symbols.create(value); } + bool readOnlyMode = true; + EvalSettings evalSettings{readOnlyMode}; EvalState state; }; diff --git a/tests/unit/libstore-support/tests/libstore.hh b/tests/unit/libstore-support/tests/libstore.hh index 267188224..84be52c23 100644 --- a/tests/unit/libstore-support/tests/libstore.hh +++ b/tests/unit/libstore-support/tests/libstore.hh @@ -8,19 +8,27 @@ namespace nix { -class LibStoreTest : public virtual ::testing::Test { - public: - static void SetUpTestSuite() { - initLibStore(false); - } +class LibStoreTest : public virtual ::testing::Test +{ +public: + static void SetUpTestSuite() + { + initLibStore(false); + } - protected: - LibStoreTest() - : store(openStore("dummy://")) - { } +protected: + LibStoreTest() + : store(openStore({ + .variant = + StoreReference::Specified{ + .scheme = "dummy", + }, + .params = {}, + })) + { + } - ref store; + ref store; }; - } /* namespace nix */ diff --git a/tests/unit/libstore/common-protocol.cc b/tests/unit/libstore/common-protocol.cc index d23805fc3..c8f6dd002 100644 --- a/tests/unit/libstore/common-protocol.cc +++ b/tests/unit/libstore/common-protocol.cc @@ -83,15 +83,15 @@ CHARACTERIZATION_TEST( "content-address", (std::tuple { ContentAddress { - .method = TextIngestionMethod {}, + .method = ContentAddressMethod::Raw::Text, .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -178,7 +178,7 @@ CHARACTERIZATION_TEST( std::nullopt, std::optional { ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, diff --git a/tests/unit/libstore/content-address.cc b/tests/unit/libstore/content-address.cc index cc1c7fcc6..72eb84fec 100644 --- a/tests/unit/libstore/content-address.cc +++ b/tests/unit/libstore/content-address.cc @@ -9,11 +9,11 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(ContentAddressMethod, testRoundTripPrintParse_1) { - for (const ContentAddressMethod & cam : { - ContentAddressMethod { TextIngestionMethod {} }, - ContentAddressMethod { FileIngestionMethod::Flat }, - ContentAddressMethod { FileIngestionMethod::Recursive }, - ContentAddressMethod { FileIngestionMethod::Git }, + for (ContentAddressMethod cam : { + ContentAddressMethod::Raw::Text, + ContentAddressMethod::Raw::Flat, + ContentAddressMethod::Raw::NixArchive, + ContentAddressMethod::Raw::Git, }) { EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam); } diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv new file mode 120000 index 000000000..353090ad8 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes-defaults.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-defaults.json b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.json new file mode 100644 index 000000000..d58e7d5b5 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.json @@ -0,0 +1,22 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "builder": "/bin/bash", + "name": "advanced-attributes-defaults", + "out": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults", + "system": "my-system" + }, + "inputDrvs": {}, + "inputSrcs": [], + "name": "advanced-attributes-defaults", + "outputs": { + "out": { + "path": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults" + } + }, + "system": "my-system" +} diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv new file mode 120000 index 000000000..11713da12 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes-structured-attrs-defaults.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json new file mode 100644 index 000000000..473d006ac --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json @@ -0,0 +1,24 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__json": "{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}", + "dev": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev", + "out": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" + }, + "inputDrvs": {}, + "inputSrcs": [], + "name": "advanced-attributes-structured-attrs-defaults", + "outputs": { + "dev": { + "path": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev" + }, + "out": { + "path": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" + } + }, + "system": "my-system" +} diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv new file mode 120000 index 000000000..962f8ea3f --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes-structured-attrs.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json new file mode 100644 index 000000000..324428124 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json @@ -0,0 +1,41 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", + "bin": "/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin", + "dev": "/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev", + "out": "/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs" + }, + "inputDrvs": { + "/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + }, + "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "inputSrcs": [], + "name": "advanced-attributes-structured-attrs", + "outputs": { + "bin": { + "path": "/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin" + }, + "dev": { + "path": "/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev" + }, + "out": { + "path": "/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs" + } + }, + "system": "my-system" +} diff --git a/tests/unit/libstore/data/derivation/advanced-attributes.drv b/tests/unit/libstore/data/derivation/advanced-attributes.drv new file mode 120000 index 000000000..2a53a05ca --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes.drv \ No newline at end of file diff --git a/tests/unit/libstore/derivation-advanced-attrs.cc b/tests/unit/libstore/derivation-advanced-attrs.cc new file mode 100644 index 000000000..26cf947a8 --- /dev/null +++ b/tests/unit/libstore/derivation-advanced-attrs.cc @@ -0,0 +1,234 @@ +#include +#include +#include + +#include "experimental-features.hh" +#include "derivations.hh" + +#include "tests/libstore.hh" +#include "tests/characterization.hh" +#include "parsed-derivations.hh" +#include "types.hh" + +namespace nix { + +using nlohmann::json; + +class DerivationAdvancedAttrsTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/derivation"; + +public: + Path goldenMaster(std::string_view testStem) const override + { + return unitTestData + "/" + testStem; + } +}; + +#define TEST_ATERM_JSON(STEM, NAME) \ + TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_from_json) \ + { \ + readTest(NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(goldenMaster(NAME ".drv")); \ + auto expected = parseDerivation(*store, std::move(aterm), NAME); \ + Derivation got = Derivation::fromJSON(*store, encoded); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_to_json) \ + { \ + writeTest( \ + NAME ".json", \ + [&]() -> json { \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(goldenMaster(NAME ".drv")); \ + return parseDerivation(*store, std::move(aterm), NAME).toJSON(*store); \ + }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } \ + \ + TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_from_aterm) \ + { \ + readTest(NAME ".drv", [&](auto encoded) { \ + /* Use JSON file instead of C++ literal as source of truth. */ \ + auto json = json::parse(readFile(goldenMaster(NAME ".json"))); \ + auto expected = Derivation::fromJSON(*store, json); \ + auto got = parseDerivation(*store, std::move(encoded), NAME); \ + EXPECT_EQ(got.toJSON(*store), expected.toJSON(*store)); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ + /* No corresponding write test, because we need to read the drv to write the json file */ + +TEST_ATERM_JSON(advancedAttributes_defaults, "advanced-attributes-defaults"); +TEST_ATERM_JSON(advancedAttributes, "advanced-attributes-defaults"); +TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attributes-structured-attrs"); +TEST_ATERM_JSON(advancedAttributes_structuredAttrs, "advanced-attributes-structured-attrs-defaults"); + +#undef TEST_ATERM_JSON + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_defaults) +{ + readTest("advanced-attributes-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), ""); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), false); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), false); + EXPECT_EQ(parsedDrv.getStringsAttr("allowedReferences"), std::nullopt); + EXPECT_EQ(parsedDrv.getStringsAttr("allowedRequisites"), std::nullopt); + EXPECT_EQ(parsedDrv.getStringsAttr("disallowedReferences"), std::nullopt); + EXPECT_EQ(parsedDrv.getStringsAttr("disallowedRequisites"), std::nullopt); + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), StringSet()); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), true); + EXPECT_EQ(parsedDrv.useUidRange(), false); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes) +{ + readTest("advanced-attributes.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + StringSet systemFeatures{"rainbow", "uid-range"}; + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), "sandcastle"); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), true); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings{"/usr/bin/ditto"}); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings{"UNICORN"}); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), true); + EXPECT_EQ( + parsedDrv.getStringsAttr("allowedReferences"), Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + EXPECT_EQ( + parsedDrv.getStringsAttr("allowedRequisites"), Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + EXPECT_EQ( + parsedDrv.getStringsAttr("disallowedReferences"), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + EXPECT_EQ( + parsedDrv.getStringsAttr("disallowedRequisites"), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), systemFeatures); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), false); + EXPECT_EQ(parsedDrv.useUidRange(), true); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttrs_defaults) +{ + readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), ""); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), false); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), false); + + { + auto structuredAttrs_ = parsedDrv.getStructuredAttrs(); + ASSERT_TRUE(structuredAttrs_); + auto & structuredAttrs = *structuredAttrs_; + + auto outputChecks_ = get(structuredAttrs, "outputChecks"); + ASSERT_FALSE(outputChecks_); + } + + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), StringSet()); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), true); + EXPECT_EQ(parsedDrv.useUidRange(), false); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttrs) +{ + readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + StringSet systemFeatures{"rainbow", "uid-range"}; + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), "sandcastle"); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), true); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings{"/usr/bin/ditto"}); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings{"UNICORN"}); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), true); + + { + auto structuredAttrs_ = parsedDrv.getStructuredAttrs(); + ASSERT_TRUE(structuredAttrs_); + auto & structuredAttrs = *structuredAttrs_; + + auto outputChecks_ = get(structuredAttrs, "outputChecks"); + ASSERT_TRUE(outputChecks_); + auto & outputChecks = *outputChecks_; + + { + auto output_ = get(outputChecks, "out"); + ASSERT_TRUE(output_); + auto & output = *output_; + EXPECT_EQ( + get(output, "allowedReferences")->get(), + Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + EXPECT_EQ( + get(output, "allowedRequisites")->get(), + Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + } + + { + auto output_ = get(outputChecks, "bin"); + ASSERT_TRUE(output_); + auto & output = *output_; + EXPECT_EQ( + get(output, "disallowedReferences")->get(), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + EXPECT_EQ( + get(output, "disallowedRequisites")->get(), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + } + + { + auto output_ = get(outputChecks, "dev"); + ASSERT_TRUE(output_); + auto & output = *output_; + EXPECT_EQ(get(output, "maxSize")->get(), 789); + EXPECT_EQ(get(output, "maxClosureSize")->get(), 5909); + } + } + + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), systemFeatures); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), false); + EXPECT_EQ(parsedDrv.useUidRange(), true); + }); +}; + +} diff --git a/tests/unit/libstore/derivation.cc b/tests/unit/libstore/derivation.cc index 7a4b1403a..71979f885 100644 --- a/tests/unit/libstore/derivation.cc +++ b/tests/unit/libstore/derivation.cc @@ -108,7 +108,7 @@ TEST_JSON(DerivationTest, inputAddressed, TEST_JSON(DerivationTest, caFixedFlat, (DerivationOutput::CAFixed { .ca = { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), @@ -117,7 +117,7 @@ TEST_JSON(DerivationTest, caFixedFlat, TEST_JSON(DerivationTest, caFixedNAR, (DerivationOutput::CAFixed { .ca = { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), @@ -126,6 +126,7 @@ TEST_JSON(DerivationTest, caFixedNAR, TEST_JSON(DynDerivationTest, caFixedText, (DerivationOutput::CAFixed { .ca = { + .method = ContentAddressMethod::Raw::Text, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), @@ -133,7 +134,7 @@ TEST_JSON(DynDerivationTest, caFixedText, TEST_JSON(CaDerivationTest, caFloating, (DerivationOutput::CAFloating { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hashAlgo = HashAlgorithm::SHA256, }), "drv-name", "output-name") @@ -144,7 +145,7 @@ TEST_JSON(DerivationTest, deferred, TEST_JSON(ImpureDerivationTest, impure, (DerivationOutput::Impure { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hashAlgo = HashAlgorithm::SHA256, }), "drv-name", "output-name") diff --git a/tests/unit/libstore/nar-info.cc b/tests/unit/libstore/nar-info.cc index bd10602e7..a6cb62de4 100644 --- a/tests/unit/libstore/nar-info.cc +++ b/tests/unit/libstore/nar-info.cc @@ -25,7 +25,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { store, "foo", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { diff --git a/tests/unit/libstore/path-info.cc b/tests/unit/libstore/path-info.cc index 06c662b74..7637cb366 100644 --- a/tests/unit/libstore/path-info.cc +++ b/tests/unit/libstore/path-info.cc @@ -32,7 +32,7 @@ static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo store, "foo", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { diff --git a/tests/unit/libstore/path.cc b/tests/unit/libstore/path.cc index 213b6e95f..c4c055abf 100644 --- a/tests/unit/libstore/path.cc +++ b/tests/unit/libstore/path.cc @@ -26,10 +26,19 @@ static std::regex nameRegex { std::string { nameRegexStr } }; TEST_F(StorePathTest, bad_ ## NAME) { \ std::string_view str = \ STORE_DIR HASH_PART "-" STR; \ - ASSERT_THROW( \ - store->parseStorePath(str), \ - BadStorePath); \ + /* ASSERT_THROW generates a duplicate goto label */ \ + /* A lambda isolates those labels. */ \ + [&](){ \ + ASSERT_THROW( \ + store->parseStorePath(str), \ + BadStorePath); \ + }(); \ std::string name { STR }; \ + [&](){ \ + ASSERT_THROW( \ + nix::checkName(name), \ + BadStorePathName); \ + }(); \ EXPECT_FALSE(std::regex_match(name, nameRegex)); \ } @@ -54,6 +63,7 @@ TEST_DONT_PARSE(dot_dash_a, ".-a") STORE_DIR HASH_PART "-" STR; \ auto p = store->parseStorePath(str); \ std::string name { p.name() }; \ + EXPECT_EQ(p.name(), STR); \ EXPECT_TRUE(std::regex_match(name, nameRegex)); \ } diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc index ebf0c52b0..2505c5a9a 100644 --- a/tests/unit/libstore/serve-protocol.cc +++ b/tests/unit/libstore/serve-protocol.cc @@ -55,15 +55,15 @@ VERSIONED_CHARACTERIZATION_TEST( defaultVersion, (std::tuple { ContentAddress { - .method = TextIngestionMethod {}, + .method = ContentAddressMethod::Raw::Text, .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -280,7 +280,7 @@ VERSIONED_CHARACTERIZATION_TEST( *LibStoreTest::store, "foo", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { .others = { @@ -398,7 +398,7 @@ VERSIONED_CHARACTERIZATION_TEST( std::nullopt, std::optional { ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc index 70e03a8ab..c15120010 100644 --- a/tests/unit/libstore/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -56,15 +56,15 @@ VERSIONED_CHARACTERIZATION_TEST( defaultVersion, (std::tuple { ContentAddress { - .method = TextIngestionMethod {}, + .method = ContentAddressMethod::Raw::Text, .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -512,7 +512,7 @@ VERSIONED_CHARACTERIZATION_TEST( *LibStoreTest::store, "foo", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { .others = { @@ -598,7 +598,7 @@ VERSIONED_CHARACTERIZATION_TEST( std::nullopt, std::optional { ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, diff --git a/tests/unit/libutil-support/.version b/tests/unit/libutil-support/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libutil-support/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libutil-support/meson.build b/tests/unit/libutil-support/meson.build new file mode 100644 index 000000000..5912c00e0 --- /dev/null +++ b/tests/unit/libutil-support/meson.build @@ -0,0 +1,110 @@ +project('nix-util-test-support', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +# See note in ../nix-util/meson.build +deps_private = [ ] + +# See note in ../nix-util/meson.build +deps_public = [ ] + +# See note in ../nix-util/meson.build +deps_other = [ ] + +add_project_arguments( + '-Wno-deprecated-declarations', + '-Wimplicit-fallthrough', + '-Werror=switch', + '-Werror=switch-enum', + '-Wdeprecated-copy', + '-Wignored-qualifiers', + # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked + # at ~1% overhead in `nix search`. + # + # FIXME: remove when we get meson 1.4.0 which will default this to on for us: + # https://mesonbuild.com/Release-notes-for-1-4-0.html#ndebug-setting-now-controls-c-stdlib-assertions + '-D_GLIBCXX_ASSERTIONS=1', + language : 'cpp', +) + +sources = files( + 'tests/hash.cc', + 'tests/string_callback.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'tests/characterization.hh', + 'tests/hash.hh', + 'tests/nix_api_util.hh', + 'tests/string_callback.hh', +) + +if host_machine.system() == 'cygwin' or host_machine.system() == 'windows' + # Windows DLLs are stricter about symbol visibility than Unix shared + # objects --- see https://gcc.gnu.org/wiki/Visibility for details. + # This is a temporary sledgehammer to export everything like on Unix, + # and not detail with this yet. + # + # TODO do not do this, and instead do fine-grained export annotations. + linker_export_flags = ['-Wl,--export-all-symbols'] +else + linker_export_flags = [] +endif + +nix_util = dependency('nix-util') +if nix_util.type_name() == 'internal' + # subproject sadly no good for pkg-config module + deps_other += nix_util +else + deps_public += nix_util +endif + +rapidcheck = dependency('rapidcheck') +deps_public += rapidcheck + +this_library = library( + 'nix-util-test-support', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326 + # is available. See also ../libutil/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +import('pkgconfig').generate( + this_library, + filebase : meson.project_name(), + name : 'Nix', + description : 'Nix Package Manager', + subdirs : ['nix'], + extra_cflags : ['-std=c++2a'], + requires : deps_public, + requires_private : deps_private, +) + +meson.override_dependency(meson.project_name(), declare_dependency( + include_directories : include_dirs, + link_with : this_library, + compile_args : ['-std=c++2a'], + dependencies : [], +)) diff --git a/tests/unit/libutil-support/package.nix b/tests/unit/libutil-support/package.nix new file mode 100644 index 000000000..0be0a9945 --- /dev/null +++ b/tests/unit/libutil-support/package.nix @@ -0,0 +1,99 @@ +{ lib +, stdenv +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util + +, rapidcheck + +# Configuration Options + +, versionSuffix ? "" + +# Check test coverage of Nix. Probably want to use with at least +# one of `doCheck` or `doInstallCheck` enabled. +, withCoverageChecks ? false +}: + +let + inherit (lib) fileset; + + version = lib.fileContents ./.version + versionSuffix; + + mkDerivation = + if withCoverageChecks + then + # TODO support `finalAttrs` args function in + # `releaseTools.coverageAnalysis`. + argsFun: + releaseTools.coverageAnalysis (let args = argsFun args; in args) + else stdenv.mkDerivation; +in + +mkDerivation (finalAttrs: { + pname = "nix-util-test-support"; + inherit version; + + src = fileset.toSource { + root = ./.; + fileset = fileset.unions [ + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + }; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-util + rapidcheck + ] + ; + + propagatedBuildInputs = [ + nix-util + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix + '' + echo ${version} > .version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 + strictDeps = !withCoverageChecks; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +} // lib.optionalAttrs withCoverageChecks { + lcovFilter = [ "*/boost/*" "*-tab.*" ]; + + hardeningDisable = [ "fortify" ]; +}) diff --git a/tests/unit/libutil-support/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh index c2f686dbf..19ba824ac 100644 --- a/tests/unit/libutil-support/tests/characterization.hh +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -5,6 +5,7 @@ #include "types.hh" #include "environment-variables.hh" +#include "file-system.hh" namespace nix { diff --git a/tests/unit/libutil/.version b/tests/unit/libutil/.version new file mode 100644 index 000000000..ad2261920 --- /dev/null +++ b/tests/unit/libutil/.version @@ -0,0 +1 @@ +2.24.0 diff --git a/tests/unit/libutil/file-content-address.cc b/tests/unit/libutil/file-content-address.cc index 294e39806..27d926a87 100644 --- a/tests/unit/libutil/file-content-address.cc +++ b/tests/unit/libutil/file-content-address.cc @@ -11,7 +11,7 @@ namespace nix { TEST(FileSerialisationMethod, testRoundTripPrintParse_1) { for (const FileSerialisationMethod fim : { FileSerialisationMethod::Flat, - FileSerialisationMethod::Recursive, + FileSerialisationMethod::NixArchive, }) { EXPECT_EQ(parseFileSerialisationMethod(renderFileSerialisationMethod(fim)), fim); } @@ -37,7 +37,7 @@ TEST(FileSerialisationMethod, testParseFileSerialisationMethodOptException) { TEST(FileIngestionMethod, testRoundTripPrintParse_1) { for (const FileIngestionMethod fim : { FileIngestionMethod::Flat, - FileIngestionMethod::Recursive, + FileIngestionMethod::NixArchive, FileIngestionMethod::Git, }) { EXPECT_EQ(parseFileIngestionMethod(renderFileIngestionMethod(fim)), fim); diff --git a/tests/unit/libutil/meson.build b/tests/unit/libutil/meson.build new file mode 100644 index 000000000..56147686b --- /dev/null +++ b/tests/unit/libutil/meson.build @@ -0,0 +1,142 @@ +project('nix-util-test', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +# See note in ../nix-util/meson.build +deps_private = [ ] + +# See note in ../nix-util/meson.build +deps_public = [ ] + +# See note in ../nix-util/meson.build +deps_other = [ ] + +configdata = configuration_data() + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util-test.h', + # '-include', 'config-store.h', + '-Wno-deprecated-declarations', + '-Wimplicit-fallthrough', + '-Werror=switch', + '-Werror=switch-enum', + '-Wdeprecated-copy', + '-Wignored-qualifiers', + # Enable assertions in libstdc++ by default. Harmless on libc++. Benchmarked + # at ~1% overhead in `nix search`. + # + # FIXME: remove when we get meson 1.4.0 which will default this to on for us: + # https://mesonbuild.com/Release-notes-for-1-4-0.html#ndebug-setting-now-controls-c-stdlib-assertions + '-D_GLIBCXX_ASSERTIONS=1', + language : 'cpp', +) + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-util-test.h', +) + +sources = files( + 'args.cc', + 'canon-path.cc', + 'chunked-vector.cc', + 'closure.cc', + 'compression.cc', + 'config.cc', + 'file-content-address.cc', + 'git.cc', + 'hash.cc', + 'hilite.cc', + 'json-utils.cc', + 'logging.cc', + 'lru-cache.cc', + 'nix_api_util.cc', + 'pool.cc', + 'references.cc', + 'spawn.cc', + 'suggestions.cc', + 'tests.cc', + 'url.cc', + 'xml-writer.cc', +) + +include_dirs = [include_directories('.')] + +if host_machine.system() == 'cygwin' or host_machine.system() == 'windows' + # Windows DLLs are stricter about symbol visibility than Unix shared + # objects --- see https://gcc.gnu.org/wiki/Visibility for details. + # This is a temporary sledgehammer to export everything like on Unix, + # and not detail with this yet. + # + # TODO do not do this, and instead do fine-grained export annotations. + linker_export_flags = ['-Wl,--export-all-symbols'] +else + linker_export_flags = [] +endif + +nix_util = dependency('nix-util') +if nix_util.type_name() == 'internal' + # subproject sadly no good for pkg-config module + deps_other += nix_util +else + deps_public += nix_util +endif + +nix_util_c = dependency('nix-util-c') +if nix_util_c.type_name() == 'internal' + # subproject sadly no good for pkg-config module + deps_other += nix_util_c +else + deps_public += nix_util_c +endif + +nix_util_test_support = dependency('nix-util-test-support') +if nix_util_test_support.type_name() == 'internal' + # subproject sadly no good for pkg-config module + deps_other += nix_util_test_support +else + deps_public += nix_util_test_support +endif + +rapidcheck = dependency('rapidcheck') +deps_public += rapidcheck + +gtest = dependency('gtest', main : true) +deps_public += gtest + +this_exe = executable( + 'nix-util-test', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + # TODO: -lrapidcheck, see ../libutil-support/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + # get main from gtest + install : true, +) + +test('nix-util-test', this_exe, env : ['_NIX_TEST_UNIT_DATA=' + meson.current_source_dir() + '/data']) + +meson.override_dependency(meson.project_name(), declare_dependency( + include_directories : include_dirs, + link_with : this_exe, + compile_args : ['-std=c++2a'], + dependencies : [], +)) diff --git a/tests/unit/libutil/nix_api_util.cc b/tests/unit/libutil/nix_api_util.cc index d2999f55b..2b7e38225 100644 --- a/tests/unit/libutil/nix_api_util.cc +++ b/tests/unit/libutil/nix_api_util.cc @@ -1,4 +1,4 @@ -#include "config.hh" +#include "config-global.hh" #include "args.hh" #include "nix_api_util.h" #include "nix_api_util_internal.h" diff --git a/tests/unit/libutil/package.nix b/tests/unit/libutil/package.nix new file mode 100644 index 000000000..391f8d853 --- /dev/null +++ b/tests/unit/libutil/package.nix @@ -0,0 +1,113 @@ +{ lib +, stdenv +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-util-c +, nix-util-test-support + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, versionSuffix ? "" + +# Check test coverage of Nix. Probably want to use with at least +# one of `doCheck` or `doInstallCheck` enabled. +, withCoverageChecks ? false +}: + +let + inherit (lib) fileset; + + version = lib.fileContents ./.version + versionSuffix; + + mkDerivation = + if withCoverageChecks + then + # TODO support `finalAttrs` args function in + # `releaseTools.coverageAnalysis`. + argsFun: + releaseTools.coverageAnalysis (let args = argsFun args; in args) + else stdenv.mkDerivation; +in + +mkDerivation (finalAttrs: { + pname = "nix-util-test"; + inherit version; + + src = fileset.toSource { + root = ./.; + fileset = fileset.unions [ + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + }; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-util + nix-util-c + nix-util-test-support + rapidcheck + gtest + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix + '' + echo ${version} > .version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + enableParallelBuilding = true; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 + strictDeps = !withCoverageChecks; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru = { + tests = { + run = runCommand "${finalAttrs.pname}-run" { + } '' + PATH="${lib.makeBinPath [ finalAttrs.finalPackage ]}:$PATH" + export _NIX_TEST_UNIT_DATA=${./data} + nix-util-test + touch $out + ''; + }; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +} // lib.optionalAttrs withCoverageChecks { + lcovFilter = [ "*/boost/*" "*-tab.*" ]; + + hardeningDisable = [ "fortify" ]; +}) diff --git a/tests/unit/libutil/spawn.cc b/tests/unit/libutil/spawn.cc new file mode 100644 index 000000000..c617acae0 --- /dev/null +++ b/tests/unit/libutil/spawn.cc @@ -0,0 +1,36 @@ +#include + +#include "processes.hh" + +namespace nix { + +#ifdef _WIN32 +TEST(SpawnTest, spawnEcho) +{ + auto output = runProgram(RunOptions{.program = "cmd.exe", .args = {"/C", "echo", "hello world"}}); + ASSERT_EQ(output.first, 0); + ASSERT_EQ(output.second, "\"hello world\"\r\n"); +} + +std::string windowsEscape(const std::string & str, bool cmd); + +TEST(SpawnTest, windowsEscape) +{ + auto empty = windowsEscape("", false); + ASSERT_EQ(empty, R"("")"); + // There's no quotes in this argument so the input should equal the output + auto backslashStr = R"(\\\\)"; + auto backslashes = windowsEscape(backslashStr, false); + ASSERT_EQ(backslashes, backslashStr); + + auto nestedQuotes = windowsEscape(R"(he said: "hello there")", false); + ASSERT_EQ(nestedQuotes, R"("he said: \"hello there\"")"); + + auto middleQuote = windowsEscape(R"( \\\" )", false); + ASSERT_EQ(middleQuote, R"(" \\\\\\\" ")"); + + auto space = windowsEscape("hello world", false); + ASSERT_EQ(space, R"("hello world")"); +} +#endif +}