diff --git a/.editorconfig b/.editorconfig index 887ecadba..86360e658 100644 --- a/.editorconfig +++ b/.editorconfig @@ -17,7 +17,7 @@ indent_style = space indent_size = 2 # Match c++/shell/perl, set indent to spaces with width of four -[*.{hpp,cc,hh,sh,pl}] +[*.{hpp,cc,hh,sh,pl,xs}] indent_style = space indent_size = 4 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index ab5908649..39d595199 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -14,5 +14,12 @@ /doc @fricklerhandwerk *.md @fricklerhandwerk +# Documentation of built-in functions +src/libexpr/primops.cc @fricklerhandwerk @roberth +# Documentation on experimental features +src/libutil/experimental-features.cc @fricklerhandwerk +# Documentation on configuration settings +src/libstore/globals.hh @fricklerhandwerk + # Libstore layer /src/libstore @thufschmitt diff --git a/.github/labeler.yml b/.github/labeler.yml index 7544f07a6..b1b18c488 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,23 +1,30 @@ "documentation": - - doc/manual/* - - src/nix/**/*.md + - changed-files: + - any-glob-to-any-file: "doc/manual/*" + - any-glob-to-any-file: "src/nix/**/*.md" "store": - - src/libstore/store-api.* - - src/libstore/*-store.* + - changed-files: + - any-glob-to-any-file: "src/libstore/store-api.*" + - any-glob-to-any-file: "src/libstore/*-store.*" "fetching": - - src/libfetchers/**/* + - changed-files: + - any-glob-to-any-file: "src/libfetchers/**/*" "repl": - - src/libcmd/repl.* - - src/nix/repl.* + - changed-files: + - any-glob-to-any-file: "src/libcmd/repl.*" + - any-glob-to-any-file: "src/nix/repl.*" "new-cli": - - src/nix/**/* + - changed-files: + - any-glob-to-any-file: "src/nix/**/*" "with-tests": - # Unit tests - - src/*/tests/**/* - # Functional and integration tests - - tests/functional/**/* + - changed-files: + # Unit tests + - any-glob-to-any-file: "src/*/tests/**/*" + # Functional and integration tests + - any-glob-to-any-file: "tests/functional/**/*" + diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 12c60c649..85ddcfad3 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v1.4.0 + uses: zeebe-io/backport-action@v2.2.0 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index afe4dc2e3..aa2551424 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,12 +20,12 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 with: # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v12 + - uses: cachix/cachix-action@v13 if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' @@ -62,10 +62,10 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - - uses: cachix/cachix-action@v12 + - uses: cachix/cachix-action@v13 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' @@ -84,7 +84,7 @@ jobs: steps: - uses: actions/checkout@v4 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -114,12 +114,12 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v23 + - uses: cachix/install-nix-action@v24 with: install_url: https://releases.nixos.org/nix/nix-2.13.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v12 + - uses: cachix/cachix-action@v13 if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index d83cb4f18..34aa4e6bd 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest if: github.repository_owner == 'NixOS' steps: - - uses: actions/labeler@v4 + - uses: actions/labeler@v5 with: repo-token: ${{ secrets.GITHUB_TOKEN }} sync-labels: false diff --git a/.gitignore b/.gitignore index 04d96ca2c..d9f9d949b 100644 --- a/.gitignore +++ b/.gitignore @@ -21,12 +21,16 @@ perl/Makefile.config /doc/manual/language.json /doc/manual/xp-features.json /doc/manual/src/SUMMARY.md +/doc/manual/src/SUMMARY-rl-next.md +/doc/manual/src/store/types/* +!/doc/manual/src/store/types/index.md.in /doc/manual/src/command-ref/new-cli /doc/manual/src/command-ref/conf-file.md /doc/manual/src/command-ref/experimental-features-shortlist.md /doc/manual/src/contributing/experimental-feature-descriptions.md /doc/manual/src/language/builtins.md /doc/manual/src/language/builtin-constants.md +/doc/manual/src/release-notes/rl-next.md # /scripts/ /scripts/nix-profile.sh @@ -41,18 +45,18 @@ perl/Makefile.config /src/libexpr/parser-tab.hh /src/libexpr/parser-tab.output /src/libexpr/nix.tbl -/src/libexpr/tests/libnixexpr-tests +/tests/unit/libexpr/libnixexpr-tests # /src/libstore/ *.gen.* -/src/libstore/tests/libnixstore-tests +/tests/unit/libstore/libnixstore-tests # /src/libutil/ -/src/libutil/tests/libnixutil-tests +/tests/unit/libutil/libnixutil-tests /src/nix/nix -/src/nix/doc +/src/nix/generated-doc # /src/nix-env/ /src/nix-env/nix-env @@ -144,3 +148,6 @@ result # clangd and possibly more .cache/ + +# Mac OS +.DS_Store diff --git a/.version b/.version index ef0f38abe..7329e21c3 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.19.0 +2.20.0 diff --git a/Makefile b/Makefile index 24af7ae2b..45215e93e 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ --include Makefile.config -clean-files += Makefile.config +include mk/build-dir.mk + +-include $(buildprefix)Makefile.config +clean-files += $(buildprefix)Makefile.config ifeq ($(ENABLE_BUILD), yes) makefiles = \ @@ -19,17 +21,17 @@ makefiles = \ misc/zsh/local.mk \ misc/systemd/local.mk \ misc/launchd/local.mk \ - misc/upstart/local.mk \ - doc/manual/local.mk \ - doc/internal-api/local.mk + misc/upstart/local.mk endif ifeq ($(ENABLE_BUILD)_$(ENABLE_TESTS), yes_yes) -UNIT_TEST_ENV = _NIX_TEST_UNIT_DATA=unit-test-data makefiles += \ - src/libutil/tests/local.mk \ - src/libstore/tests/local.mk \ - src/libexpr/tests/local.mk + tests/unit/libutil/local.mk \ + tests/unit/libutil-support/local.mk \ + tests/unit/libstore/local.mk \ + tests/unit/libstore-support/local.mk \ + tests/unit/libexpr/local.mk \ + tests/unit/libexpr-support/local.mk endif ifeq ($(ENABLE_TESTS), yes) @@ -56,4 +58,11 @@ endif include mk/lib.mk -GLOBAL_CXXFLAGS += -g -Wall -include config.h -std=c++2a -I src +# Must be included after `mk/lib.mk` so rules refer to variables defined +# by the library. Rules are not "lazy" like variables, unfortunately. +ifeq ($(ENABLE_BUILD), yes) +$(eval $(call include-sub-makefile, doc/manual/local.mk)) +$(eval $(call include-sub-makefile, doc/internal-api/local.mk)) +endif + +GLOBAL_CXXFLAGS += -g -Wall -include $(buildprefix)config.h -std=c++2a -I src diff --git a/Makefile.config.in b/Makefile.config.in index 19992fa20..c85e028c2 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -8,15 +8,19 @@ CXX = @CXX@ CXXFLAGS = @CXXFLAGS@ CXXLTO = @CXXLTO@ EDITLINE_LIBS = @EDITLINE_LIBS@ +ENABLE_BUILD = @ENABLE_BUILD@ ENABLE_S3 = @ENABLE_S3@ +ENABLE_TESTS = @ENABLE_TESTS@ GTEST_LIBS = @GTEST_LIBS@ HAVE_LIBCPUID = @HAVE_LIBCPUID@ HAVE_SECCOMP = @HAVE_SECCOMP@ HOST_OS = @host_os@ +INSTALL_UNIT_TESTS = @INSTALL_UNIT_TESTS@ LDFLAGS = @LDFLAGS@ LIBARCHIVE_LIBS = @LIBARCHIVE_LIBS@ LIBBROTLI_LIBS = @LIBBROTLI_LIBS@ LIBCURL_LIBS = @LIBCURL_LIBS@ +LIBGIT2_LIBS = @LIBGIT2_LIBS@ LIBSECCOMP_LIBS = @LIBSECCOMP_LIBS@ LOWDOWN_LIBS = @LOWDOWN_LIBS@ OPENSSL_LIBS = @OPENSSL_LIBS@ @@ -28,6 +32,8 @@ SODIUM_LIBS = @SODIUM_LIBS@ SQLITE3_LIBS = @SQLITE3_LIBS@ bash = @bash@ bindir = @bindir@ +checkbindir = @checkbindir@ +checklibdir = @checklibdir@ datadir = @datadir@ datarootdir = @datarootdir@ doc_generate = @doc_generate@ @@ -35,6 +41,7 @@ docdir = @docdir@ embedded_sandbox_shell = @embedded_sandbox_shell@ exec_prefix = @exec_prefix@ includedir = @includedir@ +internal_api_docs = @internal_api_docs@ libdir = @libdir@ libexecdir = @libexecdir@ localstatedir = @localstatedir@ @@ -46,6 +53,3 @@ sandbox_shell = @sandbox_shell@ storedir = @storedir@ sysconfdir = @sysconfdir@ system = @system@ -ENABLE_BUILD = @ENABLE_BUILD@ -ENABLE_TESTS = @ENABLE_TESTS@ -internal_api_docs = @internal_api_docs@ diff --git a/boehmgc-traceable_allocator-public.diff b/boehmgc-traceable_allocator-public.diff new file mode 100644 index 000000000..903c707a6 --- /dev/null +++ b/boehmgc-traceable_allocator-public.diff @@ -0,0 +1,12 @@ +diff --git a/include/gc_allocator.h b/include/gc_allocator.h +index 597c7f13..587286be 100644 +--- a/include/gc_allocator.h ++++ b/include/gc_allocator.h +@@ -312,6 +312,7 @@ public: + + template<> + class traceable_allocator { ++public: + typedef size_t size_type; + typedef ptrdiff_t difference_type; + typedef void* pointer; diff --git a/configure.ac b/configure.ac index 225baf6b5..f8b937eb5 100644 --- a/configure.ac +++ b/configure.ac @@ -68,6 +68,9 @@ case "$host_os" in esac +ENSURE_NO_GCC_BUG_80431 + + # Check for pubsetbuf. AC_MSG_CHECKING([for pubsetbuf]) AC_LANG_PUSH(C++) @@ -164,6 +167,18 @@ AC_ARG_ENABLE(tests, AS_HELP_STRING([--disable-tests],[Do not build the tests]), ENABLE_TESTS=$enableval, ENABLE_TESTS=yes) AC_SUBST(ENABLE_TESTS) +AC_ARG_ENABLE(install-unit-tests, AS_HELP_STRING([--enable-install-unit-tests],[Install the unit tests for running later (default no)]), + INSTALL_UNIT_TESTS=$enableval, INSTALL_UNIT_TESTS=no) +AC_SUBST(INSTALL_UNIT_TESTS) + +AC_ARG_WITH(check-bin-dir, AS_HELP_STRING([--with-check-bin-dir=PATH],[path to install unit tests for running later (defaults to $libexecdir/nix)]), + checkbindir=$withval, checkbindir=$libexecdir/nix) +AC_SUBST(checkbindir) + +AC_ARG_WITH(check-lib-dir, AS_HELP_STRING([--with-check-lib-dir=PATH],[path to install unit tests for running later (defaults to $libdir)]), + checklibdir=$withval, checklibdir=$libdir) +AC_SUBST(checklibdir) + # Building without API docs is the default as Nix' C++ interfaces are internal and unstable. AC_ARG_ENABLE(internal_api_docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), internal_api_docs=$enableval, internal_api_docs=no) @@ -267,6 +282,8 @@ case "$host_os" in esac AC_SUBST(HAVE_SECCOMP, [$have_seccomp]) +# Optional dependencies for better normalizing file system data +AC_CHECK_HEADERS[sys/xattr.h] # Look for aws-cpp-sdk-s3. AC_LANG_PUSH(C++) @@ -332,9 +349,15 @@ AC_ARG_ENABLE(doc-gen, AS_HELP_STRING([--disable-doc-gen],[disable documentation doc_generate=$enableval, doc_generate=yes) AC_SUBST(doc_generate) + # Look for lowdown library. PKG_CHECK_MODULES([LOWDOWN], [lowdown >= 0.9.0], [CXXFLAGS="$LOWDOWN_CFLAGS $CXXFLAGS"]) + +# Look for libgit2. +PKG_CHECK_MODULES([LIBGIT2], [libgit2]) + + # Setuid installations. AC_CHECK_FUNCS([setresuid setreuid lchown]) diff --git a/doc/internal-api/doxygen.cfg.in b/doc/internal-api/doxygen.cfg.in index 599be2470..ad5af97e6 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/doc/internal-api/doxygen.cfg.in @@ -39,17 +39,21 @@ INPUT = \ src/libcmd \ src/libexpr \ src/libexpr/flake \ - src/libexpr/tests \ - src/libexpr/tests/value \ + tests/unit/libexpr \ + tests/unit/libexpr/value \ + tests/unit/libexpr/test \ + tests/unit/libexpr/test/value \ src/libexpr/value \ src/libfetchers \ src/libmain \ src/libstore \ src/libstore/build \ src/libstore/builtins \ - src/libstore/tests \ + tests/unit/libstore \ + tests/unit/libstore/test \ src/libutil \ - src/libutil/tests \ + tests/unit/libutil \ + tests/unit/libutil/test \ src/nix \ src/nix-env \ src/nix-store diff --git a/doc/manual/_redirects b/doc/manual/_redirects new file mode 100644 index 000000000..62c693c97 --- /dev/null +++ b/doc/manual/_redirects @@ -0,0 +1,40 @@ +# redirect rules for paths (server-side) to prevent link rot. +# see ./redirects.js for redirects based on URL fragments (client-side) +# +# concrete user story this supports: +# - user finds URL to the manual for Nix x.y +# - Nix x.z (z > y) is the most recent release +# - updating the version in the URL will show the right thing +# +# format documentation: +# - https://docs.netlify.com/routing/redirects/#syntax-for-the-redirects-file +# - https://docs.netlify.com/routing/redirects/redirect-options/ +# +# conventions: +# - always force (!) since this allows re-using file names +# - group related paths to ease readability +# - keep in alphabetical/wildcards-last order, which will reduce version control conflicts +# - redirects that should have been there but are missing can be inserted where they belong + +/advanced-topics/advanced-topics /advanced-topics 301! + +/command-ref/command-ref /command-ref 301! + +/contributing/contributing /contributing 301! + +/expressions/expression-language /language/ 301! +/expressions/language-constructs /language/constructs 301! +/expressions/language-operators /language/operators 301! +/expressions/language-values /language/values 301! +/expressions/* /language/:splat 301! + +/installation/installation /installation 301! + +/package-management/basic-package-mgmt /command-ref/nix-env 301! +/package-management/channels /command-ref/nix-channel 301! +/package-management/package-management /package-management 301! +/package-management/s3-substituter /store/types/s3-binary-cache-store 301! + +/protocols/protocols /protocols 301! + +/release-notes/release-notes /release-notes 301! diff --git a/doc/manual/generate-builtin-constants.nix b/doc/manual/generate-builtin-constants.nix index 8af80a02c..cccd1e279 100644 --- a/doc/manual/generate-builtin-constants.nix +++ b/doc/manual/generate-builtin-constants.nix @@ -1,6 +1,6 @@ let inherit (builtins) concatStringsSep attrValues mapAttrs; - inherit (import ./utils.nix) optionalString squash; + inherit (import ) optionalString squash; in builtinsInfo: diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 813a287f5..007b698f1 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -1,6 +1,6 @@ let inherit (builtins) concatStringsSep attrValues mapAttrs; - inherit (import ./utils.nix) optionalString squash; + inherit (import ) optionalString squash; in builtinsInfo: @@ -8,7 +8,15 @@ let showBuiltin = name: { doc, args, arity, experimental-feature }: let experimentalNotice = optionalString (experimental-feature != null) '' - This function is only available if the [${experimental-feature}](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) experimental feature is enabled. + > **Note** + > + > This function is only available if the [`${experimental-feature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) is enabled. + > + > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = ${experimental-feature} + > ``` ''; in squash '' @@ -17,10 +25,9 @@ let
- ${doc} - ${experimentalNotice} + ${doc}
''; listArgs = args: concatStringsSep " " (map (s: "${s}") args); diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index 14136016d..ae31b2a1f 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -1,9 +1,29 @@ let inherit (builtins) - attrNames attrValues fromJSON listToAttrs mapAttrs groupBy - concatStringsSep concatMap length lessThan replaceStrings sort; - inherit (import ) attrsToList concatStrings optionalString filterAttrs trim squash unique; - showStoreDocs = import ./generate-store-info.nix; + attrNames + attrValues + concatMap + concatStringsSep + fromJSON + groupBy + length + lessThan + listToAttrs + mapAttrs + match + replaceStrings + sort + ; + inherit (import ) + attrsToList + concatStrings + filterAttrs + optionalString + squash + trim + unique + ; + showStoreDocs = import ; in inlineHTML: commandDump: @@ -31,7 +51,7 @@ let ${maybeSubcommands} - ${maybeStoreDocs} + ${maybeProse} ${maybeOptions} ''; @@ -71,25 +91,56 @@ let * [`${command} ${name}`](./${appendName filename name}.md) - ${subcmd.description} ''; - # FIXME: this is a hack. - # store parameters should not be part of command documentation to begin - # with, but instead be rendered on separate pages. - maybeStoreDocs = optionalString (details ? doc) - (replaceStrings [ "@stores@" ] [ (showStoreDocs inlineHTML commandInfo.stores) ] details.doc); + maybeProse = + # FIXME: this is a horrible hack to keep `nix help-stores` working. + # the correct answer to this is to remove that command and replace it + # by statically generated manpages or the output of something like `nix + # store info `. + let + help-stores = '' + ${index} - maybeOptions = let - allVisibleOptions = filterAttrs - (_: o: ! o.hiddenCategory) - (details.flags // toplevel.flags); - in optionalString (allVisibleOptions != {}) '' - # Options + ${allStores} + ''; + index = replaceStrings + [ "@store-types@" "./local-store.md" "./local-daemon-store.md" ] + [ storesOverview "#local-store" "#local-daemon-store" ] + details.doc; + storesOverview = + let + showEntry = store: + "- [${store.name}](#${store.slug})"; + in + concatStringsSep "\n" (map showEntry storesList) + "\n"; + allStores = concatStringsSep "\n" (attrValues storePages); + storePages = listToAttrs + (map (s: { name = s.filename; value = s.page; }) storesList); + storesList = showStoreDocs { + storeInfo = commandInfo.stores; + inherit inlineHTML; + }; + in + optionalString (details ? doc) ( + if match "@store-types@" details.doc != [ ] + then help-stores + else details.doc + ); - ${showOptions inlineHTML allVisibleOptions} + maybeOptions = + let + allVisibleOptions = filterAttrs + (_: o: ! o.hiddenCategory) + (details.flags // toplevel.flags); + in + optionalString (allVisibleOptions != { }) '' + # Options - > **Note** - > - > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. - ''; + ${showOptions inlineHTML allVisibleOptions} + + > **Note** + > + > See [`man nix.conf`](@docroot@/command-ref/conf-file.md#command-line-flags) for overriding configuration settings with command line flags. + ''; showOptions = inlineHTML: allOptions: let @@ -97,7 +148,7 @@ let ${optionalString (cat != "") "## ${cat}"} ${concatStringsSep "\n" (attrValues (mapAttrs showOption opts))} - ''; + ''; showOption = name: option: let result = trim '' diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 8736bb793..504cda362 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -1,6 +1,6 @@ let inherit (builtins) attrValues concatStringsSep isAttrs isBool mapAttrs; - inherit (import ./utils.nix) concatStrings indent optionalString squash; + inherit (import ) concatStrings indent optionalString squash; in # `inlineHTML` is a hack to accommodate inconsistent output from `lowdown` @@ -20,10 +20,10 @@ let else "`${setting}`"; # separate body to cleanly handle indentation body = '' - ${description} - ${experimentalFeatureNote} + ${description} + **Default:** ${showDefault documentDefault defaultValue} ${showAliases aliases} @@ -31,18 +31,19 @@ let experimentalFeatureNote = optionalString (experimentalFeature != null) '' > **Warning** + > > This setting is part of an > [experimental feature](@docroot@/contributing/experimental-features.md). - - To change this setting, you need to make sure the corresponding experimental feature, - [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), - is enabled. - For example, include the following in [`nix.conf`](#): - - ``` - extra-experimental-features = ${experimentalFeature} - ${setting} = ... - ``` + > + > To change this setting, make sure the + > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > is enabled. + > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = ${experimentalFeature} + > ${setting} = ... + > ``` ''; showDefault = documentDefault: defaultValue: diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index 36215aadf..c311c3c39 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -1,45 +1,57 @@ let - inherit (builtins) attrValues mapAttrs; - inherit (import ./utils.nix) concatStrings optionalString; - showSettings = import ./generate-settings.nix; + inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings; + inherit (import ) optionalString filterAttrs trim squash toLower unique indent; + showSettings = import ; in -inlineHTML: storesInfo: +{ + # data structure describing all stores and their parameters + storeInfo, + # whether to add inline HTML tags + # `lowdown` does not eat those for one of the output modes + inlineHTML, +}: let - showStore = name: { settings, doc, experimentalFeature }: + showStore = { name, slug }: { settings, doc, experimentalFeature }: let + result = squash '' + # ${name} - result = '' - ## ${name} + ${experimentalFeatureNote} - ${doc} + ${doc} - ${experimentalFeatureNote} + ## Settings - ### Settings + ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} + ''; - ${showSettings { prefix = "store-${slug}"; inherit inlineHTML; } settings} - ''; + experimentalFeatureNote = optionalString (experimentalFeature != null) '' + > **Warning** + > + > This store is part of an + > [experimental feature](@docroot@/contributing/experimental-features.md). + > + > To use this store, make sure the + > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > is enabled. + > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): + > + > ``` + > extra-experimental-features = ${experimentalFeature} + > ``` + ''; + in result; - # markdown doesn't like spaces in URLs - slug = builtins.replaceStrings [ " " ] [ "-" ] name; + storesList = map + (name: rec { + inherit name; + slug = replaceStrings [ " " ] [ "-" ] (toLower name); + filename = "${slug}.md"; + page = showStore { inherit name slug; } storeInfo.${name}; + }) + (attrNames storeInfo); - experimentalFeatureNote = optionalString (experimentalFeature != null) '' - > **Warning** - > This store is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). - - To use this store, you need to make sure the corresponding experimental feature, - [`${experimentalFeature}`](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}), - is enabled. - For example, include the following in [`nix.conf`](#): - - ``` - extra-experimental-features = ${experimentalFeature} - ``` - ''; - in result; - -in concatStrings (attrValues (mapAttrs showStore storesInfo)) +in storesList diff --git a/doc/manual/generate-store-types.nix b/doc/manual/generate-store-types.nix new file mode 100644 index 000000000..3b78a0e1b --- /dev/null +++ b/doc/manual/generate-store-types.nix @@ -0,0 +1,39 @@ +let + inherit (builtins) attrNames listToAttrs concatStringsSep readFile replaceStrings; + showSettings = import ; + showStoreDocs = import ; +in + +storeInfo: + +let + storesList = showStoreDocs { + inherit storeInfo; + inlineHTML = true; + }; + + index = + let + showEntry = store: + "- [${store.name}](./${store.filename})"; + in + concatStringsSep "\n" (map showEntry storesList); + + "index.md" = replaceStrings + [ "@store-types@" ] [ index ] + (readFile ./src/store/types/index.md.in); + + tableOfContents = + let + showEntry = store: + " - [${store.name}](store/types/${store.filename})"; + in + concatStringsSep "\n" (map showEntry storesList) + "\n"; + + "SUMMARY.md" = tableOfContents; + + storePages = listToAttrs + (map (s: { name = s.filename; value = s.page; }) storesList); + +in +storePages // { inherit "index.md" "SUMMARY.md"; } diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix index 30e211c96..ec09f4b75 100644 --- a/doc/manual/generate-xp-features-shortlist.nix +++ b/doc/manual/generate-xp-features-shortlist.nix @@ -1,5 +1,5 @@ with builtins; -with import ./utils.nix; +with import ; let showExperimentalFeature = name: doc: diff --git a/doc/manual/generate-xp-features.nix b/doc/manual/generate-xp-features.nix index adb94355c..0eec0e1da 100644 --- a/doc/manual/generate-xp-features.nix +++ b/doc/manual/generate-xp-features.nix @@ -1,5 +1,5 @@ with builtins; -with import ./utils.nix; +with import ; let showExperimentalFeature = name: doc: @@ -8,4 +8,6 @@ let ${doc} ''; -in xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) +in + +xps: (concatStringsSep "\n" (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 8bf16e9dd..456000d3d 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -1,5 +1,10 @@ ifeq ($(doc_generate),yes) +# The version of Nix used to generate the doc. Can also be +# `$(nix_INSTALL_PATH)` or just `nix` (to grap ambient from the `PATH`), +# if one prefers. +doc_nix = $(nix_PATH) + MANUAL_SRCS := \ $(call rwildcard, $(d)/src, *.md) \ $(call rwildcard, $(d)/src, */*.md) @@ -24,7 +29,7 @@ man-pages += $(foreach subcommand, \ clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 # Provide a dummy environment for nix, so that it will not access files outside the macOS sandbox. -# Set cores to 0 because otherwise nix show-config resolves the cores based on the current machine +# Set cores to 0 because otherwise `nix config show` resolves the cores based on the current machine dummy-env = env -i \ HOME=/dummy \ NIX_CONF_DIR=/dummy \ @@ -32,7 +37,7 @@ dummy-env = env -i \ NIX_STATE_DIR=/dummy \ NIX_CONFIG='cores = 0' -nix-eval = $(dummy-env) $(bindir)/nix eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw +nix-eval = $(dummy-env) $(doc_nix) eval --experimental-features nix-command -I nix=doc/manual --store dummy:// --impure --raw # re-implement mdBook's include directive to make it usable for terminal output and for proper @docroot@ substitution define process-includes @@ -92,58 +97,83 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) -$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(bindir)/nix +$(d)/src/store/types: $(d)/nix.json $(d)/utils.nix $(d)/generate-store-info.nix $(d)/generate-store-types.nix $(d)/src/store/types/index.md.in $(doc_nix) + @# FIXME: build out of tree! + @rm -rf $@.tmp + $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-store-types.nix (builtins.fromJSON (builtins.readFile $<)).stores' + @# do not destroy existing contents + @mv $@.tmp/* $@/ + +$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)' @mv $@.tmp $@ -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(bindir)/nix +$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(doc_nix) @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "opt-"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ -$(d)/nix.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix __dump-cli > $@.tmp +$(d)/nix.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-cli > $@.tmp @mv $@.tmp $@ -$(d)/conf-file.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix show-config --json --experimental-features nix-command > $@.tmp +$(d)/conf-file.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) config show --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(bindir)/nix +$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ -$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(bindir)/nix +$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ -$(d)/xp-features.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix __dump-xp-features > $@.tmp +$(d)/xp-features.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-xp-features > $@.tmp @mv $@.tmp $@ -$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(bindir)/nix +$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(doc_nix) @cat doc/manual/src/language/builtins-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp; @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ -$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(bindir)/nix +$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(doc_nix) @cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp; @cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp @mv $@.tmp $@ -$(d)/language.json: $(bindir)/nix - $(trace-gen) $(dummy-env) $(bindir)/nix __dump-language > $@.tmp +$(d)/language.json: $(doc_nix) + $(trace-gen) $(dummy-env) $(doc_nix) __dump-language > $@.tmp @mv $@.tmp $@ +# Generate "Upcoming release" notes (or clear it and remove from menu) +$(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/* + @if type -p changelog-d > /dev/null; then \ + echo " GEN " $@; \ + changelog-d doc/manual/rl-next > $@; \ + else \ + echo " NULL " $@; \ + true > $@; \ + fi + +$(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md + $(trace-gen) true + @if [ -s $< ]; then \ + echo ' - [Upcoming release](release-notes/rl-next.md)' > $@; \ + else \ + true > $@; \ + fi + # Generate the HTML manual. .PHONY: manual-html manual-html: $(docdir)/manual/index.html @@ -177,7 +207,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli # `@docroot@` is to be preserved for documenting the mechanism # FIXME: maybe contributing guides should live right next to the code # instead of in the manual -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index d1b10109d..3b507adf3 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -1,7 +1,9 @@ -// redirect rules for anchors ensure backwards compatibility of URLs. -// this must be done on the client side, as web servers do not see the anchor part of the URL. +// redirect rules for URL fragments (client-side) to prevent link rot. +// this must be done on the client side, as web servers do not see the fragment part of the URL. +// it will only work with JavaScript enabled in the browser, but this is the best we can do here. +// see ./_redirects for path redirects (client-side) -// redirections are declared as follows: +// redirects are declared as follows: // each entry has as its key a path matching the requested URL path, relative to the mdBook document root. // // IMPORTANT: it must specify the full path with file name and suffix diff --git a/doc/manual/rl-next/config b/doc/manual/rl-next/config new file mode 100644 index 000000000..b3c2e868f --- /dev/null +++ b/doc/manual/rl-next/config @@ -0,0 +1,2 @@ +organization: NixOS +repository: nix diff --git a/doc/manual/rl-next/hash-format-nix32.md b/doc/manual/rl-next/hash-format-nix32.md new file mode 100644 index 000000000..73e6fbb24 --- /dev/null +++ b/doc/manual/rl-next/hash-format-nix32.md @@ -0,0 +1,23 @@ +--- +synopsis: Rename hash format `base32` to `nix32` +prs: 9452 +--- + +Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for +[Base32](https://en.wikipedia.org/wiki/Base32). + +## Deprecation: Use `nix32` instead of `base32` as `toHashFormat` + +For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` +parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value +remains as a deprecated alias for `"base32"`. Please convert your code from: + +```nix +builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} +``` + +to + +```nix +builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} +``` \ No newline at end of file diff --git a/doc/manual/rl-next/mounted-ssh-store.md b/doc/manual/rl-next/mounted-ssh-store.md new file mode 100644 index 000000000..6df44dbb6 --- /dev/null +++ b/doc/manual/rl-next/mounted-ssh-store.md @@ -0,0 +1,8 @@ +--- +synopsis: Mounted SSH Store +issues: 7890 +prs: 7912 +--- + +Introduced the store [`mounted-ssh-ng://`](@docroot@/command-ref/new-cli/nix3-help-stores.md). +This store allows full access to a Nix store on a remote machine and additionally requires that the store be mounted in the local filesystem. diff --git a/doc/manual/rl-next/nix-config-show.md b/doc/manual/rl-next/nix-config-show.md new file mode 100644 index 000000000..26b961b76 --- /dev/null +++ b/doc/manual/rl-next/nix-config-show.md @@ -0,0 +1,7 @@ +--- +synopsis: Rename to `nix config show` +issues: 7672 +prs: 9477 +--- + +`nix show-config` was renamed to `nix config show`, and `nix doctor` was renamed to `nix config check`, to be more consistent with the rest of the command-line interface. diff --git a/doc/manual/rl-next/nix-env-json-drv-path.md b/doc/manual/rl-next/nix-env-json-drv-path.md new file mode 100644 index 000000000..734cefd1b --- /dev/null +++ b/doc/manual/rl-next/nix-env-json-drv-path.md @@ -0,0 +1,6 @@ +--- +synopsis: Fix `nix-env --query --drv-path --json` +prs: 9257 +--- + +Fixed a bug where `nix-env --query` ignored `--drv-path` when `--json` was set. diff --git a/doc/manual/rl-next/nix-hash-convert.md b/doc/manual/rl-next/nix-hash-convert.md new file mode 100644 index 000000000..2b718a66b --- /dev/null +++ b/doc/manual/rl-next/nix-hash-convert.md @@ -0,0 +1,47 @@ +--- +synopsis: Add `nix hash convert` +prs: 9452 +--- + +New [`nix hash convert`](https://github.com/NixOS/nix/issues/8876) sub command with a fast track +to stabilization! Examples: + +- Convert the hash to `nix32`. + + ```bash + $ nix hash convert --algo "sha1" --to nix32 "800d59cfcd3c05e900cb4e214be48f6b886a08df" + vw46m23bizj4n8afrc0fj19wrp7mj3c0 + ``` + `nix32` is a base32 encoding with a nix-specific character set. + Explicitly specify the hashing algorithm (optional with SRI hashes) but detect hash format by the length of the input + hash. +- Convert the hash to the `sri` format that includes an algorithm specification: + ```bash + nix hash convert --algo "sha1" "800d59cfcd3c05e900cb4e214be48f6b886a08df" + sha1-gA1Zz808BekAy04hS+SPa4hqCN8= + ``` + or with an explicit `-to` format: + ```bash + nix hash convert --algo "sha1" --to sri "800d59cfcd3c05e900cb4e214be48f6b886a08df" + sha1-gA1Zz808BekAy04hS+SPa4hqCN8= + ``` +- Assert the input format of the hash: + ```bash + nix hash convert --algo "sha256" --from nix32 "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" + error: input hash 'ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=' does not have the expected format '--from nix32' + nix hash convert --algo "sha256" --from nix32 "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" + sha256-ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0= + ``` + +The `--to`/`--from`/`--algo` parameters have context-sensitive auto-completion. + +## Related Deprecations + +The following commands are still available but will emit a deprecation warning. Please convert your code to +`nix hash convert`: + +- `nix hash to-base16 $hash1 $hash2`: Use `nix hash convert --to base16 $hash1 $hash2` instead. +- `nix hash to-base32 $hash1 $hash2`: Use `nix hash convert --to nix32 $hash1 $hash2` instead. +- `nix hash to-base64 $hash1 $hash2`: Use `nix hash convert --to base64 $hash1 $hash2` instead. +- `nix hash to-sri $hash1 $hash2`: : Use `nix hash convert --to sri $hash1 $hash2` + or even just `nix hash convert $hash1 $hash2` instead. diff --git a/doc/manual/rl-next/source-positions-in-errors.md b/doc/manual/rl-next/source-positions-in-errors.md new file mode 100644 index 000000000..5b210289d --- /dev/null +++ b/doc/manual/rl-next/source-positions-in-errors.md @@ -0,0 +1,42 @@ +--- +synopsis: Source locations are printed more consistently in errors +issues: 561 +prs: 9555 +--- + +Source location information is now included in error messages more +consistently. Given this code: + +```nix +let + attr = {foo = "bar";}; + key = {}; +in + attr.${key} +``` + +Previously, Nix would show this unhelpful message when attempting to evaluate +it: + +``` +error: + … while evaluating an attribute name + + error: value is a set while a string was expected +``` + +Now, the error message displays where the problematic value was found: + +``` +error: + … while evaluating an attribute name + + at bad.nix:4:11: + + 3| key = {}; + 4| in attr.${key} + | ^ + 5| + + error: value is a set while a string was expected +``` diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 2fe77d2c6..c67ddc6cb 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -2,7 +2,7 @@ - [Introduction](introduction.md) - [Quick Start](quick-start.md) -- [Installation](installation/installation.md) +- [Installation](installation/index.md) - [Supported Platforms](installation/supported-platforms.md) - [Installing a Binary Distribution](installation/installing-binary.md) - [Installing Nix from Source](installation/installing-source.md) @@ -18,6 +18,10 @@ - [Uninstalling Nix](installation/uninstall.md) - [Nix Store](store/index.md) - [File System Object](store/file-system-object.md) + - [Store Object](store/store-object.md) + - [Store Path](store/store-path.md) + - [Store Types](store/types/index.md) +{{#include ./store/types/SUMMARY.md}} - [Nix Language](language/index.md) - [Data Types](language/values.md) - [Language Constructs](language/constructs.md) @@ -29,11 +33,11 @@ - [Import From Derivation](language/import-from-derivation.md) - [Built-in Constants](language/builtin-constants.md) - [Built-in Functions](language/builtins.md) -- [Package Management](package-management/package-management.md) +- [Package Management](package-management/index.md) - [Profiles](package-management/profiles.md) - [Garbage Collection](package-management/garbage-collection.md) - [Garbage Collector Roots](package-management/garbage-collector-roots.md) -- [Advanced Topics](advanced-topics/advanced-topics.md) +- [Advanced Topics](advanced-topics/index.md) - [Sharing Packages Between Machines](package-management/sharing-packages.md) - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - [Copying Closures via SSH](package-management/copy-closure.md) @@ -43,7 +47,7 @@ - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md) - [Verifying Build Reproducibility](advanced-topics/diff-hook.md) - [Using the `post-build-hook`](advanced-topics/post-build-hook.md) -- [Command Reference](command-ref/command-ref.md) +- [Command Reference](command-ref/index.md) - [Common Options](command-ref/opt-common.md) - [Common Environment Variables](command-ref/env-common.md) - [Main Commands](command-ref/main-commands.md) @@ -100,19 +104,20 @@ - [Channels](command-ref/files/channels.md) - [Default Nix expression](command-ref/files/default-nix-expression.md) - [Architecture and Design](architecture/architecture.md) -- [Protocols](protocols/protocols.md) +- [Protocols](protocols/index.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Glossary](glossary.md) -- [Contributing](contributing/contributing.md) +- [Contributing](contributing/index.md) - [Hacking](contributing/hacking.md) - [Testing](contributing/testing.md) - [Documentation](contributing/documentation.md) - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) - [C++ style guide](contributing/cxx.md) -- [Release Notes](release-notes/release-notes.md) - - [Release X.Y (202?-??-??)](release-notes/rl-next.md) +- [Release Notes](release-notes/index.md) +{{#include ./SUMMARY-rl-next.md}} + - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) diff --git a/doc/manual/src/advanced-topics/advanced-topics.md b/doc/manual/src/advanced-topics/index.md similarity index 100% rename from doc/manual/src/advanced-topics/advanced-topics.md rename to doc/manual/src/advanced-topics/index.md diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 6e832e1f9..2fec4ed20 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -52,7 +52,7 @@ The following [concept map] shows its main components (rectangles), the objects '---------------' ``` -At the top is the [command line interface](../command-ref/command-ref.md) that drives the underlying layers. +At the top is the [command line interface](../command-ref/index.md) that drives the underlying layers. The [Nix language](../language/index.md) evaluator transforms Nix expressions into self-contained *build plans*, which are used to derive *build results* from referenced *build inputs*. @@ -63,7 +63,7 @@ The command line interface and Nix expressions are what users deal with most. > The Nix language itself does not have a notion of *packages* or *configurations*. > As far as we are concerned here, the inputs and results of a build plan are just data. -Underlying the command line interface and the Nix language evaluator is the [Nix store](../glossary.md#gloss-store), a mechanism to keep track of build plans, data, and references between them. +Underlying the command line interface and the Nix language evaluator is the [Nix store](../store/index.md), a mechanism to keep track of build plans, data, and references between them. It can also execute build plans to produce new data, which are made available to the operating system as files. A build plan itself is a series of *build tasks*, together with their build inputs. diff --git a/doc/manual/src/command-ref/command-ref.md b/doc/manual/src/command-ref/index.md similarity index 100% rename from doc/manual/src/command-ref/command-ref.md rename to doc/manual/src/command-ref/index.md diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/contributing/cli-guideline.md index e53d2d178..e90d6de8d 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/contributing/cli-guideline.md @@ -87,7 +87,7 @@ impacted the most by bad user experience. and [aligning of text](#text-alignment). - [Autocomplete](#shell-completion) of options. - Examples of such commands: `nix doctor`, `nix edit`, `nix eval`, ... + Examples of such commands: `nix edit`, `nix eval`, ... - **Utility and scripting commands** @@ -426,7 +426,7 @@ This leads to the following guidelines: ### Examples -This is bad, because all keys must be assumed to be store implementations: +This is bad, because all keys must be assumed to be store types: ```json { diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/contributing/documentation.md index f73ab2149..75226cd1a 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/contributing/documentation.md @@ -73,6 +73,17 @@ It should therefore aim to be correct, consistent, complete, and easy to navigat Non-trivial examples may need additional explanation, especially if they use concepts from outside the given context. +- Always explain code examples in the text. + + Use comments in code samples very sparingly, for instance to highlight a particular aspect. + Readers tend to glance over large amounts of code when scanning for information. + + Especially beginners will likely find reading more complex-looking code strenuous and may therefore avoid it altogether. + + If a code sample appears to require a lot of inline explanation, consider replacing it with a simpler one. + If that's not possible, break the example down into multiple parts, explain them separately, and then show the combined result at the end. + This should be a last resort, as that would amount to writing a [tutorial](https://diataxis.fr/tutorials/) on the given subject. + - Use British English. This is a somewhat arbitrary choice to force consistency, and accounts for the fact that a majority of Nix users and developers are from Europe. @@ -151,6 +162,24 @@ Please observe these guidelines to ease reviews: > This is a note. ``` + Highlight examples as such: + + ```` + > **Example** + > + > ```console + > $ nix --version + > ``` + ```` + + Highlight syntax definiions as such, using [EBNF](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form) notation: + + ```` + > **Syntax** + > + > *attribute-set* = `{` [ *attribute-name* `=` *expression* `;` ... ] `}` + ```` + ### The `@docroot@` variable `@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 38c144fcc..4d3d66397 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -10,7 +10,7 @@ $ cd nix The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. -[installation instructions]: ../installation/installation.md +[installation instructions]: ../installation/index.md ## Building Nix with flakes @@ -146,6 +146,31 @@ $ nix build .#packages.aarch64-linux.default Cross-compiled builds are available for ARMv6 (`armv6l-linux`) and ARMv7 (`armv7l-linux`). Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootstrap Nix on unsupported platforms. +### Building for multiple platforms at once + +It is useful to perform multiple cross and native builds on the same source tree, +for example to ensure that better support for one platform doesn't break the build for another. +In order to facilitate this, Nix has some support for being built out of tree – that is, placing build artefacts in a different directory than the source code: + +1. Create a directory for the build, e.g. + + ```bash + mkdir build + ``` + +2. Run the configure script from that directory, e.g. + + ```bash + cd build + ../configure + ``` + +3. Run make from the source directory, but with the build directory specified, e.g. + + ```bash + make builddir=build + ``` + ## System type Nix uses a string with he following format to identify the *system type* or *platform* it runs on: @@ -210,7 +235,7 @@ See [supported compilation environments](#compilation-environments) and instruct To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: ```console -make clean && bear -- make -j$NIX_BUILD_CORES install +make clean && bear -- make -j$NIX_BUILD_CORES default check install ``` Configure your editor to use the `clangd` from the shell, either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). @@ -220,3 +245,82 @@ Configure your editor to use the `clangd` from the shell, either by running it i > For some editors (e.g. Visual Studio Code), you may need to install a [special extension](https://open-vsx.org/extension/llvm-vs-code-extensions/vscode-clangd) for the editor to interact with `clangd`. > Some other editors (e.g. Emacs, Vim) need a plugin to support LSP servers in general (e.g. [lsp-mode](https://github.com/emacs-lsp/lsp-mode) for Emacs and [vim-lsp](https://github.com/prabirshrestha/vim-lsp) for vim). > Editor-specific setup is typically opinionated, so we will not cover it here in more detail. + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +--- +synopsis: Basically a title +issues: 1234 +prs: 1238 +--- + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. +Set `buildUnreleasedNotes = true;` in `flake.nix` to build the release notes on the fly. + +## Branches + +- [`master`](https://github.com/NixOS/nix/commits/master) + + The main development branch. All changes are approved and merged here. + When developing a change, create a branch based on the latest `master`. + + Maintainers try to [keep it in a release-worthy state](#reverting). + +- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) + + These branches are the subject of backports only, and are + also [kept](#reverting) in a release-worthy state. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) + + The latest patch release of the latest minor version. + + See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) + +- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) + + Generally branches created by the backport action. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [_other_](https://github.com/NixOS/nix/branches/all) + + Branches that do not conform to the above patterns should be feature branches. + +## Reverting + +If a change turns out to be merged by mistake, or contain a regression, it may be reverted. +A revert is not a rejection of the contribution, but merely part of an effective development process. +It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. +If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. +By embracing reverts as a good part of the development process, everyone wins. + +However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/contributing.md b/doc/manual/src/contributing/index.md similarity index 100% rename from doc/manual/src/contributing/contributing.md rename to doc/manual/src/contributing/index.md diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/contributing/testing.md index 3d75ebe7b..d8d162379 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/contributing/testing.md @@ -20,6 +20,7 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. [googletest]: https://google.github.io/googletest/ [rapidcheck]: https://github.com/emil-e/rapidcheck +[property testing]: https://en.wikipedia.org/wiki/Property_testing ### Source and header layout @@ -28,34 +29,50 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > ``` > src > ├── libexpr +> │ ├── local.mk > │ ├── value/context.hh > │ ├── value/context.cc +> │ … +> │ +> ├── tests > │ │ > │ … -> └── tests -> │ ├── value/context.hh -> │ ├── value/context.cc +> │ └── unit +> │ ├── libutil +> │ │ ├── local.mk +> │ │ … +> │ │ └── data +> │ │ ├── git/tree.txt +> │ │ … > │ │ -> │ … -> │ -> ├── unit-test-data -> │ ├── libstore -> │ │ ├── worker-protocol/content-address.bin -> │ │ … -> │ … +> │ ├── libexpr-support +> │ │ ├── local.mk +> │ │ └── tests +> │ │ ├── value/context.hh +> │ │ ├── value/context.cc +> │ │ … +> │ │ +> │ ├── libexpr +> │ … ├── local.mk +> │ ├── value/context.cc +> │ … > … > ``` -The unit tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_shortname}/tests` within the directory for the library (`src/${library_shortname}`). +The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`. +Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`. -The data is in `unit-test-data`, with one subdir per library, with the same name as where the code goes. -For example, `libnixstore` code is in `src/libstore`, and its test data is in `unit-test-data/libstore`. -The path to the `unit-test-data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`. +The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +Note that each executable only gets the data for its tests. -> **Note** -> Due to the way googletest works, downstream unit test executables will actually include and re-run upstream library tests. -> Therefore it is important that the same value for `_NIX_TEST_UNIT_DATA` be used with the tests for each library. -> That is why we have the test data nested within a single `unit-test-data` directory. +The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`. +All headers are in a `tests` subdirectory so they are included with `#include "tests/"`. + +The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing. +But organizing the tests this way has one big benefit: +there is no risk of any build-system wildcards for the library accidentally picking up test code that should not built and installed as part of the library. ### Running tests @@ -69,7 +86,7 @@ See [functional characterisation testing](#characterisation-testing-functional) Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. For example: ```shell-session -$ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN +$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN ... [ SKIPPED ] WorkerProtoTest.string_read [ SKIPPED ] WorkerProtoTest.string_write @@ -80,6 +97,18 @@ $ _NIX_TEST_ACCEPT=1 make libstore-tests-exe_RUN will regenerate the "golden master" expected result for the `libnixstore` characterisation tests. The characterisation tests will mark themselves "skipped" since they regenerated the expected result instead of actually testing anything. +### Unit test support libraries + +There are headers and code which are not just used to test the library in question, but also downstream libraries. +For example, we do [property testing] with the [rapidcheck] library. +This requires writing `Arbitrary` "instances", which are used to describe how to generate values of a given type for the sake of running property tests. +Because types contain other types, `Arbitrary` "instances" for some type are not just useful for testing that type, but also any other type that contains it. +Downstream types frequently contain upstream types, so it is very important that we share arbitrary instances so that downstream libraries' property tests can also use them. + +It is important that these testing libraries don't contain any actual tests themselves. +On some platforms they would be run as part of every test executable that uses them, which is redundant. +On other platforms they wouldn't be run at all. + ## Functional tests The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. @@ -133,17 +162,17 @@ ran test tests/functional/${testName}.sh... [PASS] or without `make`: ```shell-session -$ ./mk/run-test.sh tests/functional/${testName}.sh +$ ./mk/run-test.sh tests/functional/${testName}.sh tests/functional/init.sh ran test tests/functional/${testName}.sh... [PASS] ``` To see the complete output, one can also run: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh -+ foo +$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh ++(${testName}.sh:1) foo output from foo -+ bar ++(${testName}.sh:2) bar output from bar ... ``` @@ -175,7 +204,7 @@ edit it like so: Then, running the test with `./mk/debug-test.sh` will drop you into GDB once the script reaches that point: ```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh +$ ./mk/debug-test.sh tests/functional/${testName}.sh tests/functional/init.sh ... + gdb blash blub GNU gdb (GDB) 12.1 diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index ad3cc147b..07891175a 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -59,7 +59,7 @@ - [store]{#gloss-store} A collection of store objects, with operations to manipulate that collection. - See [Nix Store] for details. + See [Nix store](./store/index.md) for details. There are many types of stores. See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) for a complete list. @@ -86,10 +86,13 @@ - [store path]{#gloss-store-path} - The location of a [store object] in the file system, i.e., an - immediate child of the Nix store directory. + The location of a [store object](@docroot@/store/index.md#store-object) in the file system, i.e., an immediate child of the Nix store directory. - Example: `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` + > **Example** + > + > `/nix/store/a040m110amc4h71lds2jmr8qrkj2jhxd-git-2.38.1` + + See [Store Path](@docroot@/store/store-path.md) for details. [store path]: #gloss-store-path diff --git a/doc/manual/src/installation/installation.md b/doc/manual/src/installation/index.md similarity index 100% rename from doc/manual/src/installation/installation.md rename to doc/manual/src/installation/index.md diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md index a3590f55d..a82ec5960 100644 --- a/doc/manual/src/language/constructs.md +++ b/doc/manual/src/language/constructs.md @@ -132,6 +132,32 @@ a = src-set.a; b = src-set.b; c = src-set.c; when used while defining local variables in a let-expression or while defining a set. +In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example + + +```nix +let + x = { a = 1; b = 2; }; + inherit (builtins) attrNames; +in +{ + names = attrNames x; +} +``` + +is equivalent to + +```nix +let + x = { a = 1; b = 2; }; +in +{ + names = builtins.attrNames x; +} +``` + +both evaluate to `{ names = [ "a" "b" ]; }`. + ## Functions Functions have the following form: @@ -146,65 +172,65 @@ three kinds of patterns: - If a pattern is a single identifier, then the function matches any argument. Example: - + ```nix let negate = x: !x; concat = x: y: x + y; in if negate true then concat "foo" "bar" else "" ``` - + Note that `concat` is a function that takes one argument and returns a function that takes another argument. This allows partial parameterisation (i.e., only filling some of the arguments of a function); e.g., - + ```nix map (concat "foo") [ "bar" "bla" "abc" ] ``` - + evaluates to `[ "foobar" "foobla" "fooabc" ]`. - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a set containing the listed attributes, and binds the values of those attributes to variables in the function body. For example, the function - + ```nix { x, y, z }: z + y + x ``` - + can only be called with a set containing exactly the attributes `x`, `y` and `z`. No other attributes are allowed. If you want to allow additional arguments, you can use an ellipsis (`...`): - + ```nix { x, y, z, ... }: z + y + x ``` - + This works on any set that contains at least the three named attributes. - + It is possible to provide *default values* for attributes, in which case they are allowed to be missing. A default value is specified by writing `name ? e`, where *e* is an arbitrary expression. For example, - + ```nix { x, y ? "foo", z ? "bar" }: z + y + x ``` - + specifies a function that only requires an attribute named `x`, but optionally accepts `y` and `z`. - An `@`-pattern provides a means of referring to the whole value being matched: - + ```nix args@{ x, y, z, ... }: z + y + x + args.a ``` - + but can also be written as: - + ```nix { x, y, z, ... } @ args: z + y + x + args.a ``` diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index cc825b4cf..e9cbb5c92 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -25,7 +25,7 @@ | Inequality | *expr* `!=` *expr* | none | 11 | | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | -| [Logical implication] | *bool* `->` *bool* | none | 14 | +| [Logical implication] | *bool* `->` *bool* | right | 14 | [string]: ./values.md#type-string [path]: ./values.md#type-path diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index 0bb656746..aea68a441 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -116,6 +116,7 @@ [store path]: ../glossary.md#gloss-store-path Paths can include [string interpolation] and can themselves be [interpolated in other expressions]. + [interpolated in other expressions]: ./string-interpolation.md#interpolated-expressions At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. diff --git a/doc/manual/src/package-management/package-management.md b/doc/manual/src/package-management/index.md similarity index 100% rename from doc/manual/src/package-management/package-management.md rename to doc/manual/src/package-management/index.md diff --git a/doc/manual/src/protocols/protocols.md b/doc/manual/src/protocols/index.md similarity index 100% rename from doc/manual/src/protocols/protocols.md rename to doc/manual/src/protocols/index.md diff --git a/doc/manual/src/quick-start.md b/doc/manual/src/quick-start.md index 1d2688ede..04a0b7c96 100644 --- a/doc/manual/src/quick-start.md +++ b/doc/manual/src/quick-start.md @@ -1,10 +1,9 @@ # Quick Start -This chapter is for impatient people who don't like reading -documentation. For more in-depth information you are kindly referred -to subsequent chapters. +This chapter is for impatient people who don't like reading documentation. +For more in-depth information you are kindly referred to subsequent chapters. -1. Install Nix by running the following: +1. Install Nix: ```console $ curl -L https://nixos.org/nix/install | sh @@ -13,87 +12,33 @@ to subsequent chapters. The install script will use `sudo`, so make sure you have sufficient rights. On Linux, `--daemon` can be omitted for a single-user install. - For other installation methods, see [here](installation/installation.md). + For other installation methods, see the detailed [installation instructions](installation/index.md). -1. See what installable packages are currently available in the - channel: +1. Run software without installing it permanently: ```console - $ nix-env --query --available --attr-path - nixpkgs.docbook_xml_dtd_43 docbook-xml-4.3 - nixpkgs.docbook_xml_dtd_45 docbook-xml-4.5 - nixpkgs.firefox firefox-33.0.2 - nixpkgs.hello hello-2.9 - nixpkgs.libxslt libxslt-1.1.28 - … + $ nix-shell --packages cowsay lolcat ``` -1. Install some packages from the channel: + This downloads the specified packages with all their dependencies, and drops you into a Bash shell where the commands provided by those packages are present. + This will not affect your normal environment: ```console - $ nix-env --install --attr nixpkgs.hello + [nix-shell:~]$ cowsay Hello, Nix! | lolcat ``` - This should download pre-built packages; it should not build them - locally (if it does, something went wrong). - -1. Test that they work: + Exiting the shell will make the programs disappear again: ```console - $ which hello - /home/eelco/.nix-profile/bin/hello - $ hello - Hello, world! - ``` - -1. Uninstall a package: - - ```console - $ nix-env --uninstall hello - ``` - -1. You can also test a package without installing it: - - ```console - $ nix-shell --packages hello - ``` - - This builds or downloads GNU Hello and its dependencies, then drops - you into a Bash shell where the `hello` command is present, all - without affecting your normal environment: - - ```console - [nix-shell:~]$ hello - Hello, world! - [nix-shell:~]$ exit - - $ hello - hello: command not found + $ lolcat + lolcat: command not found ``` -1. To keep up-to-date with the channel, do: +1. Search for more packages on to try them out. + +1. Free up storage space: ```console - $ nix-channel --update nixpkgs - $ nix-env --upgrade '*' - ``` - - The latter command will upgrade each installed package for which - there is a “newer” version (as determined by comparing the version - numbers). - -1. If you're unhappy with the result of a `nix-env` action (e.g., an - upgraded package turned out not to work properly), you can go back: - - ```console - $ nix-env --rollback - ``` - -1. You should periodically run the Nix garbage collector to get rid of - unused packages, since uninstalls or upgrades don't actually delete - them: - - ```console - $ nix-collect-garbage --delete-old + $ nix-collect-garbage ``` diff --git a/doc/manual/src/release-notes/release-notes.md b/doc/manual/src/release-notes/index.md similarity index 100% rename from doc/manual/src/release-notes/release-notes.md rename to doc/manual/src/release-notes/index.md diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md new file mode 100644 index 000000000..ba6eb9c64 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -0,0 +1,77 @@ +# Release 2.19 (2023-11-17) + +- The experimental `nix` command can now act as a [shebang interpreter](@docroot@/command-ref/new-cli/nix.md#shebang-interpreter) + by appending the contents of any `#! nix` lines and the script's location into a single call. + +- [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. + +- [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). + +- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly. + +- There is a new flake installable syntax `flakeref#.attrPath` where the "." prefix specifies that `attrPath` is interpreted from the root of the flake outputs, with no searching of default attribute prefixes like `packages.` or `legacyPackages.`. + +- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts. + +- Add a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). + +- `nix-shell` shebang lines now support single-quoted arguments. + +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). + +- The interface for creating and updating lock files has been overhauled: + + - [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) only creates lock files and adds missing inputs now. + It will *never* update existing inputs. + + - [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) does the same, but *will* update inputs. + - Passing no arguments will update all inputs of the current flake, just like it already did. + - Passing input names as arguments will ensure only those are updated. This replaces the functionality of `nix flake lock --update-input` + - To operate on a flake outside the current directory, you must now pass `--flake path/to/flake`. + + - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. + They are superceded by `nix flake update`. + +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + +- [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) + (experimental) now returns a JSON map rather than JSON list. + The `path` field of each object has instead become the key in the outer map, since it is unique. + The `valid` field also goes away because we just use `null` instead. + + - Old way: + + ```json5 + [ + { + "path": "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15", + "valid": true, + // ... + }, + { + "path": "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path", + "valid": false + } + ] + ``` + + - New way + + ```json5 + { + "/nix/store/8fv91097mbh5049i9rglc73dx6kjg3qk-bash-5.2-p15": { + // ... + }, + "/nix/store/wffw7l0alvs3iw94cbgi1gmmbmw99sqb-home-manager-path": null, + } + ``` + + This makes it match `nix derivation show`, which also maps store paths to information. + +- When Nix is installed using the [binary installer](@docroot@/installation/installing-binary.md), in supported shells (Bash, Zsh, Fish) + [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile. + This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) + (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). + +- A new command `nix store add` has been added. It replaces `nix store add-file` and `nix store add-path` which are now deprecated. diff --git a/doc/manual/src/release-notes/rl-next.md b/doc/manual/src/release-notes/rl-next.md deleted file mode 100644 index 276252c37..000000000 --- a/doc/manual/src/release-notes/rl-next.md +++ /dev/null @@ -1,17 +0,0 @@ -# Release X.Y (202?-??-??) - -- [URL flake references](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references) now support [percent-encoded](https://datatracker.ietf.org/doc/html/rfc3986#section-2.1) characters. - -- [Path-like flake references](@docroot@/command-ref/new-cli/nix3-flake.md#path-like-syntax) now accept arbitrary unicode characters (except `#` and `?`). - -- The experimental feature `repl-flake` is no longer needed, as its functionality is now part of the `flakes` experimental feature. To get the previous behavior, use the `--file/--expr` flags accordingly. - -- Introduce new flake installable syntax `flakeref#.attrPath` where the "." prefix denotes no searching of default attribute prefixes like `packages.` or `legacyPackages.`. - -- Nix adds `apple-virt` to the default system features on macOS systems that support virtualization. This is similar to what's done for the `kvm` system feature on Linux hosts. - -- Introduce a new built-in function [`builtins.convertHash`](@docroot@/language/builtins.md#builtins-convertHash). - -- `nix-shell` shebang lines now support single-quoted arguments. - -- `builtins.fetchTree` is now marked as stable. diff --git a/doc/manual/src/store/index.md b/doc/manual/src/store/index.md index 316e04179..f1e8f1402 100644 --- a/doc/manual/src/store/index.md +++ b/doc/manual/src/store/index.md @@ -1,4 +1,5 @@ # Nix Store -The *Nix store* is an abstraction used by Nix to store immutable filesystem artifacts (such as software packages) that can have dependencies (*references*) between them. -There are multiple implementations of the Nix store, such as the actual filesystem (`/nix/store`) and binary caches. +The *Nix store* is an abstraction to store immutable file system data (such as software packages) that can have dependencies on other such data. + +There are [multiple types of Nix stores](./types/index.md) with different capabilities, such as the default one on the [local filesystem](./types/local-store.md) (`/nix/store`) or [binary caches](./types/http-binary-cache-store.md). diff --git a/doc/manual/src/store/store-object.md b/doc/manual/src/store/store-object.md new file mode 100644 index 000000000..caf5657d1 --- /dev/null +++ b/doc/manual/src/store/store-object.md @@ -0,0 +1,10 @@ +## Store Object + +A Nix store is a collection of *store objects* with *references* between them. +A store object consists of + + - A [file system object](./file-system-object.md) as data + - A set of [store paths](./store-path.md) as references to other store objects + +Store objects are [immutable](https://en.wikipedia.org/wiki/Immutable_object): +Once created, they do not change until they are deleted. diff --git a/doc/manual/src/store/store-path.md b/doc/manual/src/store/store-path.md new file mode 100644 index 000000000..b5ad0c654 --- /dev/null +++ b/doc/manual/src/store/store-path.md @@ -0,0 +1,69 @@ +# Store Path + +Nix implements references to [store objects](./index.md#store-object) as *store paths*. + +Think of a store path as an [opaque], [unique identifier]: +The only way to obtain store path is by adding or building store objects. +A store path will always reference exactly one store object. + +[opaque]: https://en.m.wikipedia.org/wiki/Opaque_data_type +[unique identifier]: https://en.m.wikipedia.org/wiki/Unique_identifier + +Store paths are pairs of + +- A 20-byte digest for identification +- A symbolic name for people to read + +> **Example** +> +> - Digest: `b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z` +> - Name: `firefox-33.1` + +To make store objects accessible to operating system processes, stores have to expose store objects through the file system. + +A store path is rendered to a file system path as the concatenation of + +- [Store directory](#store-directory) (typically `/nix/store`) +- Path separator (`/`) +- Digest rendered in a custom variant of [Base32](https://en.wikipedia.org/wiki/Base32) (20 arbitrary bytes become 32 ASCII characters) +- Hyphen (`-`) +- Name + +> **Example** +> +> ``` +> /nix/store/b6gvzjyb2pg0kjfwrjmg1vfhh54ad73z-firefox-33.1 +> |--------| |------------------------------| |----------| +> store directory digest name +> ``` + +## Store Directory + +Every [Nix store](./index.md) has a store directory. + +Not every store can be accessed through the file system. +But if the store has a file system representation, the store directory contains the store’s [file system objects], which can be addressed by [store paths](#store-path). + +[file system objects]: ./file-system-object.md + +This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. + +> **Note** +> +> The store directory defaults to `/nix/store`, but is in principle arbitrary. + +It is important which store a given store object belongs to: +Files in the store object can contain store paths, and processes may read these paths. +Nix can only guarantee referential integrity if store paths do not cross store boundaries. + +Therefore one can only copy store objects to a different store if + +- The source and target stores' directories match + + or + +- The store object in question has no references, that is, contains no store paths + +One cannot copy a store object to a store with a different store directory. +Instead, it has to be rebuilt, together with all its dependencies. +It is in general not enough to replace the store directory string in file contents, as this may render executables unusable by invalidating their internal offsets or checksums. diff --git a/src/nix/help-stores.md b/doc/manual/src/store/types/index.md.in similarity index 74% rename from src/nix/help-stores.md rename to doc/manual/src/store/types/index.md.in index 47ba9b94d..a35161ce8 100644 --- a/src/nix/help-stores.md +++ b/doc/manual/src/store/types/index.md.in @@ -1,6 +1,6 @@ -R"( +Nix supports different types of stores: -Nix supports different types of stores. These are described below. +@store-types@ ## Store URL format @@ -29,18 +29,15 @@ supported settings for each store type are documented below. The special store URL `auto` causes Nix to automatically select a store as follows: -* Use the [local store](#local-store) `/nix/store` if `/nix/var/nix` +* Use the [local store](./local-store.md) `/nix/store` if `/nix/var/nix` is writable by the current user. * Otherwise, if `/nix/var/nix/daemon-socket/socket` exists, [connect - to the Nix daemon listening on that socket](#local-daemon-store). + to the Nix daemon listening on that socket](./local-daemon-store.md). -* Otherwise, on Linux only, use the [local chroot store](#local-store) +* Otherwise, on Linux only, use the [local chroot store](./local-store.md) `~/.local/share/nix/root`, which will be created automatically if it does not exist. -* Otherwise, use the [local store](#local-store) `/nix/store`. +* Otherwise, use the [local store](./local-store.md) `/nix/store`. -@stores@ - -)" diff --git a/doc/manual/utils.nix b/doc/manual/utils.nix index 849832b2c..19ff49b64 100644 --- a/doc/manual/utils.nix +++ b/doc/manual/utils.nix @@ -1,5 +1,11 @@ with builtins; +let + lowerChars = stringToCharacters "abcdefghijklmnopqrstuvwxyz"; + upperChars = stringToCharacters "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + stringToCharacters = s: genList (p: substring p 1 s) (stringLength s); +in + rec { splitLines = s: filter (x: !isList x) (split "\n" s); @@ -18,6 +24,8 @@ rec { in if replaced == string then string else replaceStringsRec from to replaced; + toLower = replaceStrings upperChars lowerChars; + squash = replaceStringsRec "\n\n\n" "\n\n"; trim = string: diff --git a/flake.lock b/flake.lock index 56df9c3fb..3cb9e72c9 100644 --- a/flake.lock +++ b/flake.lock @@ -16,6 +16,22 @@ "type": "github" } }, + "libgit2": { + "flake": false, + "locked": { + "lastModified": 1697646580, + "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "owner": "libgit2", + "repo": "libgit2", + "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "type": "github" + }, + "original": { + "owner": "libgit2", + "repo": "libgit2", + "type": "github" + } + }, "lowdown-src": { "flake": false, "locked": { @@ -34,16 +50,16 @@ }, "nixpkgs": { "locked": { - "lastModified": 1695283060, - "narHash": "sha256-CJz71xhCLlRkdFUSQEL0pIAAfcnWFXMzd9vXhPrnrEg=", + "lastModified": 1701355166, + "narHash": "sha256-4V7XMI0Gd+y0zsi++cEHd99u3GNL0xSTGRmiWKzGnUQ=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "31ed632c692e6a36cfc18083b88ece892f863ed4", + "rev": "36c4ac09e9bebcec1fa7b7539cddb0c9e837409c", "type": "github" }, "original": { "owner": "NixOS", - "ref": "nixos-23.05-small", + "ref": "staging-23.05", "repo": "nixpkgs", "type": "github" } @@ -67,6 +83,7 @@ "root": { "inputs": { "flake-compat": "flake-compat", + "libgit2": "libgit2", "lowdown-src": "lowdown-src", "nixpkgs": "nixpkgs", "nixpkgs-regression": "nixpkgs-regression" diff --git a/flake.nix b/flake.nix index 398ba10a0..99480183a 100644 --- a/flake.nix +++ b/flake.nix @@ -1,18 +1,28 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-23.05-small"; + # TODO Go back to nixos-23.05-small once + # https://github.com/NixOS/nixpkgs/pull/271202 is merged. + # + # Also, do not grab arbitrary further staging commits. This PR was + # carefully made to be based on release-23.05 and just contain + # rebuild-causing changes to packages that Nix actually uses. + inputs.nixpkgs.url = "github:NixOS/nixpkgs/staging-23.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.lowdown-src = { url = "github:kristapsdz/lowdown"; flake = false; }; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; + inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; - outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat }: + outputs = { self, nixpkgs, nixpkgs-regression, lowdown-src, flake-compat, libgit2 }: let inherit (nixpkgs) lib; officialRelease = false; + # Set to true to build the release notes for the next release. + buildUnreleasedNotes = false; + version = lib.fileContents ./.version + versionSuffix; versionSuffix = if officialRelease @@ -89,7 +99,7 @@ ./misc ./precompiled-headers.h ./src - ./unit-test-data + ./tests/unit ./COPYING ./scripts/local.mk functionalTestFiles @@ -162,12 +172,18 @@ testConfigureFlags = [ "RAPIDCHECK_HEADERS=${lib.getDev rapidcheck}/extras/gtest/include" + ] ++ lib.optionals (stdenv.hostPlatform != stdenv.buildPlatform) [ + "--enable-install-unit-tests" + "--with-check-bin-dir=${builtins.placeholder "check"}/bin" + "--with-check-lib-dir=${builtins.placeholder "check"}/lib" ]; internalApiDocsConfigureFlags = [ "--enable-internal-api-docs" ]; + changelog-d = pkgs.buildPackages.callPackage ./misc/changelog-d.nix { }; + nativeBuildDeps = [ buildPackages.bison @@ -183,14 +199,23 @@ buildPackages.git buildPackages.mercurial # FIXME: remove? only needed for tests buildPackages.jq # Also for custom mdBook preprocessor. + buildPackages.openssh # only needed for tests (ssh-keygen) ] - ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)]; + ++ lib.optionals stdenv.hostPlatform.isLinux [(buildPackages.util-linuxMinimal or buildPackages.utillinuxMinimal)] + # Official releases don't have rl-next, so we don't need to compile a changelog + ++ lib.optional (!officialRelease && buildUnreleasedNotes) changelog-d + ; buildDeps = [ curl bzip2 xz brotli editline openssl sqlite libarchive + (pkgs.libgit2.overrideAttrs (attrs: { + src = libgit2; + version = libgit2.lastModifiedDate; + cmakeFlags = (attrs.cmakeFlags or []) ++ ["-DUSE_SSH=exec"]; + })) boost lowdown-nix libsodium @@ -219,6 +244,9 @@ }).overrideAttrs(o: { patches = (o.patches or []) ++ [ ./boehmgc-coroutine-sp-fallback.diff + + # https://github.com/ivmai/bdwgc/pull/586 + ./boehmgc-traceable_allocator-public.diff ]; }) ) @@ -401,7 +429,8 @@ src = nixSrc; VERSION_SUFFIX = versionSuffix; - outputs = [ "out" "dev" "doc" ]; + outputs = [ "out" "dev" "doc" ] + ++ lib.optional (currentStdenv.hostPlatform != currentStdenv.buildPlatform) "check"; nativeBuildInputs = nativeBuildDeps; buildInputs = buildDeps @@ -511,6 +540,8 @@ # Binary package for various platforms. build = forAllSystems (system: self.packages.${system}.nix); + shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); + buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); buildCross = forAllCrossSystems (crossSystem: @@ -662,6 +693,11 @@ perlBindings = self.hydraJobs.perlBindings.${system}; installTests = self.hydraJobs.installTests.${system}; nixpkgsLibTests = self.hydraJobs.tests.nixpkgsLibTests.${system}; + rl-next = + let pkgs = nixpkgsFor.${system}.native; + in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' + LANG=C.UTF-8 ${(commonDeps { inherit pkgs; }).changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out + ''; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; }); @@ -707,13 +743,16 @@ stdenv.mkDerivation { name = "nix"; - outputs = [ "out" "dev" "doc" ]; + outputs = [ "out" "dev" "doc" ] + ++ lib.optional (stdenv.hostPlatform != stdenv.buildPlatform) "check"; nativeBuildInputs = nativeBuildDeps ++ lib.optional stdenv.cc.isClang pkgs.buildPackages.bear ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools + # We want changelog-d in the shell even if the current build doesn't need it + ++ lib.optional (officialRelease || ! buildUnreleasedNotes) changelog-d ; buildInputs = buildDeps ++ propagatedDeps diff --git a/m4/gcc_bug_80431.m4 b/m4/gcc_bug_80431.m4 new file mode 100644 index 000000000..e42f01956 --- /dev/null +++ b/m4/gcc_bug_80431.m4 @@ -0,0 +1,64 @@ +# Ensure that this bug is not present in the C++ toolchain we are using. +# +# URL for bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=80431 +# +# The test program is from that issue, with only a slight modification +# to set an exit status instead of printing strings. +AC_DEFUN([ENSURE_NO_GCC_BUG_80431], +[ + AC_MSG_CHECKING([that GCC bug 80431 is fixed]) + AC_LANG_PUSH(C++) + AC_RUN_IFELSE( + [AC_LANG_PROGRAM( + [[ + #include + + static bool a = true; + static bool b = true; + + struct Options { }; + + struct Option + { + Option(Options * options) + { + a = false; + } + + ~Option() + { + b = false; + } + }; + + struct MyOptions : Options { }; + + struct MyOptions2 : virtual MyOptions + { + Option foo{this}; + }; + ]], + [[ + { + MyOptions2 opts; + } + return (a << 1) | b; + ]])], + [status_80431=0], + [status_80431=$?], + [ + # Assume we're bug-free when cross-compiling + ]) + AC_LANG_POP(C++) + AS_CASE([$status_80431], + [0],[ + AC_MSG_RESULT(yes) + ], + [2],[ + AC_MSG_RESULT(no) + AC_MSG_ERROR(Cannot build Nix with C++ compiler with this bug) + ], + [ + AC_MSG_RESULT(unexpected result $status_80431: not expected failure with bug, ignoring) + ]) +]) diff --git a/maintainers/README.md b/maintainers/README.md index 5be4f9d04..ee97c1195 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -2,7 +2,7 @@ ## Motivation -The team's main responsibility is to set a direction for the development of Nix and ensure that the code is in good shape. +The team's main responsibility is to guide and direct the development of Nix and ensure that the code is in good shape. We aim to achieve this by improving the contributor experience and attracting more maintainers – that is, by helping other people contributing to Nix and eventually taking responsibility – in order to scale the development process to match users' needs. diff --git a/maintainers/release-notes b/maintainers/release-notes new file mode 100755 index 000000000..34cd85a56 --- /dev/null +++ b/maintainers/release-notes @@ -0,0 +1,179 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash ../shell.nix -I nixpkgs=channel:nixos-unstable-small +# ^^^^^^^ +# Only used for bash. shell.nix goes to the flake. + +# --- CONFIGURATION --- + +# This does double duty for +# - including rl-next +# - marking where to insert new links (right after) +SUMMARY_MARKER_LINE='{{#include ./SUMMARY-rl-next.md}}' + +# --- LIB --- + +log() { + echo 1>&2 "release-notes:" "$@" +} +logcmd() { + local cmd="$1" + shift + logcmd2 "$cmd" "${*@Q}" "$cmd" "$@" +} +logcmd2() { + local fakecmd="$1" + local fakeargs="$2" + shift + shift + printf 1>&2 "release-notes: \033[34;1m$fakecmd\033[0m " + echo "$fakeargs" 1>&2 + "$@" +} +die() { + # ANSI red + printf 1>&2 "release-notes: \033[31;1merror:\033[0m" + echo 1>&2 "" "$@" + exit 1 +} +confirm() { + local answer + echo 1>&2 "$@" "[y/n]" + read -r answer + case "$answer" in + y|Y|yes|Yes|YES) + return 0 + ;; + n|N|no|No|NO) + return 1 + ;; + *) + echo 1>&2 "please answer y or n" + confirm "$@" + ;; + esac +} +report_done() { + logcmd2 "git" "show" git -c pager.show=false show + printf 1>&2 "release-notes: \033[32;1mdone\033[0m\n" +} + +# --- PARSE ARGS --- + +if [[ $# -gt 0 ]]; then + die "Release notes takes no arguments, but make sure to set VERSION." +fi + +# --- CHECKS --- + +if [[ ! -e flake.nix ]] || [[ ! -e .git ]]; then + die "must run in repo root" + exit 1 +fi + +# repo must be clean +if ! git diff --quiet; then + die "repo is dirty, please commit or stash changes" +fi + +if ! git diff --quiet --cached; then + die "repo has staged changes, please commit or stash them" +fi + +if ! grep "$SUMMARY_MARKER_LINE" doc/manual/src/SUMMARY.md.in >/dev/null; then + # would have been nice to catch this early, but won't be worth the extra infra + die "SUMMARY.md.in is missing the marker line '$SUMMARY_MARKER_LINE', which would be used for inserting a new release notes page. Please fix the script." +fi + +if [[ ! -n "${VERSION:-}" ]]; then + die "please set the VERSION environment variable before invoking this script" + exit 1 +fi + +# version_major_minor: MAJOR.MINOR +# version_full: MAJOR.MINOR.PATCH +# IS_PATCH: true if this is a patch release; append instead of create +if grep -E '^[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then + log 'is minor' + IS_PATCH=false + version_full="$VERSION.0" + version_major_minor="$VERSION" +elif grep -E '^[0-9]+\.[0-9]+\.0$' <<< "$VERSION" >/dev/null; then + log 'is minor (.0)' + IS_PATCH=false + version_full="$VERSION" + version_major_minor="$(echo "$VERSION" | sed -e 's/\.0$//')" +elif grep -E '^[0-9]+\.[0-9]+\.[0-9]+$' <<< "$VERSION" >/dev/null; then + log 'is patch' + IS_PATCH=true + version_full="$VERSION" + version_major_minor="$(echo "$VERSION" | sed -e 's/\.[0-9]*$//')" +else + die "VERSION must be MAJOR.MINOR[.PATCH], where each is a number, e.g. 2.20 or 2.20.1 (VERSION was set to $VERSION)" +fi + +unset VERSION + +log "version_major_minor=$version_major_minor" +log "version_full=$version_full" +log "IS_PATCH=$IS_PATCH" + +basename=rl-${version_major_minor}.md +file=doc/manual/src/release-notes/$basename + +if ! $IS_PATCH; then + if [[ -e $file ]]; then + die "release notes file $file already exists. If you'd like to make a minor release, pass a patch version, e.g. 2.20.1" + fi +fi + +# --- DEFAULTS --- + +if [[ ! -n "${DATE:-}" ]]; then + DATE="$(date +%Y-%m-%d)" + log "DATE not set, using $DATE" +fi + +case "$DATE" in + [0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]) + ;; + *) + die "DATE must be YYYY-MM-DD, e.g. 2021-12-31 (DATE was set to $DATE)" + ;; +esac + +# --- DO THE WORK --- + +# menu +title="Release $version_major_minor ($DATE)" +# section on page +section_title="Release $version_full ($DATE)" + +( + # TODO add minor number, and append? + echo "# $section_title" + echo + changelog-d doc/manual/rl-next | sed -e 's/ *$//' +) | tee -a $file + +log "Wrote $file" + +if ! $IS_PATCH; then + NEW_SUMMARY_LINE=" - [$title](release-notes/$basename)" + + # find the marker line, insert new link after it + escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" + escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" + logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in +fi + +for f in doc/manual/rl-next/*.md; do + if [[ config != "$(basename $f)" ]]; then + logcmd git rm $f + fi +done + +logcmd git add $file doc/manual/src/SUMMARY.md.in +logcmd git status +logcmd git commit -m "release notes: $version_full" + +report_done diff --git a/maintainers/release-process.md b/maintainers/release-process.md index d85266b81..db8b064a5 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -24,34 +24,23 @@ release: * In a checkout of the Nix repo, make sure you're on `master` and run `git pull`. -* Move the contents of `doc/manual/src/release-notes/rl-next.md` - (except the first line) to - `doc/manual/src/release-notes/rl-$VERSION.md` (where `$VERSION` is - the contents of `.version` *without* the patch level, e.g. `2.12` - rather than `2.12.0`). - -* Add a header to `doc/manual/src/release-notes/rl-$VERSION.md` like - - ``` - # Release 2.12 (2022-12-06) - ``` - -* Proof-read / edit / rearrange the release notes. Breaking changes - and highlights should go to the top. - -* Add a link to the release notes to `doc/manual/src/SUMMARY.md.in` - (*not* `SUMMARY.md`), e.g. - - ``` - - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) - ``` - -* Run +* Compile the release notes by running ```console $ git checkout -b release-notes - $ git add doc/manual/src/release-notes/rl-$VERSION.md - $ git commit -a -m 'Release notes' + $ VERSION=X.YY ./maintainers/release-notes + ``` + + where `X.YY` is *without* the patch level, e.g. `2.12` rather than ~~`2.12.0`~~. + + A commit is created. + +* Proof-read / edit / rearrange the release notes if needed. Breaking changes + and highlights should go to the top. + +* Push. + + ```console $ git push --set-upstream $REMOTE release-notes ``` @@ -67,15 +56,17 @@ release: $ git checkout -b $VERSION-maintenance ``` -* Mark the release as stable: +* Mark the release as official: ```console - $ git cherry-pick f673551e71942a52b6d7ae66af8b67140904a76a + $ sed -e 's/officialRelease = false;/officialRelease = true;/' -i flake.nix ``` This removes the link to `rl-next.md` from the manual and sets `officialRelease = true` in `flake.nix`. +* Commit + * Push the release branch: ```console @@ -159,6 +150,30 @@ release: ## Creating a point release +* Checkout. + + ```console + $ git checkout XX.YY-maintenance + ``` + +* Determine the next patch version. + + ```console + $ export VERSION=XX.YY.ZZ + ``` + +* Update release notes. + + ```console + $ ./maintainers/release-notes + ``` + +* Push. + + ```console + $ git push + ``` + * Wait for the desired evaluation of the maintenance jobset to finish building. diff --git a/misc/changelog-d.cabal.nix b/misc/changelog-d.cabal.nix new file mode 100644 index 000000000..76f9353cd --- /dev/null +++ b/misc/changelog-d.cabal.nix @@ -0,0 +1,31 @@ +{ mkDerivation, aeson, base, bytestring, cabal-install-parsers +, Cabal-syntax, containers, directory, filepath, frontmatter +, generic-lens-lite, lib, mtl, optparse-applicative, parsec, pretty +, regex-applicative, text, pkgs +}: +let rev = "f30f6969e9cd8b56242309639d58acea21c99d06"; +in +mkDerivation { + pname = "changelog-d"; + version = "0.1"; + src = pkgs.fetchurl { + name = "changelog-d-${rev}.tar.gz"; + url = "https://codeberg.org/roberth/changelog-d/archive/${rev}.tar.gz"; + hash = "sha256-8a2+i5u7YoszAgd5OIEW0eYUcP8yfhtoOIhLJkylYJ4="; + } // { inherit rev; }; + isLibrary = false; + isExecutable = true; + libraryHaskellDepends = [ + aeson base bytestring cabal-install-parsers Cabal-syntax containers + directory filepath frontmatter generic-lens-lite mtl parsec pretty + regex-applicative text + ]; + executableHaskellDepends = [ + base bytestring Cabal-syntax directory filepath + optparse-applicative + ]; + doHaddock = false; + description = "Concatenate changelog entries into a single one"; + license = lib.licenses.gpl3Plus; + mainProgram = "changelog-d"; +} diff --git a/misc/changelog-d.nix b/misc/changelog-d.nix new file mode 100644 index 000000000..1b20f4596 --- /dev/null +++ b/misc/changelog-d.nix @@ -0,0 +1,31 @@ +# Taken temporarily from +{ + callPackage, + lib, + haskell, + haskellPackages, +}: + +let + hsPkg = haskellPackages.callPackage ./changelog-d.cabal.nix { }; + + addCompletions = haskellPackages.generateOptparseApplicativeCompletions ["changelog-d"]; + + haskellModifications = + lib.flip lib.pipe [ + addCompletions + haskell.lib.justStaticExecutables + ]; + + mkDerivationOverrides = finalAttrs: oldAttrs: { + + version = oldAttrs.version + "-git-${lib.strings.substring 0 7 oldAttrs.src.rev}"; + + meta = oldAttrs.meta // { + homepage = "https://codeberg.org/roberth/changelog-d"; + maintainers = [ lib.maintainers.roberth ]; + }; + + }; +in + (haskellModifications hsPkg).overrideAttrs mkDerivationOverrides diff --git a/mk/build-dir.mk b/mk/build-dir.mk new file mode 100644 index 000000000..02f4cae60 --- /dev/null +++ b/mk/build-dir.mk @@ -0,0 +1,10 @@ +# Initialise support for build directories. +builddir ?= + +ifdef builddir + buildprefix = $(builddir)/ + buildprefixrel = $(builddir) +else + buildprefix = + buildprefixrel = . +endif diff --git a/mk/common-test.sh b/mk/common-test.sh index 7ab25febf..2783d293b 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,15 +1,27 @@ -test_dir=tests/functional +# Remove overall test dir (at most one of the two should match) and +# remove file extension. +test_name=$(echo -n "$test" | sed \ + -e "s|^tests/unit/[^/]*/data/||" \ + -e "s|^tests/functional/||" \ + -e "s|\.sh$||" \ + ) -test=$(echo -n "$test" | sed -e "s|^$test_dir/||") - -TESTS_ENVIRONMENT=("TEST_NAME=${test%.*}" 'NIX_REMOTE=') +TESTS_ENVIRONMENT=( + "TEST_NAME=$test_name" + 'NIX_REMOTE=' + 'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) ' +) : ${BASH:=/usr/bin/env bash} +run () { + cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1) +} + init_test () { - cd "$test_dir" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e init.sh 2>/dev/null > /dev/null + run "$init" 2>/dev/null > /dev/null } run_test_proper () { - cd "$test_dir/$(dirname $test)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -e $(basename $test) + run "$test" } diff --git a/mk/debug-test.sh b/mk/debug-test.sh index b5b628ecd..52482c01e 100755 --- a/mk/debug-test.sh +++ b/mk/debug-test.sh @@ -3,9 +3,12 @@ set -eu -o pipefail test=$1 +init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" -(init_test) +if [ -n "$init" ]; then + (init_test) +fi run_test_proper diff --git a/mk/install-dirs.mk b/mk/install-dirs.mk new file mode 100644 index 000000000..732b0d6fc --- /dev/null +++ b/mk/install-dirs.mk @@ -0,0 +1,11 @@ +# Default installation paths. +prefix ?= /usr/local +libdir ?= $(prefix)/lib +bindir ?= $(prefix)/bin +libexecdir ?= $(prefix)/libexec +datadir ?= $(prefix)/share +localstatedir ?= $(prefix)/var +sysconfdir ?= $(prefix)/etc +mandir ?= $(prefix)/share/man + +DESTDIR ?= diff --git a/mk/lib.mk b/mk/lib.mk index e86a7f1a4..3d503364f 100644 --- a/mk/lib.mk +++ b/mk/lib.mk @@ -43,27 +43,6 @@ define newline endef -# Default installation paths. -prefix ?= /usr/local -libdir ?= $(prefix)/lib -bindir ?= $(prefix)/bin -libexecdir ?= $(prefix)/libexec -datadir ?= $(prefix)/share -localstatedir ?= $(prefix)/var -sysconfdir ?= $(prefix)/etc -mandir ?= $(prefix)/share/man - - -# Initialise support for build directories. -builddir ?= - -ifdef builddir - buildprefix = $(builddir)/ -else - buildprefix = -endif - - # Pass -fPIC if we're building dynamic libraries. BUILD_SHARED_LIBS ?= 1 @@ -94,6 +73,8 @@ ifeq ($(BUILD_DEBUG), 1) endif +include mk/build-dir.mk +include mk/install-dirs.mk include mk/functions.mk include mk/tracing.mk include mk/clean.mk @@ -112,7 +93,7 @@ define include-sub-makefile include $(1) endef -$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile, $(mf)))) +$(foreach mf, $(makefiles), $(eval $(call include-sub-makefile,$(mf)))) # Instantiate stuff. @@ -122,14 +103,15 @@ $(foreach script, $(bin-scripts), $(eval $(call install-program-in,$(script),$(b $(foreach script, $(bin-scripts), $(eval programs-list += $(script))) $(foreach script, $(noinst-scripts), $(eval programs-list += $(script))) $(foreach template, $(template-files), $(eval $(call instantiate-template,$(template)))) +install_test_init=tests/functional/init.sh $(foreach test, $(install-tests), \ - $(eval $(call run-install-test,$(test))) \ + $(eval $(call run-test,$(test),$(install_test_init))) \ $(eval installcheck: $(test).test)) $(foreach test-group, $(install-tests-groups), \ - $(eval $(call run-install-test-group,$(test-group))) \ + $(eval $(call run-test-group,$(test-group),$(install_test_init))) \ $(eval installcheck: $(test-group).test-group) \ $(foreach test, $($(test-group)-tests), \ - $(eval $(call run-install-test,$(test))) \ + $(eval $(call run-test,$(test),$(install_test_init))) \ $(eval $(test-group).test-group: $(test).test))) $(foreach file, $(man-pages), $(eval $(call install-data-in, $(file), $(mandir)/man$(patsubst .%,%,$(suffix $(file)))))) diff --git a/mk/programs.mk b/mk/programs.mk index a88d9d949..6235311e9 100644 --- a/mk/programs.mk +++ b/mk/programs.mk @@ -87,6 +87,6 @@ define build-program # Phony target to run this program (typically as a dependency of 'check'). .PHONY: $(1)_RUN $(1)_RUN: $$($(1)_PATH) - $(trace-test) $$(UNIT_TEST_ENV) $$($(1)_PATH) + $(trace-test) $$($(1)_ENV) $$($(1)_PATH) endef diff --git a/mk/run-test.sh b/mk/run-test.sh index 1a1d65930..da9c5a473 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -8,6 +8,7 @@ yellow="" normal="" test=$1 +init=${2-} dir="$(dirname "${BASH_SOURCE[0]}")" source "$dir/common-test.sh" @@ -21,7 +22,9 @@ if [ -t 1 ]; then fi run_test () { - (init_test 2>/dev/null > /dev/null) + if [ -n "$init" ]; then + (init_test 2>/dev/null > /dev/null) + fi log="$(run_test_proper 2>&1)" && status=0 || status=$? } diff --git a/mk/templates.mk b/mk/templates.mk index c7ac7afbf..866bdc17f 100644 --- a/mk/templates.mk +++ b/mk/templates.mk @@ -10,10 +10,10 @@ endef ifneq ($(MAKECMDGOALS), clean) -%.h: %.h.in - $(trace-gen) rm -f $@ && ./config.status --quiet --header=$@ +$(buildprefix)%.h: %.h.in + $(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --header=$(@:$(buildprefix)%=%) -%: %.in - $(trace-gen) rm -f $@ && ./config.status --quiet --file=$@ +$(buildprefix)%: %.in + $(trace-gen) rm -f $@ && cd $(buildprefixrel) && ./config.status --quiet --file=$(@:$(buildprefix)%=%) endif diff --git a/mk/tests.mk b/mk/tests.mk index ec8128bdf..bac9b704a 100644 --- a/mk/tests.mk +++ b/mk/tests.mk @@ -2,19 +2,22 @@ test-deps = -define run-install-test +define run-bash - .PHONY: $1.test - $1.test: $1 $(test-deps) - @env BASH=$(bash) $(bash) mk/run-test.sh $1 < /dev/null - - .PHONY: $1.test-debug - $1.test-debug: $1 $(test-deps) - @env BASH=$(bash) $(bash) mk/debug-test.sh $1 < /dev/null + .PHONY: $1 + $1: $2 + @env BASH=$(bash) $(bash) $3 < /dev/null endef -define run-install-test-group +define run-test + + $(eval $(call run-bash,$1.test,$1 $(test-deps),mk/run-test.sh $1 $2)) + $(eval $(call run-bash,$1.test-debug,$1 $(test-deps),mk/debug-test.sh $1 $2)) + +endef + +define run-test-group .PHONY: $1.test-group diff --git a/perl/lib/Nix/Store.xs b/perl/lib/Nix/Store.xs index 08f812b31..82c7db608 100644 --- a/perl/lib/Nix/Store.xs +++ b/perl/lib/Nix/Store.xs @@ -9,9 +9,9 @@ #undef do_close #include "derivations.hh" +#include "realisation.hh" #include "globals.hh" #include "store-api.hh" -#include "util.hh" #include "crypto.hh" #include @@ -78,7 +78,7 @@ SV * queryReferences(char * path) SV * queryPathHash(char * path) PPCODE: try { - auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Base32, true); + auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(HashFormat::Nix32, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -104,7 +104,7 @@ SV * queryPathInfo(char * path, int base32) XPUSHs(&PL_sv_undef); else XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0))); - auto s = info->narHash.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, true); + auto s = info->narHash.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, true); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); mXPUSHi(info->registrationTime); mXPUSHi(info->narSize); @@ -205,8 +205,8 @@ void importPaths(int fd, int dontCheckSigs) SV * hashPath(char * algo, int base32, char * path) PPCODE: try { - Hash h = hashPath(parseHashType(algo), path).first; - auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); + Hash h = hashPath(parseHashAlgo(algo), path).first; + auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -216,8 +216,8 @@ SV * hashPath(char * algo, int base32, char * path) SV * hashFile(char * algo, int base32, char * path) PPCODE: try { - Hash h = hashFile(parseHashType(algo), path); - auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); + Hash h = hashFile(parseHashAlgo(algo), path); + auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -227,8 +227,8 @@ SV * hashFile(char * algo, int base32, char * path) SV * hashString(char * algo, int base32, char * s) PPCODE: try { - Hash h = hashString(parseHashType(algo), s); - auto s = h.to_string(base32 ? HashFormat::Base32 : HashFormat::Base16, false); + Hash h = hashString(parseHashAlgo(algo), s); + auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -238,8 +238,8 @@ SV * hashString(char * algo, int base32, char * s) SV * convertHash(char * algo, char * s, int toBase32) PPCODE: try { - auto h = Hash::parseAny(s, parseHashType(algo)); - auto s = h.to_string(toBase32 ? HashFormat::Base32 : HashFormat::Base16, false); + auto h = Hash::parseAny(s, parseHashAlgo(algo)); + auto s = h.to_string(toBase32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -281,7 +281,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashType(algo)); + auto path = store()->addToStore(std::string(baseNameOf(srcPath)), srcPath, method, parseHashAlgo(algo)); XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(path).c_str(), 0))); } catch (Error & e) { croak("%s", e.what()); @@ -291,7 +291,7 @@ SV * addToStore(char * srcPath, int recursive, char * algo) SV * makeFixedOutputPath(int recursive, char * algo, char * hash, char * name) PPCODE: try { - auto h = Hash::parseAny(hash, parseHashType(algo)); + auto h = Hash::parseAny(hash, parseHashAlgo(algo)); auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; auto path = store()->makeFixedOutputPath(name, FixedOutputInfo { .method = method, diff --git a/scripts/nix-profile-daemon.fish.in b/scripts/nix-profile-daemon.fish.in index 400696812..c23aa64f0 100644 --- a/scripts/nix-profile-daemon.fish.in +++ b/scripts/nix-profile-daemon.fish.in @@ -19,6 +19,14 @@ set __ETC_PROFILE_NIX_SOURCED 1 set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" +# Populate bash completions, .desktop files, etc +if test -z "$XDG_DATA_DIRS" + # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default + set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:/nix/var/nix/profiles/default/share" +else + set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:/nix/var/nix/profiles/default/share" +end + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if test -n "$NIX_SSH_CERT_FILE" : # Allow users to override the NIX_SSL_CERT_FILE diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 8cfd3149e..d256b24ed 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -30,6 +30,14 @@ fi export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" +# Populate bash completions, .desktop files, etc +if [ -z "${XDG_DATA_DIRS-}" ]; then + # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default + export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" +else + export XDG_DATA_DIRS="$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share" +fi + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -n "${NIX_SSL_CERT_FILE:-}" ]; then : # Allow users to override the NIX_SSL_CERT_FILE diff --git a/scripts/nix-profile.fish.in b/scripts/nix-profile.fish.in index 731498c76..619df52b8 100644 --- a/scripts/nix-profile.fish.in +++ b/scripts/nix-profile.fish.in @@ -20,6 +20,14 @@ if test -n "$HOME" && test -n "$USER" # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix set --export NIX_PROFILES "@localstatedir@/nix/profiles/default $HOME/.nix-profile" + # Populate bash completions, .desktop files, etc + if test -z "$XDG_DATA_DIRS" + # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default + set --export XDG_DATA_DIRS "/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + else + set --export XDG_DATA_DIRS "$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + end + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if test -n "$NIX_SSH_CERT_FILE" : # Allow users to override the NIX_SSL_CERT_FILE diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index c4d60cf37..44bc96e89 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -32,6 +32,14 @@ if [ -n "$HOME" ] && [ -n "$USER" ]; then # This part should be kept in sync with nixpkgs:nixos/modules/programs/environment.nix export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" + # Populate bash completions, .desktop files, etc + if [ -z "${XDG_DATA_DIRS-}" ]; then + # According to XDG spec the default is /usr/local/share:/usr/share, don't set something that prevents that default + export XDG_DATA_DIRS="/usr/local/share:/usr/share:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + else + export XDG_DATA_DIRS="$XDG_DATA_DIRS:$NIX_LINK/share:/nix/var/nix/profiles/default/share" + fi + # Set $NIX_SSL_CERT_FILE so that Nixpkgs applications like curl work. if [ -e /etc/ssl/certs/ca-certificates.crt ]; then # NixOS, Ubuntu, Debian, Gentoo, Arch export NIX_SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index 9a2dce806..8e2efc7c3 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -80,7 +80,7 @@ SingleDerivedPath SingleBuiltPath::discardOutputPath() const ); } -nlohmann::json BuiltPath::Built::toJSON(const Store & store) const +nlohmann::json BuiltPath::Built::toJSON(const StoreDirConfig & store) const { nlohmann::json res; res["drvPath"] = drvPath->toJSON(store); @@ -90,7 +90,7 @@ nlohmann::json BuiltPath::Built::toJSON(const Store & store) const return res; } -nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const +nlohmann::json SingleBuiltPath::Built::toJSON(const StoreDirConfig & store) const { nlohmann::json res; res["drvPath"] = drvPath->toJSON(store); @@ -100,14 +100,14 @@ nlohmann::json SingleBuiltPath::Built::toJSON(const Store & store) const return res; } -nlohmann::json SingleBuiltPath::toJSON(const Store & store) const +nlohmann::json SingleBuiltPath::toJSON(const StoreDirConfig & store) const { return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw()); } -nlohmann::json BuiltPath::toJSON(const Store & store) const +nlohmann::json BuiltPath::toJSON(const StoreDirConfig & store) const { return std::visit([&](const auto & buildable) { return buildable.toJSON(store); diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index e677bc810..99917e0ee 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "derived-path.hh" #include "realisation.hh" @@ -11,9 +14,9 @@ struct SingleBuiltPathBuilt { SingleDerivedPathBuilt discardOutputPath() const; - std::string to_string(const Store & store) const; - static SingleBuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); - nlohmann::json toJSON(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; + static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; DECLARE_CMP(SingleBuiltPathBuilt); }; @@ -38,8 +41,8 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { SingleDerivedPath discardOutputPath() const; - static SingleBuiltPath parse(const Store & store, std::string_view); - nlohmann::json toJSON(const Store & store) const; + static SingleBuiltPath parse(const StoreDirConfig & store, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; }; static inline ref staticDrv(StorePath drvPath) @@ -56,9 +59,9 @@ struct BuiltPathBuilt { ref drvPath; std::map outputs; - std::string to_string(const Store & store) const; - static BuiltPathBuilt parse(const Store & store, std::string_view, std::string_view); - nlohmann::json toJSON(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; + static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; DECLARE_CMP(BuiltPathBuilt); }; @@ -86,7 +89,7 @@ struct BuiltPath : _BuiltPathRaw { StorePathSet outPaths() const; RealisedPath::Set toRealisedPaths(Store & store) const; - nlohmann::json toJSON(const Store & store) const; + nlohmann::json toJSON(const StoreDirConfig & store) const; }; typedef std::vector BuiltPaths; diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index a88ba8134..369fa6004 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -1,4 +1,5 @@ #include "command.hh" +#include "markdown.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "derivations.hh" @@ -34,6 +35,19 @@ nlohmann::json NixMultiCommand::toJSON() return MultiCommand::toJSON(); } +void NixMultiCommand::run() +{ + if (!command) { + std::set subCommandTextLines; + for (auto & [name, _] : commands) + subCommandTextLines.insert(fmt("- `%s`", name)); + std::string markdownError = fmt("`nix %s` requires a sub-command. Available sub-commands:\n\n%s\n", + commandName, concatStringsSep("\n", subCommandTextLines)); + throw UsageError(renderMarkdownToTerminal(markdownError)); + } + command->second->run(); +} + StoreCommand::StoreCommand() { } @@ -175,7 +189,7 @@ void BuiltPathsCommand::run(ref store, Installables && installables) throw UsageError("'--all' does not expect arguments"); // XXX: Only uses opaque paths, ignores all the realisations for (auto & p : store->queryAllValidPaths()) - paths.push_back(BuiltPath::Opaque{p}); + paths.emplace_back(BuiltPath::Opaque{p}); } else { paths = Installable::toBuiltPaths(getEvalStore(), store, realiseMode, operateOn, installables); if (recursive) { @@ -188,7 +202,7 @@ void BuiltPathsCommand::run(ref store, Installables && installables) } store->computeFSClosure(pathsRoots, pathsClosure); for (auto & path : pathsClosure) - paths.push_back(BuiltPath::Opaque{path}); + paths.emplace_back(BuiltPath::Opaque{path}); } } diff --git a/src/libcmd/command.hh b/src/libcmd/command.hh index dafc0db3b..4a72627ed 100644 --- a/src/libcmd/command.hh +++ b/src/libcmd/command.hh @@ -26,9 +26,13 @@ static constexpr Command::Category catNixInstallation = 102; static constexpr auto installablesCategory = "Options that change the interpretation of [installables](@docroot@/command-ref/new-cli/nix.md#installables)"; -struct NixMultiCommand : virtual MultiCommand, virtual Command +struct NixMultiCommand : MultiCommand, virtual Command { nlohmann::json toJSON() override; + + using MultiCommand::MultiCommand; + + virtual void run() override; }; // For the overloaded run methods @@ -326,6 +330,12 @@ struct MixEnvironment : virtual Args { void setEnviron(); }; +void completeFlakeInputPath( + AddCompletions & completions, + ref evalState, + const std::vector & flakeRefs, + std::string_view prefix); + void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix); void completeFlakeRefWithFragment( diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index e53bc4c01..193972272 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -2,7 +2,6 @@ #include "common-eval-args.hh" #include "shared.hh" #include "filetransfer.hh" -#include "util.hh" #include "eval.hh" #include "fetchers.hh" #include "registry.hh" @@ -142,7 +141,7 @@ MixEvalArgs::MixEvalArgs() .longName = "eval-store", .description = R"( - The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format) to use for evaluation, i.e. to store derivations (`.drv` files) and inputs referenced by them. )", .category = category, @@ -165,7 +164,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) return res.finish(); } -SourcePath lookupFileArg(EvalState & state, std::string_view s) +SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir) { if (EvalSettings::isPseudoUrl(s)) { auto storePath = fetchers::downloadTarball( @@ -186,7 +185,7 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s) } else - return state.rootPath(CanonPath::fromCwd(s)); + return state.rootPath(CanonPath(s, baseDir)); } } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 6359b2579..4b403d936 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -2,6 +2,7 @@ ///@file #include "args.hh" +#include "canon-path.hh" #include "common-args.hh" #include "search-path.hh" @@ -28,6 +29,6 @@ private: std::map autoArgs; }; -SourcePath lookupFileArg(EvalState & state, std::string_view s); +SourcePath lookupFileArg(EvalState & state, std::string_view s, CanonPath baseDir = CanonPath::fromCwd()); } diff --git a/src/libcmd/editor-for.cc b/src/libcmd/editor-for.cc index a17c6f12a..619d3673f 100644 --- a/src/libcmd/editor-for.cc +++ b/src/libcmd/editor-for.cc @@ -1,5 +1,5 @@ -#include "util.hh" #include "editor-for.hh" +#include "environment-variables.hh" namespace nix { diff --git a/src/libcmd/installable-attr-path.hh b/src/libcmd/installable-attr-path.hh index e9f0c33da..86c2f8219 100644 --- a/src/libcmd/installable-attr-path.hh +++ b/src/libcmd/installable-attr-path.hh @@ -4,7 +4,6 @@ #include "globals.hh" #include "installable-value.hh" #include "outputs-spec.hh" -#include "util.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index eff18bbf6..6b3c82374 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -4,6 +4,7 @@ #include "installable-attr-path.hh" #include "installable-flake.hh" #include "outputs-spec.hh" +#include "users.hh" #include "util.hh" #include "command.hh" #include "attr-path.hh" @@ -28,7 +29,7 @@ namespace nix { -static void completeFlakeInputPath( +void completeFlakeInputPath( AddCompletions & completions, ref evalState, const std::vector & flakeRefs, @@ -48,9 +49,18 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "recreate-lock-file", - .description = "Recreate the flake's lock file from scratch.", + .description = R"( + Recreate the flake's lock file from scratch. + + > **DEPRECATED** + > + > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. + )", .category = category, - .handler = {&lockFlags.recreateLockFile, true} + .handler = {[&]() { + lockFlags.recreateLockFile = true; + warn("'--recreate-lock-file' is deprecated and will be removed in a future version; use 'nix flake update' instead."); + }} }); addFlag({ @@ -69,8 +79,13 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "no-registries", - .description = - "Don't allow lookups in the flake registries. This option is deprecated; use `--no-use-registries`.", + .description = R"( + Don't allow lookups in the flake registries. + + > **DEPRECATED** + > + > Use [`--no-use-registries`](#opt-no-use-registries) instead. + )", .category = category, .handler = {[&]() { lockFlags.useRegistries = false; @@ -87,10 +102,17 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "update-input", - .description = "Update a specific flake input (ignoring its previous entry in the lock file).", + .description = R"( + Update a specific flake input (ignoring its previous entry in the lock file). + + > **DEPRECATED** + > + > Use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) instead. + )", .category = category, .labels = {"input-path"}, .handler = {[&](std::string s) { + warn("'--update-input' is a deprecated alias for 'flake update' and will be removed in a future version."); lockFlags.inputUpdates.insert(flake::parseInputPath(s)); }}, .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { @@ -107,7 +129,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath("."), true)); + parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); }}, .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { if (n == 0) { @@ -149,7 +171,7 @@ MixFlakeOptions::MixFlakeOptions() auto evalState = getEvalState(); auto flake = flake::lockFlake( *evalState, - parseFlakeRef(flakeRef, absPath(".")), + parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes @@ -238,9 +260,10 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s evalSettings.pureEval = false; auto state = getEvalState(); - Expr *e = state->parseExprFromFile( - resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file))) - ); + auto e = + state->parseExprFromFile( + resolveExprPath( + lookupFileArg(*state, *file))); Value root; state->eval(e, root); @@ -313,6 +336,8 @@ void completeFlakeRefWithFragment( prefixRoot = "."; } auto flakeRefS = std::string(prefix.substr(0, hash)); + + // TODO: ideally this would use the command base directory instead of assuming ".". auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); auto evalCache = openEvalCache(*evalState, @@ -461,10 +486,12 @@ Installables SourceExprCommand::parseInstallables( auto e = state->parseStdin(); state->eval(e, *vFile); } - else if (file) - state->evalFile(lookupFileArg(*state, *file), *vFile); + else if (file) { + state->evalFile(lookupFileArg(*state, *file, CanonPath::fromCwd(getCommandBaseDir())), *vFile); + } else { - auto e = state->parseExprFromString(*expr, state->rootPath(CanonPath::fromCwd())); + CanonPath dir(CanonPath::fromCwd(getCommandBaseDir())); + auto e = state->parseExprFromString(*expr, state->rootPath(dir)); state->eval(e, *vFile); } @@ -499,7 +526,7 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(".")); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), @@ -683,7 +710,7 @@ BuiltPaths Installable::toBuiltPaths( BuiltPaths res; for (auto & drvPath : Installable::toDerivations(store, installables, true)) - res.push_back(BuiltPath::Opaque{drvPath}); + res.emplace_back(BuiltPath::Opaque{drvPath}); return res; } } @@ -773,7 +800,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() for (auto i : rawInstallables) res.push_back(parseFlakeRefWithFragment( expandTilde(i), - absPath(".")).first); + absPath(getCommandBaseDir())).first); return res; } @@ -795,7 +822,7 @@ std::vector InstallableCommand::getFlakeRefsForCompletion() return { parseFlakeRefWithFragment( expandTilde(_installable), - absPath(".")).first + absPath(getCommandBaseDir())).first }; } diff --git a/src/libcmd/installables.hh b/src/libcmd/installables.hh index b0dc0dc02..e087f935c 100644 --- a/src/libcmd/installables.hh +++ b/src/libcmd/installables.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "util.hh" #include "path.hh" #include "outputs-spec.hh" #include "derived-path.hh" diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index 541a7d2ba..afd35af08 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -12,4 +12,4 @@ libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) -pthread libcmd_LIBS = libstore libutil libexpr libmain libfetchers -$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 668a07763..8b3bbc1b5 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,6 +1,7 @@ #include "markdown.hh" #include "util.hh" #include "finally.hh" +#include "terminal.hh" #include #include diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 2e17a29a7..0986296ad 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -22,6 +22,7 @@ extern "C" { #include "repl.hh" #include "ansicolor.hh" +#include "signals.hh" #include "shared.hh" #include "eval.hh" #include "eval-cache.hh" @@ -36,11 +37,12 @@ extern "C" { #include "globals.hh" #include "flake/flake.hh" #include "flake/lockfile.hh" +#include "users.hh" +#include "terminal.hh" #include "editor-for.hh" #include "finally.hh" #include "markdown.hh" #include "local-fs-store.hh" -#include "progress-bar.hh" #include "print.hh" #if HAVE_BOEHMGC @@ -259,13 +261,11 @@ void NixRepl::mainLoop() rl_set_list_possib_func(listPossibleCallback); #endif - /* Stop the progress bar because it interferes with the display of - the repl. */ - stopProgressBar(); - std::string input; while (true) { + // Hide the progress bar while waiting for user input, so that it won't interfere. + logger->pause(); // When continuing input from previous lines, don't print a prompt, just align to the same // number of chars as the prompt. if (!getLine(input, input.empty() ? "nix-repl> " : " ")) { @@ -275,6 +275,7 @@ void NixRepl::mainLoop() logger->cout(""); break; } + logger->resume(); try { if (!removeWhitespace(input).empty() && !processLine(input)) return; } catch (ParseError & e) { diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index d12345710..7481a2232 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -1,6 +1,5 @@ #include "attr-path.hh" #include "eval-inline.hh" -#include "util.hh" namespace nix { diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 10fc799a9..5808d58b6 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -1,3 +1,4 @@ +#include "users.hh" #include "eval-cache.hh" #include "sqlite.hh" #include "eval.hh" @@ -21,7 +22,7 @@ struct AttrDb { std::atomic_bool failed{false}; - const Store & cfg; + const StoreDirConfig & cfg; struct State { @@ -38,7 +39,7 @@ struct AttrDb SymbolTable & symbols; AttrDb( - const Store & cfg, + const StoreDirConfig & cfg, const Hash & fingerprint, SymbolTable & symbols) : cfg(cfg) @@ -322,7 +323,7 @@ struct AttrDb }; static std::shared_ptr makeAttrDb( - const Store & cfg, + const StoreDirConfig & cfg, const Hash & fingerprint, SymbolTable & symbols) { diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index a988fa40c..c37b1d62b 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -103,8 +103,10 @@ void EvalState::forceValue(Value & v, Callable getPos) throw; } } - else if (v.isApp()) - callFunction(*v.app.left, *v.app.right, v, noPos); + else if (v.isApp()) { + PosIdx pos = getPos(); + callFunction(*v.app.left, *v.app.right, v, pos); + } else if (v.isBlackhole()) error("infinite recursion encountered").atPos(getPos()).template debugThrow(); } @@ -121,9 +123,9 @@ template [[gnu::always_inline]] inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view errorCtx) { - forceValue(v, noPos); + PosIdx pos = getPos(); + forceValue(v, pos); if (v.type() != nAttrs) { - PosIdx pos = getPos(); error("value is %1% while a set was expected", showType(v)).withTrace(pos, errorCtx).debugThrow(); } } @@ -132,7 +134,7 @@ inline void EvalState::forceAttrs(Value & v, Callable getPos, std::string_view e [[gnu::always_inline]] inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view errorCtx) { - forceValue(v, noPos); + forceValue(v, pos); if (!v.isList()) { error("value is %1% while a list was expected", showType(v)).withTrace(pos, errorCtx).debugThrow(); } diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 93b4a5289..444a7d7d6 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -1,3 +1,4 @@ +#include "users.hh" #include "globals.hh" #include "profiles.hh" #include "eval.hh" diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 5473d688e..db2971acb 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -1,4 +1,6 @@ #pragma once +///@file + #include "config.hh" namespace nix { diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d26cde423..9e494148e 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1,6 +1,7 @@ #include "eval.hh" #include "eval-settings.hh" #include "hash.hh" +#include "primops.hh" #include "types.hh" #include "util.hh" #include "store-api.hh" @@ -13,7 +14,10 @@ #include "profiles.hh" #include "print.hh" #include "fs-input-accessor.hh" +#include "filtering-input-accessor.hh" #include "memory-input-accessor.hh" +#include "signals.hh" +#include "gc-small-vector.hh" #include #include @@ -29,6 +33,7 @@ #include #include +#include #if HAVE_BOEHMGC @@ -340,7 +345,7 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) } else { Value nameValue; name.expr->eval(state, env, nameValue); - state.forceStringNoCtx(nameValue, noPos, "while evaluating an attribute name"); + state.forceStringNoCtx(nameValue, name.expr->getPos(), "while evaluating an attribute name"); return state.symbols.create(nameValue.string_view()); } } @@ -505,7 +510,16 @@ EvalState::EvalState( , sOutputSpecified(symbols.create("outputSpecified")) , repair(NoRepair) , emptyBindings(0) - , rootFS(makeFSInputAccessor(CanonPath::root)) + , rootFS( + evalSettings.restrictEval || evalSettings.pureEval + ? ref(AllowListInputAccessor::create(makeFSInputAccessor(CanonPath::root), {}, + [](const CanonPath & path) -> RestrictedPathError { + auto modeInformation = evalSettings.pureEval + ? "in pure evaluation mode (use '--impure' to override)" + : "in restricted mode"; + throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); + })) + : makeFSInputAccessor(CanonPath::root)) , corepkgsFS(makeMemoryInputAccessor()) , internalFS(makeMemoryInputAccessor()) , derivationInternal{corepkgsFS->addFile( @@ -530,6 +544,9 @@ EvalState::EvalState( , baseEnv(allocEnv(128)) , staticBaseEnv{std::make_shared(false, nullptr)} { + corepkgsFS->setPathDisplay(""); + internalFS->setPathDisplay("«nix-internal»", ""); + countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; assert(gcInitialised); @@ -544,28 +561,10 @@ EvalState::EvalState( searchPath.elements.emplace_back(SearchPath::Elem::parse(i)); } - if (evalSettings.restrictEval || evalSettings.pureEval) { - allowedPaths = PathSet(); - - for (auto & i : searchPath.elements) { - auto r = resolveSearchPathPath(i.path); - if (!r) continue; - - auto path = std::move(*r); - - if (store->isInStore(path)) { - try { - StorePathSet closure; - store->computeFSClosure(store->toStorePath(path).first, closure); - for (auto & path : closure) - allowPath(path); - } catch (InvalidPath &) { - allowPath(path); - } - } else - allowPath(path); - } - } + /* Allow access to all paths in the search path. */ + if (rootFS.dynamic_pointer_cast()) + for (auto & i : searchPath.elements) + resolveSearchPathPath(i.path, true); corepkgsFS->addFile( CanonPath("fetchurl.nix"), @@ -583,14 +582,14 @@ EvalState::~EvalState() void EvalState::allowPath(const Path & path) { - if (allowedPaths) - allowedPaths->insert(path); + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->allowPath(CanonPath(path)); } void EvalState::allowPath(const StorePath & storePath) { - if (allowedPaths) - allowedPaths->insert(store->toRealPath(storePath)); + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->allowPath(CanonPath(store->toRealPath(storePath))); } void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & v) @@ -600,54 +599,6 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & mkStorePathString(storePath, v); } -SourcePath EvalState::checkSourcePath(const SourcePath & path_) -{ - // Don't check non-rootFS accessors, they're in a different namespace. - if (path_.accessor != ref(rootFS)) return path_; - - if (!allowedPaths) return path_; - - auto i = resolvedPaths.find(path_.path.abs()); - if (i != resolvedPaths.end()) - return i->second; - - bool found = false; - - /* First canonicalize the path without symlinks, so we make sure an - * attacker can't append ../../... to a path that would be in allowedPaths - * and thus leak symlink targets. - */ - Path abspath = canonPath(path_.path.abs()); - - for (auto & i : *allowedPaths) { - if (isDirOrInDir(abspath, i)) { - found = true; - break; - } - } - - if (!found) { - auto modeInformation = evalSettings.pureEval - ? "in pure eval mode (use '--impure' to override)" - : "in restricted mode"; - throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", abspath, modeInformation); - } - - /* Resolve symlinks. */ - debug("checking access to '%s'", abspath); - SourcePath path = rootPath(CanonPath(canonPath(abspath, true))); - - for (auto & i : *allowedPaths) { - if (isDirOrInDir(path.path.abs(), i)) { - resolvedPaths.insert_or_assign(path_.path.abs(), path); - return path; - } - } - - throw RestrictedPathError("access to canonical path '%1%' is forbidden in restricted mode", path); -} - - void EvalState::checkURI(const std::string & uri) { if (!evalSettings.restrictEval) return; @@ -667,12 +618,14 @@ void EvalState::checkURI(const std::string & uri) /* If the URI is a path, then check it against allowedPaths as well. */ if (hasPrefix(uri, "/")) { - checkSourcePath(rootPath(CanonPath(uri))); + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->checkAccess(CanonPath(uri)); return; } if (hasPrefix(uri, "file://")) { - checkSourcePath(rootPath(CanonPath(std::string(uri, 7)))); + if (auto rootFS2 = rootFS.dynamic_pointer_cast()) + rootFS2->checkAccess(CanonPath(uri.substr(7))); return; } @@ -721,6 +674,23 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info) } +void PrimOp::check() +{ + if (arity > maxPrimOpArity) { + throw Error("primop arity must not exceed %1%", maxPrimOpArity); + } +} + + +void Value::mkPrimOp(PrimOp * p) +{ + p->check(); + clearValue(); + internalType = tPrimOp; + primOp = p; +} + + Value * EvalState::addPrimOp(PrimOp && primOp) { /* Hack to make constants lazy: turn them into a application of @@ -1157,10 +1127,8 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env) } -void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial) +void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) { - auto path = checkSourcePath(path_); - FileEvalCache::iterator i; if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) { v = i->second; @@ -1181,7 +1149,7 @@ void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial e = j->second; if (!e) - e = parseExprFromFile(checkSourcePath(resolvedPath)); + e = parseExprFromFile(resolvedPath); fileParseCache[resolvedPath] = e; @@ -1490,7 +1458,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) e->eval(state, env, vTmp); for (auto & i : attrPath) { - state.forceValue(*vAttrs, noPos); + state.forceValue(*vAttrs, getPos()); Bindings::iterator j; auto name = getName(i, state, env); if (vAttrs->type() != nAttrs || @@ -1659,7 +1627,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & if (countCalls) primOpCalls[name]++; try { - vCur.primOp->fun(*this, noPos, args, vCur); + vCur.primOp->fun(*this, vCur.determinePos(noPos), args, vCur); } catch (Error & e) { addErrorTrace(e, pos, "while calling the '%1%' builtin", name); throw; @@ -1690,7 +1658,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & /* We have all the arguments, so call the primop with the previous and new arguments. */ - Value * vArgs[arity]; + Value * vArgs[maxPrimOpArity]; auto n = argsDone; for (Value * arg = &vCur; arg->isPrimOpApp(); arg = arg->primOpApp.left) vArgs[--n] = arg->primOpApp.right; @@ -1707,7 +1675,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & // 1. Unify this and above code. Heavily redundant. // 2. Create a fake env (arg1, arg2, etc.) and a fake expr (arg1: arg2: etc: builtins.name arg1 arg2 etc) // so the debugger allows to inspect the wrong parameters passed to the builtin. - primOp->primOp->fun(*this, noPos, vArgs, vCur); + primOp->primOp->fun(*this, vCur.determinePos(noPos), vArgs, vCur); } catch (Error & e) { addErrorTrace(e, pos, "while calling the '%1%' builtin", name); throw; @@ -1747,11 +1715,17 @@ void ExprCall::eval(EvalState & state, Env & env, Value & v) Value vFun; fun->eval(state, env, vFun); - Value * vArgs[args.size()]; + // Empirical arity of Nixpkgs lambdas by regex e.g. ([a-zA-Z]+:(\s|(/\*.*\/)|(#.*\n))*){5} + // 2: over 4000 + // 3: about 300 + // 4: about 60 + // 5: under 10 + // This excluded attrset lambdas (`{...}:`). Contributions of mixed lambdas appears insignificant at ~150 total. + SmallValueVector<4> vArgs(args.size()); for (size_t i = 0; i < args.size(); ++i) vArgs[i] = args[i]->maybeThunk(state, env); - state.callFunction(vFun, args.size(), vArgs, v, pos); + state.callFunction(vFun, args.size(), vArgs.data(), v, pos); } @@ -1809,7 +1783,7 @@ https://nixos.org/manual/nix/stable/language/constructs.html#functions.)", symbo } } - callFunction(fun, allocValue()->mkAttrs(attrs), res, noPos); + callFunction(fun, allocValue()->mkAttrs(attrs), res, pos); } @@ -1845,7 +1819,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) void ExprOpNot::eval(EvalState & state, Env & env, Value & v) { - v.mkBool(!state.evalBool(env, e, noPos, "in the argument of the not operator")); // XXX: FIXME: ! + v.mkBool(!state.evalBool(env, e, getPos(), "in the argument of the not operator")); // XXX: FIXME: ! } @@ -1990,8 +1964,9 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) return result; }; - Value values[es->size()]; - Value * vTmpP = values; + // List of returned strings. References to these Values must NOT be persisted. + SmallTemporaryValueVector values(es->size()); + Value * vTmpP = values.data(); for (auto & [i_pos, i] : *es) { Value & vTmp = *vTmpP++; @@ -2285,7 +2260,7 @@ BackedStringView EvalState::coerceToString( std::string result; for (auto [n, v2] : enumerate(v.listItems())) { try { - result += *coerceToString(noPos, *v2, context, + result += *coerceToString(pos, *v2, context, "while evaluating one element of the list", coerceMore, copyToStore, canonicalizePath); } catch (Error & e) { @@ -2432,8 +2407,8 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) { - forceValue(v1, noPos); - forceValue(v2, noPos); + forceValue(v1, pos); + forceValue(v2, pos); /* !!! Hack to support some old broken code that relies on pointer equality tests between sets. (Specifically, builderDefs calls diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 048dff42b..f3f6d35b9 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -18,13 +18,18 @@ namespace nix { +/** + * We put a limit on primop arity because it lets us use a fixed size array on + * the stack. 8 is already an impractical number of arguments. Use an attrset + * argument for such overly complicated functions. + */ +constexpr size_t maxPrimOpArity = 8; class Store; class EvalState; class StorePath; struct SingleDerivedPath; enum RepairFlag : bool; -struct FSInputAccessor; struct MemoryInputAccessor; @@ -71,6 +76,12 @@ struct PrimOp * Optional experimental for this to be gated on. */ std::optional experimentalFeature; + + /** + * Validity check to be performed by functions that introduce primops, + * such as RegisterPrimOp() and Value::mkPrimOp(). + */ + void check(); }; /** @@ -205,18 +216,12 @@ public: */ RepairFlag repair; - /** - * The allowed filesystem paths in restricted or pure evaluation - * mode. - */ - std::optional allowedPaths; - Bindings emptyBindings; /** * The accessor for the root filesystem. */ - const ref rootFS; + const ref rootFS; /** * The in-memory filesystem for paths. @@ -384,12 +389,6 @@ public: */ void allowAndSetStorePathString(const StorePath & storePath, Value & v); - /** - * Check whether access to a path is allowed and throw an error if - * not. Otherwise return the canonicalised path. - */ - SourcePath checkSourcePath(const SourcePath & path); - void checkURI(const std::string & uri); /** @@ -433,13 +432,15 @@ public: SourcePath findFile(const SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); /** - * Try to resolve a search path value (not the optional key part) + * Try to resolve a search path value (not the optional key part). * * If the specified search path element is a URI, download it. * * If it is not found, return `std::nullopt` */ - std::optional resolveSearchPathPath(const SearchPath::Path & path); + std::optional resolveSearchPathPath( + const SearchPath::Path & elem, + bool initAccessControl = false); /** * Evaluate an expression to normal form @@ -744,6 +745,13 @@ public: */ [[nodiscard]] StringMap realiseContext(const NixStringContext & context); + /* Call the binary path filter predicate used builtins.path etc. */ + bool callPathFilter( + Value * filterFun, + const SourcePath & path, + std::string_view pathArg, + PosIdx pos); + private: /** @@ -827,7 +835,7 @@ std::string showType(const Value & v); /** * If `path` refers to a directory, then append "/default.nix". */ -SourcePath resolveExprPath(const SourcePath & path); +SourcePath resolveExprPath(SourcePath path); struct InvalidPathError : EvalError { diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc index e89014862..3c7ed5d8a 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libexpr/flake/config.cc @@ -1,6 +1,7 @@ -#include "flake.hh" +#include "users.hh" #include "globals.hh" #include "fetch-settings.hh" +#include "flake.hh" #include diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index be2cf014c..fee58792b 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -1,3 +1,4 @@ +#include "terminal.hh" #include "flake.hh" #include "eval.hh" #include "eval-settings.hh" @@ -8,6 +9,7 @@ #include "fetchers.hh" #include "finally.hh" #include "fetch-settings.hh" +#include "value-to-json.hh" namespace nix { @@ -140,8 +142,13 @@ static FlakeInput parseFlakeInput(EvalState & state, attrs.emplace(state.symbols[attr.name], (long unsigned int)attr.value->integer); break; default: - throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", - state.symbols[attr.name], showType(*attr.value)); + if (attr.name == state.symbols.create("publicKeys")) { + experimentalFeatureSettings.require(Xp::VerifiedFetches); + NixStringContext emptyContext = {}; + attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, emptyContext).dump()); + } else + throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", + state.symbols[attr.name], showType(*attr.value)); } #pragma GCC diagnostic pop } @@ -205,8 +212,16 @@ static Flake getFlake( auto [storePath, resolvedRef, lockedRef] = fetchOrSubstituteTree( state, originalRef, allowLookup, flakeCache); + // We need to guard against symlink attacks, but before we start doing + // filesystem operations we should make sure there's a flake.nix in the + // first place. + auto unsafeFlakeDir = state.store->toRealPath(storePath) + "/" + lockedRef.subdir; + auto unsafeFlakeFile = unsafeFlakeDir + "/flake.nix"; + if (!pathExists(unsafeFlakeFile)) + throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); + // Guard against symlink attacks. - auto flakeDir = canonPath(state.store->toRealPath(storePath) + "/" + lockedRef.subdir, true); + auto flakeDir = canonPath(unsafeFlakeDir, true); auto flakeFile = canonPath(flakeDir + "/flake.nix", true); if (!isInDir(flakeFile, state.store->toRealPath(storePath))) throw Error("'flake.nix' file of flake '%s' escapes from '%s'", @@ -219,9 +234,6 @@ static Flake getFlake( .storePath = storePath, }; - if (!pathExists(flakeFile)) - throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); - Value vInfo; state.evalFile(state.rootPath(CanonPath(flakeFile)), vInfo, true); // FIXME: symlink attack @@ -351,10 +363,13 @@ LockedFlake lockFlake( debug("old lock file: %s", oldLockFile); std::map overrides; + std::set explicitCliOverrides; std::set overridesUsed, updatesUsed; - for (auto & i : lockFlags.inputOverrides) + for (auto & i : lockFlags.inputOverrides) { overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); + explicitCliOverrides.insert(i.first); + } LockFile newLockFile; @@ -425,6 +440,7 @@ LockedFlake lockFlake( ancestors? */ auto i = overrides.find(inputPath); bool hasOverride = i != overrides.end(); + bool hasCliOverride = explicitCliOverrides.contains(inputPath); if (hasOverride) { overridesUsed.insert(inputPath); // Respect the “flakeness” of the input even if we @@ -447,8 +463,8 @@ LockedFlake lockFlake( assert(input.ref); - /* Do we have an entry in the existing lock file? And we - don't have a --update-input flag for this input? */ + /* Do we have an entry in the existing lock file? + And the input is not in updateInputs? */ std::shared_ptr oldLock; updatesUsed.insert(inputPath); @@ -460,7 +476,7 @@ LockedFlake lockFlake( if (oldLock && oldLock->originalRef == *input.ref - && !hasOverride) + && !hasCliOverride) { debug("keeping existing input '%s'", inputPathS); @@ -472,9 +488,8 @@ LockedFlake lockFlake( node->inputs.insert_or_assign(id, childNode); - /* If we have an --update-input flag for an input - of this input, then we must fetch the flake to - update it. */ + /* If we have this input in updateInputs, then we + must fetch the flake to update it. */ auto lb = lockFlags.inputUpdates.lower_bound(inputPath); auto mustRefetch = @@ -541,7 +556,7 @@ LockedFlake lockFlake( nuked the next time we update the lock file. That is, overrides are sticky unless you use --no-write-lock-file. */ - auto ref = input2.ref ? *input2.ref : *input.ref; + auto ref = (input2.ref && explicitCliOverrides.contains(inputPath)) ? *input2.ref : *input.ref; if (input.isFlake) { Path localPath = parentPath; @@ -616,19 +631,14 @@ LockedFlake lockFlake( for (auto & i : lockFlags.inputUpdates) if (!updatesUsed.count(i)) - warn("the flag '--update-input %s' does not match any input", printInputPath(i)); + warn("'%s' does not match any input of this flake", printInputPath(i)); /* Check 'follows' inputs. */ newLockFile.check(); debug("new lock file: %s", newLockFile); - auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; auto sourcePath = topRef.input.getSourcePath(); - auto outputLockFilePath = sourcePath ? std::optional{*sourcePath + "/" + relPath} : std::nullopt; - if (lockFlags.outputLockFilePath) { - outputLockFilePath = lockFlags.outputLockFilePath; - } /* Check whether we need to / can write the new lock file. */ if (newLockFile != oldLockFile || lockFlags.outputLockFilePath) { @@ -636,7 +646,7 @@ LockedFlake lockFlake( auto diff = LockFile::diff(oldLockFile, newLockFile); if (lockFlags.writeLockFile) { - if (outputLockFilePath) { + if (sourcePath || lockFlags.outputLockFilePath) { if (auto unlockedInput = newLockFile.isUnlocked()) { if (fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); @@ -644,41 +654,48 @@ LockedFlake lockFlake( if (!lockFlags.updateLockFile) throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); - bool lockFileExists = pathExists(*outputLockFilePath); + auto newLockFileS = fmt("%s\n", newLockFile); + + if (lockFlags.outputLockFilePath) { + if (lockFlags.commitLockFile) + throw Error("'--commit-lock-file' and '--output-lock-file' are incompatible"); + writeFile(*lockFlags.outputLockFilePath, newLockFileS); + } else { + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + auto outputLockFilePath = *sourcePath + "/" + relPath; + + bool lockFileExists = pathExists(outputLockFilePath); - if (lockFileExists) { auto s = chomp(diff); - if (s.empty()) - warn("updating lock file '%s'", *outputLockFilePath); - else - warn("updating lock file '%s':\n%s", *outputLockFilePath, s); - } else - warn("creating lock file '%s'", *outputLockFilePath); + if (lockFileExists) { + if (s.empty()) + warn("updating lock file '%s'", outputLockFilePath); + else + warn("updating lock file '%s':\n%s", outputLockFilePath, s); + } else + warn("creating lock file '%s': \n%s", outputLockFilePath, s); - newLockFile.write(*outputLockFilePath); + std::optional commitMessage = std::nullopt; - std::optional commitMessage = std::nullopt; - if (lockFlags.commitLockFile) { - if (lockFlags.outputLockFilePath) { - throw Error("--commit-lock-file and --output-lock-file are currently incompatible"); - } - std::string cm; + if (lockFlags.commitLockFile) { + std::string cm; - cm = fetchSettings.commitLockFileSummary.get(); + cm = fetchSettings.commitLockFileSummary.get(); - if (cm == "") { - cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); + if (cm == "") { + cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); + } + + cm += "\n\nFlake lock file updates:\n\n"; + cm += filterANSIEscapes(diff, true); + commitMessage = cm; } - cm += "\n\nFlake lock file updates:\n\n"; - cm += filterANSIEscapes(diff, true); - commitMessage = cm; + topRef.input.putFile( + CanonPath((topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"), + newLockFileS, commitMessage); } - topRef.input.markChangedFile( - (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", - commitMessage); - /* Rewriting the lockfile changed the top-level repo, so we should re-read it. FIXME: we could also just clear the 'rev' field... */ @@ -887,7 +904,7 @@ Fingerprint LockedFlake::getFingerprint() const // FIXME: as an optimization, if the flake contains a lock file // and we haven't changed it, then it's sufficient to use // flake.sourceInfo.storePath for the fingerprint. - return hashString(htSHA256, + return hashString(HashAlgorithm::SHA256, fmt("%s;%s;%d;%d;%s", flake.storePath.to_string(), flake.lockedRef.subdir, diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc index f3ea9063f..3e99fb2d4 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libexpr/flake/lockfile.cc @@ -214,12 +214,6 @@ std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) return stream; } -void LockFile::write(const Path & path) const -{ - createDirs(dirOf(path)); - writeFile(path, fmt("%s\n", *this)); -} - std::optional LockFile::isUnlocked() const { std::set> nodes; diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh index ba4c0c848..5a1493404 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libexpr/flake/lockfile.hh @@ -65,8 +65,6 @@ struct LockFile static LockFile read(const Path & path); - void write(const Path & path) const; - /** * Check whether this lock file has any unlocked inputs. */ diff --git a/src/libexpr/gc-small-vector.hh b/src/libexpr/gc-small-vector.hh new file mode 100644 index 000000000..7f4f08fc7 --- /dev/null +++ b/src/libexpr/gc-small-vector.hh @@ -0,0 +1,42 @@ +#pragma once + +#include + +#if HAVE_BOEHMGC + +#include +#include +#include + +#endif + +namespace nix { + +struct Value; + +/** + * A GC compatible vector that may used a reserved portion of `nItems` on the stack instead of allocating on the heap. + */ +#if HAVE_BOEHMGC +template +using SmallVector = boost::container::small_vector>; +#else +template +using SmallVector = boost::container::small_vector; +#endif + +/** + * A vector of value pointers. See `SmallVector`. + */ +template +using SmallValueVector = SmallVector; + +/** + * A vector of values that must not be referenced after the vector is destroyed. + * + * See also `SmallValueVector`. + */ +template +using SmallTemporaryValueVector = SmallVector; + +} \ No newline at end of file diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index fe3e6f7ee..d4e946d81 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -1,5 +1,4 @@ #include "get-drvs.hh" -#include "util.hh" #include "eval-inline.hh" #include "derivations.hh" #include "store-api.hh" diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index d243b9cec..ed6bc761a 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -36,15 +36,15 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh -$(eval $(call install-file-in, $(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-expr.pc, $(libdir)/pkgconfig, 0644)) $(foreach i, $(wildcard src/libexpr/value/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644))) $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) -$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh +$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh -$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh +$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh -src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = +$(buildprefix)src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index 10099d49e..020286815 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -405,6 +405,7 @@ struct ExprOpNot : Expr { Expr * e; ExprOpNot(Expr * e) : e(e) { }; + PosIdx getPos() const override { return e->getPos(); } COMMON_METHODS }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 607795937..16ad8af2e 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -19,6 +19,7 @@ #include #include "util.hh" +#include "users.hh" #include "nixexpr.hh" #include "eval.hh" @@ -685,17 +686,26 @@ Expr * EvalState::parse( } -SourcePath resolveExprPath(const SourcePath & path) +SourcePath resolveExprPath(SourcePath path) { + unsigned int followCount = 0, maxFollow = 1024; + /* If `path' is a symlink, follow it. This is so that relative path references work. */ - auto path2 = path.resolveSymlinks(); + while (!path.path.isRoot()) { + // Basic cycle/depth limit to avoid infinite loops. + if (++followCount >= maxFollow) + throw Error("too many symbolic links encountered while traversing the path '%s'", path); + auto p = path.parent().resolveSymlinks() + path.baseName(); + if (p.lstat().type != InputAccessor::tSymlink) break; + path = {path.accessor, CanonPath(p.readLink(), path.path.parent().value_or(CanonPath::root))}; + } /* If `path' refers to a directory, append `/default.nix'. */ - if (path2.lstat().type == InputAccessor::tDirectory) - return path2 + "default.nix"; + if (path.resolveSymlinks().lstat().type == InputAccessor::tDirectory) + return path + "default.nix"; - return path2; + return path; } @@ -707,7 +717,7 @@ Expr * EvalState::parseExprFromFile(const SourcePath & path) Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr & staticEnv) { - auto buffer = path.readFile(); + auto buffer = path.resolveSymlinks().readFile(); // readFile hopefully have left some extra space for terminators buffer.append("\0\0", 2); return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv); @@ -774,7 +784,7 @@ SourcePath EvalState::findFile(const SearchPath & searchPath, const std::string_ } -std::optional EvalState::resolveSearchPathPath(const SearchPath::Path & value0) +std::optional EvalState::resolveSearchPathPath(const SearchPath::Path & value0, bool initAccessControl) { auto & value = value0.s; auto i = searchPathResolved.find(value); @@ -791,7 +801,6 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa logWarning({ .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) }); - res = std::nullopt; } } @@ -805,6 +814,20 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa else { auto path = absPath(value); + + /* Allow access to paths in the search path. */ + if (initAccessControl) { + allowPath(path); + if (store->isInStore(path)) { + try { + StorePathSet closure; + store->computeFSClosure(store->toStorePath(path).first, closure); + for (auto & p : closure) + allowPath(p); + } catch (InvalidPath &) { } + } + } + if (pathExists(path)) res = { path }; else { @@ -820,7 +843,7 @@ std::optional EvalState::resolveSearchPathPath(const SearchPath::Pa else debug("failed to resolve search path element '%s'", value); - searchPathResolved[value] = res; + searchPathResolved.emplace(value, res); return res; } diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 704e7007b..89d5492da 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -4,15 +4,18 @@ #include "eval-inline.hh" #include "eval.hh" #include "eval-settings.hh" +#include "gc-small-vector.hh" #include "globals.hh" #include "json-to-value.hh" #include "names.hh" #include "path-references.hh" #include "store-api.hh" #include "util.hh" +#include "processes.hh" #include "value-to-json.hh" #include "value-to-xml.hh" #include "primops.hh" +#include "fs-input-accessor.hh" #include #include @@ -28,7 +31,6 @@ #include - namespace nix { @@ -89,9 +91,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context) for (auto & [outputName, outputPath] : outputs) { /* Add the output of this derivations to the allowed paths. */ - if (allowedPaths) { - allowPath(outputPath); - } + allowPath(store->toRealPath(outputPath)); + /* Get all the output paths corresponding to the placeholders we had */ if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { res.insert_or_assign( @@ -109,27 +110,19 @@ StringMap EvalState::realiseContext(const NixStringContext & context) return res; } -struct RealisePathFlags { - // Whether to check that the path is allowed in pure eval mode - bool checkForPureEval = true; -}; - -static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, const RealisePathFlags flags = {}) +static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, bool resolveSymlinks = true) { NixStringContext context; auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path"); try { - if (!context.empty()) { + if (!context.empty() && path.accessor == state.rootFS) { auto rewrites = state.realiseContext(context); auto realPath = state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context); - return {path.accessor, CanonPath(realPath)}; + path = {path.accessor, CanonPath(realPath)}; } - - return flags.checkForPureEval - ? state.checkSourcePath(path) - : path; + return resolveSymlinks ? path.resolveSymlinks() : path; } catch (Error & e) { e.addTrace(state.positions[pos], "while realising the context of path '%s'", path); throw; @@ -169,7 +162,7 @@ static void mkOutputString( argument. */ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v) { - auto path = realisePath(state, pos, vPath); + auto path = realisePath(state, pos, vPath, false); auto path2 = path.path.abs(); // FIXME @@ -824,7 +817,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * auto message = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.addErrorContext", false, false).toOwned(); - e.addTrace(nullptr, message, true); + e.addTrace(nullptr, hintfmt(message), true); throw; } } @@ -1316,7 +1309,7 @@ drvName, Bindings * attrs, Value & v) .errPos = state.positions[noPos] })); - auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); + auto h = newHashAllowEmpty(*outputHash, parseHashAlgoOpt(outputHashAlgo)); auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); @@ -1338,7 +1331,7 @@ drvName, Bindings * attrs, Value & v) .errPos = state.positions[noPos] }); - auto ht = parseHashTypeOpt(outputHashAlgo).value_or(htSHA256); + auto ha = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256); auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); for (auto & i : outputs) { @@ -1347,13 +1340,13 @@ drvName, Bindings * attrs, Value & v) drv.outputs.insert_or_assign(i, DerivationOutput::Impure { .method = method, - .hashType = ht, + .hashAlgo = ha, }); else drv.outputs.insert_or_assign(i, DerivationOutput::CAFloating { .method = method, - .hashType = ht, + .hashAlgo = ha, }); } } @@ -1492,7 +1485,7 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, })); NixStringContext context; - auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'")).path; + auto path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to 'builtins.storePath'").path; /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ @@ -1532,31 +1525,19 @@ static RegisterPrimOp primop_storePath({ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto & arg = *args[0]; - - /* We don’t check the path right now, because we don’t want to - throw if the path isn’t allowed, but just return false (and we - can’t just catch the exception here because we still want to - throw if something in the evaluation of `arg` tries to - access an unauthorized path). */ - auto path = realisePath(state, pos, arg, { .checkForPureEval = false }); - - /* SourcePath doesn't know about trailing slash. */ - auto mustBeDir = arg.type() == nString - && (arg.string_view().ends_with("/") - || arg.string_view().ends_with("/.")); - try { - auto checked = state.checkSourcePath(path); - auto exists = checked.pathExists(); - if (exists && mustBeDir) { - exists = checked.lstat().type == InputAccessor::tDirectory; - } + auto & arg = *args[0]; + + auto path = realisePath(state, pos, arg); + + /* SourcePath doesn't know about trailing slash. */ + auto mustBeDir = arg.type() == nString + && (arg.string_view().ends_with("/") + || arg.string_view().ends_with("/.")); + + auto st = path.maybeLstat(); + auto exists = st && (!mustBeDir || st->type == SourceAccessor::tDirectory); v.mkBool(exists); - } catch (SysError & e) { - /* Don't give away info from errors while canonicalising - ‘path’ in restricted mode. */ - v.mkBool(false); } catch (RestrictedPathError & e) { v.mkBool(false); } @@ -1700,7 +1681,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V auto path = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.findFile"); - v.mkPath(state.checkSourcePath(state.findFile(searchPath, path, pos))); + v.mkPath(state.findFile(searchPath, path, pos)); } static RegisterPrimOp primop_findFile(PrimOp { @@ -1755,17 +1736,17 @@ static RegisterPrimOp primop_findFile(PrimOp { /* Return the cryptographic hash of a file in base-16. */ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile"); - std::optional ht = parseHashType(type); - if (!ht) + auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashFile"); + std::optional ha = parseHashAlgo(algo); + if (!ha) state.debugThrowLastTrace(Error({ - .msg = hintfmt("unknown hash type '%1%'", type), + .msg = hintfmt("unknown hash algo '%1%'", algo), .errPos = state.positions[pos] })); auto path = realisePath(state, pos, *args[1]); - v.mkString(hashString(*ht, path.readFile()).to_string(HashFormat::Base16, false)); + v.mkString(hashString(*ha, path.readFile()).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashFile({ @@ -1790,7 +1771,7 @@ static std::string_view fileTypeToString(InputAccessor::Type type) static void prim_readFileType(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto path = realisePath(state, pos, *args[0]); + auto path = realisePath(state, pos, *args[0], false); /* Retrieve the directory entry type and stringize it. */ v.mkString(fileTypeToString(path.lstat().type)); } @@ -2179,11 +2160,35 @@ static RegisterPrimOp primop_toFile({ .fun = prim_toFile, }); +bool EvalState::callPathFilter( + Value * filterFun, + const SourcePath & path, + std::string_view pathArg, + PosIdx pos) +{ + auto st = path.lstat(); + + /* Call the filter function. The first argument is the path, the + second is a string indicating the type of the file. */ + Value arg1; + arg1.mkString(pathArg); + + Value arg2; + // assert that type is not "unknown" + arg2.mkString(fileTypeToString(st.type)); + + Value * args []{&arg1, &arg2}; + Value res; + callFunction(*filterFun, 2, args, res, pos); + + return forceBool(res, pos, "while evaluating the return value of the path filter function"); +} + static void addPath( EvalState & state, const PosIdx pos, std::string_view name, - Path path, + SourcePath path, Value * filterFun, FileIngestionMethod method, const std::optional expectedHash, @@ -2191,48 +2196,29 @@ static void addPath( const NixStringContext & context) { try { - // FIXME: handle CA derivation outputs (where path needs to - // be rewritten to the actual output). - auto rewrites = state.realiseContext(context); - path = state.toRealPath(rewriteStrings(path, rewrites), context); - StorePathSet refs; - if (state.store->isInStore(path)) { + if (path.accessor == state.rootFS && state.store->isInStore(path.path.abs())) { + // FIXME: handle CA derivation outputs (where path needs to + // be rewritten to the actual output). + auto rewrites = state.realiseContext(context); + path = {state.rootFS, CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))}; + try { - auto [storePath, subPath] = state.store->toStorePath(path); + auto [storePath, subPath] = state.store->toStorePath(path.path.abs()); // FIXME: we should scanForReferences on the path before adding it refs = state.store->queryPathInfo(storePath)->references; - path = state.store->toRealPath(storePath) + subPath; + path = {state.rootFS, CanonPath(state.store->toRealPath(storePath) + subPath)}; } catch (Error &) { // FIXME: should be InvalidPathError } } - path = evalSettings.pureEval && expectedHash - ? path - : state.checkSourcePath(state.rootPath(CanonPath(path))).path.abs(); - - PathFilter filter = filterFun ? ([&](const Path & path) { - auto st = lstat(path); - - /* Call the filter function. The first argument is the path, - the second is a string indicating the type of the file. */ - Value arg1; - arg1.mkString(path); - - Value arg2; - arg2.mkString( - S_ISREG(st.st_mode) ? "regular" : - S_ISDIR(st.st_mode) ? "directory" : - S_ISLNK(st.st_mode) ? "symlink" : - "unknown" /* not supported, will fail! */); - - Value * args []{&arg1, &arg2}; - Value res; - state.callFunction(*filterFun, 2, args, res, pos); - - return state.forceBool(res, pos, "while evaluating the return value of the path filter function"); - }) : defaultPathFilter; + std::unique_ptr filter; + if (filterFun) + filter = std::make_unique([&](const Path & p) { + auto p2 = CanonPath(p); + return state.callPathFilter(filterFun, {path.accessor, p2}, p2.abs(), pos); + }); std::optional expectedStorePath; if (expectedHash) @@ -2243,7 +2229,7 @@ static void addPath( }); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { - auto dstPath = state.rootPath(CanonPath(path)).fetchToStore(state.store, name, method, &filter, state.repair); + auto dstPath = path.fetchToStore(state.store, name, method, filter.get(), state.repair); if (expectedHash && expectedStorePath != dstPath) state.debugThrowLastTrace(Error("store path mismatch in (possibly filtered) path added from '%s'", path)); state.allowAndSetStorePathString(dstPath, v); @@ -2262,7 +2248,8 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg auto path = state.coerceToPath(pos, *args[1], context, "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); - addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); + + addPath(state, pos, path.baseName(), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ @@ -2342,7 +2329,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value else if (n == "recursive") method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") }; else if (n == "sha256") - expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), htSHA256); + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256); else state.debugThrowLastTrace(EvalError({ .msg = hintfmt("unsupported argument '%1%' to 'addPath'", state.symbols[attr.name]), @@ -2357,7 +2344,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value if (name.empty()) name = path->baseName(); - addPath(state, pos, name, path->path.abs(), filterFun, method, expectedHash, v, context); + addPath(state, pos, name, *path, filterFun, method, expectedHash, v, context); } static RegisterPrimOp primop_path({ @@ -2376,7 +2363,7 @@ static RegisterPrimOp primop_path({ like `@`. - filter\ - A function of the type expected by `builtins.filterSource`, + A function of the type expected by [`builtins.filterSource`](#builtins-filterSource), with the same semantics. - recursive\ @@ -2551,6 +2538,7 @@ static void prim_removeAttrs(EvalState & state, const PosIdx pos, Value * * args /* Get the attribute names to be removed. We keep them as Attrs instead of Symbols so std::set_difference can be used to remove them from attrs[0]. */ + // 64: large enough to fit the attributes of a derivation boost::container::small_vector names; names.reserve(args[1]->listSize()); for (auto elem : args[1]->listItems()) { @@ -2730,8 +2718,8 @@ static void prim_catAttrs(EvalState & state, const PosIdx pos, Value * * args, V auto attrName = state.symbols.create(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.catAttrs")); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.catAttrs"); - Value * res[args[1]->listSize()]; - unsigned int found = 0; + SmallValueVector res(args[1]->listSize()); + size_t found = 0; for (auto v2 : args[1]->listItems()) { state.forceAttrs(*v2, pos, "while evaluating an element in the list passed as second argument to builtins.catAttrs"); @@ -3065,9 +3053,8 @@ static void prim_filter(EvalState & state, const PosIdx pos, Value * * args, Val state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filter"); - // FIXME: putting this on the stack is risky. - Value * vs[args[1]->listSize()]; - unsigned int k = 0; + SmallValueVector vs(args[1]->listSize()); + size_t k = 0; bool same = true; for (unsigned int n = 0; n < args[1]->listSize(); ++n) { @@ -3180,9 +3167,16 @@ static RegisterPrimOp primop_foldlStrict({ .doc = R"( Reduce a list by applying a binary operator, from left to right, e.g. `foldl' op nul [x0 x1 x2 ...] = op (op (op nul x0) x1) x2) - ...`. For example, `foldl' (x: y: x + y) 0 [1 2 3]` evaluates to 6. - The return value of each application of `op` is evaluated immediately, - even for intermediate values. + ...`. + + For example, `foldl' (acc: elem: acc + elem) 0 [1 2 3]` evaluates + to `6` and `foldl' (acc: elem: { "${elem}" = elem; } // acc) {} + ["a" "b"]` evaluates to `{ a = "a"; b = "b"; }`. + + The first argument of `op` is the accumulator whereas the second + argument is the current element being processed. The return value + of each application of `op` is evaluated immediately, even for + intermediate values. )", .fun = prim_foldlStrict, }); @@ -3192,10 +3186,14 @@ static void anyOrAll(bool any, EvalState & state, const PosIdx pos, Value * * ar state.forceFunction(*args[0], pos, std::string("while evaluating the first argument passed to builtins.") + (any ? "any" : "all")); state.forceList(*args[1], pos, std::string("while evaluating the second argument passed to builtins.") + (any ? "any" : "all")); + std::string_view errorCtx = any + ? "while evaluating the return value of the function passed to builtins.any" + : "while evaluating the return value of the function passed to builtins.all"; + Value vTmp; for (auto elem : args[1]->listItems()) { state.callFunction(*args[0], *elem, vTmp, pos); - bool res = state.forceBool(vTmp, pos, std::string("while evaluating the return value of the function passed to builtins.") + (any ? "any" : "all")); + bool res = state.forceBool(vTmp, pos, errorCtx); if (res == any) { v.mkBool(any); return; @@ -3451,13 +3449,14 @@ static void prim_concatMap(EvalState & state, const PosIdx pos, Value * * args, state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.concatMap"); auto nrLists = args[1]->listSize(); - Value lists[nrLists]; + // List of returned lists before concatenation. References to these Values must NOT be persisted. + SmallTemporaryValueVector lists(nrLists); size_t len = 0; for (unsigned int n = 0; n < nrLists; ++n) { Value * vElem = args[1]->listElems()[n]; state.callFunction(*args[0], *vElem, lists[n], pos); - state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to buitlins.concatMap"); + state.forceList(lists[n], lists[n].determinePos(args[0]->determinePos(pos)), "while evaluating the return value of the function passed to builtins.concatMap"); len += lists[n].listSize(); } @@ -3755,18 +3754,18 @@ static RegisterPrimOp primop_stringLength({ /* Return the cryptographic hash of a string in base-16. */ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto type = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString"); - std::optional ht = parseHashType(type); - if (!ht) + auto algo = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hashString"); + std::optional ha = parseHashAlgo(algo); + if (!ha) state.debugThrowLastTrace(Error({ - .msg = hintfmt("unknown hash type '%1%'", type), + .msg = hintfmt("unknown hash algo '%1%'", algo), .errPos = state.positions[pos] })); NixStringContext context; // discarded auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); - v.mkString(hashString(*ht, s).to_string(HashFormat::Base16, false)); + v.mkString(hashString(*ha, s).to_string(HashFormat::Base16, false)); } static RegisterPrimOp primop_hashString({ @@ -3789,15 +3788,15 @@ static void prim_convertHash(EvalState & state, const PosIdx pos, Value * * args auto hash = state.forceStringNoCtx(*iteratorHash->value, pos, "while evaluating the attribute 'hash'"); Bindings::iterator iteratorHashAlgo = inputAttrs->find(state.symbols.create("hashAlgo")); - std::optional ht = std::nullopt; + std::optional ha = std::nullopt; if (iteratorHashAlgo != inputAttrs->end()) { - ht = parseHashType(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'")); + ha = parseHashAlgo(state.forceStringNoCtx(*iteratorHashAlgo->value, pos, "while evaluating the attribute 'hashAlgo'")); } Bindings::iterator iteratorToHashFormat = getAttr(state, state.symbols.create("toHashFormat"), args[0]->attrs, "while locating the attribute 'toHashFormat'"); HashFormat hf = parseHashFormat(state.forceStringNoCtx(*iteratorToHashFormat->value, pos, "while evaluating the attribute 'toHashFormat'")); - v.mkString(Hash::parseAny(hash, ht).to_string(hf, hf == HashFormat::SRI)); + v.mkString(Hash::parseAny(hash, ha).to_string(hf, hf == HashFormat::SRI)); } static RegisterPrimOp primop_convertHash({ @@ -3826,7 +3825,8 @@ static RegisterPrimOp primop_convertHash({ The format of the resulting hash. Must be one of - `"base16"` - - `"base32"` + - `"nix32"` + - `"base32"` (deprecated alias for `"nix32"`) - `"base64"` - `"sri"` @@ -4389,7 +4389,7 @@ void EvalState::createBaseEnv() addConstant("__currentSystem", v, { .type = nString, .doc = R"( - The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-pure-eval). + The value of the [`system` configuration option](@docroot@/command-ref/conf-file.md#conf-system). It can be used to set the `system` attribute for [`builtins.derivation`](@docroot@/language/derivations.md) such that the resulting derivation can be built on the same system that evaluates the Nix expression: @@ -4438,7 +4438,7 @@ void EvalState::createBaseEnv() .doc = R"( Logical file system location of the [Nix store](@docroot@/glossary.md#gloss-store) currently in use. - This value is determined by the `store` parameter in [Store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md): + This value is determined by the `store` parameter in [Store URLs](@docroot@/store/types/index.md#store-url-format): ```shell-session $ nix-instantiate --store 'dummy://?store=/blah' --eval --expr builtins.storeDir diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh index 930e7f32a..45486608f 100644 --- a/src/libexpr/primops.hh +++ b/src/libexpr/primops.hh @@ -8,6 +8,22 @@ namespace nix { +/** + * For functions where we do not expect deep recursion, we can use a sizable + * part of the stack a free allocation space. + * + * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. + */ +constexpr size_t nonRecursiveStackReservation = 128; + +/** + * Functions that maybe applied to self-similar inputs, such as concatMap on a + * tree, should reserve a smaller part of the stack for allocation. + * + * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. + */ +constexpr size_t conservativeStackReservation = 16; + struct RegisterPrimOp { typedef std::vector PrimOps; diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index b86ef6b93..27147a5d1 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -1,5 +1,6 @@ #include "primops.hh" #include "store-api.hh" +#include "realisation.hh" #include "make-content-addressed.hh" #include "url.hh" diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index e76ce455d..58fe6f173 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -31,7 +31,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a // be both a revision or a branch/tag name. auto value = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `rev` attribute passed to builtins.fetchMercurial"); if (std::regex_match(value.begin(), value.end(), revRegex)) - rev = Hash::parseAny(value, htSHA1); + rev = Hash::parseAny(value, HashAlgorithm::SHA1); else ref = value; } @@ -79,7 +79,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to // 0000000000000000000000000000000000000000 for a dirty tree. - auto rev2 = input2.getRev().value_or(Hash(htSHA1)); + auto rev2 = input2.getRev().value_or(Hash(HashAlgorithm::SHA1)); attrs2.alloc("rev").mkString(rev2.gitRev()); attrs2.alloc("shortRev").mkString(rev2.gitRev().substr(0, 12)); if (auto revCount = input2.getRevCount()) diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index a99b0e500..eb2df8626 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -7,6 +7,7 @@ #include "registry.hh" #include "tarball.hh" #include "url.hh" +#include "value-to-json.hh" #include #include @@ -45,7 +46,7 @@ void emitTreeAttrs( attrs.alloc("shortRev").mkString(rev->gitShortRev()); } else if (emptyRevFallback) { // Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev - auto emptyHash = Hash(htSHA1); + auto emptyHash = Hash(HashAlgorithm::SHA1); attrs.alloc("rev").mkString(emptyHash.gitRev()); attrs.alloc("shortRev").mkString(emptyHash.gitShortRev()); } @@ -125,6 +126,10 @@ static void fetchTree( attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean}); else if (attr.value->type() == nInt) attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer)); + else if (state.symbols[attr.name] == "publicKeys") { + experimentalFeatureSettings.require(Xp::VerifiedFetches); + attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump()); + } else state.debugThrowLastTrace(TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", state.symbols[attr.name], showType(*attr.value))); @@ -182,47 +187,218 @@ static RegisterPrimOp primop_fetchTree({ .name = "fetchTree", .args = {"input"}, .doc = R"( - Fetch a source tree or a plain file using one of the supported backends. - *input* must be a [flake reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references), either in attribute set representation or in the URL-like syntax. - The input should be "locked", that is, it should contain a commit hash or content hash unless impure evaluation (`--impure`) is enabled. + Fetch a file system tree or a plain file using one of the supported backends and return an attribute set with: - > **Note** + - the resulting fixed-output [store path](@docroot@/glossary.md#gloss-store-path) + - the corresponding [NAR](@docroot@/glossary.md#gloss-nar) hash + - backend-specific metadata (currently not documented). + + *input* must be an attribute set with the following attributes: + + - `type` (String, required) + + One of the [supported source types](#source-types). + This determines other required and allowed input attributes. + + - `narHash` (String, optional) + + The `narHash` parameter can be used to substitute the source of the tree. + It also allows for verification of tree contents that may not be provided by the underlying transfer mechanism. + If `narHash` is set, the source is first looked up is the Nix store and [substituters](@docroot@/command-ref/conf-file.md#conf-substituters), and only fetched if not available. + + A subset of the output attributes of `fetchTree` can be re-used for subsequent calls to `fetchTree` to produce the same result again. + That is, `fetchTree` is idempotent. + + Downloads are cached in `$XDG_CACHE_HOME/nix`. + The remote source will be fetched from the network if both are true: + - A NAR hash is supplied and the corresponding store path is not [valid](@docroot@/glossary.md#gloss-validity), that is, not available in the store + + > **Note** + > + > [Substituters](@docroot@/command-ref/conf-file.md#conf-substituters) are not used in fetching. + + - There is no cache entry or the cache entry is older than [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) + + ## Source types + + The following source types and associated input attributes are supported. + + + + - `"file"` + + Place a plain file into the Nix store. + This is similar to [`builtins.fetchurl`](@docroot@/language/builtins.md#builtins-fetchurl) + + - `url` (String, required) + + Supported protocols: + + - `https` + + > **Example** + > + > ```nix + > fetchTree { + > type = "file"; + > url = "https://example.com/index.html"; + > } + > ``` + + - `http` + + Insecure HTTP transfer for legacy sources. + + > **Warning** + > + > HTTP performs no encryption or authentication. + > Use a `narHash` known in advance to ensure the output has expected contents. + + - `file` + + A file on the local file system. + + > **Example** + > + > ```nix + > fetchTree { + > type = "file"; + > url = "file:///home/eelco/nix/README.md"; + > } + > ``` + + - `"tarball"` + + Download a tar archive and extract it into the Nix store. + This has the same underyling implementation as [`builtins.fetchTarball`](@docroot@/language/builtins.md#builtins-fetchTarball) + + - `url` (String, required) + + > **Example** + > + > ```nix + > fetchTree { + > type = "tarball"; + > url = "https://github.com/NixOS/nixpkgs/tarball/nixpkgs-23.11"; + > } + > ``` + + - `"git"` + + Fetch a Git tree and copy it to the Nix store. + This is similar to [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit). + + - `url` (String, required) + + The URL formats supported are the same as for Git itself. + + > **Example** + > + > ```nix + > fetchTree { + > type = "git"; + > url = "git@github.com:NixOS/nixpkgs.git"; + > } + > ``` + + > **Note** + > + > If the URL points to a local directory, and no `ref` or `rev` is given, Nix will only consider files added to the Git index, as listed by `git ls-files` but use the *current file contents* of the Git working directory. + + - `ref` (String, optional) + + A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name. + + Default: `"HEAD"` + + - `rev` (String, optional) + + A Git revision; a commit hash. + + Default: the tip of `ref` + + - `shallow` (Bool, optional) + + Make a shallow clone when fetching the Git tree. + + Default: `false` + + - `submodules` (Bool, optional) + + Also fetch submodules if available. + + Default: `false` + + - `allRefs` (Bool, optional) + + If set to `true`, always fetch the entire repository, even if the latest commit is still in the cache. + Otherwise, only the latest commit is fetched if it is not already cached. + + Default: `false` + + - `lastModified` (Integer, optional) + + Unix timestamp of the fetched commit. + + If set, pass through the value to the output attribute set. + Otherwise, generated from the fetched Git tree. + + - `revCount` (Integer, optional) + + Number of revisions in the history of the Git repository before the fetched commit. + + If set, pass through the value to the output attribute set. + Otherwise, generated from the fetched Git tree. + + The following input types are still subject to change: + + - `"path"` + - `"github"` + - `"gitlab"` + - `"sourcehut"` + - `"mercurial"` + + *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references). + The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + + > **Example** > - > The URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + > Fetch a GitHub repository using the attribute set representation: + > + > ```nix + > builtins.fetchTree { + > type = "github"; + > owner = "NixOS"; + > repo = "nixpkgs"; + > rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + > } + > ``` + > + > This evaluates to the following attribute set: + > + > ```nix + > { + > lastModified = 1686503798; + > lastModifiedDate = "20230611171638"; + > narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc="; + > outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source"; + > rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; + > shortRev = "ae2e6b3"; + > } + > ``` - Here are some examples of how to use `fetchTree`: - - - Fetch a GitHub repository using the attribute set representation: - - ```nix - builtins.fetchTree { - type = "github"; - owner = "NixOS"; - repo = "nixpkgs"; - rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; - } - ``` - - This evaluates to the following attribute set: - - ``` - { - lastModified = 1686503798; - lastModifiedDate = "20230611171638"; - narHash = "sha256-rA9RqKP9OlBrgGCPvfd5HVAXDOy8k2SmPtB/ijShNXc="; - outPath = "/nix/store/l5m6qlvfs9sdw14ja3qbzpglcjlb6j1x-source"; - rev = "ae2e6b3958682513d28f7d633734571fb18285dd"; - shortRev = "ae2e6b3"; - } - ``` - - - Fetch the same GitHub repository using the URL-like syntax: - - ``` - builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd" - ``` + > **Example** + > + > Fetch the same GitHub repository using the URL-like syntax: + > + > ```nix + > builtins.fetchTree "github:NixOS/nixpkgs/ae2e6b3958682513d28f7d633734571fb18285dd" + > ``` )", .fun = prim_fetchTree, + .experimentalFeature = Xp::FetchTree, }); static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v, @@ -240,7 +416,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v if (n == "url") url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch"); else if (n == "sha256") - expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), htSHA256); + expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), HashAlgorithm::SHA256); else if (n == "name") name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch"); else @@ -270,7 +446,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v state.debugThrowLastTrace(EvalError("in pure evaluation mode, '%s' requires a 'sha256' argument", who)); // early exit if pinned and already in the store - if (expectedHash && expectedHash->type == htSHA256) { + if (expectedHash && expectedHash->algo == HashAlgorithm::SHA256) { auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { @@ -295,10 +471,10 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v if (expectedHash) { auto hash = unpack ? state.store->queryPathInfo(storePath)->narHash - : hashFile(htSHA256, state.store->toRealPath(storePath)); + : hashFile(HashAlgorithm::SHA256, state.store->toRealPath(storePath)); if (hash != *expectedHash) state.debugThrowLastTrace(EvalError((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s", - *url, expectedHash->to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true))); + *url, expectedHash->to_string(HashFormat::Nix32, true), hash.to_string(HashFormat::Nix32, true))); } state.allowAndSetStorePathString(storePath, v); @@ -392,7 +568,7 @@ static RegisterPrimOp primop_fetchGit({ The URL of the repo. - - `name` (default: *basename of the URL*) + - `name` (default: `source`) The name of the directory the repo should be exported to in the store. @@ -419,7 +595,8 @@ static RegisterPrimOp primop_fetchGit({ - `shallow` (default: `false`) - A Boolean parameter that specifies whether fetching a shallow clone is allowed. + A Boolean parameter that specifies whether fetching from a shallow remote repository is allowed. + This still performs a full clone of what is available on the remote. - `allRefs` @@ -427,6 +604,42 @@ static RegisterPrimOp primop_fetchGit({ With this argument being true, it's possible to load a `rev` from *any* `ref` (by default only `rev`s from the specified `ref` are supported). + - `verifyCommit` (default: `true` if `publicKey` or `publicKeys` are provided, otherwise `false`) + + Whether to check `rev` for a signature matching `publicKey` or `publicKeys`. + If `verifyCommit` is enabled, then `fetchGit` cannot use a local repository with uncommitted changes. + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + - `publicKey` + + The public key against which `rev` is verified if `verifyCommit` is enabled. + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + - `keytype` (default: `"ssh-ed25519"`) + + The key type of `publicKey`. + Possible values: + - `"ssh-dsa"` + - `"ssh-ecdsa"` + - `"ssh-ecdsa-sk"` + - `"ssh-ed25519"` + - `"ssh-ed25519-sk"` + - `"ssh-rsa"` + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + - `publicKeys` + + The public keys against which `rev` is verified if `verifyCommit` is enabled. + Must be given as a list of attribute sets with the following form: + ```nix + { + key = ""; + type = ""; # optional, default: "ssh-ed25519" + } + ``` + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + + Here are some examples of how to use `fetchGit`. - To fetch a private repository over SSH: @@ -501,6 +714,21 @@ static RegisterPrimOp primop_fetchGit({ } ``` + - To verify the commit signature: + + ```nix + builtins.fetchGit { + url = "ssh://git@github.com/nixos/nix.git"; + verifyCommit = true; + publicKeys = [ + { + type = "ssh-ed25519"; + key = "AAAAC3NzaC1lZDI1NTE5AAAAIArPKULJOid8eS6XETwUjO48/HKBWl7FTCK0Z//fplDi"; + } + ]; + } + ``` + Nix will refetch the branch according to the [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) setting. This behavior is disabled in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). diff --git a/src/libexpr/search-path.cc b/src/libexpr/search-path.cc index 180d5f8b1..a25767496 100644 --- a/src/libexpr/search-path.cc +++ b/src/libexpr/search-path.cc @@ -1,5 +1,4 @@ #include "search-path.hh" -#include "util.hh" namespace nix { diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk deleted file mode 100644 index 331a5ead6..000000000 --- a/src/libexpr/tests/local.mk +++ /dev/null @@ -1,19 +0,0 @@ -check: libexpr-tests_RUN - -programs += libexpr-tests - -libexpr-tests_NAME := libnixexpr-tests - -libexpr-tests_DIR := $(d) - -libexpr-tests_INSTALL_DIR := - -libexpr-tests_SOURCES := \ - $(wildcard $(d)/*.cc) \ - $(wildcard $(d)/value/*.cc) - -libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers - -libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers - -libexpr-tests_LDFLAGS := $(GTEST_LIBS) -lgmock diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index cbc91f509..74b3ebf13 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -1,7 +1,7 @@ #include "value-to-json.hh" #include "eval-inline.hh" -#include "util.hh" #include "store-api.hh" +#include "signals.hh" #include #include diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index bd7a4ae30..5032115bb 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -1,7 +1,7 @@ #include "value-to-xml.hh" #include "xml-writer.hh" #include "eval-inline.hh" -#include "util.hh" +#include "signals.hh" #include diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 622e613ea..30b3d4934 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -3,6 +3,7 @@ #include #include +#include #include "symbol-table.hh" #include "value/context.hh" @@ -66,7 +67,6 @@ class Symbol; class PosIdx; struct Pos; class StorePath; -class Store; class EvalState; class XMLWriter; @@ -158,42 +158,60 @@ public: inline bool isPrimOp() const { return internalType == tPrimOp; }; inline bool isPrimOpApp() const { return internalType == tPrimOpApp; }; + /** + * Strings in the evaluator carry a so-called `context` which + * is a list of strings representing store paths. This is to + * allow users to write things like + * + * "--with-freetype2-library=" + freetype + "/lib" + * + * where `freetype` is a derivation (or a source to be copied + * to the store). If we just concatenated the strings without + * keeping track of the referenced store paths, then if the + * string is used as a derivation attribute, the derivation + * will not have the correct dependencies in its inputDrvs and + * inputSrcs. + + * The semantics of the context is as follows: when a string + * with context C is used as a derivation attribute, then the + * derivations in C will be added to the inputDrvs of the + * derivation, and the other store paths in C will be added to + * the inputSrcs of the derivations. + + * For canonicity, the store paths should be in sorted order. + */ + struct StringWithContext { + const char * c_str; + const char * * context; // must be in sorted order + }; + + struct Path { + InputAccessor * accessor; + const char * path; + }; + + struct ClosureThunk { + Env * env; + Expr * expr; + }; + + struct FunctionApplicationThunk { + Value * left, * right; + }; + + struct Lambda { + Env * env; + ExprLambda * fun; + }; + union { NixInt integer; bool boolean; - /** - * Strings in the evaluator carry a so-called `context` which - * is a list of strings representing store paths. This is to - * allow users to write things like + StringWithContext string; - * "--with-freetype2-library=" + freetype + "/lib" - - * where `freetype` is a derivation (or a source to be copied - * to the store). If we just concatenated the strings without - * keeping track of the referenced store paths, then if the - * string is used as a derivation attribute, the derivation - * will not have the correct dependencies in its inputDrvs and - * inputSrcs. - - * The semantics of the context is as follows: when a string - * with context C is used as a derivation attribute, then the - * derivations in C will be added to the inputDrvs of the - * derivation, and the other store paths in C will be added to - * the inputSrcs of the derivations. - - * For canonicity, the store paths should be in sorted order. - */ - struct { - const char * c_str; - const char * * context; // must be in sorted order - } string; - - struct { - InputAccessor * accessor; - const char * path; - } _path; + Path _path; Bindings * attrs; struct { @@ -201,21 +219,11 @@ public: Value * * elems; } bigList; Value * smallList[2]; - struct { - Env * env; - Expr * expr; - } thunk; - struct { - Value * left, * right; - } app; - struct { - Env * env; - ExprLambda * fun; - } lambda; + ClosureThunk thunk; + FunctionApplicationThunk app; + Lambda lambda; PrimOp * primOp; - struct { - Value * left, * right; - } primOpApp; + FunctionApplicationThunk primOpApp; ExternalValueBase * external; NixFloat fpoint; }; @@ -354,13 +362,7 @@ public: // Value will be overridden anyways } - inline void mkPrimOp(PrimOp * p) - { - clearValue(); - internalType = tPrimOp; - primOp = p; - } - + void mkPrimOp(PrimOp * p); inline void mkPrimOpApp(Value * l, Value * r) { @@ -393,7 +395,13 @@ public: return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } - const Value * const * listElems() const + std::span listItems() const + { + assert(isList()); + return std::span(listElems(), listSize()); + } + + Value * const * listElems() const { return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } @@ -412,41 +420,12 @@ public: */ bool isTrivial() const; - auto listItems() - { - struct ListIterable - { - typedef Value * const * iterator; - iterator _begin, _end; - iterator begin() const { return _begin; } - iterator end() const { return _end; } - }; - assert(isList()); - auto begin = listElems(); - return ListIterable { begin, begin + listSize() }; - } - - auto listItems() const - { - struct ConstListIterable - { - typedef const Value * const * iterator; - iterator _begin, _end; - iterator begin() const { return _begin; } - iterator end() const { return _end; } - }; - assert(isList()); - auto begin = listElems(); - return ConstListIterable { begin, begin + listSize() }; - } - SourcePath path() const { assert(internalType == tPath); - return SourcePath { - .accessor = ref(_path.accessor->shared_from_this()), - .path = CanonPath(CanonPath::unchecked_t(), _path.path) - }; + return SourcePath( + ref(_path.accessor->shared_from_this()), + CanonPath(CanonPath::unchecked_t(), _path.path)); } std::string_view string_view() const diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index 22361d8fa..6d9633268 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -1,3 +1,4 @@ +#include "util.hh" #include "value/context.hh" #include diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 9f1d59317..51fd30a44 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "util.hh" #include "comparator.hh" #include "derived-path.hh" #include "variant-wrapper.hh" diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 0c8ecac9d..63b05bdab 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -1,4 +1,5 @@ #include "cache.hh" +#include "users.hh" #include "sqlite.hh" #include "sync.hh" #include "store-api.hh" @@ -19,6 +20,9 @@ create table if not exists Cache ( ); )sql"; +// FIXME: we should periodically purge/nuke this cache to prevent it +// from growing too big. + struct CacheImpl : Cache { struct State @@ -47,6 +51,60 @@ struct CacheImpl : Cache "select info, path, immutable, timestamp from Cache where input = ?"); } + void upsert( + const Attrs & inAttrs, + const Attrs & infoAttrs) override + { + _state.lock()->add.use() + (attrsToJSON(inAttrs).dump()) + (attrsToJSON(infoAttrs).dump()) + ("") // no path + (false) + (time(0)).exec(); + } + + std::optional lookup(const Attrs & inAttrs) override + { + if (auto res = lookupExpired(inAttrs)) + return std::move(res->infoAttrs); + return {}; + } + + std::optional lookupWithTTL(const Attrs & inAttrs) override + { + if (auto res = lookupExpired(inAttrs)) { + if (!res->expired) + return std::move(res->infoAttrs); + debug("ignoring expired cache entry '%s'", + attrsToJSON(inAttrs).dump()); + } + return {}; + } + + std::optional lookupExpired(const Attrs & inAttrs) override + { + auto state(_state.lock()); + + auto inAttrsJSON = attrsToJSON(inAttrs).dump(); + + auto stmt(state->lookup.use()(inAttrsJSON)); + if (!stmt.next()) { + debug("did not find cache entry for '%s'", inAttrsJSON); + return {}; + } + + auto infoJSON = stmt.getStr(0); + auto locked = stmt.getInt(2) != 0; + auto timestamp = stmt.getInt(3); + + debug("using cache entry '%s' -> '%s'", inAttrsJSON, infoJSON); + + return Result2 { + .expired = !locked && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)), + .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)), + }; + } + void add( ref store, const Attrs & inAttrs, diff --git a/src/libfetchers/cache.hh b/src/libfetchers/cache.hh index ae398d040..f70589267 100644 --- a/src/libfetchers/cache.hh +++ b/src/libfetchers/cache.hh @@ -2,13 +2,53 @@ ///@file #include "fetchers.hh" +#include "path.hh" namespace nix::fetchers { +/** + * A cache for arbitrary `Attrs` -> `Attrs` mappings with a timestamp + * for expiration. + */ struct Cache { virtual ~Cache() { } + /** + * Add a value to the cache. The cache is an arbitrary mapping of + * Attrs to Attrs. + */ + virtual void upsert( + const Attrs & inAttrs, + const Attrs & infoAttrs) = 0; + + /** + * Look up a key with infinite TTL. + */ + virtual std::optional lookup( + const Attrs & inAttrs) = 0; + + /** + * Look up a key. Return nothing if its TTL has exceeded + * `settings.tarballTTL`. + */ + virtual std::optional lookupWithTTL( + const Attrs & inAttrs) = 0; + + struct Result2 + { + bool expired = false; + Attrs infoAttrs; + }; + + /** + * Look up a key. Return a bool denoting whether its TTL has + * exceeded `settings.tarballTTL`. + */ + virtual std::optional lookupExpired( + const Attrs & inAttrs) = 0; + + /* Old cache for things that have a store path. */ virtual void add( ref store, const Attrs & inAttrs, diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 6108a179c..f095963a8 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -3,7 +3,6 @@ #include "types.hh" #include "config.hh" -#include "util.hh" #include #include diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 5688c4dc1..7ec1f9802 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -1,16 +1,36 @@ #include "fetchers.hh" #include "store-api.hh" +#include "input-accessor.hh" #include namespace nix::fetchers { -std::unique_ptr>> inputSchemes = nullptr; +using InputSchemeMap = std::map>; + +std::unique_ptr inputSchemes = nullptr; void registerInputScheme(std::shared_ptr && inputScheme) { - if (!inputSchemes) inputSchemes = std::make_unique>>(); - inputSchemes->push_back(std::move(inputScheme)); + if (!inputSchemes) + inputSchemes = std::make_unique(); + auto schemeName = inputScheme->schemeName(); + if (inputSchemes->count(schemeName) > 0) + throw Error("Input scheme with name %s already registered", schemeName); + inputSchemes->insert_or_assign(schemeName, std::move(inputScheme)); +} + +nlohmann::json dumpRegisterInputSchemeInfo() { + using nlohmann::json; + + auto res = json::object(); + + for (auto & [name, scheme] : *inputSchemes) { + auto & r = res[name] = json::object(); + r["allowedAttrs"] = scheme->allowedAttrs(); + } + + return res; } Input Input::fromURL(const std::string & url, bool requireTree) @@ -33,7 +53,7 @@ static void fixupInput(Input & input) Input Input::fromURL(const ParsedURL & url, bool requireTree) { - for (auto & inputScheme : *inputSchemes) { + for (auto & [_, inputScheme] : *inputSchemes) { auto res = inputScheme->inputFromURL(url, requireTree); if (res) { experimentalFeatureSettings.require(inputScheme->experimentalFeature()); @@ -48,20 +68,49 @@ Input Input::fromURL(const ParsedURL & url, bool requireTree) Input Input::fromAttrs(Attrs && attrs) { - for (auto & inputScheme : *inputSchemes) { - auto res = inputScheme->inputFromAttrs(attrs); - if (res) { - experimentalFeatureSettings.require(inputScheme->experimentalFeature()); - res->scheme = inputScheme; - fixupInput(*res); - return std::move(*res); - } - } + auto schemeName = ({ + auto schemeNameOpt = maybeGetStrAttr(attrs, "type"); + if (!schemeNameOpt) + throw Error("'type' attribute to specify input scheme is required but not provided"); + *std::move(schemeNameOpt); + }); - Input input; - input.attrs = attrs; - fixupInput(input); - return input; + auto raw = [&]() { + // Return an input without a scheme; most operations will fail, + // but not all of them. Doing this is to support those other + // operations which are supposed to be robust on + // unknown/uninterpretable inputs. + Input input; + input.attrs = attrs; + fixupInput(input); + return input; + }; + + std::shared_ptr inputScheme = ({ + auto i = inputSchemes->find(schemeName); + i == inputSchemes->end() ? nullptr : i->second; + }); + + if (!inputScheme) return raw(); + + experimentalFeatureSettings.require(inputScheme->experimentalFeature()); + + auto allowedAttrs = inputScheme->allowedAttrs(); + + for (auto & [name, _] : attrs) + if (name != "type" && allowedAttrs.count(name) == 0) + throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName); + + auto res = inputScheme->inputFromAttrs(attrs); + if (!res) return raw(); + res->scheme = inputScheme; + fixupInput(*res); + return std::move(*res); +} + +std::optional Input::getFingerprint(ref store) const +{ + return scheme ? scheme->getFingerprint(store, *this) : std::nullopt; } ParsedURL Input::toURL() const @@ -176,6 +225,16 @@ std::pair Input::fetch(ref store) const return {std::move(storePath), input}; } +std::pair, Input> Input::getAccessor(ref store) const +{ + try { + return scheme->getAccessor(store, *this); + } catch (Error & e) { + e.addTrace({}, "while fetching the input '%s'", to_string()); + throw; + } +} + Input Input::applyOverrides( std::optional ref, std::optional rev) const @@ -196,12 +255,13 @@ std::optional Input::getSourcePath() const return scheme->getSourcePath(*this); } -void Input::markChangedFile( - std::string_view file, +void Input::putFile( + const CanonPath & path, + std::string_view contents, std::optional commitMsg) const { assert(scheme); - return scheme->markChangedFile(*this, file, commitMsg); + return scheme->putFile(*this, path, contents, commitMsg); } std::string Input::getName() const @@ -229,8 +289,8 @@ std::string Input::getType() const std::optional Input::getNarHash() const { if (auto s = maybeGetStrAttr(attrs, "narHash")) { - auto hash = s->empty() ? Hash(htSHA256) : Hash::parseSRI(*s); - if (hash.type != htSHA256) + auto hash = s->empty() ? Hash(HashAlgorithm::SHA256) : Hash::parseSRI(*s); + if (hash.algo != HashAlgorithm::SHA256) throw UsageError("narHash must use SHA-256"); return hash; } @@ -254,7 +314,7 @@ std::optional Input::getRev() const } catch (BadHash &e) { // Default to sha1 for backwards compatibility with existing // usages (e.g. `builtins.fetchTree` calls or flake inputs). - hash = Hash::parseAny(*s, htSHA1); + hash = Hash::parseAny(*s, HashAlgorithm::SHA1); } } @@ -292,14 +352,18 @@ Input InputScheme::applyOverrides( return input; } -std::optional InputScheme::getSourcePath(const Input & input) +std::optional InputScheme::getSourcePath(const Input & input) const { return {}; } -void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) +void InputScheme::putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const { - assert(false); + throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path); } void InputScheme::clone(const Input & input, const Path & destDir) const @@ -307,9 +371,26 @@ void InputScheme::clone(const Input & input, const Path & destDir) const throw Error("do not know how to clone input '%s'", input.to_string()); } -std::optional InputScheme::experimentalFeature() +std::pair InputScheme::fetch(ref store, const Input & input) +{ + auto [accessor, input2] = getAccessor(store, input); + auto storePath = SourcePath(accessor).fetchToStore(store, input2.getName()); + return {storePath, input2}; +} + +std::pair, Input> InputScheme::getAccessor(ref store, const Input & input) const +{ + throw UnimplementedError("InputScheme must implement fetch() or getAccessor()"); +} + +std::optional InputScheme::experimentalFeature() const { return {}; } +std::string publicKeys_to_string(const std::vector& publicKeys) +{ + return ((nlohmann::json) publicKeys).dump(); +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index ac605ff8e..5f3254b6d 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -3,13 +3,14 @@ #include "types.hh" #include "hash.hh" -#include "path.hh" +#include "canon-path.hh" #include "attrs.hh" #include "url.hh" #include +#include -namespace nix { class Store; } +namespace nix { class Store; class StorePath; struct InputAccessor; } namespace nix::fetchers { @@ -82,6 +83,8 @@ public: */ std::pair fetch(ref store) const; + std::pair, Input> getAccessor(ref store) const; + Input applyOverrides( std::optional ref, std::optional rev) const; @@ -90,8 +93,13 @@ public: std::optional getSourcePath() const; - void markChangedFile( - std::string_view file, + /** + * Write a file to this input, for input types that support + * writing. Optionally commit the change (for e.g. Git inputs). + */ + void putFile( + const CanonPath & path, + std::string_view contents, std::optional commitMsg) const; std::string getName() const; @@ -105,6 +113,12 @@ public: std::optional getRev() const; std::optional getRevCount() const; std::optional getLastModified() const; + + /** + * For locked inputs, return a string that uniquely specifies the + * content of the input (typically a commit hash or content hash). + */ + std::optional getFingerprint(ref store) const; }; @@ -126,6 +140,24 @@ struct InputScheme virtual std::optional inputFromAttrs(const Attrs & attrs) const = 0; + /** + * What is the name of the scheme? + * + * The `type` attribute is used to select which input scheme is + * used, and then the other fields are forwarded to that input + * scheme. + */ + virtual std::string_view schemeName() const = 0; + + /** + * Allowed attributes in an attribute set that is converted to an + * input. + * + * `type` is not included from this set, because the `type` field is + parsed first to choose which scheme; `type` is always required. + */ + virtual StringSet allowedAttrs() const = 0; + virtual ParsedURL toURL(const Input & input) const; virtual Input applyOverrides( @@ -135,21 +167,41 @@ struct InputScheme virtual void clone(const Input & input, const Path & destDir) const; - virtual std::optional getSourcePath(const Input & input); + virtual std::optional getSourcePath(const Input & input) const; - virtual void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg); + virtual void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const; - virtual std::pair fetch(ref store, const Input & input) = 0; + virtual std::pair fetch(ref store, const Input & input); + + virtual std::pair, Input> getAccessor(ref store, const Input & input) const; /** * Is this `InputScheme` part of an experimental feature? */ - virtual std::optional experimentalFeature(); + virtual std::optional experimentalFeature() const; virtual bool isDirect(const Input & input) const { return true; } + + virtual std::optional getFingerprint(ref store, const Input & input) const + { return std::nullopt; } }; void registerInputScheme(std::shared_ptr && fetcher); +nlohmann::json dumpRegisterInputSchemeInfo(); + +struct PublicKey +{ + std::string type = "ssh-ed25519"; + std::string key; +}; +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(PublicKey, type, key) + +std::string publicKeys_to_string(const std::vector&); + } diff --git a/src/libfetchers/filtering-input-accessor.cc b/src/libfetchers/filtering-input-accessor.cc new file mode 100644 index 000000000..5ae416fd3 --- /dev/null +++ b/src/libfetchers/filtering-input-accessor.cc @@ -0,0 +1,83 @@ +#include "filtering-input-accessor.hh" + +namespace nix { + +std::string FilteringInputAccessor::readFile(const CanonPath & path) +{ + checkAccess(path); + return next->readFile(prefix + path); +} + +bool FilteringInputAccessor::pathExists(const CanonPath & path) +{ + return isAllowed(path) && next->pathExists(prefix + path); +} + +std::optional FilteringInputAccessor::maybeLstat(const CanonPath & path) +{ + checkAccess(path); + return next->maybeLstat(prefix + path); +} + +InputAccessor::DirEntries FilteringInputAccessor::readDirectory(const CanonPath & path) +{ + checkAccess(path); + DirEntries entries; + for (auto & entry : next->readDirectory(prefix + path)) { + if (isAllowed(path + entry.first)) + entries.insert(std::move(entry)); + } + return entries; +} + +std::string FilteringInputAccessor::readLink(const CanonPath & path) +{ + checkAccess(path); + return next->readLink(prefix + path); +} + +std::string FilteringInputAccessor::showPath(const CanonPath & path) +{ + return next->showPath(prefix + path); +} + +void FilteringInputAccessor::checkAccess(const CanonPath & path) +{ + if (!isAllowed(path)) + throw makeNotAllowedError + ? makeNotAllowedError(path) + : RestrictedPathError("access to path '%s' is forbidden", showPath(path)); +} + +struct AllowListInputAccessorImpl : AllowListInputAccessor +{ + std::set allowedPaths; + + AllowListInputAccessorImpl( + ref next, + std::set && allowedPaths, + MakeNotAllowedError && makeNotAllowedError) + : AllowListInputAccessor(SourcePath(next), std::move(makeNotAllowedError)) + , allowedPaths(std::move(allowedPaths)) + { } + + bool isAllowed(const CanonPath & path) override + { + return path.isAllowed(allowedPaths); + } + + void allowPath(CanonPath path) override + { + allowedPaths.insert(std::move(path)); + } +}; + +ref AllowListInputAccessor::create( + ref next, + std::set && allowedPaths, + MakeNotAllowedError && makeNotAllowedError) +{ + return make_ref(next, std::move(allowedPaths), std::move(makeNotAllowedError)); +} + +} diff --git a/src/libfetchers/filtering-input-accessor.hh b/src/libfetchers/filtering-input-accessor.hh new file mode 100644 index 000000000..e1b83c929 --- /dev/null +++ b/src/libfetchers/filtering-input-accessor.hh @@ -0,0 +1,73 @@ +#pragma once + +#include "input-accessor.hh" + +namespace nix { + +/** + * A function that should throw an exception of type + * `RestrictedPathError` explaining that access to `path` is + * forbidden. + */ +typedef std::function MakeNotAllowedError; + +/** + * An abstract wrapping `InputAccessor` that performs access + * control. Subclasses should override `isAllowed()` to implement an + * access control policy. The error message is customized at construction. + */ +struct FilteringInputAccessor : InputAccessor +{ + ref next; + CanonPath prefix; + MakeNotAllowedError makeNotAllowedError; + + FilteringInputAccessor(const SourcePath & src, MakeNotAllowedError && makeNotAllowedError) + : next(src.accessor) + , prefix(src.path) + , makeNotAllowedError(std::move(makeNotAllowedError)) + { } + + std::string readFile(const CanonPath & path) override; + + bool pathExists(const CanonPath & path) override; + + std::optional maybeLstat(const CanonPath & path) override; + + DirEntries readDirectory(const CanonPath & path) override; + + std::string readLink(const CanonPath & path) override; + + std::string showPath(const CanonPath & path) override; + + /** + * Call `makeNotAllowedError` to throw a `RestrictedPathError` + * exception if `isAllowed()` returns `false` for `path`. + */ + void checkAccess(const CanonPath & path); + + /** + * Return `true` iff access to path is allowed. + */ + virtual bool isAllowed(const CanonPath & path) = 0; +}; + +/** + * A wrapping `InputAccessor` that checks paths against an allow-list. + */ +struct AllowListInputAccessor : public FilteringInputAccessor +{ + /** + * Grant access to the specified path. + */ + virtual void allowPath(CanonPath path) = 0; + + static ref create( + ref next, + std::set && allowedPaths, + MakeNotAllowedError && makeNotAllowedError); + + using FilteringInputAccessor::FilteringInputAccessor; +}; + +} diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index 7638d2d82..c3d8d273c 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -4,20 +4,14 @@ namespace nix { -struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor +struct FSInputAccessor : InputAccessor, PosixSourceAccessor { CanonPath root; - std::optional> allowedPaths; - MakeNotAllowedError makeNotAllowedError; - FSInputAccessorImpl( - const CanonPath & root, - std::optional> && allowedPaths, - MakeNotAllowedError && makeNotAllowedError) + FSInputAccessor(const CanonPath & root) : root(root) - , allowedPaths(std::move(allowedPaths)) - , makeNotAllowedError(std::move(makeNotAllowedError)) { + displayPrefix = root.isRoot() ? "" : root.abs(); } void readFile( @@ -26,39 +20,30 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor std::function sizeCallback) override { auto absPath = makeAbsPath(path); - checkAllowed(absPath); PosixSourceAccessor::readFile(absPath, sink, sizeCallback); } bool pathExists(const CanonPath & path) override { - auto absPath = makeAbsPath(path); - return isAllowed(absPath) && PosixSourceAccessor::pathExists(absPath); + return PosixSourceAccessor::pathExists(makeAbsPath(path)); } - Stat lstat(const CanonPath & path) override + std::optional maybeLstat(const CanonPath & path) override { - auto absPath = makeAbsPath(path); - checkAllowed(absPath); - return PosixSourceAccessor::lstat(absPath); + return PosixSourceAccessor::maybeLstat(makeAbsPath(path)); } DirEntries readDirectory(const CanonPath & path) override { - auto absPath = makeAbsPath(path); - checkAllowed(absPath); DirEntries res; - for (auto & entry : PosixSourceAccessor::readDirectory(absPath)) - if (isAllowed(absPath + entry.first)) - res.emplace(entry); + for (auto & entry : PosixSourceAccessor::readDirectory(makeAbsPath(path))) + res.emplace(entry); return res; } std::string readLink(const CanonPath & path) override { - auto absPath = makeAbsPath(path); - checkAllowed(absPath); - return PosixSourceAccessor::readLink(absPath); + return PosixSourceAccessor::readLink(makeAbsPath(path)); } CanonPath makeAbsPath(const CanonPath & path) @@ -66,59 +51,22 @@ struct FSInputAccessorImpl : FSInputAccessor, PosixSourceAccessor return root + path; } - void checkAllowed(const CanonPath & absPath) override - { - if (!isAllowed(absPath)) - throw makeNotAllowedError - ? makeNotAllowedError(absPath) - : RestrictedPathError("access to path '%s' is forbidden", absPath); - } - - bool isAllowed(const CanonPath & absPath) - { - if (!absPath.isWithin(root)) - return false; - - if (allowedPaths) { - auto p = absPath.removePrefix(root); - if (!p.isAllowed(*allowedPaths)) - return false; - } - - return true; - } - - void allowPath(CanonPath path) override - { - if (allowedPaths) - allowedPaths->insert(std::move(path)); - } - - bool hasAccessControl() override - { - return (bool) allowedPaths; - } - std::optional getPhysicalPath(const CanonPath & path) override { return makeAbsPath(path); } }; -ref makeFSInputAccessor( - const CanonPath & root, - std::optional> && allowedPaths, - MakeNotAllowedError && makeNotAllowedError) +ref makeFSInputAccessor(const CanonPath & root) { - return make_ref(root, std::move(allowedPaths), std::move(makeNotAllowedError)); + return make_ref(root); } -ref makeStorePathAccessor( +ref makeStorePathAccessor( ref store, - const StorePath & storePath, - MakeNotAllowedError && makeNotAllowedError) + const StorePath & storePath) { - return makeFSInputAccessor(CanonPath(store->toRealPath(storePath)), {}, std::move(makeNotAllowedError)); + return makeFSInputAccessor(CanonPath(store->toRealPath(storePath))); } SourcePath getUnfilteredRootPath(CanonPath path) diff --git a/src/libfetchers/fs-input-accessor.hh b/src/libfetchers/fs-input-accessor.hh index 19a5211c8..ba5af5887 100644 --- a/src/libfetchers/fs-input-accessor.hh +++ b/src/libfetchers/fs-input-accessor.hh @@ -7,26 +7,12 @@ namespace nix { class StorePath; class Store; -struct FSInputAccessor : InputAccessor -{ - virtual void checkAllowed(const CanonPath & absPath) = 0; +ref makeFSInputAccessor( + const CanonPath & root); - virtual void allowPath(CanonPath path) = 0; - - virtual bool hasAccessControl() = 0; -}; - -typedef std::function MakeNotAllowedError; - -ref makeFSInputAccessor( - const CanonPath & root, - std::optional> && allowedPaths = {}, - MakeNotAllowedError && makeNotAllowedError = {}); - -ref makeStorePathAccessor( +ref makeStorePathAccessor( ref store, - const StorePath & storePath, - MakeNotAllowedError && makeNotAllowedError = {}); + const StorePath & storePath); SourcePath getUnfilteredRootPath(CanonPath path); diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc new file mode 100644 index 000000000..65f7b45ef --- /dev/null +++ b/src/libfetchers/git-utils.cc @@ -0,0 +1,677 @@ +#include "git-utils.hh" +#include "input-accessor.hh" +#include "cache.hh" +#include "finally.hh" +#include "processes.hh" +#include "signals.hh" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace std { + +template<> struct hash +{ + size_t operator()(const git_oid & oid) const + { + return * (size_t *) oid.id; + } +}; + +} + +std::ostream & operator << (std::ostream & str, const git_oid & oid) +{ + str << git_oid_tostr_s(&oid); + return str; +} + +bool operator == (const git_oid & oid1, const git_oid & oid2) +{ + return git_oid_equal(&oid1, &oid2); +} + +namespace nix { + +// Some wrapper types that ensure that the git_*_free functions get called. +template +struct Deleter +{ + template + void operator()(T * p) const { del(p); }; +}; + +typedef std::unique_ptr> Repository; +typedef std::unique_ptr> TreeEntry; +typedef std::unique_ptr> Tree; +typedef std::unique_ptr> TreeBuilder; +typedef std::unique_ptr> Blob; +typedef std::unique_ptr> Object; +typedef std::unique_ptr> Commit; +typedef std::unique_ptr> Reference; +typedef std::unique_ptr> DescribeResult; +typedef std::unique_ptr> StatusList; +typedef std::unique_ptr> Remote; +typedef std::unique_ptr> GitConfig; +typedef std::unique_ptr> ConfigIterator; + +// A helper to ensure that we don't leak objects returned by libgit2. +template +struct Setter +{ + T & t; + typename T::pointer p = nullptr; + + Setter(T & t) : t(t) { } + + ~Setter() { if (p) t = T(p); } + + operator typename T::pointer * () { return &p; } +}; + +Hash toHash(const git_oid & oid) +{ + #ifdef GIT_EXPERIMENTAL_SHA256 + assert(oid.type == GIT_OID_SHA1); + #endif + Hash hash(HashAlgorithm::SHA1); + memcpy(hash.hash, oid.id, hash.hashSize); + return hash; +} + +static void initLibGit2() +{ + if (git_libgit2_init() < 0) + throw Error("initialising libgit2: %s", git_error_last()->message); +} + +git_oid hashToOID(const Hash & hash) +{ + git_oid oid; + if (git_oid_fromstr(&oid, hash.gitRev().c_str())) + throw Error("cannot convert '%s' to a Git OID", hash.gitRev()); + return oid; +} + +Object lookupObject(git_repository * repo, const git_oid & oid) +{ + Object obj; + if (git_object_lookup(Setter(obj), repo, &oid, GIT_OBJECT_ANY)) { + auto err = git_error_last(); + throw Error("getting Git object '%s': %s", oid, err->message); + } + return obj; +} + +template +T peelObject(git_repository * repo, git_object * obj, git_object_t type) +{ + T obj2; + if (git_object_peel((git_object * *) (typename T::pointer *) Setter(obj2), obj, type)) { + auto err = git_error_last(); + throw Error("peeling Git object '%s': %s", git_object_id(obj), err->message); + } + return obj2; +} + +struct GitRepoImpl : GitRepo, std::enable_shared_from_this +{ + CanonPath path; + Repository repo; + + GitRepoImpl(CanonPath _path, bool create, bool bare) + : path(std::move(_path)) + { + initLibGit2(); + + if (pathExists(path.abs())) { + if (git_repository_open(Setter(repo), path.c_str())) + throw Error("opening Git repository '%s': %s", path, git_error_last()->message); + } else { + if (git_repository_init(Setter(repo), path.c_str(), bare)) + throw Error("creating Git repository '%s': %s", path, git_error_last()->message); + } + } + + operator git_repository * () + { + return repo.get(); + } + + uint64_t getRevCount(const Hash & rev) override + { + std::unordered_set done; + std::queue todo; + + todo.push(peelObject(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT)); + + while (auto commit = pop(todo)) { + if (!done.insert(*git_commit_id(commit->get())).second) continue; + + for (size_t n = 0; n < git_commit_parentcount(commit->get()); ++n) { + git_commit * parent; + if (git_commit_parent(&parent, commit->get(), n)) + throw Error("getting parent of Git commit '%s': %s", *git_commit_id(commit->get()), git_error_last()->message); + todo.push(Commit(parent)); + } + } + + return done.size(); + } + + uint64_t getLastModified(const Hash & rev) override + { + auto commit = peelObject(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT); + + return git_commit_time(commit.get()); + } + + bool isShallow() override + { + return git_repository_is_shallow(*this); + } + + Hash resolveRef(std::string ref) override + { + // Handle revisions used as refs. + { + git_oid oid; + if (git_oid_fromstr(&oid, ref.c_str()) == 0) + return toHash(oid); + } + + // Resolve short names like 'master'. + Reference ref2; + if (!git_reference_dwim(Setter(ref2), *this, ref.c_str())) + ref = git_reference_name(ref2.get()); + + // Resolve full references like 'refs/heads/master'. + Reference ref3; + if (git_reference_lookup(Setter(ref3), *this, ref.c_str())) + throw Error("resolving Git reference '%s': %s", ref, git_error_last()->message); + + auto oid = git_reference_target(ref3.get()); + if (!oid) + throw Error("cannot get OID for Git reference '%s'", git_reference_name(ref3.get())); + + return toHash(*oid); + } + + std::vector parseSubmodules(const CanonPath & configFile) + { + GitConfig config; + if (git_config_open_ondisk(Setter(config), configFile.abs().c_str())) + throw Error("parsing .gitmodules file: %s", git_error_last()->message); + + ConfigIterator it; + if (git_config_iterator_glob_new(Setter(it), config.get(), "^submodule\\..*\\.(path|url|branch)$")) + throw Error("iterating over .gitmodules: %s", git_error_last()->message); + + std::map entries; + + while (true) { + git_config_entry * entry = nullptr; + if (auto err = git_config_next(&entry, it.get())) { + if (err == GIT_ITEROVER) break; + throw Error("iterating over .gitmodules: %s", git_error_last()->message); + } + entries.emplace(entry->name + 10, entry->value); + } + + std::vector result; + + for (auto & [key, value] : entries) { + if (!hasSuffix(key, ".path")) continue; + std::string key2(key, 0, key.size() - 5); + auto path = CanonPath(value); + result.push_back(Submodule { + .path = path, + .url = entries[key2 + ".url"], + .branch = entries[key2 + ".branch"], + }); + } + + return result; + } + + // Helper for statusCallback below. + static int statusCallbackTrampoline(const char * path, unsigned int statusFlags, void * payload) + { + return (*((std::function *) payload))(path, statusFlags); + } + + WorkdirInfo getWorkdirInfo() override + { + WorkdirInfo info; + + /* Get the head revision, if any. */ + git_oid headRev; + if (auto err = git_reference_name_to_id(&headRev, *this, "HEAD")) { + if (err != GIT_ENOTFOUND) + throw Error("resolving HEAD: %s", git_error_last()->message); + } else + info.headRev = toHash(headRev); + + /* Get all tracked files and determine whether the working + directory is dirty. */ + std::function statusCallback = [&](const char * path, unsigned int statusFlags) + { + if (!(statusFlags & GIT_STATUS_INDEX_DELETED) && + !(statusFlags & GIT_STATUS_WT_DELETED)) + info.files.insert(CanonPath(path)); + if (statusFlags != GIT_STATUS_CURRENT) + info.isDirty = true; + return 0; + }; + + git_status_options options = GIT_STATUS_OPTIONS_INIT; + options.flags |= GIT_STATUS_OPT_INCLUDE_UNMODIFIED; + options.flags |= GIT_STATUS_OPT_EXCLUDE_SUBMODULES; + if (git_status_foreach_ext(*this, &options, &statusCallbackTrampoline, &statusCallback)) + throw Error("getting working directory status: %s", git_error_last()->message); + + /* Get submodule info. */ + auto modulesFile = path + ".gitmodules"; + if (pathExists(modulesFile.abs())) + info.submodules = parseSubmodules(modulesFile); + + return info; + } + + std::optional getWorkdirRef() override + { + Reference ref; + if (git_reference_lookup(Setter(ref), *this, "HEAD")) + throw Error("looking up HEAD: %s", git_error_last()->message); + + if (auto target = git_reference_symbolic_target(ref.get())) + return target; + + return std::nullopt; + } + + std::vector> getSubmodules(const Hash & rev) override; + + std::string resolveSubmoduleUrl( + const std::string & url, + const std::string & base) override + { + git_buf buf = GIT_BUF_INIT; + if (git_submodule_resolve_url(&buf, *this, url.c_str())) + throw Error("resolving Git submodule URL '%s'", url); + Finally cleanup = [&]() { git_buf_dispose(&buf); }; + + std::string res(buf.ptr); + + if (!hasPrefix(res, "/") && res.find("://") == res.npos) + res = parseURL(base + "/" + res).canonicalise().to_string(); + + return res; + } + + bool hasObject(const Hash & oid_) override + { + auto oid = hashToOID(oid_); + + Object obj; + if (auto errCode = git_object_lookup(Setter(obj), *this, &oid, GIT_OBJECT_ANY)) { + if (errCode == GIT_ENOTFOUND) return false; + auto err = git_error_last(); + throw Error("getting Git object '%s': %s", oid, err->message); + } + + return true; + } + + ref getAccessor(const Hash & rev) override; + + static int sidebandProgressCallback(const char * str, int len, void * payload) + { + auto act = (Activity *) payload; + act->result(resFetchStatus, trim(std::string_view(str, len))); + return _isInterrupted ? -1 : 0; + } + + static int transferProgressCallback(const git_indexer_progress * stats, void * payload) + { + auto act = (Activity *) payload; + act->result(resFetchStatus, + fmt("%d/%d objects received, %d/%d deltas indexed, %.1f MiB", + stats->received_objects, + stats->total_objects, + stats->indexed_deltas, + stats->total_deltas, + stats->received_bytes / (1024.0 * 1024.0))); + return _isInterrupted ? -1 : 0; + } + + void fetch( + const std::string & url, + const std::string & refspec, + bool shallow) override + { + Activity act(*logger, lvlTalkative, actFetchTree, fmt("fetching Git repository '%s'", url)); + + Remote remote; + + if (git_remote_create_anonymous(Setter(remote), *this, url.c_str())) + throw Error("cannot create Git remote '%s': %s", url, git_error_last()->message); + + char * refspecs[] = {(char *) refspec.c_str()}; + git_strarray refspecs2 { + .strings = refspecs, + .count = 1 + }; + + git_fetch_options opts = GIT_FETCH_OPTIONS_INIT; + // FIXME: for some reason, shallow fetching over ssh barfs + // with "could not read from remote repository". + opts.depth = shallow && parseURL(url).scheme != "ssh" ? 1 : GIT_FETCH_DEPTH_FULL; + opts.callbacks.payload = &act; + opts.callbacks.sideband_progress = sidebandProgressCallback; + opts.callbacks.transfer_progress = transferProgressCallback; + + if (git_remote_fetch(remote.get(), &refspecs2, &opts, nullptr)) + throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message); + } + + void verifyCommit( + const Hash & rev, + const std::vector & publicKeys) override + { + // Create ad-hoc allowedSignersFile and populate it with publicKeys + auto allowedSignersFile = createTempFile().second; + std::string allowedSigners; + for (const fetchers::PublicKey & k : publicKeys) { + if (k.type != "ssh-dsa" + && k.type != "ssh-ecdsa" + && k.type != "ssh-ecdsa-sk" + && k.type != "ssh-ed25519" + && k.type != "ssh-ed25519-sk" + && k.type != "ssh-rsa") + throw Error("Unknown key type '%s'.\n" + "Please use one of\n" + "- ssh-dsa\n" + " ssh-ecdsa\n" + " ssh-ecdsa-sk\n" + " ssh-ed25519\n" + " ssh-ed25519-sk\n" + " ssh-rsa", k.type); + allowedSigners += "* " + k.type + " " + k.key + "\n"; + } + writeFile(allowedSignersFile, allowedSigners); + + // Run verification command + auto [status, output] = runProgram(RunOptions { + .program = "git", + .args = { + "-c", + "gpg.ssh.allowedSignersFile=" + allowedSignersFile, + "-C", path.abs(), + "verify-commit", + rev.gitRev() + }, + .mergeStderrToStdout = true, + }); + + /* Evaluate result through status code and checking if public + key fingerprints appear on stderr. This is neccessary + because the git command might also succeed due to the + commit being signed by gpg keys that are present in the + users key agent. */ + std::string re = R"(Good "git" signature for \* with .* key SHA256:[)"; + for (const fetchers::PublicKey & k : publicKeys){ + // Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally + auto fingerprint = trim(hashString(HashAlgorithm::SHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "="); + auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" ); + re += "(" + escaped_fingerprint + ")"; + } + re += "]"; + if (status == 0 && std::regex_search(output, std::regex(re))) + printTalkative("Signature verification on commit %s succeeded.", rev.gitRev()); + else + throw Error("Commit signature verification on commit %s failed: %s", rev.gitRev(), output); + } +}; + +ref GitRepo::openRepo(const CanonPath & path, bool create, bool bare) +{ + return make_ref(path, create, bare); +} + +struct GitInputAccessor : InputAccessor +{ + ref repo; + Tree root; + + GitInputAccessor(ref repo_, const Hash & rev) + : repo(repo_) + , root(peelObject(*repo, lookupObject(*repo, hashToOID(rev)).get(), GIT_OBJECT_TREE)) + { + } + + std::string readBlob(const CanonPath & path, bool symlink) + { + auto blob = getBlob(path, symlink); + + auto data = std::string_view((const char *) git_blob_rawcontent(blob.get()), git_blob_rawsize(blob.get())); + + return std::string(data); + } + + std::string readFile(const CanonPath & path) override + { + return readBlob(path, false); + } + + bool pathExists(const CanonPath & path) override + { + return path.isRoot() ? true : (bool) lookup(path); + } + + std::optional maybeLstat(const CanonPath & path) override + { + if (path.isRoot()) + return Stat { .type = tDirectory }; + + auto entry = lookup(path); + if (!entry) + return std::nullopt; + + auto mode = git_tree_entry_filemode(entry); + + if (mode == GIT_FILEMODE_TREE) + return Stat { .type = tDirectory }; + + else if (mode == GIT_FILEMODE_BLOB) + return Stat { .type = tRegular }; + + else if (mode == GIT_FILEMODE_BLOB_EXECUTABLE) + return Stat { .type = tRegular, .isExecutable = true }; + + else if (mode == GIT_FILEMODE_LINK) + return Stat { .type = tSymlink }; + + else if (mode == GIT_FILEMODE_COMMIT) + // Treat submodules as an empty directory. + return Stat { .type = tDirectory }; + + else + throw Error("file '%s' has an unsupported Git file type"); + } + + DirEntries readDirectory(const CanonPath & path) override + { + return std::visit(overloaded { + [&](Tree tree) { + DirEntries res; + + auto count = git_tree_entrycount(tree.get()); + + for (size_t n = 0; n < count; ++n) { + auto entry = git_tree_entry_byindex(tree.get(), n); + // FIXME: add to cache + res.emplace(std::string(git_tree_entry_name(entry)), DirEntry{}); + } + + return res; + }, + [&](Submodule) { + return DirEntries(); + } + }, getTree(path)); + } + + std::string readLink(const CanonPath & path) override + { + return readBlob(path, true); + } + + Hash getSubmoduleRev(const CanonPath & path) + { + auto entry = need(path); + + if (git_tree_entry_type(entry) != GIT_OBJECT_COMMIT) + throw Error("'%s' is not a submodule", showPath(path)); + + return toHash(*git_tree_entry_id(entry)); + } + + std::unordered_map lookupCache; + + /* Recursively look up 'path' relative to the root. */ + git_tree_entry * lookup(const CanonPath & path) + { + if (path.isRoot()) return nullptr; + + auto i = lookupCache.find(path); + if (i == lookupCache.end()) { + TreeEntry entry; + if (auto err = git_tree_entry_bypath(Setter(entry), root.get(), std::string(path.rel()).c_str())) { + if (err != GIT_ENOTFOUND) + throw Error("looking up '%s': %s", showPath(path), git_error_last()->message); + } + + i = lookupCache.emplace(path, std::move(entry)).first; + } + + return &*i->second; + } + + git_tree_entry * need(const CanonPath & path) + { + auto entry = lookup(path); + if (!entry) + throw Error("'%s' does not exist", showPath(path)); + return entry; + } + + struct Submodule { }; + + std::variant getTree(const CanonPath & path) + { + if (path.isRoot()) { + Tree tree; + if (git_tree_dup(Setter(tree), root.get())) + throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message); + return tree; + } + + auto entry = need(path); + + if (git_tree_entry_type(entry) == GIT_OBJECT_COMMIT) + return Submodule(); + + if (git_tree_entry_type(entry) != GIT_OBJECT_TREE) + throw Error("'%s' is not a directory", showPath(path)); + + Tree tree; + if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(tree), *repo, entry)) + throw Error("looking up directory '%s': %s", showPath(path), git_error_last()->message); + + return tree; + } + + Blob getBlob(const CanonPath & path, bool expectSymlink) + { + auto notExpected = [&]() + { + throw Error( + expectSymlink + ? "'%s' is not a symlink" + : "'%s' is not a regular file", + showPath(path)); + }; + + if (path.isRoot()) notExpected(); + + auto entry = need(path); + + if (git_tree_entry_type(entry) != GIT_OBJECT_BLOB) + notExpected(); + + auto mode = git_tree_entry_filemode(entry); + if (expectSymlink) { + if (mode != GIT_FILEMODE_LINK) + notExpected(); + } else { + if (mode != GIT_FILEMODE_BLOB && mode != GIT_FILEMODE_BLOB_EXECUTABLE) + notExpected(); + } + + Blob blob; + if (git_tree_entry_to_object((git_object * *) (git_blob * *) Setter(blob), *repo, entry)) + throw Error("looking up file '%s': %s", showPath(path), git_error_last()->message); + + return blob; + } +}; + +ref GitRepoImpl::getAccessor(const Hash & rev) +{ + return make_ref(ref(shared_from_this()), rev); +} + +std::vector> GitRepoImpl::getSubmodules(const Hash & rev) +{ + /* Read the .gitmodules files from this revision. */ + CanonPath modulesFile(".gitmodules"); + + auto accessor = getAccessor(rev); + if (!accessor->pathExists(modulesFile)) return {}; + + /* Parse it and get the revision of each submodule. */ + auto configS = accessor->readFile(modulesFile); + + auto [fdTemp, pathTemp] = createTempFile("nix-git-submodules"); + writeFull(fdTemp.get(), configS); + + std::vector> result; + + for (auto & submodule : parseSubmodules(CanonPath(pathTemp))) { + auto rev = accessor.dynamic_pointer_cast()->getSubmoduleRev(submodule.path); + result.push_back({std::move(submodule), rev}); + } + + return result; +} + + +} diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh new file mode 100644 index 000000000..1def82071 --- /dev/null +++ b/src/libfetchers/git-utils.hh @@ -0,0 +1,90 @@ +#pragma once + +#include "input-accessor.hh" + +namespace nix { + +namespace fetchers { struct PublicKey; } + +struct GitRepo +{ + virtual ~GitRepo() + { } + + static ref openRepo(const CanonPath & path, bool create = false, bool bare = false); + + virtual uint64_t getRevCount(const Hash & rev) = 0; + + virtual uint64_t getLastModified(const Hash & rev) = 0; + + virtual bool isShallow() = 0; + + /* Return the commit hash to which a ref points. */ + virtual Hash resolveRef(std::string ref) = 0; + + /** + * Info about a submodule. + */ + struct Submodule + { + CanonPath path; + std::string url; + std::string branch; + }; + + struct WorkdirInfo + { + bool isDirty = false; + + /* The checked out commit, or nullopt if there are no commits + in the repo yet. */ + std::optional headRev; + + /* All files in the working directory that are unchanged, + modified or added, but excluding deleted files. */ + std::set files; + + /* The submodules listed in .gitmodules of this workdir. */ + std::vector submodules; + }; + + virtual WorkdirInfo getWorkdirInfo() = 0; + + /* Get the ref that HEAD points to. */ + virtual std::optional getWorkdirRef() = 0; + + /** + * Return the submodules of this repo at the indicated revision, + * along with the revision of each submodule. + */ + virtual std::vector> getSubmodules(const Hash & rev) = 0; + + virtual std::string resolveSubmoduleUrl( + const std::string & url, + const std::string & base) = 0; + + struct TarballInfo + { + Hash treeHash; + time_t lastModified; + }; + + virtual bool hasObject(const Hash & oid) = 0; + + virtual ref getAccessor(const Hash & rev) = 0; + + virtual void fetch( + const std::string & url, + const std::string & refspec, + bool shallow) = 0; + + /** + * Verify that commit `rev` is signed by one of the keys in + * `publicKeys`. Throw an error if it isn't. + */ + virtual void verifyCommit( + const Hash & rev, + const std::vector & publicKeys) = 0; +}; + +} diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 26b8987d6..5dac66930 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -1,12 +1,19 @@ #include "fetchers.hh" +#include "users.hh" #include "cache.hh" #include "globals.hh" #include "tarfile.hh" #include "store-api.hh" #include "url-parts.hh" #include "pathlocks.hh" -#include "util.hh" +#include "processes.hh" #include "git.hh" +#include "fs-input-accessor.hh" +#include "filtering-input-accessor.hh" +#include "mounted-input-accessor.hh" +#include "git-utils.hh" +#include "logging.hh" +#include "finally.hh" #include "fetch-settings.hh" @@ -46,7 +53,7 @@ bool touchCacheFile(const Path & path, time_t touch_time) Path getCachePath(std::string_view key) { return getCacheDir() + "/nix/gitv3/" + - hashString(htSHA256, key).to_string(HashFormat::Base32, false); + hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false); } // Returns the name of the HEAD branch. @@ -132,126 +139,19 @@ std::optional readHeadCached(const std::string & actualUrl) return std::nullopt; } -bool isNotDotGitDirectory(const Path & path) +std::vector getPublicKeys(const Attrs & attrs) { - return baseNameOf(path) != ".git"; + std::vector publicKeys; + if (attrs.contains("publicKeys")) { + nlohmann::json publicKeysJson = nlohmann::json::parse(getStrAttr(attrs, "publicKeys")); + ensureType(publicKeysJson, nlohmann::json::value_t::array); + publicKeys = publicKeysJson.get>(); + } + if (attrs.contains("publicKey")) + publicKeys.push_back(PublicKey{maybeGetStrAttr(attrs, "keytype").value_or("ssh-ed25519"),getStrAttr(attrs, "publicKey")}); + return publicKeys; } -struct WorkdirInfo -{ - bool clean = false; - bool hasHead = false; -}; - -// Returns whether a git workdir is clean and has commits. -WorkdirInfo getWorkdirInfo(const Input & input, const Path & workdir) -{ - const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); - std::string gitDir(".git"); - - auto env = getEnv(); - // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong - // that way unknown errors can lead to a failure instead of continuing through the wrong code path - env["LC_ALL"] = "C"; - - /* Check whether HEAD points to something that looks like a commit, - since that is the refrence we want to use later on. */ - auto result = runProgram(RunOptions { - .program = "git", - .args = { "-C", workdir, "--git-dir", gitDir, "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" }, - .environment = env, - .mergeStderrToStdout = true - }); - auto exitCode = WEXITSTATUS(result.first); - auto errorMessage = result.second; - - if (errorMessage.find("fatal: not a git repository") != std::string::npos) { - throw Error("'%s' is not a Git repository", workdir); - } else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) { - // indicates that the repo does not have any commits - // we want to proceed and will consider it dirty later - } else if (exitCode != 0) { - // any other errors should lead to a failure - throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", workdir, exitCode, errorMessage); - } - - bool clean = false; - bool hasHead = exitCode == 0; - - try { - if (hasHead) { - // Using git diff is preferrable over lower-level operations here, - // because its conceptually simpler and we only need the exit code anyways. - auto gitDiffOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "diff", "HEAD", "--quiet"}); - if (!submodules) { - // Changes in submodules should only make the tree dirty - // when those submodules will be copied as well. - gitDiffOpts.emplace_back("--ignore-submodules"); - } - gitDiffOpts.emplace_back("--"); - runProgram("git", true, gitDiffOpts); - - clean = true; - } - } catch (ExecError & e) { - if (!WIFEXITED(e.status) || WEXITSTATUS(e.status) != 1) throw; - } - - return WorkdirInfo { .clean = clean, .hasHead = hasHead }; -} - -std::pair fetchFromWorkdir(ref store, Input & input, const Path & workdir, const WorkdirInfo & workdirInfo) -{ - const bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); - auto gitDir = ".git"; - - if (!fetchSettings.allowDirty) - throw Error("Git tree '%s' is dirty", workdir); - - if (fetchSettings.warnDirty) - warn("Git tree '%s' is dirty", workdir); - - auto gitOpts = Strings({ "-C", workdir, "--git-dir", gitDir, "ls-files", "-z" }); - if (submodules) - gitOpts.emplace_back("--recurse-submodules"); - - auto files = tokenizeString>( - runProgram("git", true, gitOpts), "\0"s); - - Path actualPath(absPath(workdir)); - - PathFilter filter = [&](const Path & p) -> bool { - assert(hasPrefix(p, actualPath)); - std::string file(p, actualPath.size() + 1); - - auto st = lstat(p); - - if (S_ISDIR(st.st_mode)) { - auto prefix = file + "/"; - auto i = files.lower_bound(prefix); - return i != files.end() && hasPrefix(*i, prefix); - } - - return files.count(file); - }; - - auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); - - // FIXME: maybe we should use the timestamp of the last - // modified dirty file? - input.attrs.insert_or_assign( - "lastModified", - workdirInfo.hasHead ? std::stoull(runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0); - - if (workdirInfo.hasHead) { - input.attrs.insert_or_assign("dirtyRev", chomp( - runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "HEAD" })) + "-dirty"); - input.attrs.insert_or_assign("dirtyShortRev", chomp( - runProgram("git", true, { "-C", actualPath, "--git-dir", gitDir, "rev-parse", "--verify", "--short", "HEAD" })) + "-dirty"); - } - - return {std::move(storePath), input}; -} } // end namespace struct GitInputScheme : InputScheme @@ -272,9 +172,9 @@ struct GitInputScheme : InputScheme attrs.emplace("type", "git"); for (auto & [name, value] : url.query) { - if (name == "rev" || name == "ref") + if (name == "rev" || name == "ref" || name == "keytype" || name == "publicKey" || name == "publicKeys") attrs.emplace(name, value); - else if (name == "shallow" || name == "submodules" || name == "allRefs") + else if (name == "shallow" || name == "submodules" || name == "allRefs" || name == "verifyCommit") attrs.emplace(name, Explicit { value == "1" }); else url2.query.emplace(name, value); @@ -285,17 +185,44 @@ struct GitInputScheme : InputScheme return inputFromAttrs(attrs); } + + std::string_view schemeName() const override + { + return "git"; + } + + StringSet allowedAttrs() const override + { + return { + "url", + "ref", + "rev", + "shallow", + "submodules", + "lastModified", + "revCount", + "narHash", + "allRefs", + "name", + "dirtyRev", + "dirtyShortRev", + "verifyCommit", + "keytype", + "publicKey", + "publicKeys", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override { - if (maybeGetStrAttr(attrs, "type") != "git") return {}; + for (auto & [name, _] : attrs) + if (name == "verifyCommit" + || name == "keytype" + || name == "publicKey" + || name == "publicKeys") + experimentalFeatureSettings.require(Xp::VerifiedFetches); - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name" && name != "dirtyRev" && name != "dirtyShortRev") - throw Error("unsupported Git input attribute '%s'", name); - - maybeGetBoolAttr(attrs, "shallow"); - maybeGetBoolAttr(attrs, "submodules"); - maybeGetBoolAttr(attrs, "allRefs"); + maybeGetBoolAttr(attrs, "verifyCommit"); if (auto ref = maybeGetStrAttr(attrs, "ref")) { if (std::regex_search(*ref, badGitRefRegex)) @@ -307,6 +234,9 @@ struct GitInputScheme : InputScheme auto url = fixGitURL(getStrAttr(attrs, "url")); parseURL(url); input.attrs["url"] = url; + getShallowAttr(input); + getSubmodulesAttr(input); + getAllRefsAttr(input); return input; } @@ -316,8 +246,19 @@ struct GitInputScheme : InputScheme if (url.scheme != "git") url.scheme = "git+" + url.scheme; if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev()); if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref); - if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false)) + if (getShallowAttr(input)) url.query.insert_or_assign("shallow", "1"); + if (getSubmodulesAttr(input)) + url.query.insert_or_assign("submodules", "1"); + if (maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(false)) + url.query.insert_or_assign("verifyCommit", "1"); + auto publicKeys = getPublicKeys(input.attrs); + if (publicKeys.size() == 1) { + url.query.insert_or_assign("keytype", publicKeys.at(0).type); + url.query.insert_or_assign("publicKey", publicKeys.at(0).key); + } + else if (publicKeys.size() > 1) + url.query.insert_or_assign("publicKeys", publicKeys_to_string(publicKeys)); return url; } @@ -336,11 +277,11 @@ struct GitInputScheme : InputScheme void clone(const Input & input, const Path & destDir) const override { - auto [isLocal, actualUrl] = getActualUrl(input); + auto repoInfo = getRepoInfo(input); Strings args = {"clone"}; - args.push_back(actualUrl); + args.push_back(repoInfo.url); if (auto ref = input.getRef()) { args.push_back("--branch"); @@ -354,30 +295,90 @@ struct GitInputScheme : InputScheme runProgram("git", true, args, {}, true); } - std::optional getSourcePath(const Input & input) override + std::optional getSourcePath(const Input & input) const override { - auto url = parseURL(getStrAttr(input.attrs, "url")); - if (url.scheme == "file" && !input.getRef() && !input.getRev()) - return url.path; - return {}; + auto repoInfo = getRepoInfo(input); + if (repoInfo.isLocal) return repoInfo.url; + return std::nullopt; } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - auto sourcePath = getSourcePath(input); - assert(sourcePath); - auto gitDir = ".git"; + auto repoInfo = getRepoInfo(input); + if (!repoInfo.isLocal) + throw Error("cannot commit '%s' to Git repository '%s' because it's not a working tree", path, input.to_string()); + + writeFile((CanonPath(repoInfo.url) + path).abs(), contents); runProgram("git", true, - { "-C", *sourcePath, "--git-dir", gitDir, "add", "--intent-to-add", "--", std::string(file) }); + { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "add", "--intent-to-add", "--", std::string(path.rel()) }); + // Pause the logger to allow for user input (such as a gpg passphrase) in `git commit` + logger->pause(); + Finally restoreLogger([]() { logger->resume(); }); if (commitMsg) runProgram("git", true, - { "-C", *sourcePath, "--git-dir", gitDir, "commit", std::string(file), "-m", *commitMsg }); + { "-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "commit", std::string(path.rel()), "-m", *commitMsg }); } - std::pair getActualUrl(const Input & input) const + struct RepoInfo { + /* Whether this is a local, non-bare repository. */ + bool isLocal = false; + + /* Working directory info: the complete list of files, and + whether the working directory is dirty compared to HEAD. */ + GitRepo::WorkdirInfo workdirInfo; + + /* URL of the repo, or its path if isLocal. Never a `file` URL. */ + std::string url; + + void warnDirty() const + { + if (workdirInfo.isDirty) { + if (!fetchSettings.allowDirty) + throw Error("Git tree '%s' is dirty", url); + + if (fetchSettings.warnDirty) + warn("Git tree '%s' is dirty", url); + } + } + + std::string gitDir = ".git"; + }; + + bool getShallowAttr(const Input & input) const + { + return maybeGetBoolAttr(input.attrs, "shallow").value_or(false); + } + + bool getSubmodulesAttr(const Input & input) const + { + return maybeGetBoolAttr(input.attrs, "submodules").value_or(false); + } + + bool getAllRefsAttr(const Input & input) const + { + return maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); + } + + RepoInfo getRepoInfo(const Input & input) const + { + auto checkHashType = [&](const std::optional & hash) + { + if (hash.has_value() && !(hash->algo == HashAlgorithm::SHA1 || hash->algo == HashAlgorithm::SHA256)) + throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true)); + }; + + if (auto rev = input.getRev()) + checkHashType(rev); + + RepoInfo repoInfo; + // file:// URIs are normally not cloned (but otherwise treated the // same as remote URIs, i.e. we don't use the working tree or // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git @@ -385,153 +386,132 @@ struct GitInputScheme : InputScheme static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing auto url = parseURL(getStrAttr(input.attrs, "url")); bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git"); - bool isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; - return {isLocal, isLocal ? url.path : url.base}; + repoInfo.isLocal = url.scheme == "file" && !forceHttp && !isBareRepository; + repoInfo.url = repoInfo.isLocal ? url.path : url.base; + + // If this is a local directory and no ref or revision is + // given, then allow the use of an unclean working tree. + if (!input.getRef() && !input.getRev() && repoInfo.isLocal) + repoInfo.workdirInfo = GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirInfo(); + + return repoInfo; } - std::pair fetch(ref store, const Input & _input) override + uint64_t getLastModified(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const { - Input input(_input); - auto gitDir = ".git"; + Attrs key{{"_what", "gitLastModified"}, {"rev", rev.gitRev()}}; + + auto cache = getCache(); + + if (auto res = cache->lookup(key)) + return getIntAttr(*res, "lastModified"); + + auto lastModified = GitRepo::openRepo(CanonPath(repoDir))->getLastModified(rev); + + cache->upsert(key, Attrs{{"lastModified", lastModified}}); + + return lastModified; + } + + uint64_t getRevCount(const RepoInfo & repoInfo, const std::string & repoDir, const Hash & rev) const + { + Attrs key{{"_what", "gitRevCount"}, {"rev", rev.gitRev()}}; + + auto cache = getCache(); + + if (auto revCountAttrs = cache->lookup(key)) + return getIntAttr(*revCountAttrs, "revCount"); + + Activity act(*logger, lvlChatty, actUnknown, fmt("getting Git revision count of '%s'", repoInfo.url)); + + auto revCount = GitRepo::openRepo(CanonPath(repoDir))->getRevCount(rev); + + cache->upsert(key, Attrs{{"revCount", revCount}}); + + return revCount; + } + + std::string getDefaultRef(const RepoInfo & repoInfo) const + { + auto head = repoInfo.isLocal + ? GitRepo::openRepo(CanonPath(repoInfo.url))->getWorkdirRef() + : readHeadCached(repoInfo.url); + if (!head) { + warn("could not read HEAD ref from repo at '%s', using 'master'", repoInfo.url); + return "master"; + } + return *head; + } + + static MakeNotAllowedError makeNotAllowedError(std::string url) + { + return [url{std::move(url)}](const CanonPath & path) -> RestrictedPathError + { + if (nix::pathExists(path.abs())) + return RestrictedPathError("access to path '%s' is forbidden because it is not under Git control; maybe you should 'git add' it to the repository '%s'?", path, url); + else + return RestrictedPathError("path '%s' does not exist in Git repository '%s'", path, url); + }; + } + + void verifyCommit(const Input & input, std::shared_ptr repo) const + { + auto publicKeys = getPublicKeys(input.attrs); + auto verifyCommit = maybeGetBoolAttr(input.attrs, "verifyCommit").value_or(!publicKeys.empty()); + + if (verifyCommit) { + if (input.getRev() && repo) + repo->verifyCommit(*input.getRev(), publicKeys); + else + throw Error("commit verification is required for Git repository '%s', but it's dirty", input.to_string()); + } + } + + std::pair, Input> getAccessorFromCommit( + ref store, + RepoInfo & repoInfo, + Input && input) const + { + assert(!repoInfo.workdirInfo.isDirty); + + auto origRev = input.getRev(); std::string name = input.getName(); - bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); - bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); - bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); - - std::string cacheType = "git"; - if (shallow) cacheType += "-shallow"; - if (submodules) cacheType += "-submodules"; - if (allRefs) cacheType += "-all-refs"; - - auto checkHashType = [&](const std::optional & hash) - { - if (hash.has_value() && !(hash->type == htSHA1 || hash->type == htSHA256)) - throw Error("Hash '%s' is not supported by Git. Supported types are sha1 and sha256.", hash->to_string(HashFormat::Base16, true)); - }; - - auto getLockedAttrs = [&]() - { - checkHashType(input.getRev()); - - return Attrs({ - {"type", cacheType}, - {"name", name}, - {"rev", input.getRev()->gitRev()}, - }); - }; - - auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath) - -> std::pair - { - assert(input.getRev()); - assert(!_input.getRev() || _input.getRev() == input.getRev()); - if (!shallow) - input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); - input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); - return {std::move(storePath), input}; - }; - - if (input.getRev()) { - if (auto res = getCache()->lookup(store, getLockedAttrs())) - return makeResult(res->first, std::move(res->second)); - } - - auto [isLocal, actualUrl_] = getActualUrl(input); - auto actualUrl = actualUrl_; // work around clang bug - - /* If this is a local directory and no ref or revision is given, - allow fetching directly from a dirty workdir. */ - if (!input.getRef() && !input.getRev() && isLocal) { - auto workdirInfo = getWorkdirInfo(input, actualUrl); - if (!workdirInfo.clean) { - return fetchFromWorkdir(store, input, actualUrl, workdirInfo); - } - } - - Attrs unlockedAttrs({ - {"type", cacheType}, - {"name", name}, - {"url", actualUrl}, - }); + auto originalRef = input.getRef(); + auto ref = originalRef ? *originalRef : getDefaultRef(repoInfo); + input.attrs.insert_or_assign("ref", ref); Path repoDir; - if (isLocal) { - if (!input.getRef()) { - auto head = readHead(actualUrl); - if (!head) { - warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl); - head = "master"; - } - input.attrs.insert_or_assign("ref", *head); - unlockedAttrs.insert_or_assign("ref", *head); - } - + if (repoInfo.isLocal) { + repoDir = repoInfo.url; if (!input.getRev()) - input.attrs.insert_or_assign("rev", - Hash::parseAny(chomp(runProgram("git", true, { "-C", actualUrl, "--git-dir", gitDir, "rev-parse", *input.getRef() })), htSHA1).gitRev()); - - repoDir = actualUrl; + input.attrs.insert_or_assign("rev", GitRepo::openRepo(CanonPath(repoDir))->resolveRef(ref).gitRev()); } else { - const bool useHeadRef = !input.getRef(); - if (useHeadRef) { - auto head = readHeadCached(actualUrl); - if (!head) { - warn("could not read HEAD ref from repo at '%s', using 'master'", actualUrl); - head = "master"; - } - input.attrs.insert_or_assign("ref", *head); - unlockedAttrs.insert_or_assign("ref", *head); - } else { - if (!input.getRev()) { - unlockedAttrs.insert_or_assign("ref", input.getRef().value()); - } - } - - if (auto res = getCache()->lookup(store, unlockedAttrs)) { - auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); - if (!input.getRev() || input.getRev() == rev2) { - input.attrs.insert_or_assign("rev", rev2.gitRev()); - return makeResult(res->first, std::move(res->second)); - } - } - - Path cacheDir = getCachePath(actualUrl); + Path cacheDir = getCachePath(repoInfo.url); repoDir = cacheDir; - gitDir = "."; + repoInfo.gitDir = "."; createDirs(dirOf(cacheDir)); - PathLocks cacheDirLock({cacheDir + ".lock"}); + PathLocks cacheDirLock({cacheDir}); - if (!pathExists(cacheDir)) { - runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir }); - } + auto repo = GitRepo::openRepo(CanonPath(cacheDir), true, true); Path localRefFile = - input.getRef()->compare(0, 5, "refs/") == 0 - ? cacheDir + "/" + *input.getRef() - : cacheDir + "/refs/heads/" + *input.getRef(); + ref.compare(0, 5, "refs/") == 0 + ? cacheDir + "/" + ref + : cacheDir + "/refs/heads/" + ref; bool doFetch; time_t now = time(0); /* If a rev was specified, we need to fetch if it's not in the repo. */ - if (input.getRev()) { - try { - runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "cat-file", "-e", input.getRev()->gitRev() }); - doFetch = false; - } catch (ExecError & e) { - if (WIFEXITED(e.status)) { - doFetch = true; - } else { - throw; - } - } + if (auto rev = input.getRev()) { + doFetch = !repo->hasObject(*rev); } else { - if (allRefs) { + if (getAllRefsAttr(input)) { doFetch = true; } else { /* If the local ref is older than ‘tarball-ttl’ seconds, do a @@ -543,160 +523,213 @@ struct GitInputScheme : InputScheme } if (doFetch) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Git repository '%s'", actualUrl)); - - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. try { - auto ref = input.getRef(); - auto fetchRef = allRefs + auto fetchRef = + getAllRefsAttr(input) ? "refs/*" - : ref->compare(0, 5, "refs/") == 0 - ? *ref - : ref == "HEAD" - ? *ref - : "refs/heads/" + *ref; - runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }, {}, true); + : input.getRev() + ? input.getRev()->gitRev() + : ref.compare(0, 5, "refs/") == 0 + ? ref + : ref == "HEAD" + ? ref + : "refs/heads/" + ref; + + repo->fetch(repoInfo.url, fmt("%s:%s", fetchRef, fetchRef), getShallowAttr(input)); } catch (Error & e) { if (!pathExists(localRefFile)) throw; - warn("could not update local clone of Git repository '%s'; continuing with the most recent version", actualUrl); + logError(e.info()); + warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.url); } if (!touchCacheFile(localRefFile, now)) warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno)); - if (useHeadRef && !storeCachedHead(actualUrl, *input.getRef())) - warn("could not update cached head '%s' for '%s'", *input.getRef(), actualUrl); + if (!originalRef && !storeCachedHead(repoInfo.url, ref)) + warn("could not update cached head '%s' for '%s'", ref, repoInfo.url); } - if (!input.getRev()) - input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev()); + if (auto rev = input.getRev()) { + if (!repo->hasObject(*rev)) + throw Error( + "Cannot find Git revision '%s' in ref '%s' of repository '%s'! " + "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " + ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD + "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", + rev->gitRev(), + ref, + repoInfo.url + ); + } else + input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), HashAlgorithm::SHA1).gitRev()); // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder } - bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-parse", "--is-shallow-repository" })) == "true"; + auto repo = GitRepo::openRepo(CanonPath(repoDir)); - if (isShallow && !shallow) - throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified.", actualUrl); + auto isShallow = repo->isShallow(); - // FIXME: check whether rev is an ancestor of ref. + if (isShallow && !getShallowAttr(input)) + throw Error("'%s' is a shallow Git repository, but shallow repositories are only allowed when `shallow = true;` is specified", repoInfo.url); - printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl); + // FIXME: check whether rev is an ancestor of ref? - /* Now that we know the ref, check again whether we have it in - the store. */ - if (auto res = getCache()->lookup(store, getLockedAttrs())) - return makeResult(res->first, std::move(res->second)); - - Path tmpDir = createTempDir(); - AutoDelete delTmpDir(tmpDir, true); - PathFilter filter = defaultPathFilter; - - auto result = runProgram(RunOptions { - .program = "git", - .args = { "-C", repoDir, "--git-dir", gitDir, "cat-file", "commit", input.getRev()->gitRev() }, - .mergeStderrToStdout = true - }); - if (WEXITSTATUS(result.first) == 128 - && result.second.find("bad file") != std::string::npos) - { - throw Error( - "Cannot find Git revision '%s' in ref '%s' of repository '%s'! " - "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " - ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD - "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", - input.getRev()->gitRev(), - *input.getRef(), - actualUrl - ); - } - - if (submodules) { - Path tmpGitDir = createTempDir(); - AutoDelete delTmpGitDir(tmpGitDir, true); - - runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", tmpDir, "--separate-git-dir", tmpGitDir }); - - { - // TODO: repoDir might lack the ref (it only checks if rev - // exists, see FIXME above) so use a big hammer and fetch - // everything to ensure we get the rev. - Activity act(*logger, lvlTalkative, actUnknown, fmt("making temporary clone of '%s'", repoDir)); - runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force", - "--update-head-ok", "--", repoDir, "refs/*:refs/*" }, {}, true); - } - - runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() }); - - /* Ensure that we use the correct origin for fetching - submodules. This matters for submodules with relative - URLs. */ - if (isLocal) { - writeFile(tmpGitDir + "/config", readFile(repoDir + "/" + gitDir + "/config")); - - /* Restore the config.bare setting we may have just - copied erroneously from the user's repo. */ - runProgram("git", true, { "-C", tmpDir, "config", "core.bare", "false" }); - } else - runProgram("git", true, { "-C", tmpDir, "config", "remote.origin.url", actualUrl }); - - /* As an optimisation, copy the modules directory of the - source repo if it exists. */ - auto modulesPath = repoDir + "/" + gitDir + "/modules"; - if (pathExists(modulesPath)) { - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying submodules of '%s'", actualUrl)); - runProgram("cp", true, { "-R", "--", modulesPath, tmpGitDir + "/modules" }); - } - - { - Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching submodules of '%s'", actualUrl)); - runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" }, {}, true); - } - - filter = isNotDotGitDirectory; - } else { - // FIXME: should pipe this, or find some better way to extract a - // revision. - auto source = sinkToSource([&](Sink & sink) { - runProgram2({ - .program = "git", - .args = { "-C", repoDir, "--git-dir", gitDir, "archive", input.getRev()->gitRev() }, - .standardOut = &sink - }); - }); - - unpackTarfile(*source, tmpDir); - } - - auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter); - - auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "log", "-1", "--format=%ct", "--no-show-signature", input.getRev()->gitRev() })); + auto rev = *input.getRev(); Attrs infoAttrs({ - {"rev", input.getRev()->gitRev()}, - {"lastModified", lastModified}, + {"rev", rev.gitRev()}, + {"lastModified", getLastModified(repoInfo, repoDir, rev)}, }); - if (!shallow) + if (!getShallowAttr(input)) infoAttrs.insert_or_assign("revCount", - std::stoull(runProgram("git", true, { "-C", repoDir, "--git-dir", gitDir, "rev-list", "--count", input.getRev()->gitRev() }))); + getRevCount(repoInfo, repoDir, rev)); - if (!_input.getRev()) - getCache()->add( - store, - unlockedAttrs, - infoAttrs, - storePath, - false); + printTalkative("using revision %s of repo '%s'", rev.gitRev(), repoInfo.url); - getCache()->add( - store, - getLockedAttrs(), - infoAttrs, - storePath, - true); + verifyCommit(input, repo); - return makeResult(infoAttrs, std::move(storePath)); + auto accessor = repo->getAccessor(rev); + + accessor->setPathDisplay("«" + input.to_string() + "»"); + + /* If the repo has submodules, fetch them and return a mounted + input accessor consisting of the accessor for the top-level + repo and the accessors for the submodules. */ + if (getSubmodulesAttr(input)) { + std::map> mounts; + + for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev)) { + auto resolved = repo->resolveSubmoduleUrl(submodule.url, repoInfo.url); + debug("Git submodule %s: %s %s %s -> %s", + submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved); + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "git"); + attrs.insert_or_assign("url", resolved); + if (submodule.branch != "") + attrs.insert_or_assign("ref", submodule.branch); + attrs.insert_or_assign("rev", submoduleRev.gitRev()); + auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); + auto [submoduleAccessor, submoduleInput2] = + submoduleInput.getAccessor(store); + mounts.insert_or_assign(submodule.path, submoduleAccessor); + } + + if (!mounts.empty()) { + mounts.insert_or_assign(CanonPath::root, accessor); + accessor = makeMountedInputAccessor(std::move(mounts)); + } + } + + assert(!origRev || origRev == rev); + if (!getShallowAttr(input)) + input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount")); + input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified")); + + return {accessor, std::move(input)}; + } + + std::pair, Input> getAccessorFromWorkdir( + ref store, + RepoInfo & repoInfo, + Input && input) const + { + if (getSubmodulesAttr(input)) + /* Create mountpoints for the submodules. */ + for (auto & submodule : repoInfo.workdirInfo.submodules) + repoInfo.workdirInfo.files.insert(submodule.path); + + ref accessor = + AllowListInputAccessor::create( + makeFSInputAccessor(CanonPath(repoInfo.url)), + std::move(repoInfo.workdirInfo.files), + makeNotAllowedError(repoInfo.url)); + + /* If the repo has submodules, return a mounted input accessor + consisting of the accessor for the top-level repo and the + accessors for the submodule workdirs. */ + if (getSubmodulesAttr(input) && !repoInfo.workdirInfo.submodules.empty()) { + std::map> mounts; + + for (auto & submodule : repoInfo.workdirInfo.submodules) { + auto submodulePath = CanonPath(repoInfo.url) + submodule.path; + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "git"); + attrs.insert_or_assign("url", submodulePath.abs()); + auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); + auto [submoduleAccessor, submoduleInput2] = + submoduleInput.getAccessor(store); + + /* If the submodule is dirty, mark this repo dirty as + well. */ + if (!submoduleInput2.getRev()) + repoInfo.workdirInfo.isDirty = true; + + mounts.insert_or_assign(submodule.path, submoduleAccessor); + } + + mounts.insert_or_assign(CanonPath::root, accessor); + accessor = makeMountedInputAccessor(std::move(mounts)); + } + + if (!repoInfo.workdirInfo.isDirty) { + auto repo = GitRepo::openRepo(CanonPath(repoInfo.url)); + + if (auto ref = repo->getWorkdirRef()) + input.attrs.insert_or_assign("ref", *ref); + + auto rev = repoInfo.workdirInfo.headRev.value(); + + input.attrs.insert_or_assign("rev", rev.gitRev()); + input.attrs.insert_or_assign("revCount", getRevCount(repoInfo, repoInfo.url, rev)); + + verifyCommit(input, repo); + } else { + repoInfo.warnDirty(); + + if (repoInfo.workdirInfo.headRev) { + input.attrs.insert_or_assign("dirtyRev", + repoInfo.workdirInfo.headRev->gitRev() + "-dirty"); + input.attrs.insert_or_assign("dirtyShortRev", + repoInfo.workdirInfo.headRev->gitShortRev() + "-dirty"); + } + + verifyCommit(input, nullptr); + } + + input.attrs.insert_or_assign( + "lastModified", + repoInfo.workdirInfo.headRev + ? getLastModified(repoInfo, repoInfo.url, *repoInfo.workdirInfo.headRev) + : 0); + + input.locked = true; // FIXME + + return {accessor, std::move(input)}; + } + + std::pair, Input> getAccessor(ref store, const Input & _input) const override + { + Input input(_input); + + auto repoInfo = getRepoInfo(input); + + auto [accessor, final] = + input.getRef() || input.getRev() || !repoInfo.isLocal + ? getAccessorFromCommit(store, repoInfo, std::move(input)) + : getAccessorFromWorkdir(store, repoInfo, std::move(input)); + + accessor->fingerprint = final.getFingerprint(store); + + return {accessor, std::move(final)}; + } + + std::optional getFingerprint(ref store, const Input & input) const override + { + if (auto rev = input.getRev()) + return rev->gitRev() + (getSubmodulesAttr(input) ? ";s" : ""); + else + return std::nullopt; } }; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 617fc7468..70acb9354 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -27,13 +27,11 @@ std::regex hostRegex(hostRegexS, std::regex::ECMAScript); struct GitArchiveInputScheme : InputScheme { - virtual std::string type() const = 0; - virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override { - if (url.scheme != type()) return {}; + if (url.scheme != schemeName()) return {}; auto path = tokenizeString>(url.path, "/"); @@ -44,7 +42,7 @@ struct GitArchiveInputScheme : InputScheme auto size = path.size(); if (size == 3) { if (std::regex_match(path[2], revRegex)) - rev = Hash::parseAny(path[2], htSHA1); + rev = Hash::parseAny(path[2], HashAlgorithm::SHA1); else if (std::regex_match(path[2], refRegex)) ref = path[2]; else @@ -70,7 +68,7 @@ struct GitArchiveInputScheme : InputScheme if (name == "rev") { if (rev) throw BadURL("URL '%s' contains multiple commit hashes", url.url); - rev = Hash::parseAny(value, htSHA1); + rev = Hash::parseAny(value, HashAlgorithm::SHA1); } else if (name == "ref") { if (!std::regex_match(value, refRegex)) @@ -91,7 +89,7 @@ struct GitArchiveInputScheme : InputScheme throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev()); Input input; - input.attrs.insert_or_assign("type", type()); + input.attrs.insert_or_assign("type", std::string { schemeName() }); input.attrs.insert_or_assign("owner", path[0]); input.attrs.insert_or_assign("repo", path[1]); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); @@ -101,14 +99,21 @@ struct GitArchiveInputScheme : InputScheme return input; } + StringSet allowedAttrs() const override + { + return { + "owner", + "repo", + "ref", + "rev", + "narHash", + "lastModified", + "host", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override { - if (maybeGetStrAttr(attrs, "type") != type()) return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified" && name != "host") - throw Error("unsupported input attribute '%s'", name); - getStrAttr(attrs, "owner"); getStrAttr(attrs, "repo"); @@ -128,7 +133,7 @@ struct GitArchiveInputScheme : InputScheme if (ref) path += "/" + *ref; if (rev) path += "/" + rev->to_string(HashFormat::Base16, false); return ParsedURL { - .scheme = type(), + .scheme = std::string { schemeName() }, .path = path, }; } @@ -220,15 +225,23 @@ struct GitArchiveInputScheme : InputScheme return {result.storePath, input}; } - std::optional experimentalFeature() override + std::optional experimentalFeature() const override { return Xp::Flakes; } + + std::optional getFingerprint(ref store, const Input & input) const override + { + if (auto rev = input.getRev()) + return rev->gitRev(); + else + return std::nullopt; + } }; struct GitHubInputScheme : GitArchiveInputScheme { - std::string type() const override { return "github"; } + std::string_view schemeName() const override { return "github"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -271,7 +284,7 @@ struct GitHubInputScheme : GitArchiveInputScheme readFile( store->toRealPath( downloadFile(store, url, "source", false, headers).storePath))); - auto rev = Hash::parseAny(std::string { json["sha"] }, htSHA1); + auto rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); return rev; } @@ -309,7 +322,7 @@ struct GitHubInputScheme : GitArchiveInputScheme struct GitLabInputScheme : GitArchiveInputScheme { - std::string type() const override { return "gitlab"; } + std::string_view schemeName() const override { return "gitlab"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -343,7 +356,7 @@ struct GitLabInputScheme : GitArchiveInputScheme readFile( store->toRealPath( downloadFile(store, url, "source", false, headers).storePath))); - auto rev = Hash::parseAny(std::string(json[0]["id"]), htSHA1); + auto rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1); debug("HEAD revision for '%s' is %s", url, rev.gitRev()); return rev; } @@ -377,7 +390,7 @@ struct GitLabInputScheme : GitArchiveInputScheme struct SourceHutInputScheme : GitArchiveInputScheme { - std::string type() const override { return "sourcehut"; } + std::string_view schemeName() const override { return "sourcehut"; } std::optional> accessHeaderFromToken(const std::string & token) const override { @@ -435,7 +448,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme if(!id) throw BadURL("in '%d', couldn't find ref '%d'", input.to_string(), ref); - auto rev = Hash::parseAny(*id, htSHA1); + auto rev = Hash::parseAny(*id, HashAlgorithm::SHA1); debug("HEAD revision for '%s' is %s", fmt("%s/%s", base_url, ref), rev.gitRev()); return rev; } diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 9a71df3d4..002c0c292 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -1,5 +1,6 @@ #include "fetchers.hh" #include "url-parts.hh" +#include "path.hh" namespace nix::fetchers { @@ -19,7 +20,7 @@ struct IndirectInputScheme : InputScheme if (path.size() == 1) { } else if (path.size() == 2) { if (std::regex_match(path[1], revRegex)) - rev = Hash::parseAny(path[1], htSHA1); + rev = Hash::parseAny(path[1], HashAlgorithm::SHA1); else if (std::regex_match(path[1], refRegex)) ref = path[1]; else @@ -30,7 +31,7 @@ struct IndirectInputScheme : InputScheme ref = path[1]; if (!std::regex_match(path[2], revRegex)) throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); - rev = Hash::parseAny(path[2], htSHA1); + rev = Hash::parseAny(path[2], HashAlgorithm::SHA1); } else throw BadURL("GitHub URL '%s' is invalid", url.url); @@ -49,14 +50,23 @@ struct IndirectInputScheme : InputScheme return input; } + std::string_view schemeName() const override + { + return "indirect"; + } + + StringSet allowedAttrs() const override + { + return { + "id", + "ref", + "rev", + "narHash", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override { - if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash") - throw Error("unsupported indirect input attribute '%s'", name); - auto id = getStrAttr(attrs, "id"); if (!std::regex_match(id, flakeRegex)) throw BadURL("'%s' is not a valid flake ID", id); @@ -92,7 +102,7 @@ struct IndirectInputScheme : InputScheme throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); } - std::optional experimentalFeature() override + std::optional experimentalFeature() const override { return Xp::Flakes; } diff --git a/src/libfetchers/input-accessor.cc b/src/libfetchers/input-accessor.cc index d1d450cf7..1f793bf1d 100644 --- a/src/libfetchers/input-accessor.cc +++ b/src/libfetchers/input-accessor.cc @@ -1,5 +1,6 @@ #include "input-accessor.hh" #include "store-api.hh" +#include "cache.hh" namespace nix { @@ -11,6 +12,27 @@ StorePath InputAccessor::fetchToStore( PathFilter * filter, RepairFlag repair) { + // FIXME: add an optimisation for the case where the accessor is + // an FSInputAccessor pointing to a store path. + + std::optional cacheKey; + + if (!filter && fingerprint) { + cacheKey = fetchers::Attrs{ + {"_what", "fetchToStore"}, + {"store", store->storeDir}, + {"name", std::string(name)}, + {"fingerprint", *fingerprint}, + {"method", (uint8_t) method}, + {"path", path.abs()} + }; + if (auto res = fetchers::getCache()->lookup(store, *cacheKey)) { + debug("store path cache hit for '%s'", showPath(path)); + return res->second; + } + } else + debug("source path '%s' is uncacheable", showPath(path)); + Activity act(*logger, lvlChatty, actUnknown, fmt("copying '%s' to the store", showPath(path))); auto source = sinkToSource([&](Sink & sink) { @@ -22,17 +44,15 @@ StorePath InputAccessor::fetchToStore( auto storePath = settings.readOnlyMode - ? store->computeStorePathFromDump(*source, name, method, htSHA256).first - : store->addToStoreFromDump(*source, name, method, htSHA256, repair); + ? store->computeStorePathFromDump(*source, name, method, HashAlgorithm::SHA256).first + : store->addToStoreFromDump(*source, name, method, HashAlgorithm::SHA256, repair); + + if (cacheKey) + fetchers::getCache()->add(store, *cacheKey, {}, storePath, true); return storePath; } -SourcePath InputAccessor::root() -{ - return {ref(shared_from_this()), CanonPath::root}; -} - std::ostream & operator << (std::ostream & str, const SourcePath & path) { str << path.to_string(); @@ -63,7 +83,7 @@ SourcePath SourcePath::parent() const SourcePath SourcePath::resolveSymlinks() const { - auto res = accessor->root(); + auto res = SourcePath(accessor); int linksAllowed = 1024; diff --git a/src/libfetchers/input-accessor.hh b/src/libfetchers/input-accessor.hh index 5dc05a363..d5ac238b1 100644 --- a/src/libfetchers/input-accessor.hh +++ b/src/libfetchers/input-accessor.hh @@ -1,8 +1,10 @@ #pragma once +///@file #include "source-accessor.hh" #include "ref.hh" #include "types.hh" +#include "file-system.hh" #include "repair-flag.hh" #include "content-address.hh" @@ -14,8 +16,10 @@ struct SourcePath; class StorePath; class Store; -struct InputAccessor : SourceAccessor, std::enable_shared_from_this +struct InputAccessor : virtual SourceAccessor, std::enable_shared_from_this { + std::optional fingerprint; + /** * Return the maximum last-modified time of the files in this * tree, if available. @@ -32,8 +36,6 @@ struct InputAccessor : SourceAccessor, std::enable_shared_from_this accessor; CanonPath path; + SourcePath(ref accessor, CanonPath path = CanonPath::root) + : accessor(std::move(accessor)) + , path(std::move(path)) + { } + std::string_view baseName() const; /** diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk index 2e8869d83..266e7a211 100644 --- a/src/libfetchers/local.mk +++ b/src/libfetchers/local.mk @@ -8,6 +8,6 @@ libfetchers_SOURCES := $(wildcard $(d)/*.cc) libfetchers_CXXFLAGS += -I src/libutil -I src/libstore -libfetchers_LDFLAGS += -pthread +libfetchers_LDFLAGS += -pthread $(LIBGIT2_LIBS) -larchive libfetchers_LIBS = libutil libstore diff --git a/src/libfetchers/memory-input-accessor.cc b/src/libfetchers/memory-input-accessor.cc index 817d063ba..057f3e37f 100644 --- a/src/libfetchers/memory-input-accessor.cc +++ b/src/libfetchers/memory-input-accessor.cc @@ -1,48 +1,16 @@ #include "memory-input-accessor.hh" +#include "memory-source-accessor.hh" namespace nix { -struct MemoryInputAccessorImpl : MemoryInputAccessor +struct MemoryInputAccessorImpl : MemoryInputAccessor, MemorySourceAccessor { - std::map files; - - std::string readFile(const CanonPath & path) override - { - auto i = files.find(path); - if (i == files.end()) - throw Error("file '%s' does not exist", path); - return i->second; - } - - bool pathExists(const CanonPath & path) override - { - auto i = files.find(path); - return i != files.end(); - } - - Stat lstat(const CanonPath & path) override - { - auto i = files.find(path); - if (i != files.end()) - return Stat { .type = tRegular, .isExecutable = false }; - throw Error("file '%s' does not exist", path); - } - - DirEntries readDirectory(const CanonPath & path) override - { - return {}; - } - - std::string readLink(const CanonPath & path) override - { - throw UnimplementedError("MemoryInputAccessor::readLink"); - } - SourcePath addFile(CanonPath path, std::string && contents) override { - files.emplace(path, std::move(contents)); - - return {ref(shared_from_this()), std::move(path)}; + return { + ref(shared_from_this()), + MemorySourceAccessor::addFile(path, std::move(contents)) + }; } }; diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index f830a3271..6056b9a3c 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -1,4 +1,6 @@ #include "fetchers.hh" +#include "processes.hh" +#include "users.hh" #include "cache.hh" #include "globals.hh" #include "tarfile.hh" @@ -69,14 +71,25 @@ struct MercurialInputScheme : InputScheme return inputFromAttrs(attrs); } + std::string_view schemeName() const override + { + return "hg"; + } + + StringSet allowedAttrs() const override + { + return { + "url", + "ref", + "rev", + "revCount", + "narHash", + "name", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override { - if (maybeGetStrAttr(attrs, "type") != "hg") return {}; - - for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name") - throw Error("unsupported Mercurial input attribute '%s'", name); - parseURL(getStrAttr(attrs, "url")); if (auto ref = maybeGetStrAttr(attrs, "ref")) { @@ -109,7 +122,7 @@ struct MercurialInputScheme : InputScheme return res; } - std::optional getSourcePath(const Input & input) override + std::optional getSourcePath(const Input & input) const override { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme == "file" && !input.getRef() && !input.getRev()) @@ -117,18 +130,27 @@ struct MercurialInputScheme : InputScheme return {}; } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - auto sourcePath = getSourcePath(input); - assert(sourcePath); + auto [isLocal, repoPath] = getActualUrl(input); + if (!isLocal) + throw Error("cannot commit '%s' to Mercurial repository '%s' because it's not a working tree", path, input.to_string()); + + auto absPath = CanonPath(repoPath) + path; + + writeFile(absPath.abs(), contents); // FIXME: shut up if file is already tracked. runHg( - { "add", *sourcePath + "/" + std::string(file) }); + { "add", absPath.abs() }); if (commitMsg) runHg( - { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); + { "commit", absPath.abs(), "-m", *commitMsg }); } std::pair getActualUrl(const Input & input) const @@ -188,7 +210,7 @@ struct MercurialInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(input.getName(), actualPath, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, filter); return {std::move(storePath), input}; } @@ -198,7 +220,7 @@ struct MercurialInputScheme : InputScheme auto checkHashType = [&](const std::optional & hash) { - if (hash.has_value() && hash->type != htSHA1) + if (hash.has_value() && hash->algo != HashAlgorithm::SHA1) throw Error("Hash '%s' is not supported by Mercurial. Only sha1 is supported.", hash->to_string(HashFormat::Base16, true)); }; @@ -238,14 +260,14 @@ struct MercurialInputScheme : InputScheme }); if (auto res = getCache()->lookup(store, unlockedAttrs)) { - auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), htSHA1); + auto rev2 = Hash::parseAny(getStrAttr(res->first, "rev"), HashAlgorithm::SHA1); if (!input.getRev() || input.getRev() == rev2) { input.attrs.insert_or_assign("rev", rev2.gitRev()); return makeResult(res->first, std::move(res->second)); } } - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(htSHA256, actualUrl).to_string(HashFormat::Base32, false)); + Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Nix32, false)); /* If this is a commit hash that we already have, we don't have to pull again. */ @@ -279,7 +301,7 @@ struct MercurialInputScheme : InputScheme runHg({ "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" })); assert(tokens.size() == 3); - input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev()); + input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], HashAlgorithm::SHA1).gitRev()); auto revCount = std::stoull(tokens[1]); input.attrs.insert_or_assign("ref", tokens[2]); @@ -317,6 +339,14 @@ struct MercurialInputScheme : InputScheme return makeResult(infoAttrs, std::move(storePath)); } + + std::optional getFingerprint(ref store, const Input & input) const override + { + if (auto rev = input.getRev()) + return rev->gitRev(); + else + return std::nullopt; + } }; static auto rMercurialInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libfetchers/mounted-input-accessor.cc b/src/libfetchers/mounted-input-accessor.cc new file mode 100644 index 000000000..6f397eb17 --- /dev/null +++ b/src/libfetchers/mounted-input-accessor.cc @@ -0,0 +1,77 @@ +#include "mounted-input-accessor.hh" + +namespace nix { + +struct MountedInputAccessor : InputAccessor +{ + std::map> mounts; + + MountedInputAccessor(std::map> _mounts) + : mounts(std::move(_mounts)) + { + // Currently we require a root filesystem. This could be relaxed. + assert(mounts.contains(CanonPath::root)); + + // FIXME: return dummy parent directories automatically? + } + + std::string readFile(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->readFile(subpath); + } + + bool pathExists(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->pathExists(subpath); + } + + std::optional maybeLstat(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->maybeLstat(subpath); + } + + DirEntries readDirectory(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->readDirectory(subpath); + } + + std::string readLink(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->readLink(subpath); + } + + std::string showPath(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->showPath(subpath); + } + + std::pair, CanonPath> resolve(CanonPath path) + { + // Find the nearest parent of `path` that is a mount point. + std::vector subpath; + while (true) { + auto i = mounts.find(path); + if (i != mounts.end()) { + std::reverse(subpath.begin(), subpath.end()); + return {i->second, CanonPath(subpath)}; + } + + assert(!path.isRoot()); + subpath.push_back(std::string(*path.baseName())); + path.pop(); + } + } +}; + +ref makeMountedInputAccessor(std::map> mounts) +{ + return make_ref(std::move(mounts)); +} + +} diff --git a/src/libfetchers/mounted-input-accessor.hh b/src/libfetchers/mounted-input-accessor.hh new file mode 100644 index 000000000..b557c5dad --- /dev/null +++ b/src/libfetchers/mounted-input-accessor.hh @@ -0,0 +1,9 @@ +#pragma once + +#include "input-accessor.hh" + +namespace nix { + +ref makeMountedInputAccessor(std::map> mounts); + +} diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index d829609b5..f9b973320 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -32,23 +32,30 @@ struct PathInputScheme : InputScheme return input; } + std::string_view schemeName() const override + { + return "path"; + } + + StringSet allowedAttrs() const override + { + return { + "path", + /* Allow the user to pass in "fake" tree info + attributes. This is useful for making a pinned tree work + the same as the repository from which is exported (e.g. + path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). + */ + "rev", + "revCount", + "lastModified", + "narHash", + }; + } std::optional inputFromAttrs(const Attrs & attrs) const override { - if (maybeGetStrAttr(attrs, "type") != "path") return {}; - getStrAttr(attrs, "path"); - for (auto & [name, value] : attrs) - /* Allow the user to pass in "fake" tree info - attributes. This is useful for making a pinned tree - work the same as the repository from which is exported - (e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */ - if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path") - // checked in Input::fromAttrs - ; - else - throw Error("unsupported path input attribute '%s'", name); - Input input; input.attrs = attrs; return input; @@ -66,14 +73,28 @@ struct PathInputScheme : InputScheme }; } - std::optional getSourcePath(const Input & input) override + std::optional getSourcePath(const Input & input) const override { return getStrAttr(input.attrs, "path"); } - void markChangedFile(const Input & input, std::string_view file, std::optional commitMsg) override + void putFile( + const Input & input, + const CanonPath & path, + std::string_view contents, + std::optional commitMsg) const override { - // nothing to do + writeFile((CanonPath(getAbsPath(input)) + path).abs(), contents); + } + + CanonPath getAbsPath(const Input & input) const + { + auto path = getStrAttr(input.attrs, "path"); + + if (path[0] == '/') + return CanonPath(path); + + throw Error("cannot fetch input '%s' because it uses a relative path", input.to_string()); } std::pair fetch(ref store, const Input & _input) override @@ -121,7 +142,7 @@ struct PathInputScheme : InputScheme return {std::move(*storePath), input}; } - std::optional experimentalFeature() override + std::optional experimentalFeature() const override { return Xp::Flakes; } diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index a0fff9ceb..9c7bc0cfe 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,6 +1,6 @@ #include "registry.hh" #include "tarball.hh" -#include "util.hh" +#include "users.hh" #include "globals.hh" #include "store-api.hh" #include "local-fs-store.hh" diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index e1ea9b58b..086366180 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -73,7 +73,7 @@ DownloadFileResult downloadFile( } else { StringSink sink; dumpString(res.data, sink); - auto hash = hashString(htSHA256, res.data); + auto hash = hashString(HashAlgorithm::SHA256, res.data); ValidPathInfo info { *store, name, @@ -82,7 +82,7 @@ DownloadFileResult downloadFile( .hash = hash, .references = {}, }, - hashString(htSHA256, sink.s), + hashString(HashAlgorithm::SHA256, sink.s), }; info.narSize = sink.s.size(); auto source = StringSource { sink.s }; @@ -156,7 +156,7 @@ DownloadTarballResult downloadTarball( throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url); auto topDir = tmpDir + "/" + members.begin()->name; lastModified = lstat(topDir).st_mtime; - unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair); + unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, HashAlgorithm::SHA256, defaultPathFilter, NoRepair); } Attrs infoAttrs({ @@ -184,7 +184,6 @@ DownloadTarballResult downloadTarball( // An input scheme corresponding to a curl-downloadable resource. struct CurlInputScheme : InputScheme { - virtual const std::string inputType() const = 0; const std::set transportUrlSchemes = {"file", "http", "https"}; const bool hasTarballExtension(std::string_view path) const @@ -222,22 +221,27 @@ struct CurlInputScheme : InputScheme url.query.erase("rev"); url.query.erase("revCount"); - input.attrs.insert_or_assign("type", inputType()); + input.attrs.insert_or_assign("type", std::string { schemeName() }); input.attrs.insert_or_assign("url", url.to_string()); return input; } + StringSet allowedAttrs() const override + { + return { + "type", + "url", + "narHash", + "name", + "unpack", + "rev", + "revCount", + "lastModified", + }; + } + std::optional inputFromAttrs(const Attrs & attrs) const override { - auto type = maybeGetStrAttr(attrs, "type"); - if (type != inputType()) return {}; - - // FIXME: some of these only apply to TarballInputScheme. - std::set allowedNames = {"type", "url", "narHash", "name", "unpack", "rev", "revCount", "lastModified"}; - for (auto & [name, value] : attrs) - if (!allowedNames.count(name)) - throw Error("unsupported %s input attribute '%s'", *type, name); - Input input; input.attrs = attrs; @@ -258,14 +262,14 @@ struct CurlInputScheme : InputScheme struct FileInputScheme : CurlInputScheme { - const std::string inputType() const override { return "file"; } + std::string_view schemeName() const override { return "file"; } bool isValidURL(const ParsedURL & url, bool requireTree) const override { auto parsedUrlScheme = parseUrlScheme(url.scheme); return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() + ? parsedUrlScheme.application.value() == schemeName() : (!requireTree && !hasTarballExtension(url.path))); } @@ -278,7 +282,7 @@ struct FileInputScheme : CurlInputScheme struct TarballInputScheme : CurlInputScheme { - const std::string inputType() const override { return "tarball"; } + std::string_view schemeName() const override { return "tarball"; } bool isValidURL(const ParsedURL & url, bool requireTree) const override { @@ -286,7 +290,7 @@ struct TarballInputScheme : CurlInputScheme return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application - ? parsedUrlScheme.application.value() == inputType() + ? parsedUrlScheme.application.value() == schemeName() : (requireTree || hasTarballExtension(url.path))); } diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 205b77808..5b49aaabc 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,7 +1,9 @@ #include "common-args.hh" #include "args/root.hh" #include "globals.hh" +#include "logging.hh" #include "loggers.hh" +#include "util.hh" namespace nix { diff --git a/src/libmain/local.mk b/src/libmain/local.mk index 99da95e27..5c7061863 100644 --- a/src/libmain/local.mk +++ b/src/libmain/local.mk @@ -14,4 +14,4 @@ libmain_LIBS = libstore libutil libmain_ALLOW_UNDEFINED = 1 -$(eval $(call install-file-in, $(d)/nix-main.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-main.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index cda5cb939..9829859de 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -1,6 +1,6 @@ #include "loggers.hh" +#include "environment-variables.hh" #include "progress-bar.hh" -#include "util.hh" namespace nix { diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 45b1fdfd1..3aa012ee1 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -1,5 +1,5 @@ #include "progress-bar.hh" -#include "util.hh" +#include "terminal.hh" #include "sync.hh" #include "store-api.hh" #include "names.hh" @@ -340,6 +340,14 @@ public: state->activitiesByType[type].expected += j; update(*state); } + + else if (type == resFetchStatus) { + auto i = state->its.find(act); + assert(i != state->its.end()); + ActInfo & actInfo = *i->second; + actInfo.lastLine = getS(fields, 0); + update(*state); + } } void update(State & state) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 9c2ad039a..862ef355b 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -1,10 +1,11 @@ #include "globals.hh" +#include "current-process.hh" #include "shared.hh" #include "store-api.hh" #include "gc-store.hh" -#include "util.hh" #include "loggers.hh" #include "progress-bar.hh" +#include "signals.hh" #include #include diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index 3159fe479..c68f6cd83 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "util.hh" +#include "processes.hh" #include "args.hh" #include "args/root.hh" #include "common-args.hh" diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 2a91233ec..2837e8934 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -2,7 +2,7 @@ #include "binary-cache-store.hh" #include "compression.hh" #include "derivations.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "globals.hh" #include "nar-info.hh" #include "sync.hh" @@ -11,6 +11,7 @@ #include "nar-accessor.hh" #include "thread-pool.hh" #include "callback.hh" +#include "signals.hh" #include #include @@ -142,9 +143,9 @@ ref BinaryCacheStore::addToStoreCommon( /* Read the NAR simultaneously into a CompressionSink+FileSink (to write the compressed NAR to disk), into a HashSink (to get the NAR hash), and into a NarAccessor (to get the NAR listing). */ - HashSink fileHashSink { htSHA256 }; - std::shared_ptr narAccessor; - HashSink narHashSink { htSHA256 }; + HashSink fileHashSink { HashAlgorithm::SHA256 }; + std::shared_ptr narAccessor; + HashSink narHashSink { HashAlgorithm::SHA256 }; { FdSink fileSink(fdTemp.get()); TeeSink teeSinkCompressed { fileSink, fileHashSink }; @@ -164,8 +165,8 @@ ref BinaryCacheStore::addToStoreCommon( auto [fileHash, fileSize] = fileHashSink.finish(); narInfo->fileHash = fileHash; narInfo->fileSize = fileSize; - narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Base32, false) + ".nar" - + (compression == "xz" ? ".xz" : + narInfo->url = "nar/" + narInfo->fileHash->to_string(HashFormat::Nix32, false) + ".nar" + + (compression == "xz" ? ".xz" : compression == "bzip2" ? ".bz2" : compression == "zstd" ? ".zst" : compression == "lzip" ? ".lzip" : @@ -195,7 +196,7 @@ ref BinaryCacheStore::addToStoreCommon( if (writeNARListing) { nlohmann::json j = { {"version", 1}, - {"root", listNar(ref(narAccessor), "", true)}, + {"root", listNar(ref(narAccessor), CanonPath::root, true)}, }; upsertFile(std::string(info.path.hashPart()) + ".ls", j.dump(), "application/json"); @@ -206,9 +207,9 @@ ref BinaryCacheStore::addToStoreCommon( specify the NAR file and member containing the debug info. */ if (writeDebugInfo) { - std::string buildIdDir = "/lib/debug/.build-id"; + CanonPath buildIdDir("lib/debug/.build-id"); - if (narAccessor->stat(buildIdDir).type == FSAccessor::tDirectory) { + if (auto st = narAccessor->maybeLstat(buildIdDir); st && st->type == SourceAccessor::tDirectory) { ThreadPool threadPool(25); @@ -231,17 +232,17 @@ ref BinaryCacheStore::addToStoreCommon( std::regex regex1("^[0-9a-f]{2}$"); std::regex regex2("^[0-9a-f]{38}\\.debug$"); - for (auto & s1 : narAccessor->readDirectory(buildIdDir)) { - auto dir = buildIdDir + "/" + s1; + for (auto & [s1, _type] : narAccessor->readDirectory(buildIdDir)) { + auto dir = buildIdDir + s1; - if (narAccessor->stat(dir).type != FSAccessor::tDirectory + if (narAccessor->lstat(dir).type != SourceAccessor::tDirectory || !std::regex_match(s1, regex1)) continue; - for (auto & s2 : narAccessor->readDirectory(dir)) { - auto debugPath = dir + "/" + s2; + for (auto & [s2, _type] : narAccessor->readDirectory(dir)) { + auto debugPath = dir + s2; - if (narAccessor->stat(debugPath).type != FSAccessor::tRegular + if (narAccessor->lstat(debugPath).type != SourceAccessor::tRegular || !std::regex_match(s2, regex2)) continue; @@ -250,7 +251,7 @@ ref BinaryCacheStore::addToStoreCommon( std::string key = "debuginfo/" + buildId; std::string target = "../" + narInfo->url; - threadPool.enqueue(std::bind(doFile, std::string(debugPath, 1), key, target)); + threadPool.enqueue(std::bind(doFile, std::string(debugPath.rel()), key, target)); } } @@ -300,9 +301,9 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource } StorePath BinaryCacheStore::addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) + FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) { - if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) + if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) unsupported("addToStoreFromDump"); return addToStoreCommon(dump, repair, CheckSigs, [&](HashResult nar) { ValidPathInfo info { @@ -398,13 +399,13 @@ void BinaryCacheStore::queryPathInfoUncached(const StorePath & storePath, } StorePath BinaryCacheStore::addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashType hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) { /* FIXME: Make BinaryCacheStore::addToStoreCommon support non-recursive+sha256 so we can just use the default @@ -447,7 +448,7 @@ StorePath BinaryCacheStore::addTextToStore( const StorePathSet & references, RepairFlag repair) { - auto textHash = hashString(htSHA256, s); + auto textHash = hashString(HashAlgorithm::SHA256, s); auto path = makeTextPath(name, TextInfo { { textHash }, references }); if (!repair && isValidPath(path)) @@ -503,9 +504,9 @@ void BinaryCacheStore::registerDrvOutput(const Realisation& info) { upsertFile(filePath, info.toJSON().dump(), "application/json"); } -ref BinaryCacheStore::getFSAccessor() +ref BinaryCacheStore::getFSAccessor(bool requireValidPath) { - return make_ref(ref(shared_from_this()), localNarCache); + return make_ref(ref(shared_from_this()), requireValidPath, localNarCache); } void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 49f271d24..395e1b479 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -17,28 +17,28 @@ struct BinaryCacheStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting compression{(StoreConfig*) this, "xz", "compression", + const Setting compression{this, "xz", "compression", "NAR compression method (`xz`, `bzip2`, `gzip`, `zstd`, or `none`)."}; - const Setting writeNARListing{(StoreConfig*) this, false, "write-nar-listing", + const Setting writeNARListing{this, false, "write-nar-listing", "Whether to write a JSON file that lists the files in each NAR."}; - const Setting writeDebugInfo{(StoreConfig*) this, false, "index-debug-info", + const Setting writeDebugInfo{this, false, "index-debug-info", R"( Whether to index DWARF debug info files by build ID. This allows [`dwarffs`](https://github.com/edolstra/dwarffs) to fetch debug info on demand )"}; - const Setting secretKeyFile{(StoreConfig*) this, "", "secret-key", + const Setting secretKeyFile{this, "", "secret-key", "Path to the secret key used to sign the binary cache."}; - const Setting localNarCache{(StoreConfig*) this, "", "local-nar-cache", + const Setting localNarCache{this, "", "local-nar-cache", "Path to a local cache of NARs fetched from this binary cache, used by commands such as `nix store cat`."}; - const Setting parallelCompression{(StoreConfig*) this, false, "parallel-compression", + const Setting parallelCompression{this, false, "parallel-compression", "Enable multi-threaded compression of NARs. This is currently only available for `xz` and `zstd`."}; - const Setting compressionLevel{(StoreConfig*) this, -1, "compression-level", + const Setting compressionLevel{this, -1, "compression-level", R"( The *preset level* to be used when compressing NARs. The meaning and accepted values depend on the compression method selected. @@ -124,16 +124,16 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; + FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override; StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashType hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override; + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) override; StorePath addTextToStore( std::string_view name, @@ -148,7 +148,7 @@ public: void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor() override; + ref getFSAccessor(bool requireValidPath) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; diff --git a/src/libstore/build/child.cc b/src/libstore/build/child.cc new file mode 100644 index 000000000..aa31c3caf --- /dev/null +++ b/src/libstore/build/child.cc @@ -0,0 +1,37 @@ +#include "child.hh" +#include "current-process.hh" +#include "logging.hh" + +#include +#include + +namespace nix { + +void commonChildInit() +{ + logger = makeSimpleLogger(); + + const static std::string pathNullDevice = "/dev/null"; + restoreProcessContext(false); + + /* Put the child in a separate session (and thus a separate + process group) so that it has no controlling terminal (meaning + that e.g. ssh cannot open /dev/tty) and it doesn't receive + terminal signals. */ + if (setsid() == -1) + throw SysError("creating a new session"); + + /* Dup stderr to stdout. */ + if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) + throw SysError("cannot dup stderr into stdout"); + + /* Reroute stdin to /dev/null. */ + int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); + if (fdDevNull == -1) + throw SysError("cannot open '%1%'", pathNullDevice); + if (dup2(fdDevNull, STDIN_FILENO) == -1) + throw SysError("cannot dup null device into stdin"); + close(fdDevNull); +} + +} diff --git a/src/libstore/build/child.hh b/src/libstore/build/child.hh new file mode 100644 index 000000000..3dfc552b9 --- /dev/null +++ b/src/libstore/build/child.hh @@ -0,0 +1,11 @@ +#pragma once +///@file + +namespace nix { + +/** + * Common initialisation performed in child processes. + */ +void commonChildInit(); + +} diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 360c6b70b..d4da374ba 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -558,7 +558,7 @@ void DerivationGoal::inputsRealised() inputDrvOutputs statefully, sometimes it gets out of sync with the real source of truth (store). So we query the store directly if there's a problem. */ - attempt = fullDrv.tryResolve(worker.store); + attempt = fullDrv.tryResolve(worker.store, &worker.evalStore); } assert(attempt); Derivation drvResolved { std::move(*attempt) }; @@ -1317,9 +1317,26 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data) auto s = handleJSONLogMessage(*json, worker.act, hook->activities, true); // ensure that logs from a builder using `ssh-ng://` as protocol // are also available to `nix log`. - if (s && !isWrittenToLog && logSink && (*json)["type"] == resBuildLogLine) { - auto f = (*json)["fields"]; - (*logSink)((f.size() > 0 ? f.at(0).get() : "") + "\n"); + if (s && !isWrittenToLog && logSink) { + const auto type = (*json)["type"]; + const auto fields = (*json)["fields"]; + if (type == resBuildLogLine) { + (*logSink)((fields.size() > 0 ? fields[0].get() : "") + "\n"); + } else if (type == resSetPhase && ! fields.is_null()) { + const auto phase = fields[0]; + if (! phase.is_null()) { + // nixpkgs' stdenv produces lines in the log to signal + // phase changes. + // We want to get the same lines in case of remote builds. + // The format is: + // @nix { "action": "setPhase", "phase": "$curPhase" } + const auto logLine = nlohmann::json::object({ + {"action", "setPhase"}, + {"phase", phase} + }); + (*logSink)("@nix " + logLine.dump(-1, ' ', false, nlohmann::json::error_handler_t::replace) + "\n"); + } + } } } currentHookLine.clear(); @@ -1474,6 +1491,7 @@ void DerivationGoal::done( SingleDrvOutputs builtOutputs, std::optional ex) { + outputLocks.unlock(); buildResult.status = status; if (ex) buildResult.errorMsg = fmt("%s", normaltxt(ex->info().msg)); diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 13ff22f45..74eca63f3 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -15,7 +15,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod worker.run(goals); - StorePathSet failed; + StringSet failed; std::optional ex; for (auto & i : goals) { if (i->ex) { @@ -26,9 +26,9 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod } if (i->exitCode != Goal::ecSuccess) { if (auto i2 = dynamic_cast(i.get())) - failed.insert(i2->drvPath); + failed.insert(std::string { i2->drvPath.to_string() }); else if (auto i2 = dynamic_cast(i.get())) - failed.insert(i2->storePath); + failed.insert(std::string { i2->storePath.to_string()}); } } @@ -37,7 +37,7 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod throw std::move(*ex); } else if (!failed.empty()) { if (ex) logError(ex->info()); - throw Error(worker.failingExitStatus(), "build of %s failed", showPaths(failed)); + throw Error(worker.failingExitStatus(), "build of %s failed", concatStringsSep(", ", quoteStrings(failed))); } } diff --git a/src/libstore/build/hook-instance.cc b/src/libstore/build/hook-instance.cc index 337c60bd4..5d045ec3d 100644 --- a/src/libstore/build/hook-instance.cc +++ b/src/libstore/build/hook-instance.cc @@ -1,5 +1,7 @@ #include "globals.hh" #include "hook-instance.hh" +#include "file-system.hh" +#include "child.hh" namespace nix { diff --git a/src/libstore/build/hook-instance.hh b/src/libstore/build/hook-instance.hh index d84f62877..61cf534f4 100644 --- a/src/libstore/build/hook-instance.hh +++ b/src/libstore/build/hook-instance.hh @@ -3,6 +3,7 @@ #include "logging.hh" #include "serialise.hh" +#include "processes.hh" namespace nix { diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index b5b060d95..802b39f84 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -15,7 +15,11 @@ #include "json-utils.hh" #include "cgroup.hh" #include "personality.hh" +#include "current-process.hh" #include "namespaces.hh" +#include "child.hh" +#include "unix-domain-socket.hh" +#include "posix-fs-canonicalise.hh" #include #include @@ -227,7 +231,7 @@ void LocalDerivationGoal::tryLocalBuild() if (!buildUser) { if (!actLock) actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, - fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); + fmt("waiting for a free build user ID for '%s'", yellowtxt(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); return; } @@ -649,8 +653,8 @@ void LocalDerivationGoal::startBuilder() #if __linux__ /* Create a temporary directory in which we set up the chroot environment using bind-mounts. We put it in the Nix store - to ensure that we can create hard-links to non-directory - inputs in the fake Nix store in the chroot (see below). */ + so that the build outputs can be moved efficiently from the + chroot to their final location. */ chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; deletePath(chrootRootDir); @@ -1062,8 +1066,8 @@ void LocalDerivationGoal::initTmpDir() { if (passAsFile.find(i.first) == passAsFile.end()) { env[i.first] = i.second; } else { - auto hash = hashString(htSHA256, i.first); - std::string fn = ".attr-" + hash.to_string(HashFormat::Base32, false); + auto hash = hashString(HashAlgorithm::SHA256, i.first); + std::string fn = ".attr-" + hash.to_string(HashFormat::Nix32, false); Path p = tmpDir + "/" + fn; writeFile(p, rewriteStrings(i.second, inputRewrites)); chownToBuilder(p); @@ -1286,13 +1290,13 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In { throw Error("queryPathFromHashPart"); } StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashType hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) override { throw Error("addToStore"); } void addToStore(const ValidPathInfo & info, Source & narSource, @@ -1314,12 +1318,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual In } StorePath addToStoreFromDump( - Source & dump, - std::string_view name, - FileIngestionMethod method, - HashType hashAlgo, - RepairFlag repair, - const StorePathSet & references) override + Source & dump, + std::string_view name, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + RepairFlag repair, + const StorePathSet & references) override { auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair, references); goal.addDependency(path); @@ -1563,10 +1567,11 @@ void LocalDerivationGoal::addDependency(const StorePath & path) Path source = worker.store.Store::toRealPath(path); Path target = chrootRootDir + worker.store.printStorePath(path); - if (pathExists(target)) + if (pathExists(target)) { // There is a similar debug message in doBind, so only run it in this block to not have double messages. debug("bind-mounting %s -> %s", target, source); throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path)); + } /* Bind-mount the path into the sandbox. This requires entering its mount namespace, which is not possible @@ -1619,6 +1624,8 @@ void setupSeccomp() seccomp_release(ctx); }); + constexpr std::string_view nativeSystem = SYSTEM; + if (nativeSystem == "x86_64-linux" && seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0) throw SysError("unable to add 32-bit seccomp architecture"); @@ -2459,7 +2466,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() rewriteOutput(outputRewrites); /* FIXME optimize and deduplicate with addToStore */ std::string oldHashPart { scratchPath->hashPart() }; - HashModuloSink caSink { outputHash.hashType, oldHashPart }; + HashModuloSink caSink {outputHash.hashAlgo, oldHashPart }; std::visit(overloaded { [&](const TextIngestionMethod &) { readFile(actualPath, caSink); @@ -2504,7 +2511,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() std::string(newInfo0.path.hashPart())}}); } - HashResult narHashAndSize = hashPath(htSHA256, actualPath); + HashResult narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath); newInfo0.narHash = narHashAndSize.first; newInfo0.narSize = narHashAndSize.second; @@ -2524,7 +2531,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() std::string { scratchPath->hashPart() }, std::string { requiredFinalPath.hashPart() }); rewriteOutput(outputRewrites); - auto narHashAndSize = hashPath(htSHA256, actualPath); + auto narHashAndSize = hashPath(HashAlgorithm::SHA256, actualPath); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; auto refs = rewriteRefs(); @@ -2539,7 +2546,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() auto newInfo0 = newInfoFromCA(DerivationOutput::CAFloating { .method = dof.ca.method, - .hashType = wanted.type, + .hashAlgo = wanted.algo, }); /* Check wanted hash */ @@ -2576,7 +2583,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() [&](const DerivationOutput::Impure & doi) { return newInfoFromCA(DerivationOutput::CAFloating { .method = doi.method, - .hashType = doi.hashType, + .hashAlgo = doi.hashAlgo, }); }, @@ -2938,7 +2945,7 @@ StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName) { return worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), - Hash(htSHA256), outputPathName(drv->name, outputName)); + Hash(HashAlgorithm::SHA256), outputPathName(drv->name, outputName)); } @@ -2946,7 +2953,7 @@ StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path) { return worker.store.makeStorePath( "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), - Hash(htSHA256), path.name()); + Hash(HashAlgorithm::SHA256), path.name()); } diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index 1cb68a869..88152a645 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -3,6 +3,7 @@ #include "derivation-goal.hh" #include "local-store.hh" +#include "processes.hh" namespace nix { diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc index 37cb86b91..9b8c36286 100644 --- a/src/libstore/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -4,6 +4,7 @@ #include "drv-output-substitution-goal.hh" #include "local-derivation-goal.hh" #include "hook-instance.hh" +#include "signals.hh" #include @@ -198,8 +199,16 @@ void Worker::childStarted(GoalPtr goal, const std::set & fds, child.respectTimeouts = respectTimeouts; children.emplace_back(child); if (inBuildSlot) { - if (goal->jobCategory() == JobCategory::Substitution) nrSubstitutions++; - else nrLocalBuilds++; + switch (goal->jobCategory()) { + case JobCategory::Substitution: + nrSubstitutions++; + break; + case JobCategory::Build: + nrLocalBuilds++; + break; + default: + abort(); + } } } @@ -211,12 +220,17 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers) if (i == children.end()) return; if (i->inBuildSlot) { - if (goal->jobCategory() == JobCategory::Substitution) { + switch (goal->jobCategory()) { + case JobCategory::Substitution: assert(nrSubstitutions > 0); nrSubstitutions--; - } else { + break; + case JobCategory::Build: assert(nrLocalBuilds > 0); nrLocalBuilds--; + break; + default: + abort(); } } @@ -505,8 +519,8 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - HashResult current = hashPath(info->narHash.type, store.printStorePath(path)); - Hash nullHash(htSHA256); + HashResult current = hashPath(info->narHash.algo, store.printStorePath(path)); + Hash nullHash(HashAlgorithm::SHA256); res = info->narHash == nullHash || info->narHash == current.first; } pathContentsGoodCache.insert_or_assign(path, res); diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index c8911d153..9283251ac 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -1,4 +1,5 @@ #include "buildenv.hh" +#include "derivations.hh" #include #include diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index 0923c2adb..8bebd390d 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "derivations.hh" #include "store-api.hh" namespace nix { diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 357800333..2086bd0b9 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -63,9 +63,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData) for (auto hashedMirror : settings.hashedMirrors.get()) try { if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/'; - std::optional ht = parseHashTypeOpt(getAttr("outputHashAlgo")); + std::optional ht = parseHashAlgoOpt(getAttr("outputHashAlgo")); Hash h = newHashAllowEmpty(getAttr("outputHash"), ht); - fetch(hashedMirror + printHashType(h.type) + "/" + h.to_string(HashFormat::Base16, false)); + fetch(hashedMirror + printHashAlgo(h.algo) + "/" + h.to_string(HashFormat::Base16, false)); return; } catch (Error & e) { debug(e.what()); diff --git a/src/libstore/common-protocol-impl.hh b/src/libstore/common-protocol-impl.hh index 079c182b8..360882c02 100644 --- a/src/libstore/common-protocol-impl.hh +++ b/src/libstore/common-protocol-impl.hh @@ -16,11 +16,11 @@ namespace nix { /* protocol-agnostic templates */ #define COMMON_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ - TEMPLATE T CommonProto::Serialise< T >::read(const Store & store, CommonProto::ReadConn conn) \ + TEMPLATE T CommonProto::Serialise< T >::read(const StoreDirConfig & store, CommonProto::ReadConn conn) \ { \ return LengthPrefixedProtoHelper::read(store, conn); \ } \ - TEMPLATE void CommonProto::Serialise< T >::write(const Store & store, CommonProto::WriteConn conn, const T & t) \ + TEMPLATE void CommonProto::Serialise< T >::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & t) \ { \ LengthPrefixedProtoHelper::write(store, conn, t); \ } diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index f906814bc..fc2b5ac6f 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -1,5 +1,4 @@ #include "serialise.hh" -#include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" #include "build-result.hh" @@ -14,40 +13,40 @@ namespace nix { /* protocol-agnostic definitions */ -std::string CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +std::string CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return readString(conn.from); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const std::string & str) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::string & str) { conn.to << str; } -StorePath CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +StorePath CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return store.parseStorePath(readString(conn.from)); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const StorePath & storePath) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const StorePath & storePath) { conn.to << store.printStorePath(storePath); } -ContentAddress CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +ContentAddress CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return ContentAddress::parse(readString(conn.from)); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const ContentAddress & ca) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const ContentAddress & ca) { conn.to << renderContentAddress(ca); } -Realisation CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +Realisation CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { std::string rawInput = readString(conn.from); return Realisation::fromJSON( @@ -56,41 +55,41 @@ Realisation CommonProto::Serialise::read(const Store & store, Commo ); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const Realisation & realisation) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation) { conn.to << realisation.toJSON().dump(); } -DrvOutput CommonProto::Serialise::read(const Store & store, CommonProto::ReadConn conn) +DrvOutput CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return DrvOutput::parse(readString(conn.from)); } -void CommonProto::Serialise::write(const Store & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) +void CommonProto::Serialise::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const DrvOutput & drvOutput) { conn.to << drvOutput.to_string(); } -std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { auto s = readString(conn.from); return s == "" ? std::optional {} : store.parseStorePath(s); } -void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & storePathOpt) +void CommonProto::Serialise>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional & storePathOpt) { conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); } -std::optional CommonProto::Serialise>::read(const Store & store, CommonProto::ReadConn conn) +std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { return ContentAddress::parseOpt(readString(conn.from)); } -void CommonProto::Serialise>::write(const Store & store, CommonProto::WriteConn conn, const std::optional & caOpt) +void CommonProto::Serialise>::write(const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional & caOpt) { conn.to << (caOpt ? renderContentAddress(*caOpt) : ""); } diff --git a/src/libstore/common-protocol.hh b/src/libstore/common-protocol.hh index f3f28972a..a878e84c9 100644 --- a/src/libstore/common-protocol.hh +++ b/src/libstore/common-protocol.hh @@ -5,7 +5,7 @@ namespace nix { -class Store; +struct StoreDirConfig; struct Source; // items being serialized @@ -48,7 +48,7 @@ struct CommonProto * infer the type instead of having to write it down explicitly. */ template - static void write(const Store & store, WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, WriteConn conn, const T & t) { CommonProto::Serialise::write(store, conn, t); } @@ -57,8 +57,8 @@ struct CommonProto #define DECLARE_COMMON_SERIALISER(T) \ struct CommonProto::Serialise< T > \ { \ - static T read(const Store & store, CommonProto::ReadConn conn); \ - static void write(const Store & store, CommonProto::WriteConn conn, const T & str); \ + static T read(const StoreDirConfig & store, CommonProto::ReadConn conn); \ + static void write(const StoreDirConfig & store, CommonProto::WriteConn conn, const T & str); \ } template<> diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index a5f7cdf81..f42a13126 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -38,14 +38,14 @@ ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) return FileIngestionMethod::Flat; } -std::string ContentAddressMethod::render(HashType ht) const +std::string ContentAddressMethod::render(HashAlgorithm ha) const { return std::visit(overloaded { [&](const TextIngestionMethod & th) { - return std::string{"text:"} + printHashType(ht); + return std::string{"text:"} + printHashAlgo(ha); }, [&](const FileIngestionMethod & fim) { - return "fixed:" + makeFileIngestionPrefix(fim) + printHashType(ht); + return "fixed:" + makeFileIngestionPrefix(fim) + printHashAlgo(ha); } }, raw); } @@ -61,13 +61,13 @@ std::string ContentAddress::render() const + makeFileIngestionPrefix(method); }, }, method.raw) - + this->hash.to_string(HashFormat::Base32, true); + + this->hash.to_string(HashFormat::Nix32, true); } /** * Parses content address strings up to the hash. */ -static std::pair parseContentAddressMethodPrefix(std::string_view & rest) +static std::pair parseContentAddressMethodPrefix(std::string_view & rest) { std::string_view wholeInput { rest }; @@ -83,27 +83,27 @@ static std::pair parseContentAddressMethodPrefix auto hashTypeRaw = splitPrefixTo(rest, ':'); if (!hashTypeRaw) throw UsageError("content address hash must be in form ':', but found: %s", wholeInput); - HashType hashType = parseHashType(*hashTypeRaw); - return hashType; + HashAlgorithm hashAlgo = parseHashAlgo(*hashTypeRaw); + return hashAlgo; }; // Switch on prefix if (prefix == "text") { // No parsing of the ingestion method, "text" only support flat. - HashType hashType = parseHashType_(); + HashAlgorithm hashAlgo = parseHashType_(); return { TextIngestionMethod {}, - std::move(hashType), + std::move(hashAlgo), }; } else if (prefix == "fixed") { // Parse method auto method = FileIngestionMethod::Flat; if (splitPrefix(rest, "r:")) method = FileIngestionMethod::Recursive; - HashType hashType = parseHashType_(); + HashAlgorithm hashAlgo = parseHashType_(); return { std::move(method), - std::move(hashType), + std::move(hashAlgo), }; } else throw UsageError("content address prefix '%s' is unrecognized. Recogonized prefixes are 'text' or 'fixed'", prefix); @@ -113,15 +113,15 @@ ContentAddress ContentAddress::parse(std::string_view rawCa) { auto rest = rawCa; - auto [caMethod, hashType] = parseContentAddressMethodPrefix(rest); + auto [caMethod, hashAlgo] = parseContentAddressMethodPrefix(rest); return ContentAddress { .method = std::move(caMethod), - .hash = Hash::parseNonSRIUnprefixed(rest, hashType), + .hash = Hash::parseNonSRIUnprefixed(rest, hashAlgo), }; } -std::pair ContentAddressMethod::parse(std::string_view caMethod) +std::pair ContentAddressMethod::parse(std::string_view caMethod) { std::string asPrefix = std::string{caMethod} + ":"; // parseContentAddressMethodPrefix takes its argument by reference @@ -144,7 +144,7 @@ std::string renderContentAddress(std::optional ca) std::string ContentAddress::printMethodAlgo() const { return method.renderPrefix() - + printHashType(hash.type); + + printHashAlgo(hash.algo); } bool StoreReferences::empty() const diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index c4d619bdc..05234da38 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -39,12 +39,12 @@ enum struct FileIngestionMethod : uint8_t { /** * Flat-file hashing. Directly ingest the contents of a single file */ - Flat = false, + Flat = 0, /** * Recursive (or NAR) hashing. Serializes the file-system object in Nix * Archive format and ingest that */ - Recursive = true + Recursive = 1 }; /** @@ -94,7 +94,7 @@ struct ContentAddressMethod /** * Parse a content addressing method and hash type. */ - static std::pair parse(std::string_view rawCaMethod); + static std::pair parse(std::string_view rawCaMethod); /** * Render a content addressing method and hash type in a @@ -102,7 +102,7 @@ struct ContentAddressMethod * * The rough inverse of `parse()`. */ - std::string render(HashType ht) const; + std::string render(HashAlgorithm ha) const; }; diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 1027469c9..1b705733c 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -1,4 +1,5 @@ #include "crypto.hh" +#include "file-system.hh" #include "util.hh" #include "globals.hh" diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 007ffc05a..a112d6d31 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -400,22 +400,22 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); - auto [contentAddressMethod, hashType_] = ContentAddressMethod::parse(camStr); - auto hashType = hashType_; // work around clang bug + auto [contentAddressMethod, hashAlgo_] = ContentAddressMethod::parse(camStr); + auto hashAlgo = hashAlgo_; // work around clang bug FramedSource source(from); // TODO this is essentially RemoteStore::addCAToStore. Move it up to Store. return std::visit(overloaded { [&](const TextIngestionMethod &) { - if (hashType != htSHA256) + if (hashAlgo != HashAlgorithm::SHA256) throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", - name, printHashType(hashType)); + name, printHashAlgo(hashAlgo)); // We could stream this by changing Store std::string contents = source.drain(); auto path = store->addTextToStore(name, contents, refs, repair); return store->queryPathInfo(path); }, [&](const FileIngestionMethod & fim) { - auto path = store->addToStoreFromDump(source, name, fim, hashType, repair, refs); + auto path = store->addToStoreFromDump(source, name, fim, hashAlgo, repair, refs); return store->queryPathInfo(path); }, }, contentAddressMethod.raw); @@ -424,7 +424,7 @@ static void performOp(TunnelLogger * logger, ref store, WorkerProto::Serialise::write(*store, wconn, *pathInfo); } else { - HashType hashAlgo; + HashAlgorithm hashAlgo; std::string baseName; FileIngestionMethod method; { @@ -440,7 +440,7 @@ static void performOp(TunnelLogger * logger, ref store, hashAlgoRaw = "sha256"; method = FileIngestionMethod::Recursive; } - hashAlgo = parseHashType(hashAlgoRaw); + hashAlgo = parseHashAlgo(hashAlgoRaw); } auto dumpSource = sinkToSource([&](Sink & saved) { @@ -454,13 +454,13 @@ static void performOp(TunnelLogger * logger, ref store, eagerly consume the entire stream it's given, past the length of the Nar. */ TeeSource savedNARSource(from, saved); - ParseSink sink; /* null sink; just parse the NAR */ + NullParseSink sink; /* just parse the NAR */ parseDump(sink, savedNARSource); } else { /* Incrementally parse the NAR file, stripping the metadata, and streaming the sole file we expect into `saved`. */ - RetrieveRegularNARSink savedRegular { saved }; + RegularFileSink savedRegular { saved }; parseDump(savedRegular, from); if (!savedRegular.regular) throw Error("regular file expected"); } @@ -574,6 +574,15 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::BuildDerivation: { auto drvPath = store->parseStorePath(readString(from)); BasicDerivation drv; + /* + * Note: unlike wopEnsurePath, this operation reads a + * derivation-to-be-realized from the client with + * readDerivation(Source,Store) rather than reading it from + * the local store with Store::readDerivation(). Since the + * derivation-to-be-realized is not registered in the store + * it cannot be trusted that its outPath was calculated + * correctly. + */ readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath)); BuildMode buildMode = (BuildMode) readInt(from); logger->startWork(); @@ -657,6 +666,21 @@ static void performOp(TunnelLogger * logger, ref store, break; } + case WorkerProto::Op::AddPermRoot: { + if (!trusted) + throw Error( + "you are not privileged to create perm roots\n\n" + "hint: you can just do this client-side without special privileges, and probably want to do that instead."); + auto storePath = WorkerProto::Serialise::read(*store, rconn); + Path gcRoot = absPath(readString(from)); + logger->startWork(); + auto & localFSStore = require(*store); + localFSStore.addPermRoot(storePath, gcRoot); + logger->stopWork(); + to << gcRoot; + break; + } + case WorkerProto::Op::AddIndirectRoot: { Path path = absPath(readString(from)); @@ -868,7 +892,7 @@ static void performOp(TunnelLogger * logger, ref store, bool repair, dontCheckSigs; auto path = store->parseStorePath(readString(from)); auto deriver = readString(from); - auto narHash = Hash::parseAny(readString(from), htSHA256); + auto narHash = Hash::parseAny(readString(from), HashAlgorithm::SHA256); ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); @@ -899,7 +923,7 @@ static void performOp(TunnelLogger * logger, ref store, source = std::make_unique(from, to); else { TeeSource tee { from, saved }; - ParseSink ether; + NullParseSink ether; parseDump(ether, tee); source = std::make_unique(saved.s); } diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index a5ceb29dc..c35150b57 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -6,13 +6,12 @@ #include "split.hh" #include "common-protocol.hh" #include "common-protocol-impl.hh" -#include "fs-accessor.hh" #include #include namespace nix { -std::optional DerivationOutput::path(const Store & store, std::string_view drvName, OutputNameView outputName) const +std::optional DerivationOutput::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { return std::visit(overloaded { [](const DerivationOutput::InputAddressed & doi) -> std::optional { @@ -36,7 +35,7 @@ std::optional DerivationOutput::path(const Store & store, std::string } -StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, OutputNameView outputName) const +StorePath DerivationOutput::CAFixed::path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { return store.makeFixedOutputPathFromCA( outputPathName(drvName, outputName), @@ -152,11 +151,10 @@ StorePath writeDerivation(Store & store, /* Read string `s' from stream `str'. */ static void expect(std::istream & str, std::string_view s) { - char s2[s.size()]; - str.read(s2, s.size()); - std::string_view s2View { s2, s.size() }; - if (s2View != s) - throw FormatError("expected string '%s', got '%s'", s, s2View); + for (auto & c : s) { + if (str.get() != c) + throw FormatError("expected string '%1%'", s); + } } @@ -216,26 +214,26 @@ static StringSet parseStrings(std::istream & str, bool arePaths) static DerivationOutput parseDerivationOutput( - const Store & store, - std::string_view pathS, std::string_view hashAlgo, std::string_view hashS, + const StoreDirConfig & store, + std::string_view pathS, std::string_view hashAlgoStr, std::string_view hashS, const ExperimentalFeatureSettings & xpSettings) { - if (hashAlgo != "") { - ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgo); + if (hashAlgoStr != "") { + ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgoStr); if (method == TextIngestionMethod {}) xpSettings.require(Xp::DynamicDerivations); - const auto hashType = parseHashType(hashAlgo); + const auto hashAlgo = parseHashAlgo(hashAlgoStr); if (hashS == "impure") { xpSettings.require(Xp::ImpureDerivations); if (pathS != "") throw FormatError("impure derivation output should not specify output path"); return DerivationOutput::Impure { .method = std::move(method), - .hashType = std::move(hashType), + .hashAlgo = std::move(hashAlgo), }; } else if (hashS != "") { validatePath(pathS); - auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType); + auto hash = Hash::parseNonSRIUnprefixed(hashS, hashAlgo); return DerivationOutput::CAFixed { .ca = ContentAddress { .method = std::move(method), @@ -248,7 +246,7 @@ static DerivationOutput parseDerivationOutput( throw FormatError("content-addressed derivation output should not specify output path"); return DerivationOutput::CAFloating { .method = std::move(method), - .hashType = std::move(hashType), + .hashAlgo = std::move(hashAlgo), }; } } else { @@ -263,7 +261,7 @@ static DerivationOutput parseDerivationOutput( } static DerivationOutput parseDerivationOutput( - const Store & store, std::istringstream & str, + const StoreDirConfig & store, std::istringstream & str, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings) { expect(str, ","); const auto pathS = parseString(str); @@ -292,7 +290,7 @@ enum struct DerivationATermVersion { }; static DerivedPathMap::ChildNode parseDerivedPathMapNode( - const Store & store, + const StoreDirConfig & store, std::istringstream & str, DerivationATermVersion version) { @@ -339,7 +337,7 @@ static DerivedPathMap::ChildNode parseDerivedPathMapNode( Derivation parseDerivation( - const Store & store, std::string && s, std::string_view name, + const StoreDirConfig & store, std::string && s, std::string_view name, const ExperimentalFeatureSettings & xpSettings) { Derivation drv; @@ -353,7 +351,7 @@ Derivation parseDerivation( expect(str, "erive("); version = DerivationATermVersion::Traditional; break; - case 'r': + case 'r': { expect(str, "rvWithVersion("); auto versionS = parseString(str); if (versionS == "xp-dyn-drv") { @@ -366,6 +364,9 @@ Derivation parseDerivation( expect(str, ","); break; } + default: + throw Error("derivation does not start with 'Derive' or 'DrvWithVersion'"); + } /* Parse the list of outputs. */ expect(str, "["); @@ -469,7 +470,7 @@ static void printUnquotedStrings(std::string & res, ForwardIterator i, ForwardIt } -static void unparseDerivedPathMapNode(const Store & store, std::string & s, const DerivedPathMap::ChildNode & node) +static void unparseDerivedPathMapNode(const StoreDirConfig & store, std::string & s, const DerivedPathMap::ChildNode & node) { s += ','; if (node.childMap.empty()) { @@ -510,7 +511,7 @@ static bool hasDynamicDrvDep(const Derivation & drv) } -std::string Derivation::unparse(const Store & store, bool maskOutputs, +std::string Derivation::unparse(const StoreDirConfig & store, bool maskOutputs, DerivedPathMap::ChildNode::Map * actualInputs) const { std::string s; @@ -546,7 +547,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, }, [&](const DerivationOutput::CAFloating & dof) { s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashType(dof.hashType)); + s += ','; printUnquotedString(s, dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo)); s += ','; printUnquotedString(s, ""); }, [&](const DerivationOutput::Deferred &) { @@ -557,7 +558,7 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs, [&](const DerivationOutput::Impure & doi) { // FIXME s += ','; printUnquotedString(s, ""); - s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashType(doi.hashType)); + s += ','; printUnquotedString(s, doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo)); s += ','; printUnquotedString(s, "impure"); } }, i.second.raw); @@ -630,7 +631,7 @@ DerivationType BasicDerivation::type() const floatingCAOutputs, deferredIAOutputs, impureOutputs; - std::optional floatingHashType; + std::optional floatingHashAlgo; for (auto & i : outputs) { std::visit(overloaded { @@ -642,10 +643,10 @@ DerivationType BasicDerivation::type() const }, [&](const DerivationOutput::CAFloating & dof) { floatingCAOutputs.insert(i.first); - if (!floatingHashType) { - floatingHashType = dof.hashType; + if (!floatingHashAlgo) { + floatingHashAlgo = dof.hashAlgo; } else { - if (*floatingHashType != dof.hashType) + if (*floatingHashAlgo != dof.hashAlgo) throw Error("all floating outputs must use the same hash type"); } }, @@ -773,7 +774,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut std::map outputHashes; for (const auto & i : drv.outputs) { auto & dof = std::get(i.second.raw); - auto hash = hashString(htSHA256, "fixed:out:" + auto hash = hashString(HashAlgorithm::SHA256, "fixed:out:" + dof.ca.printMethodAlgo() + ":" + dof.ca.hash.to_string(HashFormat::Base16, false) + ":" + store.printStorePath(dof.path(store, drv.name, i.first))); @@ -824,7 +825,7 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut } } - auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2)); + auto hash = hashString(HashAlgorithm::SHA256, drv.unparse(store, maskOutputs, &inputs2)); std::map outputHashes; for (const auto & [outputName, _] : drv.outputs) { @@ -844,7 +845,7 @@ std::map staticOutputHashes(Store & store, const Derivation & } -static DerivationOutput readDerivationOutput(Source & in, const Store & store) +static DerivationOutput readDerivationOutput(Source & in, const StoreDirConfig & store) { const auto pathS = readString(in); const auto hashAlgo = readString(in); @@ -861,7 +862,7 @@ StringSet BasicDerivation::outputNames() const return names; } -DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const Store & store) const +DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const StoreDirConfig & store) const { DerivationOutputsAndOptPaths outsAndOptPaths; for (auto & [outputName, output] : outputs) @@ -883,7 +884,7 @@ std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) } -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name) +Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name) { drv.name = name; @@ -911,7 +912,7 @@ Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, } -void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv) +void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv) { out << drv.outputs.size(); for (auto & i : drv.outputs) { @@ -929,7 +930,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, [&](const DerivationOutput::CAFloating & dof) { out << "" - << (dof.method.renderPrefix() + printHashType(dof.hashType)) + << (dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo)) << ""; }, [&](const DerivationOutput::Deferred &) { @@ -939,7 +940,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr }, [&](const DerivationOutput::Impure & doi) { out << "" - << (doi.method.renderPrefix() + printHashType(doi.hashType)) + << (doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo)) << "impure"; }, }, i.second.raw); @@ -957,7 +958,7 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr std::string hashPlaceholder(const OutputNameView outputName) { // FIXME: memoize? - return "/" + hashString(htSHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Base32, false); + return "/" + hashString(HashAlgorithm::SHA256, concatStrings("nix-output:", outputName)).to_string(HashFormat::Nix32, false); } @@ -1001,13 +1002,13 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String } -std::optional Derivation::tryResolve(Store & store) const +std::optional Derivation::tryResolve(Store & store, Store * evalStore) const { std::map, StorePath> inputDrvOutputs; std::function::ChildNode &)> accum; accum = [&](auto & inputDrv, auto & node) { - for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv)) { + for (auto & [outputName, outputPath] : store.queryPartialDerivationOutputMap(inputDrv, evalStore)) { if (outputPath) { inputDrvOutputs.insert_or_assign({inputDrv, outputName}, *outputPath); if (auto p = get(node.childMap, outputName)) @@ -1149,10 +1150,10 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const } -const Hash impureOutputHash = hashString(htSHA256, "impure"); +const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure"); nlohmann::json DerivationOutput::toJSON( - const Store & store, std::string_view drvName, OutputNameView outputName) const + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const { nlohmann::json res = nlohmann::json::object(); std::visit(overloaded { @@ -1166,11 +1167,11 @@ nlohmann::json DerivationOutput::toJSON( // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { - res["hashAlgo"] = dof.method.renderPrefix() + printHashType(dof.hashType); + res["hashAlgo"] = dof.method.renderPrefix() + printHashAlgo(dof.hashAlgo); }, [&](const DerivationOutput::Deferred &) {}, [&](const DerivationOutput::Impure & doi) { - res["hashAlgo"] = doi.method.renderPrefix() + printHashType(doi.hashType); + res["hashAlgo"] = doi.method.renderPrefix() + printHashAlgo(doi.hashAlgo); res["impure"] = true; }, }, raw); @@ -1179,7 +1180,7 @@ nlohmann::json DerivationOutput::toJSON( DerivationOutput DerivationOutput::fromJSON( - const Store & store, std::string_view drvName, OutputNameView outputName, + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName, const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings) { @@ -1190,15 +1191,15 @@ DerivationOutput DerivationOutput::fromJSON( for (const auto & [key, _] : json) keys.insert(key); - auto methodAlgo = [&]() -> std::pair { - std::string hashAlgo = json["hashAlgo"]; + auto methodAlgo = [&]() -> std::pair { + std::string hashAlgoStr = json["hashAlgo"]; // remaining to parse, will be mutated by parsers - std::string_view s = hashAlgo; + std::string_view s = hashAlgoStr; ContentAddressMethod method = ContentAddressMethod::parsePrefix(s); if (method == TextIngestionMethod {}) xpSettings.require(Xp::DynamicDerivations); - auto hashType = parseHashType(s); - return { std::move(method), std::move(hashType) }; + auto hashAlgo = parseHashAlgo(s); + return { std::move(method), std::move(hashAlgo) }; }; if (keys == (std::set { "path" })) { @@ -1208,11 +1209,11 @@ DerivationOutput DerivationOutput::fromJSON( } else if (keys == (std::set { "path", "hashAlgo", "hash" })) { - auto [method, hashType] = methodAlgo(); + auto [method, hashAlgo] = methodAlgo(); auto dof = DerivationOutput::CAFixed { .ca = ContentAddress { .method = std::move(method), - .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashType), + .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashAlgo), }, }; if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) @@ -1222,10 +1223,10 @@ DerivationOutput DerivationOutput::fromJSON( else if (keys == (std::set { "hashAlgo" })) { xpSettings.require(Xp::CaDerivations); - auto [method, hashType] = methodAlgo(); + auto [method, hashAlgo] = methodAlgo(); return DerivationOutput::CAFloating { .method = std::move(method), - .hashType = std::move(hashType), + .hashAlgo = std::move(hashAlgo), }; } @@ -1235,10 +1236,10 @@ DerivationOutput DerivationOutput::fromJSON( else if (keys == (std::set { "hashAlgo", "impure" })) { xpSettings.require(Xp::ImpureDerivations); - auto [method, hashType] = methodAlgo(); + auto [method, hashAlgo] = methodAlgo(); return DerivationOutput::Impure { .method = std::move(method), - .hashType = hashType, + .hashAlgo = hashAlgo, }; } @@ -1248,7 +1249,7 @@ DerivationOutput DerivationOutput::fromJSON( } -nlohmann::json Derivation::toJSON(const Store & store) const +nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const { nlohmann::json res = nlohmann::json::object(); @@ -1301,7 +1302,7 @@ nlohmann::json Derivation::toJSON(const Store & store) const Derivation Derivation::fromJSON( - const Store & store, + const StoreDirConfig & store, const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings) { diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index fa14e7536..2a326b578 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -17,7 +17,7 @@ namespace nix { -class Store; +struct StoreDirConfig; /* Abstract syntax of derivations. */ @@ -55,7 +55,7 @@ struct DerivationOutput * @param drvName The name of the derivation this is an output of, without the `.drv`. * @param outputName The name of this output. */ - StorePath path(const Store & store, std::string_view drvName, OutputNameView outputName) const; + StorePath path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; GENERATE_CMP(CAFixed, me->ca); }; @@ -75,9 +75,9 @@ struct DerivationOutput /** * How the serialization will be hashed */ - HashType hashType; + HashAlgorithm hashAlgo; - GENERATE_CMP(CAFloating, me->method, me->hashType); + GENERATE_CMP(CAFloating, me->method, me->hashAlgo); }; /** @@ -102,9 +102,9 @@ struct DerivationOutput /** * How the serialization will be hashed */ - HashType hashType; + HashAlgorithm hashAlgo; - GENERATE_CMP(Impure, me->method, me->hashType); + GENERATE_CMP(Impure, me->method, me->hashAlgo); }; typedef std::variant< @@ -132,17 +132,17 @@ struct DerivationOutput * the safer interface provided by * BasicDerivation::outputsAndOptPaths */ - std::optional path(const Store & store, std::string_view drvName, OutputNameView outputName) const; + std::optional path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; nlohmann::json toJSON( - const Store & store, + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; /** * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivationOutput fromJSON( - const Store & store, + const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName, const nlohmann::json & json, @@ -304,7 +304,7 @@ struct BasicDerivation * augmented with knowledge of the Store paths they would be written * into. */ - DerivationOutputsAndOptPaths outputsAndOptPaths(const Store & store) const; + DerivationOutputsAndOptPaths outputsAndOptPaths(const StoreDirConfig & store) const; static std::string_view nameFromPath(const StorePath & storePath); @@ -318,6 +318,8 @@ struct BasicDerivation me->name); }; +class Store; + struct Derivation : BasicDerivation { /** @@ -328,7 +330,7 @@ struct Derivation : BasicDerivation /** * Print a derivation. */ - std::string unparse(const Store & store, bool maskOutputs, + std::string unparse(const StoreDirConfig & store, bool maskOutputs, DerivedPathMap::ChildNode::Map * actualInputs = nullptr) const; /** @@ -340,7 +342,7 @@ struct Derivation : BasicDerivation * 2. Input placeholders are replaced with realized input store * paths. */ - std::optional tryResolve(Store & store) const; + std::optional tryResolve(Store & store, Store * evalStore = nullptr) const; /** * Like the above, but instead of querying the Nix database for @@ -365,9 +367,9 @@ struct Derivation : BasicDerivation Derivation(const BasicDerivation & bd) : BasicDerivation(bd) { } Derivation(BasicDerivation && bd) : BasicDerivation(std::move(bd)) { } - nlohmann::json toJSON(const Store & store) const; + nlohmann::json toJSON(const StoreDirConfig & store) const; static Derivation fromJSON( - const Store & store, + const StoreDirConfig & store, const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); @@ -391,7 +393,7 @@ StorePath writeDerivation(Store & store, * Read a derivation from a file. */ Derivation parseDerivation( - const Store & store, + const StoreDirConfig & store, std::string && s, std::string_view name, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); @@ -493,8 +495,8 @@ extern Sync drvHashes; struct Source; struct Sink; -Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name); -void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv); +Source & readDerivation(Source & in, const StoreDirConfig & store, BasicDerivation & drv, std::string_view name); +void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDerivation & drv); /** * This creates an opaque and almost certainly unique string diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 5982c04b3..4c1ea417a 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -1,4 +1,5 @@ #include "derived-path-map.hh" +#include "util.hh" namespace nix { diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh index 4d72b301e..393cdedf7 100644 --- a/src/libstore/derived-path-map.hh +++ b/src/libstore/derived-path-map.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "types.hh" #include "derived-path.hh" diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 47d784deb..3105dbc93 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,4 +1,5 @@ #include "derived-path.hh" +#include "derivations.hh" #include "store-api.hh" #include @@ -32,7 +33,7 @@ CMP(SingleDerivedPath, DerivedPathBuilt, outputs) #undef CMP #undef CMP_ONE -nlohmann::json DerivedPath::Opaque::toJSON(const Store & store) const +nlohmann::json DerivedPath::Opaque::toJSON(const StoreDirConfig & store) const { return store.printStorePath(path); } @@ -86,50 +87,50 @@ nlohmann::json DerivedPath::toJSON(Store & store) const }, raw()); } -std::string DerivedPath::Opaque::to_string(const Store & store) const +std::string DerivedPath::Opaque::to_string(const StoreDirConfig & store) const { return store.printStorePath(path); } -std::string SingleDerivedPath::Built::to_string(const Store & store) const +std::string SingleDerivedPath::Built::to_string(const StoreDirConfig & store) const { return drvPath->to_string(store) + "^" + output; } -std::string SingleDerivedPath::Built::to_string_legacy(const Store & store) const +std::string SingleDerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const { return drvPath->to_string(store) + "!" + output; } -std::string DerivedPath::Built::to_string(const Store & store) const +std::string DerivedPath::Built::to_string(const StoreDirConfig & store) const { return drvPath->to_string(store) + '^' + outputs.to_string(); } -std::string DerivedPath::Built::to_string_legacy(const Store & store) const +std::string DerivedPath::Built::to_string_legacy(const StoreDirConfig & store) const { return drvPath->to_string_legacy(store) + "!" + outputs.to_string(); } -std::string SingleDerivedPath::to_string(const Store & store) const +std::string SingleDerivedPath::to_string(const StoreDirConfig & store) const { return std::visit( [&](const auto & req) { return req.to_string(store); }, raw()); } -std::string DerivedPath::to_string(const Store & store) const +std::string DerivedPath::to_string(const StoreDirConfig & store) const { return std::visit( [&](const auto & req) { return req.to_string(store); }, raw()); } -std::string SingleDerivedPath::to_string_legacy(const Store & store) const +std::string SingleDerivedPath::to_string_legacy(const StoreDirConfig & store) const { return std::visit(overloaded { [&](const SingleDerivedPath::Built & req) { return req.to_string_legacy(store); }, @@ -137,7 +138,7 @@ std::string SingleDerivedPath::to_string_legacy(const Store & store) const }, this->raw()); } -std::string DerivedPath::to_string_legacy(const Store & store) const +std::string DerivedPath::to_string_legacy(const StoreDirConfig & store) const { return std::visit(overloaded { [&](const DerivedPath::Built & req) { return req.to_string_legacy(store); }, @@ -146,7 +147,7 @@ std::string DerivedPath::to_string_legacy(const Store & store) const } -DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_view s) +DerivedPath::Opaque DerivedPath::Opaque::parse(const StoreDirConfig & store, std::string_view s) { return {store.parseStorePath(s)}; } @@ -166,7 +167,7 @@ void drvRequireExperiment( } SingleDerivedPath::Built SingleDerivedPath::Built::parse( - const Store & store, ref drv, + const StoreDirConfig & store, ref drv, OutputNameView output, const ExperimentalFeatureSettings & xpSettings) { @@ -178,7 +179,7 @@ SingleDerivedPath::Built SingleDerivedPath::Built::parse( } DerivedPath::Built DerivedPath::Built::parse( - const Store & store, ref drv, + const StoreDirConfig & store, ref drv, OutputNameView outputsS, const ExperimentalFeatureSettings & xpSettings) { @@ -190,7 +191,7 @@ DerivedPath::Built DerivedPath::Built::parse( } static SingleDerivedPath parseWithSingle( - const Store & store, std::string_view s, std::string_view separator, + const StoreDirConfig & store, std::string_view s, std::string_view separator, const ExperimentalFeatureSettings & xpSettings) { size_t n = s.rfind(separator); @@ -207,7 +208,7 @@ static SingleDerivedPath parseWithSingle( } SingleDerivedPath SingleDerivedPath::parse( - const Store & store, + const StoreDirConfig & store, std::string_view s, const ExperimentalFeatureSettings & xpSettings) { @@ -215,7 +216,7 @@ SingleDerivedPath SingleDerivedPath::parse( } SingleDerivedPath SingleDerivedPath::parseLegacy( - const Store & store, + const StoreDirConfig & store, std::string_view s, const ExperimentalFeatureSettings & xpSettings) { @@ -223,7 +224,7 @@ SingleDerivedPath SingleDerivedPath::parseLegacy( } static DerivedPath parseWith( - const Store & store, std::string_view s, std::string_view separator, + const StoreDirConfig & store, std::string_view s, std::string_view separator, const ExperimentalFeatureSettings & xpSettings) { size_t n = s.rfind(separator); @@ -240,7 +241,7 @@ static DerivedPath parseWith( } DerivedPath DerivedPath::parse( - const Store & store, + const StoreDirConfig & store, std::string_view s, const ExperimentalFeatureSettings & xpSettings) { @@ -248,7 +249,7 @@ DerivedPath DerivedPath::parse( } DerivedPath DerivedPath::parseLegacy( - const Store & store, + const StoreDirConfig & store, std::string_view s, const ExperimentalFeatureSettings & xpSettings) { diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index 4d7033df2..b238f844e 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -1,10 +1,10 @@ #pragma once ///@file -#include "util.hh" #include "path.hh" #include "outputs-spec.hh" #include "comparator.hh" +#include "config.hh" #include @@ -12,6 +12,9 @@ namespace nix { +struct StoreDirConfig; + +// TODO stop needing this, `toJSON` below should be pure class Store; /** @@ -24,9 +27,9 @@ class Store; struct DerivedPathOpaque { StorePath path; - std::string to_string(const Store & store) const; - static DerivedPathOpaque parse(const Store & store, std::string_view); - nlohmann::json toJSON(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; + static DerivedPathOpaque parse(const StoreDirConfig & store, std::string_view); + nlohmann::json toJSON(const StoreDirConfig & store) const; GENERATE_CMP(DerivedPathOpaque, me->path); }; @@ -59,18 +62,18 @@ struct SingleDerivedPathBuilt { /** * Uses `^` as the separator */ - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; /** * Uses `!` as the separator */ - std::string to_string_legacy(const Store & store) const; + std::string to_string_legacy(const StoreDirConfig & store) const; /** * The caller splits on the separator, so it works for both variants. * * @param xpSettings Stop-gap to avoid globals during unit tests. */ static SingleDerivedPathBuilt parse( - const Store & store, ref drvPath, + const StoreDirConfig & store, ref drvPath, OutputNameView outputs, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; @@ -120,18 +123,18 @@ struct SingleDerivedPath : _SingleDerivedPathRaw { /** * Uses `^` as the separator */ - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; /** * Uses `!` as the separator */ - std::string to_string_legacy(const Store & store) const; + std::string to_string_legacy(const StoreDirConfig & store) const; /** * Uses `^` as the separator * * @param xpSettings Stop-gap to avoid globals during unit tests. */ static SingleDerivedPath parse( - const Store & store, + const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** @@ -140,7 +143,7 @@ struct SingleDerivedPath : _SingleDerivedPathRaw { * @param xpSettings Stop-gap to avoid globals during unit tests. */ static SingleDerivedPath parseLegacy( - const Store & store, + const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; @@ -182,18 +185,18 @@ struct DerivedPathBuilt { /** * Uses `^` as the separator */ - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; /** * Uses `!` as the separator */ - std::string to_string_legacy(const Store & store) const; + std::string to_string_legacy(const StoreDirConfig & store) const; /** * The caller splits on the separator, so it works for both variants. * * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivedPathBuilt parse( - const Store & store, ref, + const StoreDirConfig & store, ref, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; @@ -242,18 +245,18 @@ struct DerivedPath : _DerivedPathRaw { /** * Uses `^` as the separator */ - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; /** * Uses `!` as the separator */ - std::string to_string_legacy(const Store & store) const; + std::string to_string_legacy(const StoreDirConfig & store) const; /** * Uses `^` as the separator * * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivedPath parse( - const Store & store, + const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); /** @@ -262,7 +265,7 @@ struct DerivedPath : _DerivedPathRaw { * @param xpSettings Stop-gap to avoid globals during unit tests. */ static DerivedPath parseLegacy( - const Store & store, + const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); diff --git a/src/libstore/downstream-placeholder.cc b/src/libstore/downstream-placeholder.cc index ca9f7476e..91d47f946 100644 --- a/src/libstore/downstream-placeholder.cc +++ b/src/libstore/downstream-placeholder.cc @@ -5,7 +5,7 @@ namespace nix { std::string DownstreamPlaceholder::render() const { - return "/" + hash.to_string(HashFormat::Base32, false); + return "/" + hash.to_string(HashFormat::Nix32, false); } @@ -19,7 +19,7 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownCaOutput( auto drvName = drvNameWithExtension.substr(0, drvNameWithExtension.size() - 4); auto clearText = "nix-upstream-output:" + std::string { drvPath.hashPart() } + ":" + outputPathName(drvName, outputName); return DownstreamPlaceholder { - hashString(htSHA256, clearText) + hashString(HashAlgorithm::SHA256, clearText) }; } @@ -31,10 +31,10 @@ DownstreamPlaceholder DownstreamPlaceholder::unknownDerivation( xpSettings.require(Xp::DynamicDerivations); auto compressed = compressHash(placeholder.hash, 20); auto clearText = "nix-computed-output:" - + compressed.to_string(HashFormat::Base32, false) + + compressed.to_string(HashFormat::Nix32, false) + ":" + std::string { outputName }; return DownstreamPlaceholder { - hashString(htSHA256, clearText) + hashString(HashAlgorithm::SHA256, clearText) }; } diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 74d6ed3b5..821cda399 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -72,7 +72,7 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store Callback> callback) noexcept override { callback(nullptr); } - virtual ref getFSAccessor() override + virtual ref getFSAccessor(bool requireValidPath) override { unsupported("getFSAccessor"); } }; diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 91b7e30db..d57b25bd7 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -30,7 +30,7 @@ void Store::exportPath(const StorePath & path, Sink & sink) { auto info = queryPathInfo(path); - HashSink hashSink(htSHA256); + HashSink hashSink(HashAlgorithm::SHA256); TeeSink teeSink(sink, hashSink); narFromPath(path, teeSink); @@ -39,9 +39,9 @@ void Store::exportPath(const StorePath & path, Sink & sink) filesystem corruption from spreading to other machines. Don't complain if the stored hash is zero (unknown). */ Hash hash = hashSink.currentHash().first; - if (hash != info->narHash && info->narHash != Hash(info->narHash.type)) + if (hash != info->narHash && info->narHash != Hash(info->narHash.algo)) throw Error("hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), info->narHash.to_string(HashFormat::Base32, true), hash.to_string(HashFormat::Base32, true)); + printStorePath(path), info->narHash.to_string(HashFormat::Nix32, true), hash.to_string(HashFormat::Nix32, true)); teeSink << exportMagic @@ -65,7 +65,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) /* Extract the NAR from the source. */ StringSink saved; TeeSource tee { source, saved }; - ParseSink ether; + NullParseSink ether; parseDump(ether, tee); uint32_t magic = readInt(source); @@ -79,7 +79,7 @@ StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) auto references = CommonProto::Serialise::read(*this, CommonProto::ReadConn { .from = source }); auto deriver = readString(source); - auto narHash = hashString(htSHA256, saved.s); + auto narHash = hashString(HashAlgorithm::SHA256, saved.s); ValidPathInfo info { path, narHash }; if (deriver != "") diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index a283af5a2..dcbec4acd 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,11 +1,12 @@ #include "filetransfer.hh" -#include "util.hh" +#include "namespaces.hh" #include "globals.hh" #include "store-api.hh" #include "s3.hh" #include "compression.hh" #include "finally.hh" #include "callback.hh" +#include "signals.hh" #if ENABLE_S3 #include diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh deleted file mode 100644 index 1df19e647..000000000 --- a/src/libstore/fs-accessor.hh +++ /dev/null @@ -1,52 +0,0 @@ -#pragma once -///@file - -#include "types.hh" - -namespace nix { - -/** - * An abstract class for accessing a filesystem-like structure, such - * as a (possibly remote) Nix store or the contents of a NAR file. - */ -class FSAccessor -{ -public: - enum Type { tMissing, tRegular, tSymlink, tDirectory }; - - struct Stat - { - Type type = tMissing; - /** - * regular files only - */ - uint64_t fileSize = 0; - /** - * regular files only - */ - bool isExecutable = false; // regular files only - /** - * regular files only - */ - uint64_t narOffset = 0; // regular files only - }; - - virtual ~FSAccessor() { } - - virtual Stat stat(const Path & path) = 0; - - virtual StringSet readDirectory(const Path & path) = 0; - - /** - * Read a file inside the store. - * - * If `requireValidPath` is set to `true` (the default), the path must be - * inside a valid store path, otherwise it just needs to be physically - * present (but not necessarily properly registered) - */ - virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0; - - virtual std::string readLink(const Path & path) = 0; -}; - -} diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ddfbac521..be3a37984 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -2,6 +2,13 @@ #include "globals.hh" #include "local-store.hh" #include "finally.hh" +#include "unix-domain-socket.hh" +#include "signals.hh" + +#if !defined(__linux__) +// For shelling out to lsof +# include "processes.hh" +#endif #include #include @@ -43,7 +50,7 @@ static void makeSymlink(const Path & link, const Path & target) void LocalStore::addIndirectRoot(const Path & path) { - std::string hash = hashString(htSHA1, path).to_string(HashFormat::Base32, false); + std::string hash = hashString(HashAlgorithm::SHA1, path).to_string(HashFormat::Nix32, false); Path realRoot = canonPath(fmt("%1%/%2%/auto/%3%", stateDir, gcRootsDir, hash)); makeSymlink(realRoot, path); } @@ -323,9 +330,7 @@ typedef std::unordered_map> UncheckedRoots static void readProcLink(const std::string & file, UncheckedRoots & roots) { - /* 64 is the starting buffer size gnu readlink uses... */ - auto bufsiz = ssize_t{64}; -try_again: + constexpr auto bufsiz = PATH_MAX; char buf[bufsiz]; auto res = readlink(file.c_str(), buf, bufsiz); if (res == -1) { @@ -334,10 +339,7 @@ try_again: throw SysError("reading symlink"); } if (res == bufsiz) { - if (SSIZE_MAX / 2 < bufsiz) - throw Error("stupidly long symlink"); - bufsiz *= 2; - goto try_again; + throw Error("overly long symlink starting with '%1%'", std::string_view(buf, bufsiz)); } if (res > 0 && buf[0] == '/') roots[std::string(static_cast(buf), res)] diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 9c25d9868..f401d076d 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,7 +1,8 @@ #include "globals.hh" -#include "util.hh" +#include "current-process.hh" #include "archive.hh" #include "args.hh" +#include "users.hh" #include "abstract-setting-to-json.hh" #include "compute-levels.hh" @@ -17,9 +18,13 @@ #include #ifdef __GLIBC__ -#include -#include -#include +# include +# include +# include +#endif + +#if __APPLE__ +# include "processes.hh" #endif #include "config-impl.hh" @@ -111,7 +116,14 @@ Settings::Settings() void loadConfFile() { - globalConfig.applyConfigFile(settings.nixConfDir + "/nix.conf"); + auto applyConfigFile = [&](const Path & path) { + try { + std::string contents = readFile(path); + globalConfig.applyConfig(contents, path); + } catch (SysError &) { } + }; + + applyConfigFile(settings.nixConfDir + "/nix.conf"); /* We only want to send overrides to the daemon, i.e. stuff from ~/.nix/nix.conf or the command line. */ @@ -119,7 +131,7 @@ void loadConfFile() auto files = settings.nixUserConfFiles; for (auto file = files.rbegin(); file != files.rend(); file++) { - globalConfig.applyConfigFile(*file); + applyConfigFile(*file); } auto nixConfEnv = getEnv("NIX_CONFIG"); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 12fb48d93..8da9e371f 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -3,7 +3,7 @@ #include "types.hh" #include "config.hh" -#include "util.hh" +#include "environment-variables.hh" #include "experimental-features.hh" #include @@ -117,10 +117,11 @@ public: Setting storeUri{this, getEnv("NIX_REMOTE").value_or("auto"), "store", R"( - The [URL of the Nix store](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + The [URL of the Nix store](@docroot@/store/types/index.md#store-url-format) to use for most operations. - See [`nix help-stores`](@docroot@/command-ref/new-cli/nix3-help-stores.md) - for supported store types and settings. + See the + [Store Types](@docroot@/store/types/index.md) + section of the manual for supported store types and settings. )"}; Setting keepFailed{this, false, "keep-failed", @@ -183,7 +184,9 @@ public: command line switch and defaults to `1`. The value `0` means that the builder should use all available CPU cores in the system. )", - {"build-cores"}, false}; + {"build-cores"}, + // Don't document the machine-specific default value + false}; /** * Read-only mode. Don't copy stuff to the store, don't change @@ -198,7 +201,7 @@ public: Nix will only build a given [derivation](@docroot@/language/derivations.md) locally when its `system` attribute equals any of the values specified here or in [`extra-platforms`](#conf-extra-platforms). The default value is set when Nix itself is compiled for the system it will run on. - The following system types are widely used, as [Nix is actively supported on these platforms](@docroot@/contributing/hacking.md#platforms): + The following system types are widely used, as Nix is actively supported on these platforms: - `x86_64-linux` - `x86_64-darwin` @@ -699,7 +702,10 @@ public: Build systems will usually detect the target platform to be the current physical system and therefore produce machine code incompatible with what may be intended in the derivation. You should design your derivation's `builder` accordingly and cross-check the results when using this option against natively-built versions of your derivation. - )", {}, false}; + )", + {}, + // Don't document the machine-specific default value + false}; Setting systemFeatures{ this, @@ -744,15 +750,18 @@ public: [nspawn]: https://github.com/NixOS/nix/blob/67bcb99700a0da1395fa063d7c6586740b304598/tests/systemd-nspawn.nix. Included by default on Linux if the [`auto-allocate-uids`](#conf-auto-allocate-uids) setting is enabled. - )", {}, false}; + )", + {}, + // Don't document the machine-specific default value + false}; Setting substituters{ this, Strings{"https://cache.nixos.org/"}, "substituters", R"( - A list of [URLs of Nix stores](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) to be used as substituters, separated by whitespace. - A substituter is an additional [store]{@docroot@/glossary.md##gloss-store} from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them. + A list of [URLs of Nix stores](@docroot@/store/types/index.md#store-url-format) to be used as substituters, separated by whitespace. + A substituter is an additional [store](@docroot@/glossary.md#gloss-store) from which Nix can obtain [store objects](@docroot@/glossary.md#gloss-store-object) instead of building them. Substituters are tried based on their priority value, which each substituter can set independently. Lower value means higher priority. @@ -770,7 +779,7 @@ public: Setting trustedSubstituters{ this, {}, "trusted-substituters", R"( - A list of [Nix store URLs](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format), separated by whitespace. + A list of [Nix store URLs](@docroot@/store/types/index.md#store-url-format), separated by whitespace. These are not used by default, but users of the Nix daemon can enable them by specifying [`substituters`](#conf-substituters). Unprivileged users (those set in only [`allowed-users`](#conf-allowed-users) but not [`trusted-users`](#conf-trusted-users)) can pass as `substituters` only those URLs listed in `trusted-substituters`. @@ -1084,6 +1093,16 @@ public: true, // document default Xp::ConfigurableImpureEnv }; + + Setting upgradeNixStorePathUrl{ + this, + "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix", + "upgrade-nix-store-path-url", + R"( + Used by `nix upgrade-nix`, the URL of the file that contains the + store paths of the latest Nix release. + )" + }; }; diff --git a/src/libstore/indirect-root-store.hh b/src/libstore/indirect-root-store.hh index 59e45af45..c11679fe8 100644 --- a/src/libstore/indirect-root-store.hh +++ b/src/libstore/indirect-root-store.hh @@ -11,6 +11,30 @@ namespace nix { * reference. * * See methods for details on the operations it represents. + * + * @note + * To understand the purpose of this class, it might help to do some + * "closed-world" rather than "open-world" reasoning, and consider the + * problem it solved for us. This class was factored out from + * `LocalFSStore` in order to support the following table, which + * contains 4 concrete store types (non-abstract classes, exposed to the + * user), and how they implemented the two GC root methods: + * + * @note + * | | `addPermRoot()` | `addIndirectRoot()` | + * |-------------------|-----------------|---------------------| + * | `LocalStore` | local | local | + * | `UDSRemoteStore` | local | remote | + * | `SSHStore` | doesn't have | doesn't have | + * | `MountedSSHStore` | remote | doesn't have | + * + * @note + * Note how only the local implementations of `addPermRoot()` need + * `addIndirectRoot()`; that is what this class enforces. Without it, + * and with `addPermRoot()` and `addIndirectRoot()` both `virtual`, we + * would accidentally be allowing for a combinatorial explosion of + * possible implementations many of which make no sense. Having this and + * that invariant enforced cuts down that space. */ struct IndirectRootStore : public virtual LocalFSStore { diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index c712f7eb1..06bef9d08 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -1,3 +1,4 @@ +#include "legacy-ssh-store.hh" #include "ssh-store-config.hh" #include "archive.hh" #include "pool.hh" @@ -13,432 +14,355 @@ namespace nix { -struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig +std::string LegacySSHStoreConfig::doc() { - using CommonSSHStoreConfig::CommonSSHStoreConfig; + return + #include "legacy-ssh-store.md" + ; +} - const Setting remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", - "Path to the `nix-store` executable on the remote machine."}; - const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", - "Maximum number of concurrent SSH connections."}; - - const std::string name() override { return "SSH Store"; } - - std::string doc() override - { - return - #include "legacy-ssh-store.md" - ; - } -}; - -struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store +struct LegacySSHStore::Connection { - // Hack for getting remote build log output. - // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in - // the documentation - const Setting logFD{(StoreConfig*) this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; - - struct Connection - { - std::unique_ptr sshConn; - FdSink to; - FdSource from; - ServeProto::Version remoteVersion; - bool good = true; - - /** - * Coercion to `ServeProto::ReadConn`. This makes it easy to use the - * factored out serve protocol searlizers with a - * `LegacySSHStore::Connection`. - * - * The serve protocol connection types are unidirectional, unlike - * this type. - */ - operator ServeProto::ReadConn () - { - return ServeProto::ReadConn { - .from = from, - .version = remoteVersion, - }; - } - - /* - * Coercion to `ServeProto::WriteConn`. This makes it easy to use the - * factored out serve protocol searlizers with a - * `LegacySSHStore::Connection`. - * - * The serve protocol connection types are unidirectional, unlike - * this type. - */ - operator ServeProto::WriteConn () - { - return ServeProto::WriteConn { - .to = to, - .version = remoteVersion, - }; - } - }; - - std::string host; - - ref> connections; - - SSHMaster master; - - static std::set uriSchemes() { return {"ssh"}; } - - LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params) - : StoreConfig(params) - , CommonSSHStoreConfig(params) - , LegacySSHStoreConfig(params) - , Store(params) - , host(host) - , connections(make_ref>( - std::max(1, (int) maxConnections), - [this]() { return openConnection(); }, - [](const ref & r) { return r->good; } - )) - , master( - host, - sshKey, - sshPublicHostKey, - // Use SSH master only if using more than 1 connection. - connections->capacity() > 1, - compress, - logFD) - { - } - - ref openConnection() - { - auto conn = make_ref(); - conn->sshConn = master.startCommand( - fmt("%s --serve --write", remoteProgram) - + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); - conn->to = FdSink(conn->sshConn->in.get()); - conn->from = FdSource(conn->sshConn->out.get()); - - try { - conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; - conn->to.flush(); - - StringSink saved; - try { - TeeSource tee(conn->from, saved); - unsigned int magic = readInt(tee); - if (magic != SERVE_MAGIC_2) - throw Error("'nix-store --serve' protocol mismatch from '%s'", host); - } catch (SerialisationError & e) { - /* In case the other side is waiting for our input, - close it. */ - conn->sshConn->in.close(); - auto msg = conn->from.drain(); - throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", - host, chomp(saved.s + msg)); - } - conn->remoteVersion = readInt(conn->from); - if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) - throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); - - } catch (EndOfFile & e) { - throw Error("cannot connect to '%1%'", host); - } - - return conn; - }; - - std::string getUri() override - { - return *uriSchemes().begin() + "://" + host; - } - - void queryPathInfoUncached(const StorePath & path, - Callback> callback) noexcept override - { - try { - auto conn(connections->get()); - - /* No longer support missing NAR hash */ - assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); - - debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); - - conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)}; - conn->to.flush(); - - auto p = readString(conn->from); - if (p.empty()) return callback(nullptr); - auto path2 = parseStorePath(p); - assert(path == path2); - /* Hash will be set below. FIXME construct ValidPathInfo at end. */ - auto info = std::make_shared(path, Hash::dummy); - - auto deriver = readString(conn->from); - if (deriver != "") - info->deriver = parseStorePath(deriver); - info->references = ServeProto::Serialise::read(*this, *conn); - readLongLong(conn->from); // download size - info->narSize = readLongLong(conn->from); - - { - auto s = readString(conn->from); - if (s == "") - throw Error("NAR hash is now mandatory"); - info->narHash = Hash::parseAnyPrefixed(s); - } - info->ca = ContentAddress::parseOpt(readString(conn->from)); - info->sigs = readStrings(conn->from); - - auto s = readString(conn->from); - assert(s == ""); - - callback(std::move(info)); - } catch (...) { callback.rethrow(); } - } - - void addToStore(const ValidPathInfo & info, Source & source, - RepairFlag repair, CheckSigsFlag checkSigs) override - { - debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); - - auto conn(connections->get()); - - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { - - conn->to - << ServeProto::Command::AddToStoreNar - << printStorePath(info.path) - << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(HashFormat::Base16, false); - ServeProto::write(*this, *conn, info.references); - conn->to - << info.registrationTime - << info.narSize - << info.ultimate - << info.sigs - << renderContentAddress(info.ca); - try { - copyNAR(source, conn->to); - } catch (...) { - conn->good = false; - throw; - } - conn->to.flush(); - - } else { - - conn->to - << ServeProto::Command::ImportPaths - << 1; - try { - copyNAR(source, conn->to); - } catch (...) { - conn->good = false; - throw; - } - conn->to - << exportMagic - << printStorePath(info.path); - ServeProto::write(*this, *conn, info.references); - conn->to - << (info.deriver ? printStorePath(*info.deriver) : "") - << 0 - << 0; - conn->to.flush(); - - } - - if (readInt(conn->from) != 1) - throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); - } - - void narFromPath(const StorePath & path, Sink & sink) override - { - auto conn(connections->get()); - - conn->to << ServeProto::Command::DumpStorePath << printStorePath(path); - conn->to.flush(); - copyNAR(conn->from, sink); - } - - std::optional queryPathFromHashPart(const std::string & hashPart) override - { unsupported("queryPathFromHashPart"); } - - StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method, - HashType hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) override - { unsupported("addToStore"); } - - StorePath addTextToStore( - std::string_view name, - std::string_view s, - const StorePathSet & references, - RepairFlag repair) override - { unsupported("addTextToStore"); } - -private: - - void putBuildSettings(Connection & conn) - { - conn.to - << settings.maxSilentTime - << settings.buildTimeout; - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 2) - conn.to - << settings.maxLogSize; - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 3) - conn.to - << 0 // buildRepeat hasn't worked for ages anyway - << 0; - - if (GET_PROTOCOL_MINOR(conn.remoteVersion) >= 7) { - conn.to << ((int) settings.keepFailed); - } - } - -public: - - BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) override - { - auto conn(connections->get()); - - conn->to - << ServeProto::Command::BuildDerivation - << printStorePath(drvPath); - writeDerivation(conn->to, *this, drv); - - putBuildSettings(*conn); - - conn->to.flush(); - - return ServeProto::Serialise::read(*this, *conn); - } - - void buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override - { - if (evalStore && evalStore.get() != this) - throw Error("building on an SSH store is incompatible with '--eval-store'"); - - auto conn(connections->get()); - - conn->to << ServeProto::Command::BuildPaths; - Strings ss; - for (auto & p : drvPaths) { - auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); - std::visit(overloaded { - [&](const StorePathWithOutputs & s) { - ss.push_back(s.to_string(*this)); - }, - [&](const StorePath & drvPath) { - throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); - }, - [&](std::monostate) { - throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://"); - }, - }, sOrDrvPath); - } - conn->to << ss; - - putBuildSettings(*conn); - - conn->to.flush(); - - BuildResult result; - result.status = (BuildResult::Status) readInt(conn->from); - - if (!result.success()) { - conn->from >> result.errorMsg; - throw Error(result.status, result.errorMsg); - } - } - - void ensurePath(const StorePath & path) override - { unsupported("ensurePath"); } - - virtual ref getFSAccessor() override - { unsupported("getFSAccessor"); } + std::unique_ptr sshConn; + FdSink to; + FdSource from; + ServeProto::Version remoteVersion; + bool good = true; /** - * The default instance would schedule the work on the client side, but - * for consistency with `buildPaths` and `buildDerivation` it should happen - * on the remote side. + * Coercion to `ServeProto::ReadConn`. This makes it easy to use the + * factored out serve protocol searlizers with a + * `LegacySSHStore::Connection`. * - * We make this fail for now so we can add implement this properly later - * without it being a breaking change. + * The serve protocol connection types are unidirectional, unlike + * this type. */ - void repairPath(const StorePath & path) override - { unsupported("repairPath"); } - - void computeFSClosure(const StorePathSet & paths, - StorePathSet & out, bool flipDirection = false, - bool includeOutputs = false, bool includeDerivers = false) override + operator ServeProto::ReadConn () { - if (flipDirection || includeDerivers) { - Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers); - return; - } - - auto conn(connections->get()); - - conn->to - << ServeProto::Command::QueryClosure - << includeOutputs; - ServeProto::write(*this, *conn, paths); - conn->to.flush(); - - for (auto & i : ServeProto::Serialise::read(*this, *conn)) - out.insert(i); + return ServeProto::ReadConn { + .from = from, + .version = remoteVersion, + }; } - StorePathSet queryValidPaths(const StorePathSet & paths, - SubstituteFlag maybeSubstitute = NoSubstitute) override - { - auto conn(connections->get()); - - conn->to - << ServeProto::Command::QueryValidPaths - << false // lock - << maybeSubstitute; - ServeProto::write(*this, *conn, paths); - conn->to.flush(); - - return ServeProto::Serialise::read(*this, *conn); - } - - void connect() override - { - auto conn(connections->get()); - } - - unsigned int getProtocol() override - { - auto conn(connections->get()); - return conn->remoteVersion; - } - - /** - * The legacy ssh protocol doesn't support checking for trusted-user. - * Try using ssh-ng:// instead if you want to know. + /* + * Coercion to `ServeProto::WriteConn`. This makes it easy to use the + * factored out serve protocol searlizers with a + * `LegacySSHStore::Connection`. + * + * The serve protocol connection types are unidirectional, unlike + * this type. */ - std::optional isTrustedClient() override + operator ServeProto::WriteConn () { - return std::nullopt; + return ServeProto::WriteConn { + .to = to, + .version = remoteVersion, + }; } - - void queryRealisationUncached(const DrvOutput &, - Callback> callback) noexcept override - // TODO: Implement - { unsupported("queryRealisation"); } }; + +LegacySSHStore::LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params) + : StoreConfig(params) + , CommonSSHStoreConfig(params) + , LegacySSHStoreConfig(params) + , Store(params) + , host(host) + , connections(make_ref>( + std::max(1, (int) maxConnections), + [this]() { return openConnection(); }, + [](const ref & r) { return r->good; } + )) + , master( + host, + sshKey, + sshPublicHostKey, + // Use SSH master only if using more than 1 connection. + connections->capacity() > 1, + compress, + logFD) +{ +} + + +ref LegacySSHStore::openConnection() +{ + auto conn = make_ref(); + conn->sshConn = master.startCommand( + fmt("%s --serve --write", remoteProgram) + + (remoteStore.get() == "" ? "" : " --store " + shellEscape(remoteStore.get()))); + conn->to = FdSink(conn->sshConn->in.get()); + conn->from = FdSource(conn->sshConn->out.get()); + + try { + conn->to << SERVE_MAGIC_1 << SERVE_PROTOCOL_VERSION; + conn->to.flush(); + + StringSink saved; + try { + TeeSource tee(conn->from, saved); + unsigned int magic = readInt(tee); + if (magic != SERVE_MAGIC_2) + throw Error("'nix-store --serve' protocol mismatch from '%s'", host); + } catch (SerialisationError & e) { + /* In case the other side is waiting for our input, + close it. */ + conn->sshConn->in.close(); + auto msg = conn->from.drain(); + throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", + host, chomp(saved.s + msg)); + } + conn->remoteVersion = readInt(conn->from); + if (GET_PROTOCOL_MAJOR(conn->remoteVersion) != 0x200) + throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); + + } catch (EndOfFile & e) { + throw Error("cannot connect to '%1%'", host); + } + + return conn; +}; + + +std::string LegacySSHStore::getUri() +{ + return *uriSchemes().begin() + "://" + host; +} + + +void LegacySSHStore::queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept +{ + try { + auto conn(connections->get()); + + /* No longer support missing NAR hash */ + assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); + + debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); + + conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)}; + conn->to.flush(); + + auto p = readString(conn->from); + if (p.empty()) return callback(nullptr); + auto path2 = parseStorePath(p); + assert(path == path2); + auto info = std::make_shared( + path, + ServeProto::Serialise::read(*this, *conn)); + + if (info->narHash == Hash::dummy) + throw Error("NAR hash is now mandatory"); + + auto s = readString(conn->from); + assert(s == ""); + + callback(std::move(info)); + } catch (...) { callback.rethrow(); } +} + + +void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) +{ + debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host); + + auto conn(connections->get()); + + if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { + + conn->to + << ServeProto::Command::AddToStoreNar + << printStorePath(info.path) + << (info.deriver ? printStorePath(*info.deriver) : "") + << info.narHash.to_string(HashFormat::Base16, false); + ServeProto::write(*this, *conn, info.references); + conn->to + << info.registrationTime + << info.narSize + << info.ultimate + << info.sigs + << renderContentAddress(info.ca); + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to.flush(); + + } else { + + conn->to + << ServeProto::Command::ImportPaths + << 1; + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; + } + conn->to + << exportMagic + << printStorePath(info.path); + ServeProto::write(*this, *conn, info.references); + conn->to + << (info.deriver ? printStorePath(*info.deriver) : "") + << 0 + << 0; + conn->to.flush(); + + } + + if (readInt(conn->from) != 1) + throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); +} + + +void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink) +{ + auto conn(connections->get()); + + conn->to << ServeProto::Command::DumpStorePath << printStorePath(path); + conn->to.flush(); + copyNAR(conn->from, sink); +} + + +void LegacySSHStore::putBuildSettings(Connection & conn) +{ + ServeProto::write(*this, conn, ServeProto::BuildOptions { + .maxSilentTime = settings.maxSilentTime, + .buildTimeout = settings.buildTimeout, + .maxLogSize = settings.maxLogSize, + .nrRepeats = 0, // buildRepeat hasn't worked for ages anyway + .enforceDeterminism = 0, + .keepFailed = settings.keepFailed, + }); +} + + +BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) +{ + auto conn(connections->get()); + + conn->to + << ServeProto::Command::BuildDerivation + << printStorePath(drvPath); + writeDerivation(conn->to, *this, drv); + + putBuildSettings(*conn); + + conn->to.flush(); + + return ServeProto::Serialise::read(*this, *conn); +} + + +void LegacySSHStore::buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) +{ + if (evalStore && evalStore.get() != this) + throw Error("building on an SSH store is incompatible with '--eval-store'"); + + auto conn(connections->get()); + + conn->to << ServeProto::Command::BuildPaths; + Strings ss; + for (auto & p : drvPaths) { + auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p); + std::visit(overloaded { + [&](const StorePathWithOutputs & s) { + ss.push_back(s.to_string(*this)); + }, + [&](const StorePath & drvPath) { + throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath)); + }, + [&](std::monostate) { + throw Error("wanted build derivation that is itself a build product, but the legacy ssh protocol doesn't support that. Try using ssh-ng://"); + }, + }, sOrDrvPath); + } + conn->to << ss; + + putBuildSettings(*conn); + + conn->to.flush(); + + BuildResult result; + result.status = (BuildResult::Status) readInt(conn->from); + + if (!result.success()) { + conn->from >> result.errorMsg; + throw Error(result.status, result.errorMsg); + } +} + + +void LegacySSHStore::computeFSClosure(const StorePathSet & paths, + StorePathSet & out, bool flipDirection, + bool includeOutputs, bool includeDerivers) +{ + if (flipDirection || includeDerivers) { + Store::computeFSClosure(paths, out, flipDirection, includeOutputs, includeDerivers); + return; + } + + auto conn(connections->get()); + + conn->to + << ServeProto::Command::QueryClosure + << includeOutputs; + ServeProto::write(*this, *conn, paths); + conn->to.flush(); + + for (auto & i : ServeProto::Serialise::read(*this, *conn)) + out.insert(i); +} + + +StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths, + SubstituteFlag maybeSubstitute) +{ + auto conn(connections->get()); + + conn->to + << ServeProto::Command::QueryValidPaths + << false // lock + << maybeSubstitute; + ServeProto::write(*this, *conn, paths); + conn->to.flush(); + + return ServeProto::Serialise::read(*this, *conn); +} + + +void LegacySSHStore::connect() +{ + auto conn(connections->get()); +} + + +unsigned int LegacySSHStore::getProtocol() +{ + auto conn(connections->get()); + return conn->remoteVersion; +} + + +/** + * The legacy ssh protocol doesn't support checking for trusted-user. + * Try using ssh-ng:// instead if you want to know. + */ +std::optional isTrustedClient() +{ + return std::nullopt; +} + + static RegisterStoreImplementation regLegacySSHStore; } diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh new file mode 100644 index 000000000..c40c256bb --- /dev/null +++ b/src/libstore/legacy-ssh-store.hh @@ -0,0 +1,132 @@ +#pragma once +///@file + +#include "ssh-store-config.hh" +#include "store-api.hh" +#include "ssh.hh" +#include "callback.hh" +#include "pool.hh" + +namespace nix { + +struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig +{ + using CommonSSHStoreConfig::CommonSSHStoreConfig; + + const Setting remoteProgram{this, "nix-store", "remote-program", + "Path to the `nix-store` executable on the remote machine."}; + + const Setting maxConnections{this, 1, "max-connections", + "Maximum number of concurrent SSH connections."}; + + const std::string name() override { return "SSH Store"; } + + std::string doc() override; +}; + +struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store +{ + // Hack for getting remote build log output. + // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in + // the documentation + const Setting logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + + struct Connection; + + std::string host; + + ref> connections; + + SSHMaster master; + + static std::set uriSchemes() { return {"ssh"}; } + + LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params); + + ref openConnection(); + + std::string getUri() override; + + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override; + + void addToStore(const ValidPathInfo & info, Source & source, + RepairFlag repair, CheckSigsFlag checkSigs) override; + + void narFromPath(const StorePath & path, Sink & sink) override; + + std::optional queryPathFromHashPart(const std::string & hashPart) override + { unsupported("queryPathFromHashPart"); } + + StorePath addToStore( + std::string_view name, + const Path & srcPath, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) override + { unsupported("addToStore"); } + + StorePath addTextToStore( + std::string_view name, + std::string_view s, + const StorePathSet & references, + RepairFlag repair) override + { unsupported("addTextToStore"); } + +private: + + void putBuildSettings(Connection & conn); + +public: + + BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, + BuildMode buildMode) override; + + void buildPaths(const std::vector & drvPaths, BuildMode buildMode, std::shared_ptr evalStore) override; + + void ensurePath(const StorePath & path) override + { unsupported("ensurePath"); } + + virtual ref getFSAccessor(bool requireValidPath) override + { unsupported("getFSAccessor"); } + + /** + * The default instance would schedule the work on the client side, but + * for consistency with `buildPaths` and `buildDerivation` it should happen + * on the remote side. + * + * We make this fail for now so we can add implement this properly later + * without it being a breaking change. + */ + void repairPath(const StorePath & path) override + { unsupported("repairPath"); } + + void computeFSClosure(const StorePathSet & paths, + StorePathSet & out, bool flipDirection = false, + bool includeOutputs = false, bool includeDerivers = false) override; + + StorePathSet queryValidPaths(const StorePathSet & paths, + SubstituteFlag maybeSubstitute = NoSubstitute) override; + + void connect() override; + + unsigned int getProtocol() override; + + /** + * The legacy ssh protocol doesn't support checking for trusted-user. + * Try using ssh-ng:// instead if you want to know. + */ + std::optional isTrustedClient() override + { + return std::nullopt; + } + + void queryRealisationUncached(const DrvOutput &, + Callback> callback) noexcept override + // TODO: Implement + { unsupported("queryRealisation"); } +}; + +} diff --git a/src/libstore/length-prefixed-protocol-helper.hh b/src/libstore/length-prefixed-protocol-helper.hh index 4061b0cd6..0cf950a47 100644 --- a/src/libstore/length-prefixed-protocol-helper.hh +++ b/src/libstore/length-prefixed-protocol-helper.hh @@ -10,7 +10,7 @@ namespace nix { -class Store; +struct StoreDirConfig; /** * Reusable serialisers for serialization container types in a @@ -44,8 +44,8 @@ struct LengthPrefixedProtoHelper; #define LENGTH_PREFIXED_PROTO_HELPER(Inner, T) \ struct LengthPrefixedProtoHelper< Inner, T > \ { \ - static T read(const Store & store, typename Inner::ReadConn conn); \ - static void write(const Store & store, typename Inner::WriteConn conn, const T & str); \ + static T read(const StoreDirConfig & store, typename Inner::ReadConn conn); \ + static void write(const StoreDirConfig & store, typename Inner::WriteConn conn, const T & str); \ private: \ template using S = typename Inner::template Serialise; \ } @@ -67,7 +67,7 @@ LENGTH_PREFIXED_PROTO_HELPER(Inner, _X); template std::vector LengthPrefixedProtoHelper>::read( - const Store & store, typename Inner::ReadConn conn) + const StoreDirConfig & store, typename Inner::ReadConn conn) { std::vector resSet; auto size = readNum(conn.from); @@ -80,7 +80,7 @@ LengthPrefixedProtoHelper>::read( template void LengthPrefixedProtoHelper>::write( - const Store & store, typename Inner::WriteConn conn, const std::vector & resSet) + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::vector & resSet) { conn.to << resSet.size(); for (auto & key : resSet) { @@ -91,7 +91,7 @@ LengthPrefixedProtoHelper>::write( template std::set LengthPrefixedProtoHelper>::read( - const Store & store, typename Inner::ReadConn conn) + const StoreDirConfig & store, typename Inner::ReadConn conn) { std::set resSet; auto size = readNum(conn.from); @@ -104,7 +104,7 @@ LengthPrefixedProtoHelper>::read( template void LengthPrefixedProtoHelper>::write( - const Store & store, typename Inner::WriteConn conn, const std::set & resSet) + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::set & resSet) { conn.to << resSet.size(); for (auto & key : resSet) { @@ -115,7 +115,7 @@ LengthPrefixedProtoHelper>::write( template std::map LengthPrefixedProtoHelper>::read( - const Store & store, typename Inner::ReadConn conn) + const StoreDirConfig & store, typename Inner::ReadConn conn) { std::map resMap; auto size = readNum(conn.from); @@ -130,7 +130,7 @@ LengthPrefixedProtoHelper>::read( template void LengthPrefixedProtoHelper>::write( - const Store & store, typename Inner::WriteConn conn, const std::map & resMap) + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::map & resMap) { conn.to << resMap.size(); for (auto & i : resMap) { @@ -142,7 +142,7 @@ LengthPrefixedProtoHelper>::write( template std::tuple LengthPrefixedProtoHelper>::read( - const Store & store, typename Inner::ReadConn conn) + const StoreDirConfig & store, typename Inner::ReadConn conn) { return std::tuple { S::read(store, conn)..., @@ -152,7 +152,7 @@ LengthPrefixedProtoHelper>::read( template void LengthPrefixedProtoHelper>::write( - const Store & store, typename Inner::WriteConn conn, const std::tuple & res) + const StoreDirConfig & store, typename Inner::WriteConn conn, const std::tuple & res) { std::apply([&](const Us &... args) { (S::write(store, conn, args), ...); diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index b224fc3e9..953f3a264 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -1,5 +1,5 @@ #include "archive.hh" -#include "fs-accessor.hh" +#include "posix-source-accessor.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" @@ -13,69 +13,53 @@ LocalFSStore::LocalFSStore(const Params & params) { } -struct LocalStoreAccessor : public FSAccessor +struct LocalStoreAccessor : PosixSourceAccessor { ref store; + bool requireValidPath; - LocalStoreAccessor(ref store) : store(store) { } + LocalStoreAccessor(ref store, bool requireValidPath) + : store(store) + , requireValidPath(requireValidPath) + { } - Path toRealPath(const Path & path, bool requireValidPath = true) + CanonPath toRealPath(const CanonPath & path) { - auto storePath = store->toStorePath(path).first; + auto [storePath, rest] = store->toStorePath(path.abs()); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); - return store->getRealStoreDir() + std::string(path, store->storeDir.size()); + return CanonPath(store->getRealStoreDir()) + storePath.to_string() + CanonPath(rest); } - FSAccessor::Stat stat(const Path & path) override + std::optional maybeLstat(const CanonPath & path) override { - auto realPath = toRealPath(path); - - struct stat st; - if (lstat(realPath.c_str(), &st)) { - if (errno == ENOENT || errno == ENOTDIR) return {Type::tMissing, 0, false}; - throw SysError("getting status of '%1%'", path); - } - - if (!S_ISREG(st.st_mode) && !S_ISDIR(st.st_mode) && !S_ISLNK(st.st_mode)) - throw Error("file '%1%' has unsupported type", path); - - return { - S_ISREG(st.st_mode) ? Type::tRegular : - S_ISLNK(st.st_mode) ? Type::tSymlink : - Type::tDirectory, - S_ISREG(st.st_mode) ? (uint64_t) st.st_size : 0, - S_ISREG(st.st_mode) && st.st_mode & S_IXUSR}; + return PosixSourceAccessor::maybeLstat(toRealPath(path)); } - StringSet readDirectory(const Path & path) override + DirEntries readDirectory(const CanonPath & path) override { - auto realPath = toRealPath(path); - - auto entries = nix::readDirectory(realPath); - - StringSet res; - for (auto & entry : entries) - res.insert(entry.name); - - return res; + return PosixSourceAccessor::readDirectory(toRealPath(path)); } - std::string readFile(const Path & path, bool requireValidPath = true) override + void readFile( + const CanonPath & path, + Sink & sink, + std::function sizeCallback) override { - return nix::readFile(toRealPath(path, requireValidPath)); + return PosixSourceAccessor::readFile(toRealPath(path), sink, sizeCallback); } - std::string readLink(const Path & path) override + std::string readLink(const CanonPath & path) override { - return nix::readLink(toRealPath(path)); + return PosixSourceAccessor::readLink(toRealPath(path)); } }; -ref LocalFSStore::getFSAccessor() +ref LocalFSStore::getFSAccessor(bool requireValidPath) { return make_ref(ref( - std::dynamic_pointer_cast(shared_from_this()))); + std::dynamic_pointer_cast(shared_from_this())), + requireValidPath); } void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 488109501..bf855b67e 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -11,25 +11,21 @@ struct LocalFSStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - // FIXME: the (StoreConfig*) cast works around a bug in gcc that causes - // it to omit the call to the Setting constructor. Clang works fine - // either way. - - const OptionalPathSetting rootDir{(StoreConfig*) this, std::nullopt, + const OptionalPathSetting rootDir{this, std::nullopt, "root", "Directory prefixed to all other paths."}; - const PathSetting stateDir{(StoreConfig*) this, + const PathSetting stateDir{this, rootDir.get() ? *rootDir.get() + "/nix/var/nix" : settings.nixStateDir, "state", "Directory where Nix will store state."}; - const PathSetting logDir{(StoreConfig*) this, + const PathSetting logDir{this, rootDir.get() ? *rootDir.get() + "/nix/var/log/nix" : settings.nixLogDir, "log", "directory where Nix will store log files."}; - const PathSetting realStoreDir{(StoreConfig*) this, + const PathSetting realStoreDir{this, rootDir.get() ? *rootDir.get() + "/nix/store" : storeDir, "real", "Physical path of the Nix store."}; }; @@ -47,7 +43,7 @@ public: LocalFSStore(const Params & params); void narFromPath(const StorePath & path, Sink & sink) override; - ref getFSAccessor() override; + ref getFSAccessor(bool requireValidPath) override; /** * Creates symlink from the `gcRoot` to the `storePath` and diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc index 732b4d6ce..f657d8341 100644 --- a/src/libstore/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -1,5 +1,7 @@ #include "local-overlay-store.hh" #include "callback.hh" +#include "realisation.hh" +#include "processes.hh" #include "url.hh" #include diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8fb2a5dba..8561d58be 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -4,12 +4,15 @@ #include "pathlocks.hh" #include "worker-protocol.hh" #include "derivations.hh" +#include "realisation.hh" #include "nar-info.hh" #include "references.hh" #include "callback.hh" #include "topo-sort.hh" #include "finally.hh" #include "compression.hh" +#include "signals.hh" +#include "posix-fs-canonicalise.hh" #include #include @@ -32,7 +35,6 @@ #include #include #include -#include #endif #ifdef __CYGWIN__ @@ -586,164 +588,6 @@ void LocalStore::makeStoreWritable() } -const time_t mtimeStore = 1; /* 1 second into the epoch */ - - -static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st) -{ - if (!S_ISLNK(st.st_mode)) { - - /* Mask out all type related bits. */ - mode_t mode = st.st_mode & ~S_IFMT; - - if (mode != 0444 && mode != 0555) { - mode = (st.st_mode & S_IFMT) - | 0444 - | (st.st_mode & S_IXUSR ? 0111 : 0); - if (chmod(path.c_str(), mode) == -1) - throw SysError("changing mode of '%1%' to %2$o", path, mode); - } - - } - - if (st.st_mtime != mtimeStore) { - struct timeval times[2]; - times[0].tv_sec = st.st_atime; - times[0].tv_usec = 0; - times[1].tv_sec = mtimeStore; - times[1].tv_usec = 0; -#if HAVE_LUTIMES - if (lutimes(path.c_str(), times) == -1) - if (errno != ENOSYS || - (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) -#else - if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) -#endif - throw SysError("changing modification time of '%1%'", path); - } -} - - -void canonicaliseTimestampAndPermissions(const Path & path) -{ - canonicaliseTimestampAndPermissions(path, lstat(path)); -} - - -static void canonicalisePathMetaData_( - const Path & path, - std::optional> uidRange, - InodesSeen & inodesSeen) -{ - checkInterrupt(); - -#if __APPLE__ - /* Remove flags, in particular UF_IMMUTABLE which would prevent - the file from being garbage-collected. FIXME: Use - setattrlist() to remove other attributes as well. */ - if (lchflags(path.c_str(), 0)) { - if (errno != ENOTSUP) - throw SysError("clearing flags of path '%1%'", path); - } -#endif - - auto st = lstat(path); - - /* Really make sure that the path is of a supported type. */ - if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) - throw Error("file '%1%' has an unsupported type", path); - -#if __linux__ - /* Remove extended attributes / ACLs. */ - ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); - - if (eaSize < 0) { - if (errno != ENOTSUP && errno != ENODATA) - throw SysError("querying extended attributes of '%s'", path); - } else if (eaSize > 0) { - std::vector eaBuf(eaSize); - - if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) - throw SysError("querying extended attributes of '%s'", path); - - for (auto & eaName: tokenizeString(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { - if (settings.ignoredAcls.get().count(eaName)) continue; - if (lremovexattr(path.c_str(), eaName.c_str()) == -1) - throw SysError("removing extended attribute '%s' from '%s'", eaName, path); - } - } -#endif - - /* Fail if the file is not owned by the build user. This prevents - us from messing up the ownership/permissions of files - hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). - However, ignore files that we chown'ed ourselves previously to - ensure that we don't fail on hard links within the same build - (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ - if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) { - if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino))) - throw BuildError("invalid ownership on file '%1%'", path); - mode_t mode = st.st_mode & ~S_IFMT; - assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); - return; - } - - inodesSeen.insert(Inode(st.st_dev, st.st_ino)); - - canonicaliseTimestampAndPermissions(path, st); - - /* Change ownership to the current uid. If it's a symlink, use - lchown if available, otherwise don't bother. Wrong ownership - of a symlink doesn't matter, since the owning user can't change - the symlink and can't delete it because the directory is not - writable. The only exception is top-level paths in the Nix - store (since that directory is group-writable for the Nix build - users group); we check for this case below. */ - if (st.st_uid != geteuid()) { -#if HAVE_LCHOWN - if (lchown(path.c_str(), geteuid(), getegid()) == -1) -#else - if (!S_ISLNK(st.st_mode) && - chown(path.c_str(), geteuid(), getegid()) == -1) -#endif - throw SysError("changing owner of '%1%' to %2%", - path, geteuid()); - } - - if (S_ISDIR(st.st_mode)) { - DirEntries entries = readDirectory(path); - for (auto & i : entries) - canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen); - } -} - - -void canonicalisePathMetaData( - const Path & path, - std::optional> uidRange, - InodesSeen & inodesSeen) -{ - canonicalisePathMetaData_(path, uidRange, inodesSeen); - - /* On platforms that don't have lchown(), the top-level path can't - be a symlink, since we can't change its ownership. */ - auto st = lstat(path); - - if (st.st_uid != geteuid()) { - assert(S_ISLNK(st.st_mode)); - throw Error("wrong ownership of top-level store path '%1%'", path); - } -} - - -void canonicalisePathMetaData(const Path & path, - std::optional> uidRange) -{ - InodesSeen inodesSeen; - canonicalisePathMetaData(path, uidRange, inodesSeen); -} - - void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) { experimentalFeatureSettings.require(Xp::CaDerivations); @@ -1117,7 +961,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) StorePathSet paths; for (auto & [_, i] : infos) { - assert(i.narHash.type == htSHA256); + assert(i.narHash.algo == HashAlgorithm::SHA256); if (isValidPath_(*state, i.path)) updatePathInfo(*state, i); else @@ -1206,7 +1050,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, bool narRead = false; Finally cleanup = [&]() { if (!narRead) { - ParseSink sink; + NullParseSink sink; parseDump(sink, source); } }; @@ -1231,7 +1075,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, /* While restoring the path from the NAR, compute the hash of the NAR. */ - HashSink hashSink(htSHA256); + HashSink hashSink(HashAlgorithm::SHA256); TeeSource wrapperSource { source, hashSink }; @@ -1242,7 +1086,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (hashResult.first != info.narHash) throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(HashFormat::Base32, true), hashResult.first.to_string(HashFormat::Base32, true)); + printStorePath(info.path), info.narHash.to_string(HashFormat::Nix32, true), hashResult.first.to_string(HashFormat::Nix32, true)); if (hashResult.second != info.narSize) throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", @@ -1252,14 +1096,14 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, auto & specified = *info.ca; auto actualHash = hashCAPath( specified.method, - specified.hash.type, + specified.hash.algo, info.path ); if (specified.hash != actualHash.hash) { throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", printStorePath(info.path), - specified.hash.to_string(HashFormat::Base32, true), - actualHash.hash.to_string(HashFormat::Base32, true)); + specified.hash.to_string(HashFormat::Nix32, true), + actualHash.hash.to_string(HashFormat::Nix32, true)); } } @@ -1278,7 +1122,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) + FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) { /* For computing the store path. */ auto hashSink = std::make_unique(hashAlgo); @@ -1382,8 +1226,8 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, std::string_view name /* For computing the nar hash. In recursive SHA-256 mode, this is the same as the store hash, so no need to do it again. */ auto narHash = std::pair { hash, size }; - if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) { - HashSink narSink { htSHA256 }; + if (method != FileIngestionMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) { + HashSink narSink { HashAlgorithm::SHA256 }; dumpPath(realPath, narSink); narHash = narSink.finish(); } @@ -1414,7 +1258,7 @@ StorePath LocalStore::addTextToStore( std::string_view s, const StorePathSet & references, RepairFlag repair) { - auto hash = hashString(htSHA256, s); + auto hash = hashString(HashAlgorithm::SHA256, s); auto dstPath = makeTextPath(name, TextInfo { .hash = hash, .references = references, @@ -1440,7 +1284,7 @@ StorePath LocalStore::addTextToStore( StringSink sink; dumpString(s, sink); - auto narHash = hashString(htSHA256, sink.s); + auto narHash = hashString(HashAlgorithm::SHA256, sink.s); optimisePath(realPath, repair); @@ -1523,7 +1367,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : readDirectory(linksDir)) { printMsg(lvlTalkative, "checking contents of '%s'", link.name); Path linkPath = linksDir + "/" + link.name; - std::string hash = hashPath(htSHA256, linkPath).first.to_string(HashFormat::Base32, false); + std::string hash = hashPath(HashAlgorithm::SHA256, linkPath).first.to_string(HashFormat::Nix32, false); if (hash != link.name) { printError("link '%s' was modified! expected hash '%s', got '%s'", linkPath, link.name, hash); @@ -1540,7 +1384,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printInfo("checking store hashes..."); - Hash nullHash(htSHA256); + Hash nullHash(HashAlgorithm::SHA256); for (auto & i : validPaths) { try { @@ -1549,14 +1393,14 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) /* Check the content hash (optionally - slow). */ printMsg(lvlTalkative, "checking contents of '%s'", printStorePath(i)); - auto hashSink = HashSink(info->narHash.type); + auto hashSink = HashSink(info->narHash.algo); dumpPath(Store::toRealPath(i), hashSink); auto current = hashSink.finish(); if (info->narHash != nullHash && info->narHash != current.first) { printError("path '%s' was modified! expected hash '%s', got '%s'", - printStorePath(i), info->narHash.to_string(HashFormat::Base32, true), current.first.to_string(HashFormat::Base32, true)); + printStorePath(i), info->narHash.to_string(HashFormat::Nix32, true), current.first.to_string(HashFormat::Nix32, true)); if (repair) repairPath(i); else errors = true; } else { @@ -1868,20 +1712,20 @@ void LocalStore::queryRealisationUncached(const DrvOutput & id, } ContentAddress LocalStore::hashCAPath( - const ContentAddressMethod & method, const HashType & hashType, + const ContentAddressMethod & method, const HashAlgorithm & hashAlgo, const StorePath & path) { - return hashCAPath(method, hashType, Store::toRealPath(path), path.hashPart()); + return hashCAPath(method, hashAlgo, Store::toRealPath(path), path.hashPart()); } ContentAddress LocalStore::hashCAPath( const ContentAddressMethod & method, - const HashType & hashType, + const HashAlgorithm & hashAlgo, const Path & path, const std::string_view pathHash ) { - HashModuloSink caSink ( hashType, std::string(pathHash) ); + HashModuloSink caSink ( hashAlgo, std::string(pathHash) ); std::visit(overloaded { [&](const TextIngestionMethod &) { readFile(path, caSink); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 88549eea5..9a9171e2f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -7,7 +7,6 @@ #include "store-api.hh" #include "indirect-root-store.hh" #include "sync.hh" -#include "util.hh" #include #include @@ -40,12 +39,12 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; - Setting requireSigs{(StoreConfig*) this, + Setting requireSigs{this, settings.requireSigs, "require-sigs", "Whether store paths copied into this store should have a trusted signature."}; - Setting readOnly{(StoreConfig*) this, + Setting readOnly{this, false, "read-only", R"( @@ -179,7 +178,7 @@ public: RepairFlag repair, CheckSigsFlag checkSigs) override; StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashAlgo, RepairFlag repair, const StorePathSet & references) override; + FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) override; StorePath addTextToStore( std::string_view name, @@ -382,12 +381,12 @@ private: // XXX: Make a generic `Store` method ContentAddress hashCAPath( const ContentAddressMethod & method, - const HashType & hashType, + const HashAlgorithm & hashAlgo, const StorePath & path); ContentAddress hashCAPath( const ContentAddressMethod & method, - const HashType & hashType, + const HashAlgorithm & hashAlgo, const Path & path, const std::string_view pathHash ); @@ -400,38 +399,4 @@ private: friend struct DerivationGoal; }; - -typedef std::pair Inode; -typedef std::set InodesSeen; - - -/** - * "Fix", or canonicalise, the meta-data of the files in a store path - * after it has been built. In particular: - * - * - the last modification date on each file is set to 1 (i.e., - * 00:00:01 1/1/1970 UTC) - * - * - the permissions are set of 444 or 555 (i.e., read-only with or - * without execute permission; setuid bits etc. are cleared) - * - * - the owner and group are set to the Nix user and group, if we're - * running as root. - * - * If uidRange is not empty, this function will throw an error if it - * encounters files owned by a user outside of the closed interval - * [uidRange->first, uidRange->second]. - */ -void canonicalisePathMetaData( - const Path & path, - std::optional> uidRange, - InodesSeen & inodesSeen); -void canonicalisePathMetaData( - const Path & path, - std::optional> uidRange); - -void canonicaliseTimestampAndPermissions(const Path & path); - -MakeError(PathInUse, Error); - } diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 0be0bf310..68ccdc409 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -59,7 +59,7 @@ $(d)/build.cc: clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh -$(eval $(call install-file-in, $(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-store.pc, $(libdir)/pkgconfig, 0644)) $(foreach i, $(wildcard src/libstore/builtins/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644))) diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 165e4969f..87f55ce49 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -1,4 +1,5 @@ #include "lock.hh" +#include "file-system.hh" #include "globals.hh" #include "pathlocks.hh" diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index e87f46980..512115893 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -1,5 +1,4 @@ #include "machines.hh" -#include "util.hh" #include "globals.hh" #include "store-api.hh" diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 253609ed2..170fe67b9 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -43,7 +43,7 @@ std::map makeContentAddressed( sink.s = rewriteStrings(sink.s, rewrites); - HashModuloSink hashModuloSink(htSHA256, oldHashPart); + HashModuloSink hashModuloSink(HashAlgorithm::SHA256, oldHashPart); hashModuloSink(sink.s); auto narModuloHash = hashModuloSink.finish().first; @@ -66,7 +66,7 @@ std::map makeContentAddressed( rsink2(sink.s); rsink2.flush(); - info.narHash = hashString(htSHA256, sink2.s); + info.narHash = hashString(HashAlgorithm::SHA256, sink2.s); info.narSize = sink.s.size(); StringSource source(sink2.s); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 1035691c7..9f63fbbb5 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -4,6 +4,7 @@ #include "local-store.hh" #include "store-api.hh" #include "thread-pool.hh" +#include "realisation.hh" #include "topo-sort.hh" #include "callback.hh" #include "closure.hh" diff --git a/src/libstore/mounted-ssh-store.md b/src/libstore/mounted-ssh-store.md new file mode 100644 index 000000000..1ebfe3081 --- /dev/null +++ b/src/libstore/mounted-ssh-store.md @@ -0,0 +1,18 @@ +R"( + +**Store URL format**: `mounted-ssh-ng://[username@]hostname` + +Experimental store type that allows full access to a Nix store on a remote machine, +and additionally requires that store be mounted in the local file system. + +The mounting of that store is not managed by Nix, and must by managed manually. +It could be accomplished with SSHFS or NFS, for example. + +The local file system is used to optimize certain operations. +For example, rather than serializing Nix archives and sending over the Nix channel, +we can directly access the file system data via the mount-point. + +The local file system is also used to make certain operations possible that wouldn't otherwise be. +For example, persistent GC roots can be created if they reside on the same file system as the remote store: +the remote side will create the symlinks necessary to avoid race conditions. +)" diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index f0dfcb19b..15b05fe25 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -11,13 +11,7 @@ namespace nix { struct NarMember { - FSAccessor::Type type = FSAccessor::Type::tMissing; - - bool isExecutable = false; - - /* If this is a regular file, position of the contents of this - file in the NAR. */ - uint64_t start = 0, size = 0; + SourceAccessor::Stat stat; std::string target; @@ -25,7 +19,7 @@ struct NarMember std::map children; }; -struct NarAccessor : public FSAccessor +struct NarAccessor : public SourceAccessor { std::optional nar; @@ -57,7 +51,7 @@ struct NarAccessor : public FSAccessor acc.root = std::move(member); parents.push(&acc.root); } else { - if (parents.top()->type != FSAccessor::Type::tDirectory) + if (parents.top()->stat.type != Type::tDirectory) throw Error("NAR file missing parent directory of path '%s'", path); auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); parents.push(&result.first->second); @@ -66,12 +60,22 @@ struct NarAccessor : public FSAccessor void createDirectory(const Path & path) override { - createMember(path, {FSAccessor::Type::tDirectory, false, 0, 0}); + createMember(path, NarMember{ .stat = { + .type = Type::tDirectory, + .fileSize = 0, + .isExecutable = false, + .narOffset = 0 + } }); } void createRegularFile(const Path & path) override { - createMember(path, {FSAccessor::Type::tRegular, false, 0, 0}); + createMember(path, NarMember{ .stat = { + .type = Type::tRegular, + .fileSize = 0, + .isExecutable = false, + .narOffset = 0 + } }); } void closeRegularFile() override @@ -79,14 +83,14 @@ struct NarAccessor : public FSAccessor void isExecutable() override { - parents.top()->isExecutable = true; + parents.top()->stat.isExecutable = true; } void preallocateContents(uint64_t size) override { - assert(size <= std::numeric_limits::max()); - parents.top()->size = (uint64_t) size; - parents.top()->start = pos; + auto & st = parents.top()->stat; + st.fileSize = size; + st.narOffset = pos; } void receiveContents(std::string_view data) override @@ -95,7 +99,9 @@ struct NarAccessor : public FSAccessor void createSymlink(const Path & path, const std::string & target) override { createMember(path, - NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target}); + NarMember{ + .stat = {.type = Type::tSymlink}, + .target = target}); } size_t read(char * data, size_t len) override @@ -130,18 +136,19 @@ struct NarAccessor : public FSAccessor std::string type = v["type"]; if (type == "directory") { - member.type = FSAccessor::Type::tDirectory; - for (auto i = v["entries"].begin(); i != v["entries"].end(); ++i) { - std::string name = i.key(); - recurse(member.children[name], i.value()); + member.stat = {.type = Type::tDirectory}; + for (const auto &[name, function] : v["entries"].items()) { + recurse(member.children[name], function); } } else if (type == "regular") { - member.type = FSAccessor::Type::tRegular; - member.size = v["size"]; - member.isExecutable = v.value("executable", false); - member.start = v["narOffset"]; + member.stat = { + .type = Type::tRegular, + .fileSize = v["size"], + .isExecutable = v.value("executable", false), + .narOffset = v["narOffset"] + }; } else if (type == "symlink") { - member.type = FSAccessor::Type::tSymlink; + member.stat = {.type = Type::tSymlink}; member.target = v.value("target", ""); } else return; }; @@ -150,134 +157,122 @@ struct NarAccessor : public FSAccessor recurse(root, v); } - NarMember * find(const Path & path) + NarMember * find(const CanonPath & path) { - Path canon = path == "" ? "" : canonPath(path); NarMember * current = &root; - auto end = path.end(); - for (auto it = path.begin(); it != end; ) { - // because it != end, the remaining component is non-empty so we need - // a directory - if (current->type != FSAccessor::Type::tDirectory) return nullptr; - // skip slash (canonPath above ensures that this is always a slash) - assert(*it == '/'); - it += 1; - - // lookup current component - auto next = std::find(it, end, '/'); - auto child = current->children.find(std::string(it, next)); + for (const auto & i : path) { + if (current->stat.type != Type::tDirectory) return nullptr; + auto child = current->children.find(std::string(i)); if (child == current->children.end()) return nullptr; current = &child->second; - - it = next; } return current; } - NarMember & get(const Path & path) { + NarMember & get(const CanonPath & path) { auto result = find(path); - if (result == nullptr) + if (!result) throw Error("NAR file does not contain path '%1%'", path); return *result; } - Stat stat(const Path & path) override + std::optional maybeLstat(const CanonPath & path) override { auto i = find(path); - if (i == nullptr) - return {FSAccessor::Type::tMissing, 0, false}; - return {i->type, i->size, i->isExecutable, i->start}; + if (!i) + return std::nullopt; + return i->stat; } - StringSet readDirectory(const Path & path) override + DirEntries readDirectory(const CanonPath & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tDirectory) + if (i.stat.type != Type::tDirectory) throw Error("path '%1%' inside NAR file is not a directory", path); - StringSet res; - for (auto & child : i.children) - res.insert(child.first); + DirEntries res; + for (const auto & child : i.children) + res.insert_or_assign(child.first, std::nullopt); return res; } - std::string readFile(const Path & path, bool requireValidPath = true) override + std::string readFile(const CanonPath & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tRegular) + if (i.stat.type != Type::tRegular) throw Error("path '%1%' inside NAR file is not a regular file", path); - if (getNarBytes) return getNarBytes(i.start, i.size); + if (getNarBytes) return getNarBytes(*i.stat.narOffset, *i.stat.fileSize); assert(nar); - return std::string(*nar, i.start, i.size); + return std::string(*nar, *i.stat.narOffset, *i.stat.fileSize); } - std::string readLink(const Path & path) override + std::string readLink(const CanonPath & path) override { auto i = get(path); - if (i.type != FSAccessor::Type::tSymlink) + if (i.stat.type != Type::tSymlink) throw Error("path '%1%' inside NAR file is not a symlink", path); return i.target; } }; -ref makeNarAccessor(std::string && nar) +ref makeNarAccessor(std::string && nar) { return make_ref(std::move(nar)); } -ref makeNarAccessor(Source & source) +ref makeNarAccessor(Source & source) { return make_ref(source); } -ref makeLazyNarAccessor(const std::string & listing, +ref makeLazyNarAccessor(const std::string & listing, GetNarBytes getNarBytes) { return make_ref(listing, getNarBytes); } using nlohmann::json; -json listNar(ref accessor, const Path & path, bool recurse) +json listNar(ref accessor, const CanonPath & path, bool recurse) { - auto st = accessor->stat(path); + auto st = accessor->lstat(path); json obj = json::object(); switch (st.type) { - case FSAccessor::Type::tRegular: + case SourceAccessor::Type::tRegular: obj["type"] = "regular"; - obj["size"] = st.fileSize; + if (st.fileSize) + obj["size"] = *st.fileSize; if (st.isExecutable) obj["executable"] = true; - if (st.narOffset) - obj["narOffset"] = st.narOffset; + if (st.narOffset && *st.narOffset) + obj["narOffset"] = *st.narOffset; break; - case FSAccessor::Type::tDirectory: + case SourceAccessor::Type::tDirectory: obj["type"] = "directory"; { obj["entries"] = json::object(); json &res2 = obj["entries"]; - for (auto & name : accessor->readDirectory(path)) { + for (const auto & [name, type] : accessor->readDirectory(path)) { if (recurse) { - res2[name] = listNar(accessor, path + "/" + name, true); + res2[name] = listNar(accessor, path + name, true); } else res2[name] = json::object(); } } break; - case FSAccessor::Type::tSymlink: + case SourceAccessor::Type::tSymlink: obj["type"] = "symlink"; obj["target"] = accessor->readLink(path); break; - case FSAccessor::Type::tMissing: - default: - throw Error("path '%s' does not exist in NAR", path); + case SourceAccessor::Type::tMisc: + assert(false); // cannot happen for NARs } return obj; } diff --git a/src/libstore/nar-accessor.hh b/src/libstore/nar-accessor.hh index 5e19bd3c7..0043897c6 100644 --- a/src/libstore/nar-accessor.hh +++ b/src/libstore/nar-accessor.hh @@ -1,10 +1,11 @@ #pragma once ///@file +#include "source-accessor.hh" + #include #include -#include "fs-accessor.hh" namespace nix { @@ -14,9 +15,9 @@ struct Source; * Return an object that provides access to the contents of a NAR * file. */ -ref makeNarAccessor(std::string && nar); +ref makeNarAccessor(std::string && nar); -ref makeNarAccessor(Source & source); +ref makeNarAccessor(Source & source); /** * Create a NAR accessor from a NAR listing (in the format produced by @@ -24,9 +25,9 @@ ref makeNarAccessor(Source & source); * readFile() method of the accessor to get the contents of files * inside the NAR. */ -typedef std::function GetNarBytes; +using GetNarBytes = std::function; -ref makeLazyNarAccessor( +ref makeLazyNarAccessor( const std::string & listing, GetNarBytes getNarBytes); @@ -34,6 +35,6 @@ ref makeLazyNarAccessor( * Write a JSON representation of the contents of a NAR (except file * contents). */ -nlohmann::json listNar(ref accessor, const Path & path, bool recurse); +nlohmann::json listNar(ref accessor, const CanonPath & path, bool recurse); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index cdbcf7e74..310105c75 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -1,4 +1,5 @@ #include "nar-info-disk-cache.hh" +#include "users.hh" #include "sync.hh" #include "sqlite.hh" #include "globals.hh" @@ -332,9 +333,9 @@ public: (std::string(info->path.name())) (narInfo ? narInfo->url : "", narInfo != 0) (narInfo ? narInfo->compression : "", narInfo != 0) - (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Base32, true) : "", narInfo && narInfo->fileHash) + (narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(HashFormat::Nix32, true) : "", narInfo && narInfo->fileHash) (narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize) - (info->narHash.to_string(HashFormat::Base32, true)) + (info->narHash.to_string(HashFormat::Nix32, true)) (info->narSize) (concatStringsSep(" ", info->shortRefs())) (info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver) diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index ee2ddfd81..d9618d04c 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -4,6 +4,15 @@ namespace nix { +GENERATE_CMP_EXT( + , + NarInfo, + me->url, + me->compression, + me->fileHash, + me->fileSize, + static_cast(*me)); + NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack { @@ -29,12 +38,12 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & while (pos < s.size()) { size_t colon = s.find(':', pos); - if (colon == std::string::npos) throw corrupt("expecting ':'"); + if (colon == s.npos) throw corrupt("expecting ':'"); std::string name(s, pos, colon - pos); size_t eol = s.find('\n', colon + 2); - if (eol == std::string::npos) throw corrupt("expecting '\\n'"); + if (eol == s.npos) throw corrupt("expecting '\\n'"); std::string value(s, colon + 2, eol - colon - 2); @@ -104,11 +113,11 @@ std::string NarInfo::to_string(const Store & store) const res += "URL: " + url + "\n"; assert(compression != ""); res += "Compression: " + compression + "\n"; - assert(fileHash && fileHash->type == htSHA256); - res += "FileHash: " + fileHash->to_string(HashFormat::Base32, true) + "\n"; + assert(fileHash && fileHash->algo == HashAlgorithm::SHA256); + res += "FileHash: " + fileHash->to_string(HashFormat::Nix32, true) + "\n"; res += "FileSize: " + std::to_string(fileSize) + "\n"; - assert(narHash.type == htSHA256); - res += "NarHash: " + narHash.to_string(HashFormat::Base32, true) + "\n"; + assert(narHash.algo == HashAlgorithm::SHA256); + res += "NarHash: " + narHash.to_string(HashFormat::Nix32, true) + "\n"; res += "NarSize: " + std::to_string(narSize) + "\n"; res += "References: " + concatStringsSep(" ", shortRefs()) + "\n"; @@ -125,4 +134,59 @@ std::string NarInfo::to_string(const Store & store) const return res; } +nlohmann::json NarInfo::toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const +{ + using nlohmann::json; + + auto jsonObject = ValidPathInfo::toJSON(store, includeImpureInfo, hashFormat); + + if (includeImpureInfo) { + if (!url.empty()) + jsonObject["url"] = url; + if (!compression.empty()) + jsonObject["compression"] = compression; + if (fileHash) + jsonObject["downloadHash"] = fileHash->to_string(hashFormat, true); + if (fileSize) + jsonObject["downloadSize"] = fileSize; + } + + return jsonObject; +} + +NarInfo NarInfo::fromJSON( + const Store & store, + const StorePath & path, + const nlohmann::json & json) +{ + using nlohmann::detail::value_t; + + NarInfo res { + ValidPathInfo { + path, + UnkeyedValidPathInfo::fromJSON(store, json), + } + }; + + if (json.contains("url")) + res.url = ensureType(valueAt(json, "url"), value_t::string); + + if (json.contains("compression")) + res.compression = ensureType(valueAt(json, "compression"), value_t::string); + + if (json.contains("downloadHash")) + res.fileHash = Hash::parseAny( + static_cast( + ensureType(valueAt(json, "downloadHash"), value_t::string)), + std::nullopt); + + if (json.contains("downloadSize")) + res.fileSize = ensureType(valueAt(json, "downloadSize"), value_t::number_integer); + + return res; +} + } diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index 5dbdafac3..fd538a7cd 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -17,14 +17,25 @@ struct NarInfo : ValidPathInfo uint64_t fileSize = 0; NarInfo() = delete; - NarInfo(const Store & store, std::string && name, ContentAddressWithReferences && ca, Hash narHash) + NarInfo(const Store & store, std::string name, ContentAddressWithReferences ca, Hash narHash) : ValidPathInfo(store, std::move(name), std::move(ca), narHash) { } - NarInfo(StorePath && path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } + NarInfo(StorePath path, Hash narHash) : ValidPathInfo(std::move(path), narHash) { } NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); + DECLARE_CMP(NarInfo); + std::string to_string(const Store & store) const; + + nlohmann::json toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const override; + static NarInfo fromJSON( + const Store & store, + const StorePath & path, + const nlohmann::json & json); }; } diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 23c6a41e4..b395453d1 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -1,6 +1,7 @@ -#include "util.hh" #include "local-store.hh" #include "globals.hh" +#include "signals.hh" +#include "posix-fs-canonicalise.hh" #include #include @@ -145,17 +146,17 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, Also note that if `path' is a symlink, then we're hashing the contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ - Hash hash = hashPath(htSHA256, path).first; - debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Base32, true)); + Hash hash = hashPath(HashAlgorithm::SHA256, path).first; + debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true)); /* Check if this is a known hash. */ - Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Base32, false); + Path linkPath = linksDir + "/" + hash.to_string(HashFormat::Nix32, false); /* Maybe delete the link, if it has been corrupted. */ if (pathExists(linkPath)) { auto stLink = lstat(linkPath); if (st.st_size != stLink.st_size - || (repair && hash != hashPath(htSHA256, linkPath).first)) + || (repair && hash != hashPath(HashAlgorithm::SHA256, linkPath).first)) { // XXX: Consider overwriting linkPath with our valid version. warn("removing corrupted link '%s'", linkPath); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index 1d900c272..72f45143d 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -132,6 +132,41 @@ bool ParsedDerivation::useUidRange() const static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); +/** + * Write a JSON representation of store object metadata, such as the + * hash and the references. + */ +static nlohmann::json pathInfoToJSON( + Store & store, + const StorePathSet & storePaths) +{ + nlohmann::json::array_t jsonList = nlohmann::json::array(); + + for (auto & storePath : storePaths) { + auto info = store.queryPathInfo(storePath); + + auto & jsonPath = jsonList.emplace_back( + info->toJSON(store, false, HashFormat::Nix32)); + + // Add the path to the object whose metadata we are including. + jsonPath["path"] = store.printStorePath(storePath); + + jsonPath["valid"] = true; + + jsonPath["closureSize"] = ({ + uint64_t totalNarSize = 0; + StorePathSet closure; + store.computeFSClosure(info->path, closure, false, false); + for (auto & p : closure) { + auto info = store.queryPathInfo(p); + totalNarSize += info->narSize; + } + totalNarSize; + }); + } + return jsonList; +} + std::optional ParsedDerivation::prepareStructuredAttrs(Store & store, const StorePathSet & inputPaths) { auto structuredAttrs = getStructuredAttrs(); @@ -152,8 +187,8 @@ std::optional ParsedDerivation::prepareStructuredAttrs(Store & s StorePathSet storePaths; for (auto & p : *i) storePaths.insert(store.parseStorePath(p.get())); - json[i.key()] = store.pathInfoToJSON( - store.exportReferences(storePaths, inputPaths), false, true); + json[i.key()] = pathInfoToJSON(store, + store.exportReferences(storePaths, inputPaths)); } } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index ab39e71f4..f58e31bfd 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -1,5 +1,8 @@ +#include + #include "path-info.hh" #include "store-api.hh" +#include "json-utils.hh" namespace nix { @@ -28,9 +31,9 @@ std::string ValidPathInfo::fingerprint(const Store & store) const throw Error("cannot calculate fingerprint of path '%s' because its size is not known", store.printStorePath(path)); return - "1;" + store.printStorePath(path) + ";" - + narHash.to_string(HashFormat::Base32, true) + ";" - + std::to_string(narSize) + ";" + "1;" + store.printStorePath(path) + ";" + + narHash.to_string(HashFormat::Nix32, true) + ";" + + std::to_string(narSize) + ";" + concatStringsSep(",", store.printStorePathSet(references)); } @@ -144,4 +147,94 @@ ValidPathInfo::ValidPathInfo( }, std::move(ca).raw); } + +nlohmann::json UnkeyedValidPathInfo::toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const +{ + using nlohmann::json; + + auto jsonObject = json::object(); + + jsonObject["narHash"] = narHash.to_string(hashFormat, true); + jsonObject["narSize"] = narSize; + + { + auto& jsonRefs = (jsonObject["references"] = json::array()); + for (auto & ref : references) + jsonRefs.emplace_back(store.printStorePath(ref)); + } + + if (ca) + jsonObject["ca"] = renderContentAddress(ca); + + if (includeImpureInfo) { + if (deriver) + jsonObject["deriver"] = store.printStorePath(*deriver); + + if (registrationTime) + jsonObject["registrationTime"] = registrationTime; + + if (ultimate) + jsonObject["ultimate"] = ultimate; + + if (!sigs.empty()) { + for (auto & sig : sigs) + jsonObject["signatures"].push_back(sig); + } + } + + return jsonObject; +} + +UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON( + const Store & store, + const nlohmann::json & json) +{ + using nlohmann::detail::value_t; + + UnkeyedValidPathInfo res { + Hash(Hash::dummy), + }; + + ensureType(json, value_t::object); + res.narHash = Hash::parseAny( + static_cast( + ensureType(valueAt(json, "narHash"), value_t::string)), + std::nullopt); + res.narSize = ensureType(valueAt(json, "narSize"), value_t::number_integer); + + try { + auto & references = ensureType(valueAt(json, "references"), value_t::array); + for (auto & input : references) + res.references.insert(store.parseStorePath(static_cast +(input))); + } catch (Error & e) { + e.addTrace({}, "while reading key 'references'"); + throw; + } + + if (json.contains("ca")) + res.ca = ContentAddress::parse( + static_cast( + ensureType(valueAt(json, "ca"), value_t::string))); + + if (json.contains("deriver")) + res.deriver = store.parseStorePath( + static_cast( + ensureType(valueAt(json, "deriver"), value_t::string))); + + if (json.contains("registrationTime")) + res.registrationTime = ensureType(valueAt(json, "registrationTime"), value_t::number_integer); + + if (json.contains("ultimate")) + res.ultimate = ensureType(valueAt(json, "ultimate"), value_t::boolean); + + if (json.contains("signatures")) + res.sigs = valueAt(json, "signatures"); + + return res; +} + } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index a82e643ae..077abc7e1 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -29,7 +29,7 @@ struct SubstitutablePathInfo uint64_t narSize; }; -typedef std::map SubstitutablePathInfos; +using SubstitutablePathInfos = std::map; struct UnkeyedValidPathInfo @@ -42,7 +42,7 @@ struct UnkeyedValidPathInfo StorePathSet references; time_t registrationTime = 0; uint64_t narSize = 0; // 0 = unknown - uint64_t id; // internal use only + uint64_t id = 0; // internal use only /** * Whether the path is ultimately trusted, that is, it's a @@ -78,6 +78,18 @@ struct UnkeyedValidPathInfo DECLARE_CMP(UnkeyedValidPathInfo); virtual ~UnkeyedValidPathInfo() { } + + /** + * @param includeImpureInfo If true, variable elements such as the + * registration time are included. + */ + virtual nlohmann::json toJSON( + const Store & store, + bool includeImpureInfo, + HashFormat hashFormat) const; + static UnkeyedValidPathInfo fromJSON( + const Store & store, + const nlohmann::json & json); }; struct ValidPathInfo : UnkeyedValidPathInfo { @@ -136,6 +148,6 @@ struct ValidPathInfo : UnkeyedValidPathInfo { virtual ~ValidPathInfo() { } }; -typedef std::map ValidPathInfos; +using ValidPathInfos = std::map; } diff --git a/src/libstore/path-references.cc b/src/libstore/path-references.cc index 33cf66ce3..15f52ec9d 100644 --- a/src/libstore/path-references.cc +++ b/src/libstore/path-references.cc @@ -1,6 +1,5 @@ #include "path-references.hh" #include "hash.hh" -#include "util.hh" #include "archive.hh" #include @@ -50,7 +49,7 @@ std::pair scanForReferences( const std::string & path, const StorePathSet & refs) { - HashSink hashSink { htSHA256 }; + HashSink hashSink { HashAlgorithm::SHA256 }; auto found = scanForReferences(hashSink, path, refs); auto hash = hashSink.finish(); return std::pair(found, hash); diff --git a/src/libstore/path-references.hh b/src/libstore/path-references.hh index 7b44e3261..0553003f8 100644 --- a/src/libstore/path-references.hh +++ b/src/libstore/path-references.hh @@ -1,4 +1,5 @@ #pragma once +///@file #include "references.hh" #include "path.hh" diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index af6837370..026e37647 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -5,7 +5,7 @@ namespace nix { -std::string StorePathWithOutputs::to_string(const Store & store) const +std::string StorePathWithOutputs::to_string(const StoreDirConfig & store) const { return outputs.empty() ? store.printStorePath(path) @@ -85,7 +85,7 @@ std::pair parsePathWithOutputs(std::string_view s) } -StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs) +StorePathWithOutputs parsePathWithOutputs(const StoreDirConfig & store, std::string_view pathWithOutputs) { auto [path, outputs] = parsePathWithOutputs(pathWithOutputs); return StorePathWithOutputs { store.parseStorePath(path), std::move(outputs) }; diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh index 57e03252d..5f76a583a 100644 --- a/src/libstore/path-with-outputs.hh +++ b/src/libstore/path-with-outputs.hh @@ -6,6 +6,8 @@ namespace nix { +struct StoreDirConfig; + /** * This is a deprecated old type just for use by the old CLI, and older * versions of the RPC protocols. In new code don't use it; you want @@ -19,7 +21,7 @@ struct StorePathWithOutputs StorePath path; std::set outputs; - std::string to_string(const Store & store) const; + std::string to_string(const StoreDirConfig & store) const; DerivedPath toDerivedPath() const; @@ -32,14 +34,14 @@ std::vector toDerivedPaths(const std::vector) std::pair parsePathWithOutputs(std::string_view s); -class Store; - /** * Split a string specifying a derivation and a set of outputs * (/nix/store/hash-foo!out1,out2,...) into the derivation path * and the outputs. */ -StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs); +StorePathWithOutputs parsePathWithOutputs(const StoreDirConfig & store, std::string_view pathWithOutputs); + +class Store; StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs); diff --git a/src/libstore/path.cc b/src/libstore/path.cc index ec3e53232..1afd10af7 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -1,4 +1,4 @@ -#include "store-api.hh" +#include "store-dir-config.hh" #include @@ -35,7 +35,7 @@ StorePath::StorePath(std::string_view _baseName) } StorePath::StorePath(const Hash & hash, std::string_view _name) - : baseName((hash.to_string(HashFormat::Base32, false) + "-").append(std::string(_name))) + : baseName((hash.to_string(HashFormat::Nix32, false) + "-").append(std::string(_name))) { checkName(baseName, name()); } @@ -49,12 +49,12 @@ StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x"); StorePath StorePath::random(std::string_view name) { - Hash hash(htSHA1); + Hash hash(HashAlgorithm::SHA1); randombytes_buf(hash.hash, hash.hashSize); return StorePath(hash, name); } -StorePath Store::parseStorePath(std::string_view path) const +StorePath StoreDirConfig::parseStorePath(std::string_view path) const { auto p = canonPath(std::string(path)); if (dirOf(p) != storeDir) @@ -62,7 +62,7 @@ StorePath Store::parseStorePath(std::string_view path) const return StorePath(baseNameOf(p)); } -std::optional Store::maybeParseStorePath(std::string_view path) const +std::optional StoreDirConfig::maybeParseStorePath(std::string_view path) const { try { return parseStorePath(path); @@ -71,24 +71,24 @@ std::optional Store::maybeParseStorePath(std::string_view path) const } } -bool Store::isStorePath(std::string_view path) const +bool StoreDirConfig::isStorePath(std::string_view path) const { return (bool) maybeParseStorePath(path); } -StorePathSet Store::parseStorePathSet(const PathSet & paths) const +StorePathSet StoreDirConfig::parseStorePathSet(const PathSet & paths) const { StorePathSet res; for (auto & i : paths) res.insert(parseStorePath(i)); return res; } -std::string Store::printStorePath(const StorePath & path) const +std::string StoreDirConfig::printStorePath(const StorePath & path) const { return (storeDir + "/").append(path.to_string()); } -PathSet Store::printStorePathSet(const StorePathSet & paths) const +PathSet StoreDirConfig::printStorePathSet(const StorePathSet & paths) const { PathSet res; for (auto & i : paths) res.insert(printStorePath(i)); diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index adc763e6a..2b5b8dfe7 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -1,6 +1,7 @@ #include "pathlocks.hh" #include "util.hh" #include "sync.hh" +#include "signals.hh" #include #include diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh index 4921df352..7fcfa2e40 100644 --- a/src/libstore/pathlocks.hh +++ b/src/libstore/pathlocks.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "util.hh" +#include "file-descriptor.hh" namespace nix { diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc new file mode 100644 index 000000000..f38fa8369 --- /dev/null +++ b/src/libstore/posix-fs-canonicalise.cc @@ -0,0 +1,171 @@ +#if HAVE_SYS_XATTR_H +# include +#endif + +#include "posix-fs-canonicalise.hh" +#include "file-system.hh" +#include "signals.hh" +#include "util.hh" +#include "globals.hh" +#include "store-api.hh" + +namespace nix { + +const time_t mtimeStore = 1; /* 1 second into the epoch */ + + +static void canonicaliseTimestampAndPermissions(const Path & path, const struct stat & st) +{ + if (!S_ISLNK(st.st_mode)) { + + /* Mask out all type related bits. */ + mode_t mode = st.st_mode & ~S_IFMT; + + if (mode != 0444 && mode != 0555) { + mode = (st.st_mode & S_IFMT) + | 0444 + | (st.st_mode & S_IXUSR ? 0111 : 0); + if (chmod(path.c_str(), mode) == -1) + throw SysError("changing mode of '%1%' to %2$o", path, mode); + } + + } + + if (st.st_mtime != mtimeStore) { + struct timeval times[2]; + times[0].tv_sec = st.st_atime; + times[0].tv_usec = 0; + times[1].tv_sec = mtimeStore; + times[1].tv_usec = 0; +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) + if (errno != ENOSYS || + (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) +#else + if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) +#endif + throw SysError("changing modification time of '%1%'", path); + } +} + + +void canonicaliseTimestampAndPermissions(const Path & path) +{ + canonicaliseTimestampAndPermissions(path, lstat(path)); +} + + +static void canonicalisePathMetaData_( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen) +{ + checkInterrupt(); + +#if __APPLE__ + /* Remove flags, in particular UF_IMMUTABLE which would prevent + the file from being garbage-collected. FIXME: Use + setattrlist() to remove other attributes as well. */ + if (lchflags(path.c_str(), 0)) { + if (errno != ENOTSUP) + throw SysError("clearing flags of path '%1%'", path); + } +#endif + + auto st = lstat(path); + + /* Really make sure that the path is of a supported type. */ + if (!(S_ISREG(st.st_mode) || S_ISDIR(st.st_mode) || S_ISLNK(st.st_mode))) + throw Error("file '%1%' has an unsupported type", path); + +#ifdef HAVE_SYS_XATTR_H + /* Remove extended attributes / ACLs. */ + ssize_t eaSize = llistxattr(path.c_str(), nullptr, 0); + + if (eaSize < 0) { + if (errno != ENOTSUP && errno != ENODATA) + throw SysError("querying extended attributes of '%s'", path); + } else if (eaSize > 0) { + std::vector eaBuf(eaSize); + + if ((eaSize = llistxattr(path.c_str(), eaBuf.data(), eaBuf.size())) < 0) + throw SysError("querying extended attributes of '%s'", path); + + for (auto & eaName: tokenizeString(std::string(eaBuf.data(), eaSize), std::string("\000", 1))) { + if (settings.ignoredAcls.get().count(eaName)) continue; + if (lremovexattr(path.c_str(), eaName.c_str()) == -1) + throw SysError("removing extended attribute '%s' from '%s'", eaName, path); + } + } +#endif + + /* Fail if the file is not owned by the build user. This prevents + us from messing up the ownership/permissions of files + hard-linked into the output (e.g. "ln /etc/shadow $out/foo"). + However, ignore files that we chown'ed ourselves previously to + ensure that we don't fail on hard links within the same build + (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ + if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) { + if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino))) + throw BuildError("invalid ownership on file '%1%'", path); + mode_t mode = st.st_mode & ~S_IFMT; + assert(S_ISLNK(st.st_mode) || (st.st_uid == geteuid() && (mode == 0444 || mode == 0555) && st.st_mtime == mtimeStore)); + return; + } + + inodesSeen.insert(Inode(st.st_dev, st.st_ino)); + + canonicaliseTimestampAndPermissions(path, st); + + /* Change ownership to the current uid. If it's a symlink, use + lchown if available, otherwise don't bother. Wrong ownership + of a symlink doesn't matter, since the owning user can't change + the symlink and can't delete it because the directory is not + writable. The only exception is top-level paths in the Nix + store (since that directory is group-writable for the Nix build + users group); we check for this case below. */ + if (st.st_uid != geteuid()) { +#if HAVE_LCHOWN + if (lchown(path.c_str(), geteuid(), getegid()) == -1) +#else + if (!S_ISLNK(st.st_mode) && + chown(path.c_str(), geteuid(), getegid()) == -1) +#endif + throw SysError("changing owner of '%1%' to %2%", + path, geteuid()); + } + + if (S_ISDIR(st.st_mode)) { + DirEntries entries = readDirectory(path); + for (auto & i : entries) + canonicalisePathMetaData_(path + "/" + i.name, uidRange, inodesSeen); + } +} + + +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen) +{ + canonicalisePathMetaData_(path, uidRange, inodesSeen); + + /* On platforms that don't have lchown(), the top-level path can't + be a symlink, since we can't change its ownership. */ + auto st = lstat(path); + + if (st.st_uid != geteuid()) { + assert(S_ISLNK(st.st_mode)); + throw Error("wrong ownership of top-level store path '%1%'", path); + } +} + + +void canonicalisePathMetaData(const Path & path, + std::optional> uidRange) +{ + InodesSeen inodesSeen; + canonicalisePathMetaData(path, uidRange, inodesSeen); +} + +} diff --git a/src/libstore/posix-fs-canonicalise.hh b/src/libstore/posix-fs-canonicalise.hh new file mode 100644 index 000000000..35644af12 --- /dev/null +++ b/src/libstore/posix-fs-canonicalise.hh @@ -0,0 +1,45 @@ +#pragma once +///@file + +#include +#include + +#include "types.hh" +#include "error.hh" + +namespace nix { + +typedef std::pair Inode; +typedef std::set InodesSeen; + + +/** + * "Fix", or canonicalise, the meta-data of the files in a store path + * after it has been built. In particular: + * + * - the last modification date on each file is set to 1 (i.e., + * 00:00:01 1/1/1970 UTC) + * + * - the permissions are set of 444 or 555 (i.e., read-only with or + * without execute permission; setuid bits etc. are cleared) + * + * - the owner and group are set to the Nix user and group, if we're + * running as root. + * + * If uidRange is not empty, this function will throw an error if it + * encounters files owned by a user outside of the closed interval + * [uidRange->first, uidRange->second]. + */ +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange, + InodesSeen & inodesSeen); +void canonicalisePathMetaData( + const Path & path, + std::optional> uidRange); + +void canonicaliseTimestampAndPermissions(const Path & path); + +MakeError(PathInUse, Error); + +} diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index 239047dd6..e8b88693d 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -1,7 +1,7 @@ #include "profiles.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "util.hh" +#include "users.hh" #include #include diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index fcfb527f5..03e57a565 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -8,8 +8,9 @@ namespace nix { -RemoteFSAccessor::RemoteFSAccessor(ref store, const Path & cacheDir) +RemoteFSAccessor::RemoteFSAccessor(ref store, bool requireValidPath, const Path & cacheDir) : store(store) + , requireValidPath(requireValidPath) , cacheDir(cacheDir) { if (cacheDir != "") @@ -22,7 +23,7 @@ Path RemoteFSAccessor::makeCacheFile(std::string_view hashPart, const std::strin return fmt("%s/%s.%s", cacheDir, hashPart, ext); } -ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) +ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::string && nar) { if (cacheDir != "") { try { @@ -38,7 +39,7 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str if (cacheDir != "") { try { - nlohmann::json j = listNar(narAccessor, "", true); + nlohmann::json j = listNar(narAccessor, CanonPath::root, true); writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { ignoreException(); @@ -48,11 +49,10 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std::str return narAccessor; } -std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath) +std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path) { - auto path = canonPath(path_); - - auto [storePath, restPath] = store->toStorePath(path); + auto [storePath, restPath_] = store->toStorePath(path.abs()); + auto restPath = CanonPath(restPath_); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); @@ -63,7 +63,7 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo std::string listing; Path cacheFile; - if (cacheDir != "" && pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { + if (cacheDir != "" && nix::pathExists(cacheFile = makeCacheFile(storePath.hashPart(), "nar"))) { try { listing = nix::readFile(makeCacheFile(storePath.hashPart(), "ls")); @@ -101,25 +101,25 @@ std::pair, Path> RemoteFSAccessor::fetch(const Path & path_, boo return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath}; } -FSAccessor::Stat RemoteFSAccessor::stat(const Path & path) +std::optional RemoteFSAccessor::maybeLstat(const CanonPath & path) { auto res = fetch(path); - return res.first->stat(res.second); + return res.first->maybeLstat(res.second); } -StringSet RemoteFSAccessor::readDirectory(const Path & path) +SourceAccessor::DirEntries RemoteFSAccessor::readDirectory(const CanonPath & path) { auto res = fetch(path); return res.first->readDirectory(res.second); } -std::string RemoteFSAccessor::readFile(const Path & path, bool requireValidPath) +std::string RemoteFSAccessor::readFile(const CanonPath & path) { - auto res = fetch(path, requireValidPath); + auto res = fetch(path); return res.first->readFile(res.second); } -std::string RemoteFSAccessor::readLink(const Path & path) +std::string RemoteFSAccessor::readLink(const CanonPath & path) { auto res = fetch(path); return res.first->readLink(res.second); diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index e2673b6f6..d09762a53 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -1,40 +1,43 @@ #pragma once ///@file -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "ref.hh" #include "store-api.hh" namespace nix { -class RemoteFSAccessor : public FSAccessor +class RemoteFSAccessor : public SourceAccessor { ref store; - std::map> nars; + std::map> nars; + + bool requireValidPath; Path cacheDir; - std::pair, Path> fetch(const Path & path_, bool requireValidPath = true); + std::pair, CanonPath> fetch(const CanonPath & path); friend class BinaryCacheStore; Path makeCacheFile(std::string_view hashPart, const std::string & ext); - ref addToCache(std::string_view hashPart, std::string && nar); + ref addToCache(std::string_view hashPart, std::string && nar); public: RemoteFSAccessor(ref store, + bool requireValidPath = true, const /* FIXME: use std::optional */ Path & cacheDir = ""); - Stat stat(const Path & path) override; + std::optional maybeLstat(const CanonPath & path) override; - StringSet readDirectory(const Path & path) override; + DirEntries readDirectory(const CanonPath & path) override; - std::string readFile(const Path & path, bool requireValidPath = true) override; + std::string readFile(const CanonPath & path) override; - std::string readLink(const Path & path) override; + std::string readLink(const CanonPath & path) override; }; } diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh index e4a9cacb9..44328b06b 100644 --- a/src/libstore/remote-store-connection.hh +++ b/src/libstore/remote-store-connection.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "remote-store.hh" #include "worker-protocol.hh" #include "pool.hh" diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 7bdc25433..cc26c2a94 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -20,7 +20,7 @@ namespace nix { -/* TODO: Separate these store impls into different files, give them better names */ +/* TODO: Separate these store types into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) : RemoteStoreConfig(params) , Store(params) @@ -417,12 +417,12 @@ std::optional RemoteStore::queryPathFromHashPart(const std::string & ref RemoteStore::addCAToStore( - Source & dump, - std::string_view name, - ContentAddressMethod caMethod, - HashType hashType, - const StorePathSet & references, - RepairFlag repair) + Source & dump, + std::string_view name, + ContentAddressMethod caMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) { std::optional conn_(getConnection()); auto & conn = *conn_; @@ -432,7 +432,7 @@ ref RemoteStore::addCAToStore( conn->to << WorkerProto::Op::AddToStore << name - << caMethod.render(hashType); + << caMethod.render(hashAlgo); WorkerProto::write(*this, *conn, references); conn->to << repair; @@ -453,9 +453,9 @@ ref RemoteStore::addCAToStore( std::visit(overloaded { [&](const TextIngestionMethod & thm) -> void { - if (hashType != htSHA256) + if (hashAlgo != HashAlgorithm::SHA256) throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", - name, printHashType(hashType)); + name, printHashAlgo(hashAlgo)); std::string s = dump.drain(); conn->to << WorkerProto::Op::AddTextToStore << name << s; WorkerProto::write(*this, *conn, references); @@ -465,9 +465,9 @@ ref RemoteStore::addCAToStore( conn->to << WorkerProto::Op::AddToStore << name - << ((hashType == htSHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ + << ((hashAlgo == HashAlgorithm::SHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ << (fim == FileIngestionMethod::Recursive ? 1 : 0) - << printHashType(hashType); + << printHashAlgo(hashAlgo); try { conn->to.written = 0; @@ -503,9 +503,9 @@ ref RemoteStore::addCAToStore( StorePath RemoteStore::addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method, HashType hashType, RepairFlag repair, const StorePathSet & references) + FileIngestionMethod method, HashAlgorithm hashAlgo, RepairFlag repair, const StorePathSet & references) { - return addCAToStore(dump, name, method, hashType, references, repair)->path; + return addCAToStore(dump, name, method, hashAlgo, references, repair)->path; } @@ -610,7 +610,7 @@ StorePath RemoteStore::addTextToStore( RepairFlag repair) { StringSource source(s); - return addCAToStore(source, name, TextIngestionMethod {}, htSHA256, references, repair)->path; + return addCAToStore(source, name, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair)->path; } void RemoteStore::registerDrvOutput(const Realisation & info) @@ -970,7 +970,7 @@ void RemoteStore::narFromPath(const StorePath & path, Sink & sink) copyNAR(conn->from, sink); } -ref RemoteStore::getFSAccessor() +ref RemoteStore::getFSAccessor(bool requireValidPath) { return make_ref(ref(shared_from_this())); } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index a1ae82a0f..f2e34c1a3 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -22,10 +22,10 @@ struct RemoteStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting maxConnections{(StoreConfig*) this, 1, "max-connections", + const Setting maxConnections{this, 1, "max-connections", "Maximum number of concurrent connections to the Nix daemon."}; - const Setting maxConnectionAge{(StoreConfig*) this, + const Setting maxConnectionAge{this, std::numeric_limits::max(), "max-connection-age", "Maximum age of a connection before it is closed."}; @@ -74,18 +74,18 @@ public: * Add a content-addressable store path. `dump` will be drained. */ ref addCAToStore( - Source & dump, - std::string_view name, - ContentAddressMethod caMethod, - HashType hashType, - const StorePathSet & references, - RepairFlag repair); + Source & dump, + std::string_view name, + ContentAddressMethod caMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair); /** * Add a content-addressable store path. Does not support references. `dump` will be drained. */ StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; + FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair, const StorePathSet & references = StorePathSet()) override; void addToStore(const ValidPathInfo & info, Source & nar, RepairFlag repair, CheckSigsFlag checkSigs) override; @@ -185,7 +185,7 @@ protected: friend struct ConnectionHandle; - virtual ref getFSAccessor() override; + virtual ref getFSAccessor(bool requireValidPath) override; virtual void narFromPath(const StorePath & path, Sink & sink) override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index d2fc6abaf..1a62d92d4 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -193,20 +193,20 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; - const Setting profile{(StoreConfig*) this, "", "profile", + const Setting profile{this, "", "profile", R"( The name of the AWS configuration profile to use. By default Nix will use the `default` profile. )"}; - const Setting region{(StoreConfig*) this, Aws::Region::US_EAST_1, "region", + const Setting region{this, Aws::Region::US_EAST_1, "region", R"( The region of the S3 bucket. If your bucket is not in `us–east-1`, you should always explicitly specify the region parameter. )"}; - const Setting scheme{(StoreConfig*) this, "", "scheme", + const Setting scheme{this, "", "scheme", R"( The scheme used for S3 requests, `https` (default) or `http`. This option allows you to disable HTTPS for binary caches which don't @@ -218,7 +218,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig > information. )"}; - const Setting endpoint{(StoreConfig*) this, "", "endpoint", + const Setting endpoint{this, "", "endpoint", R"( The URL of the endpoint of an S3-compatible service such as MinIO. Do not specify this setting if you're using Amazon S3. @@ -229,13 +229,13 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig > addressing instead of virtual host based addressing. )"}; - const Setting narinfoCompression{(StoreConfig*) this, "", "narinfo-compression", + const Setting narinfoCompression{this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; - const Setting lsCompression{(StoreConfig*) this, "", "ls-compression", + const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; - const Setting logCompression{(StoreConfig*) this, "", "log-compression", + const Setting logCompression{this, "", "log-compression", R"( Compression method for `log/*` files. It is recommended to use a compression method supported by most web browsers @@ -243,11 +243,11 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig )"}; const Setting multipartUpload{ - (StoreConfig*) this, false, "multipart-upload", + this, false, "multipart-upload", "Whether to use multi-part uploads."}; const Setting bufferSize{ - (StoreConfig*) this, 5 * 1024 * 1024, "buffer-size", + this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; const std::string name() override { return "S3 Binary Cache Store"; } diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh index a3ce81026..6f3b177ac 100644 --- a/src/libstore/serve-protocol-impl.hh +++ b/src/libstore/serve-protocol-impl.hh @@ -16,11 +16,11 @@ namespace nix { /* protocol-agnostic templates */ #define SERVE_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ - TEMPLATE T ServeProto::Serialise< T >::read(const Store & store, ServeProto::ReadConn conn) \ + TEMPLATE T ServeProto::Serialise< T >::read(const StoreDirConfig & store, ServeProto::ReadConn conn) \ { \ return LengthPrefixedProtoHelper::read(store, conn); \ } \ - TEMPLATE void ServeProto::Serialise< T >::write(const Store & store, ServeProto::WriteConn conn, const T & t) \ + TEMPLATE void ServeProto::Serialise< T >::write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t) \ { \ LengthPrefixedProtoHelper::write(store, conn, t); \ } @@ -41,12 +41,12 @@ SERVE_USE_LENGTH_PREFIX_SERIALISER( template struct ServeProto::Serialise { - static T read(const Store & store, ServeProto::ReadConn conn) + static T read(const StoreDirConfig & store, ServeProto::ReadConn conn) { return CommonProto::Serialise::read(store, CommonProto::ReadConn { .from = conn.from }); } - static void write(const Store & store, ServeProto::WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t) { CommonProto::Serialise::write(store, CommonProto::WriteConn { .to = conn.to }, diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index 97a0ddf0e..08bfad9e4 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -1,11 +1,11 @@ #include "serialise.hh" -#include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" #include "build-result.hh" #include "serve-protocol.hh" #include "serve-protocol-impl.hh" #include "archive.hh" +#include "path-info.hh" #include @@ -13,7 +13,7 @@ namespace nix { /* protocol-specific definitions */ -BuildResult ServeProto::Serialise::read(const Store & store, ServeProto::ReadConn conn) +BuildResult ServeProto::Serialise::read(const StoreDirConfig & store, ServeProto::ReadConn conn) { BuildResult status; status.status = (BuildResult::Status) readInt(conn.from); @@ -35,7 +35,7 @@ BuildResult ServeProto::Serialise::read(const Store & store, ServeP return status; } -void ServeProto::Serialise::write(const Store & store, ServeProto::WriteConn conn, const BuildResult & status) +void ServeProto::Serialise::write(const StoreDirConfig & store, ServeProto::WriteConn conn, const BuildResult & status) { conn.to << status.status @@ -55,4 +55,83 @@ void ServeProto::Serialise::write(const Store & store, ServeProto:: } } + +UnkeyedValidPathInfo ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + /* Hash should be set below unless very old `nix-store --serve`. + Caller should assert that it did set it. */ + UnkeyedValidPathInfo info { Hash::dummy }; + + auto deriver = readString(conn.from); + if (deriver != "") + info.deriver = store.parseStorePath(deriver); + info.references = ServeProto::Serialise::read(store, conn); + + readLongLong(conn.from); // download size, unused + info.narSize = readLongLong(conn.from); + + if (GET_PROTOCOL_MINOR(conn.version) >= 4) { + auto s = readString(conn.from); + if (!s.empty()) + info.narHash = Hash::parseAnyPrefixed(s); + info.ca = ContentAddress::parseOpt(readString(conn.from)); + info.sigs = readStrings(conn.from); + } + + return info; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const UnkeyedValidPathInfo & info) +{ + conn.to + << (info.deriver ? store.printStorePath(*info.deriver) : ""); + + ServeProto::write(store, conn, info.references); + // !!! Maybe we want compression? + conn.to + << info.narSize // downloadSize, lie a little + << info.narSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 4) + conn.to + << info.narHash.to_string(HashFormat::Nix32, true) + << renderContentAddress(info.ca) + << info.sigs; +} + + +ServeProto::BuildOptions ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + BuildOptions options; + options.maxSilentTime = readInt(conn.from); + options.buildTimeout = readInt(conn.from); + if (GET_PROTOCOL_MINOR(conn.version) >= 2) + options.maxLogSize = readNum(conn.from); + if (GET_PROTOCOL_MINOR(conn.version) >= 3) { + options.nrRepeats = readInt(conn.from); + options.enforceDeterminism = readInt(conn.from); + } + if (GET_PROTOCOL_MINOR(conn.version) >= 7) { + options.keepFailed = (bool) readInt(conn.from); + } + return options; +} + +void ServeProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const ServeProto::BuildOptions & options) +{ + conn.to + << options.maxSilentTime + << options.buildTimeout; + if (GET_PROTOCOL_MINOR(conn.version) >= 2) + conn.to + << options.maxLogSize; + if (GET_PROTOCOL_MINOR(conn.version) >= 3) + conn.to + << options.nrRepeats + << options.enforceDeterminism; + + if (GET_PROTOCOL_MINOR(conn.version) >= 7) { + conn.to << ((int) options.keepFailed); + } +} + } diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh index ba159f6e9..1665b935f 100644 --- a/src/libstore/serve-protocol.hh +++ b/src/libstore/serve-protocol.hh @@ -13,11 +13,12 @@ namespace nix { #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) -class Store; +struct StoreDirConfig; struct Source; // items being serialised struct BuildResult; +struct UnkeyedValidPathInfo; /** @@ -72,8 +73,8 @@ struct ServeProto // See `worker-protocol.hh` for a longer explanation. #if 0 { - static T read(const Store & store, ReadConn conn); - static void write(const Store & store, WriteConn conn, const T & t); + static T read(const StoreDirConfig & store, ReadConn conn); + static void write(const StoreDirConfig & store, WriteConn conn, const T & t); }; #endif @@ -82,10 +83,17 @@ struct ServeProto * infer the type instead of having to write it down explicitly. */ template - static void write(const Store & store, WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, WriteConn conn, const T & t) { ServeProto::Serialise::write(store, conn, t); } + + /** + * Options for building shared between + * `ServeProto::Command::BuildPaths` and + * `ServeProto::Command::BuildDerivation`. + */ + struct BuildOptions; }; enum struct ServeProto::Command : uint64_t @@ -101,6 +109,22 @@ enum struct ServeProto::Command : uint64_t AddToStoreNar = 9, }; + +struct ServeProto::BuildOptions { + /** + * Default value in this and every other field is so tests pass when + * testing older deserialisers which do not set all the fields. + */ + time_t maxSilentTime = -1; + time_t buildTimeout = -1; + size_t maxLogSize = -1; + size_t nrRepeats = -1; + bool enforceDeterminism = -1; + bool keepFailed = -1; + + bool operator == (const ServeProto::BuildOptions &) const = default; +}; + /** * Convenience for sending operation codes. * @@ -135,12 +159,16 @@ inline std::ostream & operator << (std::ostream & s, ServeProto::Command op) #define DECLARE_SERVE_SERIALISER(T) \ struct ServeProto::Serialise< T > \ { \ - static T read(const Store & store, ServeProto::ReadConn conn); \ - static void write(const Store & store, ServeProto::WriteConn conn, const T & t); \ + static T read(const StoreDirConfig & store, ServeProto::ReadConn conn); \ + static void write(const StoreDirConfig & store, ServeProto::WriteConn conn, const T & t); \ }; template<> DECLARE_SERVE_SERIALISER(BuildResult); +template<> +DECLARE_SERVE_SERIALISER(UnkeyedValidPathInfo); +template<> +DECLARE_SERVE_SERIALISER(ServeProto::BuildOptions); template DECLARE_SERVE_SERIALISER(std::vector); diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 7c8decb74..d7432a305 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -2,6 +2,7 @@ #include "globals.hh" #include "util.hh" #include "url.hh" +#include "signals.hh" #include diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh index c27a5d00f..4ce4ffc4c 100644 --- a/src/libstore/ssh-store-config.hh +++ b/src/libstore/ssh-store-config.hh @@ -9,18 +9,18 @@ struct CommonSSHStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; - const Setting sshKey{(StoreConfig*) this, "", "ssh-key", + const Setting sshKey{this, "", "ssh-key", "Path to the SSH private key used to authenticate to the remote machine."}; - const Setting sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", + const Setting sshPublicHostKey{this, "", "base64-ssh-public-host-key", "The public host key of the remote machine."}; - const Setting compress{(StoreConfig*) this, false, "compress", + const Setting compress{this, false, "compress", "Whether to enable SSH compression."}; - const Setting remoteStore{(StoreConfig*) this, "", "remote-store", + const Setting remoteStore{this, "", "remote-store", R"( - [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + [Store URL](@docroot@/store/types/index.md#store-url-format) to be used on the remote machine. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). )"}; diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 9c6c42ef4..d4c8ab5b2 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -3,9 +3,10 @@ #include "local-fs-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" -#include "remote-fs-accessor.hh" +#include "source-accessor.hh" #include "archive.hh" #include "worker-protocol.hh" +#include "worker-protocol-impl.hh" #include "pool.hh" #include "ssh.hh" @@ -16,7 +17,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig using RemoteStoreConfig::RemoteStoreConfig; using CommonSSHStoreConfig::CommonSSHStoreConfig; - const Setting remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", + const Setting remoteProgram{this, "nix-daemon", "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; const std::string name() override { return "Experimental SSH Store"; } @@ -78,6 +79,8 @@ protected: std::string host; + std::vector extraRemoteProgramArgs; + SSHMaster master; void setOptions(RemoteStore::Connection & conn) override @@ -91,6 +94,121 @@ protected: }; }; +struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig +{ + using SSHStoreConfig::SSHStoreConfig; + using LocalFSStoreConfig::LocalFSStoreConfig; + + MountedSSHStoreConfig(StringMap params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) + { + } + + const std::string name() override { return "Experimental SSH Store with filesytem mounted"; } + + std::string doc() override + { + return + #include "mounted-ssh-store.md" + ; + } + + std::optional experimentalFeature() const override + { + return ExperimentalFeature::MountedSSHStore; + } +}; + +/** + * The mounted ssh store assumes that filesystems on the remote host are + * shared with the local host. This means that the remote nix store is + * available locally and is therefore treated as a local filesystem + * store. + * + * MountedSSHStore is very similar to UDSRemoteStore --- ignoring the + * superficial differnce of SSH vs Unix domain sockets, they both are + * accessing remote stores, and they both assume the store will be + * mounted in the local filesystem. + * + * The difference lies in how they manage GC roots. See addPermRoot + * below for details. + */ +class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSHStore, public virtual LocalFSStore +{ +public: + + MountedSSHStore(const std::string & scheme, const std::string & host, const Params & params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) + , MountedSSHStoreConfig(params) + , Store(params) + , RemoteStore(params) + , SSHStore(scheme, host, params) + , LocalFSStore(params) + { + extraRemoteProgramArgs = { + "--process-ops", + }; + } + + static std::set uriSchemes() + { + return {"mounted-ssh-ng"}; + } + + std::string getUri() override + { + return *uriSchemes().begin() + "://" + host; + } + + void narFromPath(const StorePath & path, Sink & sink) override + { + return LocalFSStore::narFromPath(path, sink); + } + + ref getFSAccessor(bool requireValidPath) override + { + return LocalFSStore::getFSAccessor(requireValidPath); + } + + std::optional getBuildLogExact(const StorePath & path) override + { + return LocalFSStore::getBuildLogExact(path); + } + + /** + * This is the key difference from UDSRemoteStore: UDSRemote store + * has the client create the direct root, and the remote side create + * the indirect root. + * + * We could also do that, but the race conditions (will the remote + * side see the direct root the client made?) seems bigger. + * + * In addition, the remote-side will have a process associated with + * the authenticating user handling the connection (even if there + * is a system-wide daemon or similar). This process can safely make + * the direct and indirect roots without there being such a risk of + * privilege escalation / symlinks in directories owned by the + * originating requester that they cannot delete. + */ + Path addPermRoot(const StorePath & path, const Path & gcRoot) override + { + auto conn(getConnection()); + conn->to << WorkerProto::Op::AddPermRoot; + WorkerProto::write(*this, *conn, path); + WorkerProto::write(*this, *conn, gcRoot); + conn.processStderr(); + return readString(conn->from); + } +}; + ref SSHStore::openConnection() { auto conn = make_ref(); @@ -98,6 +216,8 @@ ref SSHStore::openConnection() std::string command = remoteProgram + " --stdio"; if (remoteStore.get() != "") command += " --store " + shellEscape(remoteStore.get()); + for (auto & arg : extraRemoteProgramArgs) + command += " " + shellEscape(arg); conn->sshConn = master.startCommand(command); conn->to = FdSink(conn->sshConn->in.get()); @@ -106,5 +226,6 @@ ref SSHStore::openConnection() } static RegisterStoreImplementation regSSHStore; +static RegisterStoreImplementation regMountedSSHStore; } diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index da32f1b79..5c8d6a504 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -1,5 +1,8 @@ #include "ssh.hh" #include "finally.hh" +#include "current-process.hh" +#include "environment-variables.hh" +#include "util.hh" namespace nix { @@ -111,8 +114,10 @@ std::unique_ptr SSHMaster::startCommand(const std::string reply = readLine(out.readSide.get()); } catch (EndOfFile & e) { } - if (reply != "started") + if (reply != "started") { + printTalkative("SSH stdout first line: %s", reply); throw Error("failed to start SSH connection to '%s'", host); + } } conn->out = std::move(out.readSide); @@ -129,7 +134,6 @@ Path SSHMaster::startMaster() if (state->sshMaster != -1) return state->socketPath; - state->socketPath = (Path) *state->tmpDir + "/ssh.sock"; Pipe out; @@ -141,7 +145,8 @@ Path SSHMaster::startMaster() logger->pause(); Finally cleanup = [&]() { logger->resume(); }; - bool wasMasterRunning = isMasterRunning(); + if (isMasterRunning()) + return state->socketPath; state->sshMaster = startProcess([&]() { restoreProcessContext(); @@ -162,14 +167,14 @@ Path SSHMaster::startMaster() out.writeSide = -1; - if (!wasMasterRunning) { - std::string reply; - try { - reply = readLine(out.readSide.get()); - } catch (EndOfFile & e) { } + std::string reply; + try { + reply = readLine(out.readSide.get()); + } catch (EndOfFile & e) { } - if (reply != "started") - throw Error("failed to start SSH master connection to '%s'", host); + if (reply != "started") { + printTalkative("SSH master stdout first line: %s", reply); + throw Error("failed to start SSH master connection to '%s'", host); } return state->socketPath; diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index 94b952af9..bfcd6f21c 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -1,8 +1,9 @@ #pragma once ///@file -#include "util.hh" #include "sync.hh" +#include "processes.hh" +#include "file-system.hh" namespace nix { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 069fad420..27a561eb9 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,6 +1,8 @@ #include "crypto.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "globals.hh" +#include "derived-path.hh" +#include "realisation.hh" #include "derivations.hh" #include "store-api.hh" #include "util.hh" @@ -14,6 +16,8 @@ // FIXME this should not be here, see TODO below on // `addMultipleToStore`. #include "worker-protocol.hh" +#include "signals.hh" +#include "users.hh" #include #include @@ -23,13 +27,13 @@ using json = nlohmann::json; namespace nix { -bool Store::isInStore(PathView path) const +bool StoreDirConfig::isInStore(PathView path) const { return isInDir(path, storeDir); } -std::pair Store::toStorePath(PathView path) const +std::pair StoreDirConfig::toStorePath(PathView path) const { if (!isInStore(path)) throw Error("path '%1%' is not in the Nix store", path); @@ -143,25 +147,25 @@ StorePath Store::followLinksToStorePath(std::string_view path) const */ -StorePath Store::makeStorePath(std::string_view type, +StorePath StoreDirConfig::makeStorePath(std::string_view type, std::string_view hash, std::string_view name) const { /* e.g., "source:sha256:1abc...:/nix/store:foo.tar.gz" */ auto s = std::string(type) + ":" + std::string(hash) + ":" + storeDir + ":" + std::string(name); - auto h = compressHash(hashString(htSHA256, s), 20); + auto h = compressHash(hashString(HashAlgorithm::SHA256, s), 20); return StorePath(h, name); } -StorePath Store::makeStorePath(std::string_view type, +StorePath StoreDirConfig::makeStorePath(std::string_view type, const Hash & hash, std::string_view name) const { return makeStorePath(type, hash.to_string(HashFormat::Base16, true), name); } -StorePath Store::makeOutputPath(std::string_view id, +StorePath StoreDirConfig::makeOutputPath(std::string_view id, const Hash & hash, std::string_view name) const { return makeStorePath("output:" + std::string { id }, hash, outputPathName(name, id)); @@ -172,7 +176,7 @@ StorePath Store::makeOutputPath(std::string_view id, hacky, but we can't put them in, say, (per the grammar above) since that would be ambiguous. */ static std::string makeType( - const Store & store, + const StoreDirConfig & store, std::string && type, const StoreReferences & references) { @@ -185,14 +189,14 @@ static std::string makeType( } -StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const +StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const { - if (info.hash.type == htSHA256 && info.method == FileIngestionMethod::Recursive) { + if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { assert(info.references.size() == 0); return makeStorePath("output:out", - hashString(htSHA256, + hashString(HashAlgorithm::SHA256, "fixed:out:" + makeFileIngestionPrefix(info.method) + info.hash.to_string(HashFormat::Base16, true) + ":"), @@ -201,9 +205,9 @@ StorePath Store::makeFixedOutputPath(std::string_view name, const FixedOutputInf } -StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) const +StorePath StoreDirConfig::makeTextPath(std::string_view name, const TextInfo & info) const { - assert(info.hash.type == htSHA256); + assert(info.hash.algo == HashAlgorithm::SHA256); return makeStorePath( makeType(*this, "text", StoreReferences { .others = info.references, @@ -214,7 +218,7 @@ StorePath Store::makeTextPath(std::string_view name, const TextInfo & info) cons } -StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const +StorePath StoreDirConfig::makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const { // New template return std::visit(overloaded { @@ -228,12 +232,12 @@ StorePath Store::makeFixedOutputPathFromCA(std::string_view name, const ContentA } -std::pair Store::computeStorePathFromDump( - Source & dump, - std::string_view name, - FileIngestionMethod method, - HashType hashAlgo, - const StorePathSet & references) const +std::pair StoreDirConfig::computeStorePathFromDump( + Source & dump, + std::string_view name, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + const StorePathSet & references) const { HashSink sink(hashAlgo); dump.drainInto(sink); @@ -247,26 +251,26 @@ std::pair Store::computeStorePathFromDump( } -StorePath Store::computeStorePathForText( +StorePath StoreDirConfig::computeStorePathForText( std::string_view name, std::string_view s, const StorePathSet & references) const { return makeTextPath(name, TextInfo { - .hash = hashString(htSHA256, s), + .hash = hashString(HashAlgorithm::SHA256, s), .references = references, }); } StorePath Store::addToStore( - std::string_view name, - const Path & _srcPath, - FileIngestionMethod method, - HashType hashAlgo, - PathFilter & filter, - RepairFlag repair, - const StorePathSet & references) + std::string_view name, + const Path & _srcPath, + FileIngestionMethod method, + HashAlgorithm hashAlgo, + PathFilter & filter, + RepairFlag repair, + const StorePathSet & references) { Path srcPath(absPath(_srcPath)); auto source = sinkToSource([&](Sink & sink) { @@ -401,19 +405,19 @@ digraph graphname { } */ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, - FileIngestionMethod method, HashType hashAlgo, - std::optional expectedCAHash) + FileIngestionMethod method, HashAlgorithm hashAlgo, + std::optional expectedCAHash) { - HashSink narHashSink { htSHA256 }; + HashSink narHashSink { HashAlgorithm::SHA256 }; HashSink caHashSink { hashAlgo }; /* Note that fileSink and unusualHashTee must be mutually exclusive, since they both write to caHashSink. Note that that requisite is currently true because the former is only used in the flat case. */ - RetrieveRegularNARSink fileSink { caHashSink }; + RegularFileSink fileSink { caHashSink }; TeeSink unusualHashTee { narHashSink, caHashSink }; - auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != htSHA256 + auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != HashAlgorithm::SHA256 ? static_cast(unusualHashTee) : narHashSink; @@ -428,10 +432,10 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, information to narSink. */ TeeSource tapped { *fileSource, narSink }; - ParseSink blank; + NullParseSink blank; auto & parseSink = method == FileIngestionMethod::Flat - ? fileSink - : blank; + ? (ParseSink &) fileSink + : (ParseSink &) blank; /* The information that flows from tapped (besides being replicated in narSink), is now put in parseSink. */ @@ -441,7 +445,7 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, finish. */ auto [narHash, narSize] = narHashSink.finish(); - auto hash = method == FileIngestionMethod::Recursive && hashAlgo == htSHA256 + auto hash = method == FileIngestionMethod::Recursive && hashAlgo == HashAlgorithm::SHA256 ? narHash : caHashSink.finish().first; @@ -543,8 +547,8 @@ std::map> Store::queryPartialDerivationOut return outputs; } -OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) { - auto resp = queryPartialDerivationOutputMap(path); +OutputPathMap Store::queryDerivationOutputMap(const StorePath & path, Store * evalStore) { + auto resp = queryPartialDerivationOutputMap(path, evalStore); OutputPathMap result; for (auto & [outName, optOutPath] : resp) { if (!optOutPath) @@ -819,7 +823,7 @@ void Store::substitutePaths(const StorePathSet & paths) std::vector paths2; for (auto & path : paths) if (!path.isDerivation()) - paths2.push_back(DerivedPath::Opaque{path}); + paths2.emplace_back(DerivedPath::Opaque{path}); uint64_t downloadSize, narSize; StorePathSet willBuild, willSubstitute, unknown; queryMissing(paths2, @@ -949,96 +953,6 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor return paths; } -json Store::pathInfoToJSON(const StorePathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, - HashFormat hashFormat, - AllowInvalidFlag allowInvalid) -{ - json::array_t jsonList = json::array(); - - for (auto & storePath : storePaths) { - auto& jsonPath = jsonList.emplace_back(json::object()); - - try { - auto info = queryPathInfo(storePath); - - jsonPath["path"] = printStorePath(info->path); - jsonPath["valid"] = true; - jsonPath["narHash"] = info->narHash.to_string(hashFormat, true); - jsonPath["narSize"] = info->narSize; - - { - auto& jsonRefs = (jsonPath["references"] = json::array()); - for (auto & ref : info->references) - jsonRefs.emplace_back(printStorePath(ref)); - } - - if (info->ca) - jsonPath["ca"] = renderContentAddress(info->ca); - - std::pair closureSizes; - - if (showClosureSize) { - closureSizes = getClosureSize(info->path); - jsonPath["closureSize"] = closureSizes.first; - } - - if (includeImpureInfo) { - - if (info->deriver) - jsonPath["deriver"] = printStorePath(*info->deriver); - - if (info->registrationTime) - jsonPath["registrationTime"] = info->registrationTime; - - if (info->ultimate) - jsonPath["ultimate"] = info->ultimate; - - if (!info->sigs.empty()) { - for (auto & sig : info->sigs) - jsonPath["signatures"].push_back(sig); - } - - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - - if (narInfo) { - if (!narInfo->url.empty()) - jsonPath["url"] = narInfo->url; - if (narInfo->fileHash) - jsonPath["downloadHash"] = narInfo->fileHash->to_string(hashFormat, true); - if (narInfo->fileSize) - jsonPath["downloadSize"] = narInfo->fileSize; - if (showClosureSize) - jsonPath["closureDownloadSize"] = closureSizes.second; - } - } - - } catch (InvalidPath &) { - jsonPath["path"] = printStorePath(storePath); - jsonPath["valid"] = false; - } - } - return jsonList; -} - - -std::pair Store::getClosureSize(const StorePath & storePath) -{ - uint64_t totalNarSize = 0, totalDownloadSize = 0; - StorePathSet closure; - computeFSClosure(storePath, closure, false, false); - for (auto & p : closure) { - auto info = queryPathInfo(p); - totalNarSize += info->narSize; - auto narInfo = std::dynamic_pointer_cast( - std::shared_ptr(info)); - if (narInfo) - totalDownloadSize += narInfo->fileSize; - } - return {totalNarSize, totalDownloadSize}; -} - const Store::Stats & Store::getStats() { @@ -1291,7 +1205,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre if (!hashGiven) { std::string s; getline(str, s); - auto narHash = Hash::parseAny(s, htSHA256); + auto narHash = Hash::parseAny(s, HashAlgorithm::SHA256); getline(str, s); auto narSize = string2Int(s); if (!narSize) throw Error("number expected"); @@ -1315,7 +1229,7 @@ std::optional decodeValidPathInfo(const Store & store, std::istre } -std::string Store::showPaths(const StorePathSet & paths) +std::string StoreDirConfig::showPaths(const StorePathSet & paths) { std::string s; for (auto & i : paths) { @@ -1338,12 +1252,12 @@ Derivation Store::derivationFromPath(const StorePath & drvPath) return readDerivation(drvPath); } -Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath) +static Derivation readDerivationCommon(Store & store, const StorePath & drvPath, bool requireValidPath) { - auto accessor = store.getFSAccessor(); + auto accessor = store.getFSAccessor(requireValidPath); try { return parseDerivation(store, - accessor->readFile(store.printStorePath(drvPath), requireValidPath), + accessor->readFile(CanonPath(store.printStorePath(drvPath))), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index e123fccc5..13e5a1446 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -1,8 +1,6 @@ #pragma once ///@file -#include "nar-info.hh" -#include "realisation.hh" #include "path.hh" #include "derived-path.hh" #include "hash.hh" @@ -14,6 +12,7 @@ #include "config.hh" #include "path-info.hh" #include "repair-flag.hh" +#include "store-dir-config.hh" #include #include @@ -30,7 +29,7 @@ namespace nix { /** - * About the class hierarchy of the store implementations: + * About the class hierarchy of the store types: * * Each store type `Foo` consists of two classes: * @@ -64,13 +63,17 @@ MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); -MakeError(BadStorePath, Error); MakeError(InvalidStoreURI, Error); +struct Realisation; +struct RealisedPath; +struct DrvOutput; + struct BasicDerivation; struct Derivation; -class FSAccessor; + +struct SourceAccessor; class NarInfoDiskCache; class Store; @@ -80,7 +83,6 @@ typedef std::map OutputPathMap; enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; -enum AllowInvalidFlag : bool { DisallowInvalid = false, AllowInvalid = true }; /** * Magic header of exportPath() output (obsolete). @@ -97,11 +99,11 @@ struct KeyedBuildResult; typedef std::map> StorePathCAMap; -struct StoreConfig : public Config +struct StoreConfig : public StoreDirConfig { typedef std::map Params; - using Config::Config; + using StoreDirConfig::StoreDirConfig; StoreConfig() = delete; @@ -131,15 +133,6 @@ struct StoreConfig : public Config return std::nullopt; } - const PathSetting storeDir_{this, settings.nixStore, - "store", - R"( - Logical location of the Nix store, usually - `/nix/store`. Note that you can only copy store paths - between stores if they have the same `store` setting. - )"}; - const Path storeDir = storeDir_; - const Setting pathInfoCacheSize{this, 65536, "path-info-cache-size", "Size of the in-memory store path metadata cache."}; @@ -168,7 +161,10 @@ struct StoreConfig : public Config Optional [system features](@docroot@/command-ref/conf-file.md#conf-system-features) available on the system this store uses to build derivations. Example: `"kvm"` - )" }; + )", + {}, + // Don't document the machine-specific default value + false}; }; class Store : public std::enable_shared_from_this, public virtual StoreConfig @@ -224,45 +220,6 @@ public: virtual std::string getUri() = 0; - StorePath parseStorePath(std::string_view path) const; - - std::optional maybeParseStorePath(std::string_view path) const; - - std::string printStorePath(const StorePath & path) const; - - /** - * Deprecated - * - * \todo remove - */ - StorePathSet parseStorePathSet(const PathSet & paths) const; - - PathSet printStorePathSet(const StorePathSet & path) const; - - /** - * Display a set of paths in human-readable form (i.e., between quotes - * and separated by commas). - */ - std::string showPaths(const StorePathSet & paths); - - /** - * @return true if ‘path’ is in the Nix store (but not the Nix - * store itself). - */ - bool isInStore(PathView path) const; - - /** - * @return true if ‘path’ is a store path, i.e. a direct child of the - * Nix store. - */ - bool isStorePath(std::string_view path) const; - - /** - * Split a path like /nix/store/-/ into - * /nix/store/- and /. - */ - std::pair toStorePath(PathView path) const; - /** * Follow symlinks until we end up with a path in the Nix store. */ @@ -274,55 +231,6 @@ public: */ StorePath followLinksToStorePath(std::string_view path) const; - /** - * Constructs a unique store path name. - */ - StorePath makeStorePath(std::string_view type, - std::string_view hash, std::string_view name) const; - StorePath makeStorePath(std::string_view type, - const Hash & hash, std::string_view name) const; - - StorePath makeOutputPath(std::string_view id, - const Hash & hash, std::string_view name) const; - - StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; - - StorePath makeTextPath(std::string_view name, const TextInfo & info) const; - - StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; - - /** - * Read-only variant of addToStoreFromDump(). It returns the store - * path to which a NAR or flat file would be written. - */ - std::pair computeStorePathFromDump( - Source & dump, - std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, - const StorePathSet & references = {}) const; - - /** - * Preparatory part of addTextToStore(). - * - * !!! Computation of the path should take the references given to - * addTextToStore() into account, otherwise we have a (relatively - * minor) security hole: a caller can register a source file with - * bogus references. If there are too many references, the path may - * not be garbage collected when it has to be (not really a problem, - * the caller could create a root anyway), or it may be garbage - * collected when it shouldn't be (more serious). - * - * Hashing the references would solve this (bogus references would - * simply yield a different store path, so other users wouldn't be - * affected), but it has some backwards compatibility issues (the - * hashing scheme changes), so I'm not doing that for now. - */ - StorePath computeStorePathForText( - std::string_view name, - std::string_view s, - const StorePathSet & references) const; - /** * Check whether a path is valid. */ @@ -461,7 +369,7 @@ public: * Query the mapping outputName=>outputPath for the given derivation. * Assume every output has a mapping and throw an exception otherwise. */ - OutputPathMap queryDerivationOutputMap(const StorePath & path); + OutputPathMap queryDerivationOutputMap(const StorePath & path, Store * evalStore = nullptr); /** * Query the full store path given the hash part of a valid store @@ -519,13 +427,13 @@ public: * libutil/archive.hh). */ virtual StorePath addToStore( - std::string_view name, - const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, - HashType hashAlgo = htSHA256, - PathFilter & filter = defaultPathFilter, - RepairFlag repair = NoRepair, - const StorePathSet & references = StorePathSet()); + std::string_view name, + const Path & srcPath, + FileIngestionMethod method = FileIngestionMethod::Recursive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + PathFilter & filter = defaultPathFilter, + RepairFlag repair = NoRepair, + const StorePathSet & references = StorePathSet()); /** * Copy the contents of a path to the store and register the @@ -533,8 +441,8 @@ public: * memory. */ ValidPathInfo addToStoreSlow(std::string_view name, const Path & srcPath, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, - std::optional expectedCAHash = {}); + FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + std::optional expectedCAHash = {}); /** * Like addToStore(), but the contents of the path are contained @@ -546,8 +454,8 @@ public: * \todo remove? */ virtual StorePath addToStoreFromDump(Source & dump, std::string_view name, - FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair, - const StorePathSet & references = StorePathSet()) + FileIngestionMethod method = FileIngestionMethod::Recursive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, RepairFlag repair = NoRepair, + const StorePathSet & references = StorePathSet()) { unsupported("addToStoreFromDump"); } /** @@ -665,28 +573,6 @@ public: std::string makeValidityRegistration(const StorePathSet & paths, bool showDerivers, bool showHash); - /** - * Write a JSON representation of store path metadata, such as the - * hash and the references. - * - * @param includeImpureInfo If true, variable elements such as the - * registration time are included. - * - * @param showClosureSize If true, the closure size of each path is - * included. - */ - nlohmann::json pathInfoToJSON(const StorePathSet & storePaths, - bool includeImpureInfo, bool showClosureSize, - HashFormat hashFormat = HashFormat::Base32, - AllowInvalidFlag allowInvalid = DisallowInvalid); - - /** - * @return the size of the closure of the specified path, that is, - * the sum of the size of the NAR serialisation of each path in the - * closure. - */ - std::pair getClosureSize(const StorePath & storePath); - /** * Optimise the disk space usage of the Nix store by hard-linking files * with the same contents. @@ -703,7 +589,7 @@ public: /** * @return An object to access files in the Nix store. */ - virtual ref getFSAccessor() = 0; + virtual ref getFSAccessor(bool requireValidPath = true) = 0; /** * Repair the contents of the given path by redownloading it using @@ -908,7 +794,7 @@ void copyStorePath( */ std::map copyPaths( Store & srcStore, Store & dstStore, - const RealisedPath::Set &, + const std::set &, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); @@ -925,7 +811,7 @@ std::map copyPaths( */ void copyClosure( Store & srcStore, Store & dstStore, - const RealisedPath::Set & paths, + const std::set & paths, RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs, SubstituteFlag substitute = NoSubstitute); @@ -982,7 +868,7 @@ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * ev * - ‘ssh://[user@]’: A remote Nix store accessed by running * ‘nix-store --serve’ via SSH. * - * You can pass parameters to the store implementation by appending + * You can pass parameters to the store type by appending * ‘?key=value&key=value&...’ to the URI. */ ref openStore(const std::string & uri = settings.storeUri.get(), diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh new file mode 100644 index 000000000..8dafca096 --- /dev/null +++ b/src/libstore/store-dir-config.hh @@ -0,0 +1,126 @@ +#pragma once + +#include "path.hh" +#include "hash.hh" +#include "content-address.hh" +#include "globals.hh" +#include "config.hh" + +#include +#include +#include + + +namespace nix { + +MakeError(BadStorePath, Error); + +struct StoreDirConfig : public Config +{ + using Config::Config; + + StoreDirConfig() = delete; + + virtual ~StoreDirConfig() = default; + + const PathSetting storeDir_{this, settings.nixStore, + "store", + R"( + Logical location of the Nix store, usually + `/nix/store`. Note that you can only copy store paths + between stores if they have the same `store` setting. + )"}; + const Path storeDir = storeDir_; + + // pure methods + + StorePath parseStorePath(std::string_view path) const; + + std::optional maybeParseStorePath(std::string_view path) const; + + std::string printStorePath(const StorePath & path) const; + + /** + * Deprecated + * + * \todo remove + */ + StorePathSet parseStorePathSet(const PathSet & paths) const; + + PathSet printStorePathSet(const StorePathSet & path) const; + + /** + * Display a set of paths in human-readable form (i.e., between quotes + * and separated by commas). + */ + std::string showPaths(const StorePathSet & paths); + + /** + * @return true if ‘path’ is in the Nix store (but not the Nix + * store itself). + */ + bool isInStore(PathView path) const; + + /** + * @return true if ‘path’ is a store path, i.e. a direct child of the + * Nix store. + */ + bool isStorePath(std::string_view path) const; + + /** + * Split a path like /nix/store/-/ into + * /nix/store/- and /. + */ + std::pair toStorePath(PathView path) const; + + /** + * Constructs a unique store path name. + */ + StorePath makeStorePath(std::string_view type, + std::string_view hash, std::string_view name) const; + StorePath makeStorePath(std::string_view type, + const Hash & hash, std::string_view name) const; + + StorePath makeOutputPath(std::string_view id, + const Hash & hash, std::string_view name) const; + + StorePath makeFixedOutputPath(std::string_view name, const FixedOutputInfo & info) const; + + StorePath makeTextPath(std::string_view name, const TextInfo & info) const; + + StorePath makeFixedOutputPathFromCA(std::string_view name, const ContentAddressWithReferences & ca) const; + + /** + * Read-only variant of addToStoreFromDump(). It returns the store + * path to which a NAR or flat file would be written. + */ + std::pair computeStorePathFromDump( + Source & dump, + std::string_view name, + FileIngestionMethod method = FileIngestionMethod::Recursive, + HashAlgorithm hashAlgo = HashAlgorithm::SHA256, + const StorePathSet & references = {}) const; + + /** + * Preparatory part of addTextToStore(). + * + * !!! Computation of the path should take the references given to + * addTextToStore() into account, otherwise we have a (relatively + * minor) security hole: a caller can register a source file with + * bogus references. If there are too many references, the path may + * not be garbage collected when it has to be (not really a problem, + * the caller could create a root anyway), or it may be garbage + * collected when it shouldn't be (more serious). + * + * Hashing the references would solve this (bogus references would + * simply yield a different store path, so other users wouldn't be + * affected), but it has some backwards compatibility issues (the + * hashing scheme changes), so I'm not doing that for now. + */ + StorePath computeStorePathForText( + std::string_view name, + std::string_view s, + const StorePathSet & references) const; +}; + +} diff --git a/src/libstore/tests/characterization.hh b/src/libstore/tests/characterization.hh deleted file mode 100644 index 5f366cb42..000000000 --- a/src/libstore/tests/characterization.hh +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once -///@file - -namespace nix { - -/** - * The path to the `unit-test-data` directory. See the contributing - * guide in the manual for further details. - */ -static Path getUnitTestData() { - return getEnv("_NIX_TEST_UNIT_DATA").value(); -} - -/** - * Whether we should update "golden masters" instead of running tests - * against them. See the contributing guide in the manual for further - * details. - */ -static bool testAccept() { - return getEnv("_NIX_TEST_ACCEPT") == "1"; -} - -} diff --git a/src/libstore/tests/derivation.cc b/src/libstore/tests/derivation.cc deleted file mode 100644 index c360c9707..000000000 --- a/src/libstore/tests/derivation.cc +++ /dev/null @@ -1,369 +0,0 @@ -#include -#include - -#include "experimental-features.hh" -#include "derivations.hh" - -#include "tests/libstore.hh" - -namespace nix { - -class DerivationTest : public LibStoreTest -{ -public: - /** - * We set these in tests rather than the regular globals so we don't have - * to worry about race conditions if the tests run concurrently. - */ - ExperimentalFeatureSettings mockXpSettings; -}; - -class CaDerivationTest : public DerivationTest -{ - void SetUp() override - { - mockXpSettings.set("experimental-features", "ca-derivations"); - } -}; - -class DynDerivationTest : public DerivationTest -{ - void SetUp() override - { - mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); - } -}; - -class ImpureDerivationTest : public DerivationTest -{ - void SetUp() override - { - mockXpSettings.set("experimental-features", "impure-derivations"); - } -}; - -TEST_F(DerivationTest, BadATerm_version) { - ASSERT_THROW( - parseDerivation( - *store, - R"(DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", - "whatever", - mockXpSettings), - FormatError); -} - -TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { - ASSERT_THROW( - parseDerivation( - *store, - R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", - "dyn-dep-derivation", - mockXpSettings), - FormatError); -} - -#define TEST_JSON(FIXTURE, NAME, STR, VAL, DRV_NAME, OUTPUT_NAME) \ - TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (DerivationOutput { VAL }).toJSON( \ - *store, \ - DRV_NAME, \ - OUTPUT_NAME)); \ - } \ - \ - TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - DerivationOutput { VAL }, \ - DerivationOutput::fromJSON( \ - *store, \ - DRV_NAME, \ - OUTPUT_NAME, \ - STR ## _json, \ - mockXpSettings)); \ - } - -TEST_JSON(DerivationTest, inputAddressed, - R"({ - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" - })", - (DerivationOutput::InputAddressed { - .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"), - }), - "drv-name", "output-name") - -TEST_JSON(DerivationTest, caFixedFlat, - R"({ - "hashAlgo": "sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" - })", - (DerivationOutput::CAFixed { - .ca = { - .method = FileIngestionMethod::Flat, - .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), - }, - }), - "drv-name", "output-name") - -TEST_JSON(DerivationTest, caFixedNAR, - R"({ - "hashAlgo": "r:sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" - })", - (DerivationOutput::CAFixed { - .ca = { - .method = FileIngestionMethod::Recursive, - .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), - }, - }), - "drv-name", "output-name") - -TEST_JSON(DynDerivationTest, caFixedText, - R"({ - "hashAlgo": "text:sha256", - "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" - })", - (DerivationOutput::CAFixed { - .ca = { - .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), - }, - }), - "drv-name", "output-name") - -TEST_JSON(CaDerivationTest, caFloating, - R"({ - "hashAlgo": "r:sha256" - })", - (DerivationOutput::CAFloating { - .method = FileIngestionMethod::Recursive, - .hashType = htSHA256, - }), - "drv-name", "output-name") - -TEST_JSON(DerivationTest, deferred, - R"({ })", - DerivationOutput::Deferred { }, - "drv-name", "output-name") - -TEST_JSON(ImpureDerivationTest, impure, - R"({ - "hashAlgo": "r:sha256", - "impure": true - })", - (DerivationOutput::Impure { - .method = FileIngestionMethod::Recursive, - .hashType = htSHA256, - }), - "drv-name", "output-name") - -#undef TEST_JSON - -#define TEST_JSON(FIXTURE, NAME, STR, VAL) \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - STR ## _json, \ - (VAL).toJSON(*store)); \ - } \ - \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \ - using nlohmann::literals::operator "" _json; \ - ASSERT_EQ( \ - (VAL), \ - Derivation::fromJSON( \ - *store, \ - STR ## _json, \ - mockXpSettings)); \ - } - -#define TEST_ATERM(FIXTURE, NAME, STR, VAL, DRV_NAME) \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \ - ASSERT_EQ( \ - STR, \ - (VAL).unparse(*store, false)); \ - } \ - \ - TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \ - auto parsed = parseDerivation( \ - *store, \ - STR, \ - DRV_NAME, \ - mockXpSettings); \ - ASSERT_EQ( \ - (VAL).toJSON(*store), \ - parsed.toJSON(*store)); \ - ASSERT_EQ( \ - (VAL), \ - parsed); \ - } - -Derivation makeSimpleDrv(const Store & store) { - Derivation drv; - drv.name = "simple-derivation"; - drv.inputSrcs = { - store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), - }; - drv.inputDrvs = { - .map = { - { - store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), - { - .value = { - "cat", - "dog", - }, - }, - }, - }, - }; - drv.platform = "wasm-sel4"; - drv.builder = "foo"; - drv.args = { - "bar", - "baz", - }; - drv.env = { - { - "BIG_BAD", - "WOLF", - }, - }; - return drv; -} - -TEST_JSON(DerivationTest, simple, - R"({ - "name": "simple-derivation", - "inputSrcs": [ - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" - ], - "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { - "dynamicOutputs": {}, - "outputs": [ - "cat", - "dog" - ] - } - }, - "system": "wasm-sel4", - "builder": "foo", - "args": [ - "bar", - "baz" - ], - "env": { - "BIG_BAD": "WOLF" - }, - "outputs": {} - })", - makeSimpleDrv(*store)) - -TEST_ATERM(DerivationTest, simple, - R"(Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", - makeSimpleDrv(*store), - "simple-derivation") - -Derivation makeDynDepDerivation(const Store & store) { - Derivation drv; - drv.name = "dyn-dep-derivation"; - drv.inputSrcs = { - store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), - }; - drv.inputDrvs = { - .map = { - { - store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), - DerivedPathMap::ChildNode { - .value = { - "cat", - "dog", - }, - .childMap = { - { - "cat", - DerivedPathMap::ChildNode { - .value = { - "kitten", - }, - }, - }, - { - "goose", - DerivedPathMap::ChildNode { - .value = { - "gosling", - }, - }, - }, - }, - }, - }, - }, - }; - drv.platform = "wasm-sel4"; - drv.builder = "foo"; - drv.args = { - "bar", - "baz", - }; - drv.env = { - { - "BIG_BAD", - "WOLF", - }, - }; - return drv; -} - -TEST_JSON(DynDerivationTest, dynDerivationDeps, - R"({ - "name": "dyn-dep-derivation", - "inputSrcs": [ - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" - ], - "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { - "dynamicOutputs": { - "cat": { - "dynamicOutputs": {}, - "outputs": ["kitten"] - }, - "goose": { - "dynamicOutputs": {}, - "outputs": ["gosling"] - } - }, - "outputs": [ - "cat", - "dog" - ] - } - }, - "system": "wasm-sel4", - "builder": "foo", - "args": [ - "bar", - "baz" - ], - "env": { - "BIG_BAD": "WOLF" - }, - "outputs": {} - })", - makeDynDepDerivation(*store)) - -TEST_ATERM(DynDerivationTest, dynDerivationDeps, - R"(DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]))", - makeDynDepDerivation(*store), - "dyn-dep-derivation") - -#undef TEST_JSON -#undef TEST_ATERM - -} diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk deleted file mode 100644 index 03becc7d1..000000000 --- a/src/libstore/tests/local.mk +++ /dev/null @@ -1,29 +0,0 @@ -check: libstore-tests-exe_RUN - -programs += libstore-tests-exe - -libstore-tests-exe_NAME = libnixstore-tests - -libstore-tests-exe_DIR := $(d) - -libstore-tests-exe_INSTALL_DIR := - -libstore-tests-exe_LIBS = libstore-tests - -libstore-tests-exe_LDFLAGS := $(GTEST_LIBS) - -libraries += libstore-tests - -libstore-tests_NAME = libnixstore-tests - -libstore-tests_DIR := $(d) - -libstore-tests_INSTALL_DIR := - -libstore-tests_SOURCES := $(wildcard $(d)/*.cc) - -libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil - -libstore-tests_LIBS = libutil-tests libstore libutil - -libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libstore/tests/protocol.hh b/src/libstore/tests/protocol.hh deleted file mode 100644 index 496915745..000000000 --- a/src/libstore/tests/protocol.hh +++ /dev/null @@ -1,91 +0,0 @@ -#include -#include - -#include "tests/libstore.hh" -#include "tests/characterization.hh" - -namespace nix { - -template -class ProtoTest : public LibStoreTest -{ -protected: - Path unitTestData = getUnitTestData() + "/libstore/" + protocolDir; - - Path goldenMaster(std::string_view testStem) { - return unitTestData + "/" + testStem + ".bin"; - } -}; - -template -class VersionedProtoTest : public ProtoTest -{ -public: - /** - * Golden test for `T` reading - */ - template - void readTest(PathView testStem, typename Proto::Version version, T value) - { - if (testAccept()) - { - GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; - } - else - { - auto expected = readFile(ProtoTest::goldenMaster(testStem)); - - T got = ({ - StringSource from { expected }; - Proto::template Serialise::read( - *LibStoreTest::store, - typename Proto::ReadConn { - .from = from, - .version = version, - }); - }); - - ASSERT_EQ(got, value); - } - } - - /** - * Golden test for `T` write - */ - template - void writeTest(PathView testStem, typename Proto::Version version, const T & value) - { - auto file = ProtoTest::goldenMaster(testStem); - - StringSink to; - Proto::write( - *LibStoreTest::store, - typename Proto::WriteConn { - .to = to, - .version = version, - }, - value); - - if (testAccept()) - { - createDirs(dirOf(file)); - writeFile(file, to.s); - GTEST_SKIP() << "Updating golden master"; - } - else - { - auto expected = readFile(file); - ASSERT_EQ(to.s, expected); - } - } -}; - -#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ - TEST_F(FIXTURE, NAME ## _read) { \ - readTest(STEM, VERSION, VALUE); \ - } \ - TEST_F(FIXTURE, NAME ## _write) { \ - writeTest(STEM, VERSION, VALUE); \ - } - -} diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 99589f8b2..226cdf717 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -1,4 +1,5 @@ #include "uds-remote-store.hh" +#include "unix-domain-socket.hh" #include "worker-protocol.hh" #include diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index cdb28a001..a5ac9080a 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -35,8 +35,8 @@ public: static std::set uriSchemes() { return {"unix"}; } - ref getFSAccessor() override - { return LocalFSStore::getFSAccessor(); } + ref getFSAccessor(bool requireValidPath) override + { return LocalFSStore::getFSAccessor(requireValidPath); } void narFromPath(const StorePath & path, Sink & sink) override { LocalFSStore::narFromPath(path, sink); } diff --git a/src/libstore/worker-protocol-impl.hh b/src/libstore/worker-protocol-impl.hh index c043588d6..026cc37bc 100644 --- a/src/libstore/worker-protocol-impl.hh +++ b/src/libstore/worker-protocol-impl.hh @@ -16,11 +16,11 @@ namespace nix { /* protocol-agnostic templates */ #define WORKER_USE_LENGTH_PREFIX_SERIALISER(TEMPLATE, T) \ - TEMPLATE T WorkerProto::Serialise< T >::read(const Store & store, WorkerProto::ReadConn conn) \ + TEMPLATE T WorkerProto::Serialise< T >::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) \ { \ return LengthPrefixedProtoHelper::read(store, conn); \ } \ - TEMPLATE void WorkerProto::Serialise< T >::write(const Store & store, WorkerProto::WriteConn conn, const T & t) \ + TEMPLATE void WorkerProto::Serialise< T >::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t) \ { \ LengthPrefixedProtoHelper::write(store, conn, t); \ } @@ -41,12 +41,12 @@ WORKER_USE_LENGTH_PREFIX_SERIALISER( template struct WorkerProto::Serialise { - static T read(const Store & store, WorkerProto::ReadConn conn) + static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { return CommonProto::Serialise::read(store, CommonProto::ReadConn { .from = conn.from }); } - static void write(const Store & store, WorkerProto::WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t) { CommonProto::Serialise::write(store, CommonProto::WriteConn { .to = conn.to }, diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index d618b9bd8..2a379e75e 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -1,5 +1,4 @@ #include "serialise.hh" -#include "util.hh" #include "path-with-outputs.hh" #include "store-api.hh" #include "build-result.hh" @@ -14,7 +13,7 @@ namespace nix { /* protocol-specific definitions */ -std::optional WorkerProto::Serialise>::read(const Store & store, WorkerProto::ReadConn conn) +std::optional WorkerProto::Serialise>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { auto temp = readNum(conn.from); switch (temp) { @@ -29,17 +28,17 @@ std::optional WorkerProto::Serialise>::r } } -void WorkerProto::Serialise>::write(const Store & store, WorkerProto::WriteConn conn, const std::optional & optTrusted) +void WorkerProto::Serialise>::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const std::optional & optTrusted) { if (!optTrusted) - conn.to << (uint8_t)0; + conn.to << uint8_t{0}; else { switch (*optTrusted) { case Trusted: - conn.to << (uint8_t)1; + conn.to << uint8_t{1}; break; case NotTrusted: - conn.to << (uint8_t)2; + conn.to << uint8_t{2}; break; default: assert(false); @@ -48,7 +47,7 @@ void WorkerProto::Serialise>::write(const Store & sto } -DerivedPath WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) +DerivedPath WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { auto s = readString(conn.from); if (GET_PROTOCOL_MINOR(conn.version) >= 30) { @@ -58,7 +57,7 @@ DerivedPath WorkerProto::Serialise::read(const Store & store, Worke } } -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const DerivedPath & req) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const DerivedPath & req) { if (GET_PROTOCOL_MINOR(conn.version) >= 30) { conn.to << req.to_string_legacy(store); @@ -82,7 +81,7 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } -KeyedBuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) +KeyedBuildResult WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { auto path = WorkerProto::Serialise::read(store, conn); auto br = WorkerProto::Serialise::read(store, conn); @@ -92,17 +91,17 @@ KeyedBuildResult WorkerProto::Serialise::read(const Store & st }; } -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const KeyedBuildResult & res) { WorkerProto::write(store, conn, res.path); WorkerProto::write(store, conn, static_cast(res)); } -BuildResult WorkerProto::Serialise::read(const Store & store, WorkerProto::ReadConn conn) +BuildResult WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { BuildResult res; - res.status = (BuildResult::Status) readInt(conn.from); + res.status = static_cast(readInt(conn.from)); conn.from >> res.errorMsg; if (GET_PROTOCOL_MINOR(conn.version) >= 29) { conn.from @@ -121,7 +120,7 @@ BuildResult WorkerProto::Serialise::read(const Store & store, Worke return res; } -void WorkerProto::Serialise::write(const Store & store, WorkerProto::WriteConn conn, const BuildResult & res) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const BuildResult & res) { conn.to << res.status @@ -142,7 +141,7 @@ void WorkerProto::Serialise::write(const Store & store, WorkerProto } -ValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) +ValidPathInfo WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) { auto path = WorkerProto::Serialise::read(store, conn); return ValidPathInfo { @@ -151,17 +150,17 @@ ValidPathInfo WorkerProto::Serialise::read(const Store & store, R }; } -void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const ValidPathInfo & pathInfo) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const ValidPathInfo & pathInfo) { WorkerProto::write(store, conn, pathInfo.path); WorkerProto::write(store, conn, static_cast(pathInfo)); } -UnkeyedValidPathInfo WorkerProto::Serialise::read(const Store & store, ReadConn conn) +UnkeyedValidPathInfo WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) { auto deriver = readString(conn.from); - auto narHash = Hash::parseAny(readString(conn.from), htSHA256); + auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256); UnkeyedValidPathInfo info(narHash); if (deriver != "") info.deriver = store.parseStorePath(deriver); info.references = WorkerProto::Serialise::read(store, conn); @@ -174,7 +173,7 @@ UnkeyedValidPathInfo WorkerProto::Serialise::read(const St return info; } -void WorkerProto::Serialise::write(const Store & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo) +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo) { conn.to << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "") diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index dcd54ad16..c26914289 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -9,7 +9,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 35) +#define PROTOCOL_VERSION (1 << 8 | 36) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -24,7 +24,7 @@ namespace nix { #define STDERR_RESULT 0x52534c54 -class Store; +struct StoreDirConfig; struct Source; // items being serialised @@ -100,8 +100,8 @@ struct WorkerProto // This makes for a quicker debug cycle, as desired. #if 0 { - static T read(const Store & store, ReadConn conn); - static void write(const Store & store, WriteConn conn, const T & t); + static T read(const StoreDirConfig & store, ReadConn conn); + static void write(const StoreDirConfig & store, WriteConn conn, const T & t); }; #endif @@ -110,7 +110,7 @@ struct WorkerProto * infer the type instead of having to write it down explicitly. */ template - static void write(const Store & store, WriteConn conn, const T & t) + static void write(const StoreDirConfig & store, WriteConn conn, const T & t) { WorkerProto::Serialise::write(store, conn, t); } @@ -161,6 +161,7 @@ enum struct WorkerProto::Op : uint64_t AddMultipleToStore = 44, AddBuildLog = 45, BuildPathsWithResults = 46, + AddPermRoot = 47, }; /** @@ -171,7 +172,7 @@ enum struct WorkerProto::Op : uint64_t */ inline Sink & operator << (Sink & sink, WorkerProto::Op op) { - return sink << (uint64_t) op; + return sink << static_cast(op); } /** @@ -181,7 +182,7 @@ inline Sink & operator << (Sink & sink, WorkerProto::Op op) */ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) { - return s << (uint64_t) op; + return s << static_cast(op); } /** @@ -197,8 +198,8 @@ inline std::ostream & operator << (std::ostream & s, WorkerProto::Op op) #define DECLARE_WORKER_SERIALISER(T) \ struct WorkerProto::Serialise< T > \ { \ - static T read(const Store & store, WorkerProto::ReadConn conn); \ - static void write(const Store & store, WorkerProto::WriteConn conn, const T & t); \ + static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn); \ + static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t); \ }; template<> diff --git a/src/libutil/abstract-setting-to-json.hh b/src/libutil/abstract-setting-to-json.hh index d506dfb74..eea687d8a 100644 --- a/src/libutil/abstract-setting-to-json.hh +++ b/src/libutil/abstract-setting-to-json.hh @@ -7,7 +7,7 @@ namespace nix { template -std::map BaseSetting::toJSONObject() +std::map BaseSetting::toJSONObject() const { auto obj = AbstractSetting::toJSONObject(); obj.emplace("value", value); diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 3b1a1e0ef..465df2073 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -5,16 +5,11 @@ #include // for strcasecmp -#include -#include -#include -#include -#include - #include "archive.hh" -#include "util.hh" #include "config.hh" #include "posix-source-accessor.hh" +#include "file-system.hh" +#include "signals.hh" namespace nix { @@ -299,7 +294,7 @@ void copyNAR(Source & source, Sink & sink) // FIXME: if 'source' is the output of dumpPath() followed by EOF, // we should just forward all data directly without parsing. - ParseSink parseSink; /* null sink; just parse the NAR */ + NullParseSink parseSink; /* just parse the NAR */ TeeSource wrapper { source, sink }; diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 3530783c1..2cf8ee891 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -73,33 +73,6 @@ time_t dumpPathAndGetMtime(const Path & path, Sink & sink, */ void dumpString(std::string_view s, Sink & sink); -/** - * If the NAR archive contains a single file at top-level, then save - * the contents of the file to `s`. Otherwise barf. - */ -struct RetrieveRegularNARSink : ParseSink -{ - bool regular = true; - Sink & sink; - - RetrieveRegularNARSink(Sink & sink) : sink(sink) { } - - void createDirectory(const Path & path) override - { - regular = false; - } - - void receiveContents(std::string_view data) override - { - sink(data); - } - - void createSymlink(const Path & path, const std::string & target) override - { - regular = false; - } -}; - void parseDump(ParseSink & sink, Source & source); void restorePath(const Path & path, Source & source); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 6bc3cae07..e2668c673 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -1,8 +1,14 @@ #include "args.hh" #include "args/root.hh" #include "hash.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "users.hh" #include "json-utils.hh" +#include +#include +#include #include namespace nix { @@ -74,7 +80,178 @@ std::optional RootArgs::needsCompletion(std::string_view s) return {}; } -void RootArgs::parseCmdline(const Strings & _cmdline) +/** + * Basically this is `typedef std::optional Parser(std::string_view s, Strings & r);` + * + * Except we can't recursively reference the Parser typedef, so we have to write a class. + */ +struct Parser { + std::string_view remaining; + + /** + * @brief Parse the next character(s) + * + * @param r + * @return std::shared_ptr + */ + virtual void operator()(std::shared_ptr & state, Strings & r) = 0; + + Parser(std::string_view s) : remaining(s) {}; + + virtual ~Parser() { }; +}; + +struct ParseQuoted : public Parser { + /** + * @brief Accumulated string + * + * Parsed argument up to this point. + */ + std::string acc; + + ParseQuoted(std::string_view s) : Parser(s) {}; + + virtual void operator()(std::shared_ptr & state, Strings & r) override; +}; + + +struct ParseUnquoted : public Parser { + /** + * @brief Accumulated string + * + * Parsed argument up to this point. Empty string is not representable in + * unquoted syntax, so we use it for the initial state. + */ + std::string acc; + + ParseUnquoted(std::string_view s) : Parser(s) {}; + + virtual void operator()(std::shared_ptr & state, Strings & r) override { + if (remaining.empty()) { + if (!acc.empty()) + r.push_back(acc); + state = nullptr; // done + return; + } + switch (remaining[0]) { + case ' ': case '\t': case '\n': case '\r': + if (!acc.empty()) + r.push_back(acc); + state = std::make_shared(ParseUnquoted(remaining.substr(1))); + return; + case '`': + if (remaining.size() > 1 && remaining[1] == '`') { + state = std::make_shared(ParseQuoted(remaining.substr(2))); + return; + } + else + throw Error("single backtick is not a supported syntax in the nix shebang."); + + // reserved characters + // meaning to be determined, or may be reserved indefinitely so that + // #!nix syntax looks unambiguous + case '$': + case '*': + case '~': + case '<': + case '>': + case '|': + case ';': + case '(': + case ')': + case '[': + case ']': + case '{': + case '}': + case '\'': + case '"': + case '\\': + throw Error("unsupported unquoted character in nix shebang: " + std::string(1, remaining[0]) + ". Use double backticks to escape?"); + + case '#': + if (acc.empty()) { + throw Error ("unquoted nix shebang argument cannot start with #. Use double backticks to escape?"); + } else { + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + + default: + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + assert(false); + } +}; + +void ParseQuoted::operator()(std::shared_ptr &state, Strings & r) { + if (remaining.empty()) { + throw Error("unterminated quoted string in nix shebang"); + } + switch (remaining[0]) { + case ' ': + if ((remaining.size() == 3 && remaining[1] == '`' && remaining[2] == '`') + || (remaining.size() > 3 && remaining[1] == '`' && remaining[2] == '`' && remaining[3] != '`')) { + // exactly two backticks mark the end of a quoted string, but a preceding space is ignored if present. + state = std::make_shared(ParseUnquoted(remaining.substr(3))); + r.push_back(acc); + return; + } + else { + // just a normal space + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + case '`': + // exactly two backticks mark the end of a quoted string + if ((remaining.size() == 2 && remaining[1] == '`') + || (remaining.size() > 2 && remaining[1] == '`' && remaining[2] != '`')) { + state = std::make_shared(ParseUnquoted(remaining.substr(2))); + r.push_back(acc); + return; + } + + // a sequence of at least 3 backticks is one escape-backtick which is ignored, followed by any number of backticks, which are verbatim + else if (remaining.size() >= 3 && remaining[1] == '`' && remaining[2] == '`') { + // ignore "escape" backtick + remaining = remaining.substr(1); + // add the rest + while (remaining.size() > 0 && remaining[0] == '`') { + acc += '`'; + remaining = remaining.substr(1); + } + return; + } + else { + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + default: + acc += remaining[0]; + remaining = remaining.substr(1); + return; + } + assert(false); +} + +Strings parseShebangContent(std::string_view s) { + Strings result; + std::shared_ptr parserState(std::make_shared(ParseUnquoted(s))); + + // trampoline == iterated strategy pattern + while (parserState) { + auto currentState = parserState; + (*currentState)(parserState, result); + } + + return result; +} + +void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) { Strings pendingArgs; bool dashDash = false; @@ -90,6 +267,45 @@ void RootArgs::parseCmdline(const Strings & _cmdline) } bool argsSeen = false; + + // Heuristic to see if we're invoked as a shebang script, namely, + // if we have at least one argument, it's the name of an + // executable file, and it starts with "#!". + Strings savedArgs; + if (allowShebang){ + auto script = *cmdline.begin(); + try { + std::ifstream stream(script); + char shebang[3]={0,0,0}; + stream.get(shebang,3); + if (strncmp(shebang,"#!",2) == 0){ + for (auto pos = std::next(cmdline.begin()); pos != cmdline.end();pos++) + savedArgs.push_back(*pos); + cmdline.clear(); + + std::string line; + std::getline(stream,line); + static const std::string commentChars("#/\\%@*-"); + std::string shebangContent; + while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){ + line = chomp(line); + + std::smatch match; + // We match one space after `nix` so that we preserve indentation. + // No space is necessary for an empty line. An empty line has basically no effect. + if (std::regex_match(line, match, std::regex("^#!\\s*nix(:? |$)(.*)$"))) + shebangContent += match[2].str() + "\n"; + } + for (const auto & word : parseShebangContent(shebangContent)) { + cmdline.push_back(word); + } + cmdline.push_back(script); + commandBaseDir = dirOf(script); + for (auto pos = savedArgs.begin(); pos != savedArgs.end();pos++) + cmdline.push_back(*pos); + } + } catch (SysError &) { } + } for (auto pos = cmdline.begin(); pos != cmdline.end(); ) { auto arg = *pos; @@ -145,6 +361,17 @@ void RootArgs::parseCmdline(const Strings & _cmdline) d.completer(*completions, d.n, d.prefix); } +Path Args::getCommandBaseDir() const +{ + assert(parent); + return parent->getCommandBaseDir(); +} + +Path RootArgs::getCommandBaseDir() const +{ + return commandBaseDir; +} + bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) { assert(pos != end); @@ -255,7 +482,18 @@ bool Args::processArgs(const Strings & args, bool finish) } if (!anyCompleted) exp.handler.fun(ss); - expectedArgs.pop_front(); + + /* Move the list element to the processedArgs. This is almost the same as + `processedArgs.push_back(expectedArgs.front()); expectedArgs.pop_front()`, + except that it will only adjust the next and prev pointers of the list + elements, meaning the actual contents don't move in memory. This is + critical to prevent invalidating internal pointers! */ + processedArgs.splice( + processedArgs.end(), + expectedArgs, + expectedArgs.begin(), + ++expectedArgs.begin()); + res = true; } @@ -306,36 +544,70 @@ nlohmann::json Args::toJSON() return res; } -static void hashTypeCompleter(AddCompletions & completions, size_t index, std::string_view prefix) +static void hashFormatCompleter(AddCompletions & completions, size_t index, std::string_view prefix) { - for (auto & type : hashTypes) - if (hasPrefix(type, prefix)) - completions.add(type); + for (auto & format : hashFormats) { + if (hasPrefix(format, prefix)) { + completions.add(format); + } + } } -Args::Flag Args::Flag::mkHashTypeFlag(std::string && longName, HashType * ht) -{ - return Flag { - .longName = std::move(longName), - .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')", - .labels = {"hash-algo"}, - .handler = {[ht](std::string s) { - *ht = parseHashType(s); - }}, - .completer = hashTypeCompleter, +Args::Flag Args::Flag::mkHashFormatFlagWithDefault(std::string &&longName, HashFormat * hf) { + assert(*hf == nix::HashFormat::SRI); + return Flag{ + .longName = std::move(longName), + .description = "hash format ('base16', 'nix32', 'base64', 'sri'). Default: 'sri'", + .labels = {"hash-format"}, + .handler = {[hf](std::string s) { + *hf = parseHashFormat(s); + }}, + .completer = hashFormatCompleter, }; } -Args::Flag Args::Flag::mkHashTypeOptFlag(std::string && longName, std::optional * oht) +Args::Flag Args::Flag::mkHashFormatOptFlag(std::string && longName, std::optional * ohf) { + return Flag{ + .longName = std::move(longName), + .description = "hash format ('base16', 'nix32', 'base64', 'sri').", + .labels = {"hash-format"}, + .handler = {[ohf](std::string s) { + *ohf = std::optional{parseHashFormat(s)}; + }}, + .completer = hashFormatCompleter, + }; +} + +static void hashAlgoCompleter(AddCompletions & completions, size_t index, std::string_view prefix) { - return Flag { - .longName = std::move(longName), - .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.", - .labels = {"hash-algo"}, - .handler = {[oht](std::string s) { - *oht = std::optional { parseHashType(s) }; - }}, - .completer = hashTypeCompleter, + for (auto & algo : hashAlgorithms) + if (hasPrefix(algo, prefix)) + completions.add(algo); +} + +Args::Flag Args::Flag::mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha) +{ + return Flag{ + .longName = std::move(longName), + .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512')", + .labels = {"hash-algo"}, + .handler = {[ha](std::string s) { + *ha = parseHashAlgo(s); + }}, + .completer = hashAlgoCompleter, + }; +} + +Args::Flag Args::Flag::mkHashAlgoOptFlag(std::string && longName, std::optional * oha) +{ + return Flag{ + .longName = std::move(longName), + .description = "hash algorithm ('md5', 'sha1', 'sha256', or 'sha512'). Optional as can also be gotten from SRI hash itself.", + .labels = {"hash-algo"}, + .handler = {[oha](std::string s) { + *oha = std::optional{parseHashAlgo(s)}; + }}, + .completer = hashAlgoCompleter, }; } @@ -384,8 +656,9 @@ std::optional Command::experimentalFeature () return { Xp::NixCommand }; } -MultiCommand::MultiCommand(const Commands & commands_) +MultiCommand::MultiCommand(std::string_view commandName, const Commands & commands_) : commands(commands_) + , commandName(commandName) { expectArgs({ .label = "subcommand", diff --git a/src/libutil/args.hh b/src/libutil/args.hh index ff2bf3cab..18b0ae583 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -2,16 +2,20 @@ ///@file #include +#include #include #include +#include #include -#include "util.hh" +#include "types.hh" +#include "experimental-features.hh" namespace nix { -enum HashType : char; +enum struct HashAlgorithm : char; +enum struct HashFormat : int; class MultiCommand; @@ -21,11 +25,12 @@ class AddCompletions; class Args { + public: /** * Return a short one-line description of the command. - */ + */ virtual std::string description() { return ""; } virtual bool forceImpureByDefault() { return false; } @@ -35,6 +40,16 @@ public: */ virtual std::string doc() { return ""; } + /** + * @brief Get the base directory for the command. + * + * @return Generally the working directory, but in case of a shebang + * interpreter, returns the directory of the script. + * + * This only returns the correct value after parseCmdline() has run. + */ + virtual Path getCommandBaseDir() const; + protected: /** @@ -161,8 +176,10 @@ protected: std::optional experimentalFeature; - static Flag mkHashTypeFlag(std::string && longName, HashType * ht); - static Flag mkHashTypeOptFlag(std::string && longName, std::optional * oht); + static Flag mkHashAlgoFlag(std::string && longName, HashAlgorithm * ha); + static Flag mkHashAlgoOptFlag(std::string && longName, std::optional * oha); + static Flag mkHashFormatFlagWithDefault(std::string && longName, HashFormat * hf); + static Flag mkHashFormatOptFlag(std::string && longName, std::optional * ohf); }; /** @@ -200,13 +217,25 @@ protected: /** * Queue of expected positional argument forms. * - * Positional arugment descriptions are inserted on the back. + * Positional argument descriptions are inserted on the back. * * As positional arguments are passed, these are popped from the * front, until there are hopefully none left as all args that were * expected in fact were passed. */ std::list expectedArgs; + /** + * List of processed positional argument forms. + * + * All items removed from `expectedArgs` are added here. After all + * arguments were processed, this list should be exactly the same as + * `expectedArgs` was before. + * + * This list is used to extend the lifetime of the argument forms. + * If this is not done, some closures that reference the command + * itself will segfault. + */ + std::list processedArgs; /** * Process some positional arugments @@ -330,13 +359,16 @@ public: */ std::optional>> command; - MultiCommand(const Commands & commands); + MultiCommand(std::string_view commandName, const Commands & commands); bool processFlag(Strings::iterator & pos, Strings::iterator end) override; bool processArgs(const Strings & args, bool finish) override; nlohmann::json toJSON() override; + +protected: + std::string commandName = ""; }; Strings argvToStrings(int argc, char * * argv); @@ -383,4 +415,6 @@ public: virtual void add(std::string completion, std::string description = "") = 0; }; +Strings parseShebangContent(std::string_view s); + } diff --git a/src/libutil/args/root.hh b/src/libutil/args/root.hh index bb98732a1..5c55c37a5 100644 --- a/src/libutil/args/root.hh +++ b/src/libutil/args/root.hh @@ -29,14 +29,26 @@ struct Completions final : AddCompletions */ class RootArgs : virtual public Args { + /** + * @brief The command's "working directory", but only set when top level. + * + * Use getCommandBaseDir() to get the directory regardless of whether this + * is a top-level command or subcommand. + * + * @see getCommandBaseDir() + */ + Path commandBaseDir = "."; + public: /** Parse the command line, throwing a UsageError if something goes * wrong. */ - void parseCmdline(const Strings & cmdline); + void parseCmdline(const Strings & cmdline, bool allowShebang = false); std::shared_ptr completions; + Path getCommandBaseDir() const override; + protected: friend class Args; diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index 040464532..1e465f1f6 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -1,5 +1,5 @@ #include "canon-path.hh" -#include "util.hh" +#include "file-system.hh" namespace nix { @@ -13,6 +13,13 @@ CanonPath::CanonPath(std::string_view raw, const CanonPath & root) : path(absPath((Path) raw, root.abs())) { } +CanonPath::CanonPath(const std::vector & elems) + : path("/") +{ + for (auto & s : elems) + push(s); +} + CanonPath CanonPath::fromCwd(std::string_view path) { return CanonPath(unchecked_t(), absPath((Path) path)); diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index eefe05ed5..6aff4ec0d 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -6,6 +6,7 @@ #include #include #include +#include namespace nix { @@ -46,6 +47,11 @@ public: : path(std::move(path)) { } + /** + * Construct a canon path from a vector of elements. + */ + CanonPath(const std::vector & elems); + static CanonPath fromCwd(std::string_view path = "."); static CanonPath root; @@ -199,8 +205,19 @@ public: * `CanonPath(this.makeRelative(x), this) == path`. */ std::string makeRelative(const CanonPath & path) const; + + friend class std::hash; }; std::ostream & operator << (std::ostream & stream, const CanonPath & path); } + +template<> +struct std::hash +{ + std::size_t operator ()(const nix::CanonPath & s) const noexcept + { + return std::hash{}(s.path); + } +}; diff --git a/src/libutil/cgroup.cc b/src/libutil/cgroup.cc index a008481ca..4c2bf31ff 100644 --- a/src/libutil/cgroup.cc +++ b/src/libutil/cgroup.cc @@ -2,6 +2,7 @@ #include "cgroup.hh" #include "util.hh" +#include "file-system.hh" #include "finally.hh" #include diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index ba0847cde..d06f1f87b 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -1,6 +1,6 @@ #include "compression.hh" +#include "signals.hh" #include "tarfile.hh" -#include "util.hh" #include "finally.hh" #include "logging.hh" diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index b9639e761..9f69e8444 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -45,13 +45,13 @@ bool BaseSetting::isAppendable() return trait::appendable; } -template<> void BaseSetting::appendOrSet(Strings && newValue, bool append); -template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append); -template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append); -template<> void BaseSetting>::appendOrSet(std::set && newValue, bool append); +template<> void BaseSetting::appendOrSet(Strings newValue, bool append); +template<> void BaseSetting::appendOrSet(StringSet newValue, bool append); +template<> void BaseSetting::appendOrSet(StringMap newValue, bool append); +template<> void BaseSetting>::appendOrSet(std::set newValue, bool append); template -void BaseSetting::appendOrSet(T && newValue, bool append) +void BaseSetting::appendOrSet(T newValue, bool append) { static_assert( !trait::appendable, diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 8e06273ee..a3310f4ec 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -2,6 +2,8 @@ #include "args.hh" #include "abstract-setting-to-json.hh" #include "experimental-features.hh" +#include "util.hh" +#include "file-system.hh" #include "config-impl.hh" @@ -34,27 +36,25 @@ bool Config::set(const std::string & name, const std::string & value) void Config::addSetting(AbstractSetting * setting) { _settings.emplace(setting->name, Config::SettingData{false, setting}); - for (auto & alias : setting->aliases) + for (const auto & alias : setting->aliases) _settings.emplace(alias, Config::SettingData{true, setting}); bool set = false; - auto i = unknownSettings.find(setting->name); - if (i != unknownSettings.end()) { - setting->set(i->second); + if (auto i = unknownSettings.find(setting->name); i != unknownSettings.end()) { + setting->set(std::move(i->second)); setting->overridden = true; unknownSettings.erase(i); set = true; } for (auto & alias : setting->aliases) { - auto i = unknownSettings.find(alias); - if (i != unknownSettings.end()) { + if (auto i = unknownSettings.find(alias); i != unknownSettings.end()) { if (set) warn("setting '%s' is set, but it's an alias of '%s' which is also set", alias, setting->name); else { - setting->set(i->second); + setting->set(std::move(i->second)); setting->overridden = true; unknownSettings.erase(i); set = true; @@ -69,7 +69,7 @@ AbstractConfig::AbstractConfig(StringMap initials) void AbstractConfig::warnUnknownSettings() { - for (auto & s : unknownSettings) + for (const auto & s : unknownSettings) warn("unknown setting '%s'", s.first); } @@ -83,15 +83,14 @@ void AbstractConfig::reapplyUnknownSettings() void Config::getSettings(std::map & res, bool overriddenOnly) { - for (auto & opt : _settings) + for (const auto & opt : _settings) if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden)) res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description}); } -void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { - unsigned int pos = 0; - std::vector> parsedContents; +static void applyConfigInner(const std::string & contents, const std::string & path, std::vector> & parsedContents) { + unsigned int pos = 0; while (pos < contents.size()) { std::string line; @@ -99,8 +98,7 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string line += contents[pos++]; pos++; - auto hash = line.find('#'); - if (hash != std::string::npos) + if (auto hash = line.find('#'); hash != line.npos) line = std::string(line, 0, hash); auto tokens = tokenizeString>(line); @@ -123,7 +121,12 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); auto p = absPath(tokens[1], dirOf(path)); if (pathExists(p)) { - applyConfigFile(p); + try { + std::string includedContents = readFile(path); + applyConfigInner(includedContents, p, parsedContents); + } catch (SysError &) { + // TODO: Do we actually want to ignore this? Or is it better to fail? + } } else if (!ignoreMissing) { throw Error("file '%1%' included from '%2%' not found", p, path); } @@ -133,36 +136,34 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string if (tokens[1] != "=") throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); - std::string name = tokens[0]; + std::string name = std::move(tokens[0]); auto i = tokens.begin(); advance(i, 2); parsedContents.push_back({ - name, + std::move(name), concatStringsSep(" ", Strings(i, tokens.end())), }); }; +} + +void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { + std::vector> parsedContents; + + applyConfigInner(contents, path, parsedContents); // First apply experimental-feature related settings - for (auto & [name, value] : parsedContents) + for (const auto & [name, value] : parsedContents) if (name == "experimental-features" || name == "extra-experimental-features") set(name, value); // Then apply other settings - for (auto & [name, value] : parsedContents) + for (const auto & [name, value] : parsedContents) if (name != "experimental-features" && name != "extra-experimental-features") set(name, value); } -void AbstractConfig::applyConfigFile(const Path & path) -{ - try { - std::string contents = readFile(path); - applyConfig(contents, path); - } catch (SysError &) { } -} - void Config::resetOverridden() { for (auto & s : _settings) @@ -172,7 +173,7 @@ void Config::resetOverridden() nlohmann::json Config::toJSON() { auto res = nlohmann::json::object(); - for (auto & s : _settings) + for (const auto & s : _settings) if (!s.second.isAlias) res.emplace(s.first, s.second.setting->toJSON()); return res; @@ -180,8 +181,8 @@ nlohmann::json Config::toJSON() std::string Config::toKeyValue() { - auto res = std::string(); - for (auto & s : _settings) + std::string res; + for (const auto & s : _settings) if (s.second.isAlias) res += fmt("%s = %s\n", s.first, s.second.setting->to_string()); return res; @@ -203,7 +204,7 @@ AbstractSetting::AbstractSetting( : name(name) , description(stripIndentation(description)) , aliases(aliases) - , experimentalFeature(experimentalFeature) + , experimentalFeature(std::move(experimentalFeature)) { } @@ -219,7 +220,7 @@ nlohmann::json AbstractSetting::toJSON() return nlohmann::json(toJSONObject()); } -std::map AbstractSetting::toJSONObject() +std::map AbstractSetting::toJSONObject() const { std::map obj; obj.emplace("description", description); @@ -282,14 +283,14 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & .longName = name, .description = fmt("Enable the `%s` setting.", name), .category = category, - .handler = {[this]() { override(true); }}, + .handler = {[this] { override(true); }}, .experimentalFeature = experimentalFeature, }); args.addFlag({ .longName = "no-" + name, .description = fmt("Disable the `%s` setting.", name), .category = category, - .handler = {[this]() { override(false); }}, + .handler = {[this] { override(false); }}, .experimentalFeature = experimentalFeature, }); } @@ -299,10 +300,11 @@ template<> Strings BaseSetting::parse(const std::string & str) const return tokenizeString(str); } -template<> void BaseSetting::appendOrSet(Strings && newValue, bool append) +template<> void BaseSetting::appendOrSet(Strings newValue, bool append) { if (!append) value.clear(); - for (auto && s : std::move(newValue)) value.push_back(std::move(s)); + value.insert(value.end(), std::make_move_iterator(newValue.begin()), + std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const @@ -315,11 +317,10 @@ template<> StringSet BaseSetting::parse(const std::string & str) cons return tokenizeString(str); } -template<> void BaseSetting::appendOrSet(StringSet && newValue, bool append) +template<> void BaseSetting::appendOrSet(StringSet newValue, bool append) { if (!append) value.clear(); - for (auto && s : std::move(newValue)) - value.insert(s); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const @@ -331,26 +332,26 @@ template<> std::set BaseSetting res; for (auto & s : tokenizeString(str)) { - auto thisXpFeature = parseExperimentalFeature(s); - if (thisXpFeature) + if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature) { res.insert(thisXpFeature.value()); - else + if (thisXpFeature.value() == Xp::Flakes) + res.insert(Xp::FetchTree); + } else warn("unknown experimental feature '%s'", s); } return res; } -template<> void BaseSetting>::appendOrSet(std::set && newValue, bool append) +template<> void BaseSetting>::appendOrSet(std::set newValue, bool append) { if (!append) value.clear(); - for (auto && s : std::move(newValue)) - value.insert(s); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting>::to_string() const { StringSet stringifiedXpFeatures; - for (auto & feature : value) + for (const auto & feature : value) stringifiedXpFeatures.insert(std::string(showExperimentalFeature(feature))); return concatStringsSep(" ", stringifiedXpFeatures); } @@ -358,28 +359,25 @@ template<> std::string BaseSetting>::to_string() c template<> StringMap BaseSetting::parse(const std::string & str) const { StringMap res; - for (auto & s : tokenizeString(str)) { - auto eq = s.find_first_of('='); - if (std::string::npos != eq) + for (const auto & s : tokenizeString(str)) { + if (auto eq = s.find_first_of('='); s.npos != eq) res.emplace(std::string(s, 0, eq), std::string(s, eq + 1)); // else ignored } return res; } -template<> void BaseSetting::appendOrSet(StringMap && newValue, bool append) +template<> void BaseSetting::appendOrSet(StringMap newValue, bool append) { if (!append) value.clear(); - for (auto && [k, v] : std::move(newValue)) - value.emplace(std::move(k), std::move(v)); + value.insert(std::make_move_iterator(newValue.begin()), std::make_move_iterator(newValue.end())); } template<> std::string BaseSetting::to_string() const { - Strings kvstrs; - std::transform(value.begin(), value.end(), back_inserter(kvstrs), - [&](auto kvpair){ return kvpair.first + "=" + kvpair.second; }); - return concatStringsSep(" ", kvstrs); + return std::transform_reduce(value.cbegin(), value.cend(), std::string{}, + [](const auto & l, const auto &r) { return l + " " + r; }, + [](const auto & kvpair){ return kvpair.first + "=" + kvpair.second; }); } template class BaseSetting; @@ -468,7 +466,7 @@ void GlobalConfig::resetOverridden() nlohmann::json GlobalConfig::toJSON() { auto res = nlohmann::json::object(); - for (auto & config : *configRegistrations) + for (const auto & config : *configRegistrations) res.update(config->toJSON()); return res; } @@ -478,7 +476,7 @@ std::string GlobalConfig::toKeyValue() std::string res; std::map settings; globalConfig.getSettings(settings); - for (auto & s : settings) + for (const auto & s : settings) res += fmt("%s = %s\n", s.first, s.second.value); return res; } diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 38c3ce0c4..07322b60d 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -82,12 +82,6 @@ public: */ void applyConfig(const std::string & contents, const std::string & path = ""); - /** - * Applies a nix configuration file - * - path: the location of the config file to apply - */ - void applyConfigFile(const Path & path); - /** * Resets the `overridden` flag of all Settings */ @@ -150,7 +144,7 @@ public: AbstractSetting * setting; }; - typedef std::map Settings; + using Settings = std::map; private: @@ -213,7 +207,7 @@ protected: nlohmann::json toJSON(); - virtual std::map toJSONObject(); + virtual std::map toJSONObject() const; virtual void convertToArg(Args & args, const std::string & category); @@ -247,7 +241,7 @@ protected: * * @param append Whether to append or overwrite. */ - virtual void appendOrSet(T && newValue, bool append); + virtual void appendOrSet(T newValue, bool append); public: @@ -306,7 +300,7 @@ public: void convertToArg(Args & args, const std::string & category) override; - std::map toJSONObject() override; + std::map toJSONObject() const override; }; template @@ -316,7 +310,7 @@ std::ostream & operator <<(std::ostream & str, const BaseSetting & opt) } template -bool operator ==(const T & v1, const BaseSetting & v2) { return v1 == (const T &) v2; } +bool operator ==(const T & v1, const BaseSetting & v2) { return v1 == static_cast(v2); } template class Setting : public BaseSetting @@ -329,7 +323,7 @@ public: const std::set & aliases = {}, const bool documentDefault = true, std::optional experimentalFeature = std::nullopt) - : BaseSetting(def, documentDefault, name, description, aliases, experimentalFeature) + : BaseSetting(def, documentDefault, name, description, aliases, std::move(experimentalFeature)) { options->addSetting(this); } diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc new file mode 100644 index 000000000..352a6a0fb --- /dev/null +++ b/src/libutil/current-process.cc @@ -0,0 +1,110 @@ +#include "current-process.hh" +#include "namespaces.hh" +#include "util.hh" +#include "finally.hh" +#include "file-system.hh" +#include "processes.hh" +#include "signals.hh" + +#ifdef __APPLE__ +# include +#endif + +#if __linux__ +# include +# include +# include "cgroup.hh" +#endif + +#include + +namespace nix { + +unsigned int getMaxCPU() +{ + #if __linux__ + try { + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) return 0; + + auto cgroups = getCgroups("/proc/self/cgroup"); + auto cgroup = cgroups[""]; + if (cgroup == "") return 0; + + auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; + + auto cpuMax = readFile(cpuFile); + auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); + auto quota = cpuMaxParts[0]; + auto period = cpuMaxParts[1]; + if (quota != "max") + return std::ceil(std::stoi(quota) / std::stof(period)); + } catch (Error &) { ignoreException(lvlDebug); } + #endif + + return 0; +} + + +////////////////////////////////////////////////////////////////////// + + +#if __linux__ +rlim_t savedStackSize = 0; +#endif + +void setStackSize(size_t stackSize) +{ + #if __linux__ + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { + savedStackSize = limit.rlim_cur; + limit.rlim_cur = stackSize; + setrlimit(RLIMIT_STACK, &limit); + } + #endif +} + +void restoreProcessContext(bool restoreMounts) +{ + restoreSignals(); + if (restoreMounts) { + restoreMountNamespace(); + } + + #if __linux__ + if (savedStackSize) { + struct rlimit limit; + if (getrlimit(RLIMIT_STACK, &limit) == 0) { + limit.rlim_cur = savedStackSize; + setrlimit(RLIMIT_STACK, &limit); + } + } + #endif +} + + +////////////////////////////////////////////////////////////////////// + + +std::optional getSelfExe() +{ + static auto cached = []() -> std::optional + { + #if __linux__ + return readLink("/proc/self/exe"); + #elif __APPLE__ + char buf[1024]; + uint32_t size = sizeof(buf); + if (_NSGetExecutablePath(buf, &size) == 0) + return buf; + else + return std::nullopt; + #else + return std::nullopt; + #endif + }(); + return cached; +} + +} diff --git a/src/libutil/current-process.hh b/src/libutil/current-process.hh new file mode 100644 index 000000000..826d6fe20 --- /dev/null +++ b/src/libutil/current-process.hh @@ -0,0 +1,34 @@ +#pragma once +///@file + +#include + +#include "types.hh" + +namespace nix { + +/** + * If cgroups are active, attempt to calculate the number of CPUs available. + * If cgroups are unavailable or if cpu.max is set to "max", return 0. + */ +unsigned int getMaxCPU(); + +/** + * Change the stack size. + */ +void setStackSize(size_t stackSize); + +/** + * Restore the original inherited Unix process context (such as signal + * masks, stack size). + + * See startSignalHandlerThread(), saveSignalMask(). + */ +void restoreProcessContext(bool restoreMounts = true); + +/** + * @return the path of the current executable. + */ +std::optional getSelfExe(); + +} diff --git a/src/libutil/environment-variables.cc b/src/libutil/environment-variables.cc new file mode 100644 index 000000000..6618d7872 --- /dev/null +++ b/src/libutil/environment-variables.cc @@ -0,0 +1,49 @@ +#include "util.hh" +#include "environment-variables.hh" + +extern char * * environ __attribute__((weak)); + +namespace nix { + +std::optional getEnv(const std::string & key) +{ + char * value = getenv(key.c_str()); + if (!value) return {}; + return std::string(value); +} + +std::optional getEnvNonEmpty(const std::string & key) { + auto value = getEnv(key); + if (value == "") return {}; + return value; +} + +std::map getEnv() +{ + std::map env; + for (size_t i = 0; environ[i]; ++i) { + auto s = environ[i]; + auto eq = strchr(s, '='); + if (!eq) + // invalid env, just keep going + continue; + env.emplace(std::string(s, eq), std::string(eq + 1)); + } + return env; +} + + +void clearEnv() +{ + for (auto & name : getEnv()) + unsetenv(name.first.c_str()); +} + +void replaceEnv(const std::map & newEnv) +{ + clearEnv(); + for (auto & newEnvVar : newEnv) + setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); +} + +} diff --git a/src/libutil/environment-variables.hh b/src/libutil/environment-variables.hh new file mode 100644 index 000000000..21eb4619b --- /dev/null +++ b/src/libutil/environment-variables.hh @@ -0,0 +1,41 @@ +#pragma once +/** + * @file + * + * Utilities for working with the current process's environment + * variables. + */ + +#include + +#include "types.hh" + +namespace nix { + +/** + * @return an environment variable. + */ +std::optional getEnv(const std::string & key); + +/** + * @return a non empty environment variable. Returns nullopt if the env + * variable is set to "" + */ +std::optional getEnvNonEmpty(const std::string & key); + +/** + * Get the entire environment. + */ +std::map getEnv(); + +/** + * Clear the environment. + */ +void clearEnv(); + +/** + * Replace the entire environment with the given one. + */ +void replaceEnv(const std::map & newEnv); + +} diff --git a/src/libutil/error.cc b/src/libutil/error.cc index dd9612471..72c346cb5 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,4 +1,7 @@ #include "error.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "terminal.hh" #include #include @@ -7,8 +10,6 @@ namespace nix { -const std::string nativeSystem = SYSTEM; - void BaseError::addTrace(std::shared_ptr && e, hintformat hint, bool frame) { err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .frame = frame }); @@ -158,11 +159,11 @@ static std::string indent(std::string_view indentFirst, std::string_view indentR /** * A development aid for finding missing positions, to improve error messages. Example use: * - * NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test + * _NIX_EVAL_SHOW_UNKNOWN_LOCATIONS=1 _NIX_TEST_ACCEPT=1 make tests/lang.sh.test * git diff -U20 tests * */ -static bool printUnknownLocations = getEnv("_NIX_DEVELOPER_SHOW_UNKNOWN_LOCATIONS").has_value(); +static bool printUnknownLocations = getEnv("_NIX_EVAL_SHOW_UNKNOWN_LOCATIONS").has_value(); /** * Print a position, if it is known. diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index c576c0d4d..595dd4301 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -12,7 +12,19 @@ struct ExperimentalFeatureDetails std::string_view description; }; -constexpr std::array xpFeatureDetails = {{ +/** + * If two different PRs both add an experimental feature, and we just + * used a number for this, we *woudln't* get merge conflict and the + * counter will be incremented once instead of twice, causing a build + * failure. + * + * By instead defining this instead as 1 + the bottom experimental + * feature, we either have no issue at all if few features are not added + * at the end of the list, or a proper merge conflict if they are. + */ +constexpr size_t numXpFeatures = 1 + static_cast(Xp::VerifiedFetches); + +constexpr std::array xpFeatureDetails = {{ { .tag = Xp::CaDerivations, .name = "ca-derivations", @@ -62,6 +74,19 @@ constexpr std::array xpFeatureDetails = {{ flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. )", }, + { + .tag = Xp::FetchTree, + .name = "fetch-tree", + .description = R"( + Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language. + + `fetchTree` exposes a generic interface for fetching remote file system trees from different types of remote sources. + The [`flakes`](#xp-feature-flakes) feature flag always enables `fetch-tree`. + This built-in was previously guarded by the `flakes` experimental feature because of that overlap. + + Enabling just this feature serves as a "release candidate", allowing users to try it out in isolation. + )", + }, { .tag = Xp::NixCommand, .name = "nix-command", @@ -70,6 +95,14 @@ constexpr std::array xpFeatureDetails = {{ [`nix`](@docroot@/command-ref/new-cli/nix.md) for details. )", }, + { + .tag = Xp::GitHashing, + .name = "git-hashing", + .description = R"( + Allow creating (content-addressed) store objects which are hashed via Git's hashing algorithm. + These store objects will not be understandable by older versions of Nix. + )", + }, { .tag = Xp::RecursiveNix, .name = "recursive-nix", @@ -218,7 +251,7 @@ constexpr std::array xpFeatureDetails = {{ .tag = Xp::ReadOnlyLocalStore, .name = "read-only-local-store", .description = R"( - Allow the use of the `read-only` parameter in [local store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-store) URIs. + Allow the use of the `read-only` parameter in [local store](@docroot@/store/types/local-store.md) URIs. )", }, { @@ -234,7 +267,21 @@ constexpr std::array xpFeatureDetails = {{ .description = R"( Allow the use of the [impure-env](@docroot@/command-ref/conf-file.md#conf-impure-env) setting. )", - } + }, + { + .tag = Xp::MountedSSHStore, + .name = "mounted-ssh-store", + .description = R"( + Allow the use of the [`mounted SSH store`](@docroot@/command-ref/new-cli/nix3-help-stores.html#experimental-ssh-store-with-filesytem-mounted). + )", + }, + { + .tag = Xp::VerifiedFetches, + .name = "verified-fetches", + .description = R"( + Enables verification of git commit signatures through the [`fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) built-in. + )", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index d006db3ef..282f0ff35 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -20,7 +20,9 @@ enum struct ExperimentalFeature CaDerivations, ImpureDerivations, Flakes, + FetchTree, NixCommand, + GitHashing, RecursiveNix, NoUrlLiterals, FetchClosure, @@ -33,6 +35,8 @@ enum struct ExperimentalFeature ReadOnlyLocalStore, LocalOverlayStore, ConfigurableImpureEnv, + MountedSSHStore, + VerifiedFetches, }; /** diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc new file mode 100644 index 000000000..38dd70c8e --- /dev/null +++ b/src/libutil/file-descriptor.cc @@ -0,0 +1,254 @@ +#include "file-system.hh" +#include "signals.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include + +namespace nix { + +std::string readFile(int fd) +{ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + return drainFD(fd, true, st.st_size); +} + + +void readFull(int fd, char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = read(fd, buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("reading from file"); + } + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + + +void writeFull(int fd, std::string_view s, bool allowInterrupts) +{ + while (!s.empty()) { + if (allowInterrupts) checkInterrupt(); + ssize_t res = write(fd, s.data(), s.size()); + if (res == -1 && errno != EINTR) + throw SysError("writing to file"); + if (res > 0) + s.remove_prefix(res); + } +} + + +std::string readLine(int fd) +{ + std::string s; + while (1) { + checkInterrupt(); + char ch; + // FIXME: inefficient + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +void writeLine(int fd, std::string s) +{ + s += '\n'; + writeFull(fd, s); +} + + +std::string drainFD(int fd, bool block, const size_t reserveSize) +{ + // the parser needs two extra bytes to append terminating characters, other users will + // not care very much about the extra memory. + StringSink sink(reserveSize + 2); + drainFD(fd, sink, block); + return std::move(sink.s); +} + + +void drainFD(int fd, Sink & sink, bool block) +{ + // silence GCC maybe-uninitialized warning in finally + int saved = 0; + + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } + + Finally finally([&]() { + if (!block) { + if (fcntl(fd, F_SETFL, saved) == -1) + throw SysError("making file descriptor blocking"); + } + }); + + std::vector buf(64 * 1024); + while (1) { + checkInterrupt(); + ssize_t rd = read(fd, buf.data(), buf.size()); + if (rd == -1) { + if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) + break; + if (errno != EINTR) + throw SysError("reading from file"); + } + else if (rd == 0) break; + else sink({(char *) buf.data(), (size_t) rd}); + } +} + +////////////////////////////////////////////////////////////////////// + +AutoCloseFD::AutoCloseFD() : fd{-1} {} + + +AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} + + +AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} +{ + that.fd = -1; +} + + +AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) +{ + close(); + fd = that.fd; + that.fd = -1; + return *this; +} + + +AutoCloseFD::~AutoCloseFD() +{ + try { + close(); + } catch (...) { + ignoreException(); + } +} + + +int AutoCloseFD::get() const +{ + return fd; +} + + +void AutoCloseFD::close() +{ + if (fd != -1) { + if (::close(fd) == -1) + /* This should never happen. */ + throw SysError("closing file descriptor %1%", fd); + fd = -1; + } +} + +void AutoCloseFD::fsync() +{ + if (fd != -1) { + int result; +#if __APPLE__ + result = ::fcntl(fd, F_FULLFSYNC); +#else + result = ::fsync(fd); +#endif + if (result == -1) + throw SysError("fsync file descriptor %1%", fd); + } +} + + +AutoCloseFD::operator bool() const +{ + return fd != -1; +} + + +int AutoCloseFD::release() +{ + int oldFD = fd; + fd = -1; + return oldFD; +} + + +void Pipe::create() +{ + int fds[2]; +#if HAVE_PIPE2 + if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); +#else + if (pipe(fds) != 0) throw SysError("creating pipe"); + closeOnExec(fds[0]); + closeOnExec(fds[1]); +#endif + readSide = fds[0]; + writeSide = fds[1]; +} + + +void Pipe::close() +{ + readSide.close(); + writeSide.close(); +} + +////////////////////////////////////////////////////////////////////// + +void closeMostFDs(const std::set & exceptions) +{ +#if __linux__ + try { + for (auto & s : readDirectory("/proc/self/fd")) { + auto fd = std::stoi(s.name); + if (!exceptions.count(fd)) { + debug("closing leaked FD %d", fd); + close(fd); + } + } + return; + } catch (SysError &) { + } +#endif + + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = 0; fd < maxFD; ++fd) + if (!exceptions.count(fd)) + close(fd); /* ignore result */ +} + + +void closeOnExec(int fd) +{ + int prev; + if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || + fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) + throw SysError("setting close-on-exec flag"); +} + +} diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh new file mode 100644 index 000000000..80ec86135 --- /dev/null +++ b/src/libutil/file-descriptor.hh @@ -0,0 +1,84 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" + +namespace nix { + +struct Sink; +struct Source; + +/** + * Read the contents of a resource into a string. + */ +std::string readFile(int fd); + +/** + * Wrappers arount read()/write() that read/write exactly the + * requested number of bytes. + */ +void readFull(int fd, char * buf, size_t count); + +void writeFull(int fd, std::string_view s, bool allowInterrupts = true); + +/** + * Read a line from a file descriptor. + */ +std::string readLine(int fd); + +/** + * Write a line to a file descriptor. + */ +void writeLine(int fd, std::string s); + +/** + * Read a file descriptor until EOF occurs. + */ +std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); + +void drainFD(int fd, Sink & sink, bool block = true); + +/** + * Automatic cleanup of resources. + */ +class AutoCloseFD +{ + int fd; +public: + AutoCloseFD(); + AutoCloseFD(int fd); + AutoCloseFD(const AutoCloseFD & fd) = delete; + AutoCloseFD(AutoCloseFD&& fd); + ~AutoCloseFD(); + AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; + AutoCloseFD& operator =(AutoCloseFD&& fd); + int get() const; + explicit operator bool() const; + int release(); + void close(); + void fsync(); +}; + +class Pipe +{ +public: + AutoCloseFD readSide, writeSide; + void create(); + void close(); +}; + +/** + * Close all file descriptors except those listed in the given set. + * Good practice in child processes. + */ +void closeMostFDs(const std::set & exceptions); + +/** + * Set the close-on-exec flag for the given file descriptor. + */ +void closeOnExec(int fd); + +MakeError(EndOfFile, Error); + +} diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc new file mode 100644 index 000000000..c96effff9 --- /dev/null +++ b/src/libutil/file-system.cc @@ -0,0 +1,647 @@ +#include "environment-variables.hh" +#include "file-system.hh" +#include "signals.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace fs = std::filesystem; + +namespace nix { + +Path absPath(Path path, std::optional dir, bool resolveSymlinks) +{ + if (path[0] != '/') { + if (!dir) { +#ifdef __GNU__ + /* GNU (aka. GNU/Hurd) doesn't have any limitation on path + lengths and doesn't define `PATH_MAX'. */ + char *buf = getcwd(NULL, 0); + if (buf == NULL) +#else + char buf[PATH_MAX]; + if (!getcwd(buf, sizeof(buf))) +#endif + throw SysError("cannot get cwd"); + path = concatStrings(buf, "/", path); +#ifdef __GNU__ + free(buf); +#endif + } else + path = concatStrings(*dir, "/", path); + } + return canonPath(path, resolveSymlinks); +} + + +Path canonPath(PathView path, bool resolveSymlinks) +{ + assert(path != ""); + + std::string s; + s.reserve(256); + + if (path[0] != '/') + throw Error("not an absolute path: '%1%'", path); + + std::string temp; + + /* Count the number of times we follow a symlink and stop at some + arbitrary (but high) limit to prevent infinite loops. */ + unsigned int followCount = 0, maxFollow = 1024; + + while (1) { + + /* Skip slashes. */ + while (!path.empty() && path[0] == '/') path.remove_prefix(1); + if (path.empty()) break; + + /* Ignore `.'. */ + if (path == "." || path.substr(0, 2) == "./") + path.remove_prefix(1); + + /* If `..', delete the last component. */ + else if (path == ".." || path.substr(0, 3) == "../") + { + if (!s.empty()) s.erase(s.rfind('/')); + path.remove_prefix(2); + } + + /* Normal component; copy it. */ + else { + s += '/'; + if (const auto slash = path.find('/'); slash == std::string::npos) { + s += path; + path = {}; + } else { + s += path.substr(0, slash); + path = path.substr(slash); + } + + /* If s points to a symlink, resolve it and continue from there */ + if (resolveSymlinks && isLink(s)) { + if (++followCount >= maxFollow) + throw Error("infinite symlink recursion in path '%1%'", path); + temp = concatStrings(readLink(s), path); + path = temp; + if (!temp.empty() && temp[0] == '/') { + s.clear(); /* restart for symlinks pointing to absolute path */ + } else { + s = dirOf(s); + if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / + s.clear(); + } + } + } + } + } + + return s.empty() ? "/" : std::move(s); +} + + +Path dirOf(const PathView path) +{ + Path::size_type pos = path.rfind('/'); + if (pos == std::string::npos) + return "."; + return pos == 0 ? "/" : Path(path, 0, pos); +} + + +std::string_view baseNameOf(std::string_view path) +{ + if (path.empty()) + return ""; + + auto last = path.size() - 1; + if (path[last] == '/' && last > 0) + last -= 1; + + auto pos = path.rfind('/', last); + if (pos == std::string::npos) + pos = 0; + else + pos += 1; + + return path.substr(pos, last - pos + 1); +} + + +bool isInDir(std::string_view path, std::string_view dir) +{ + return path.substr(0, 1) == "/" + && path.substr(0, dir.size()) == dir + && path.size() >= dir.size() + 2 + && path[dir.size()] == '/'; +} + + +bool isDirOrInDir(std::string_view path, std::string_view dir) +{ + return path == dir || isInDir(path, dir); +} + + +struct stat stat(const Path & path) +{ + struct stat st; + if (stat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + + +struct stat lstat(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError("getting status of '%1%'", path); + return st; +} + + +bool pathExists(const Path & path) +{ + int res; + struct stat st; + res = lstat(path.c_str(), &st); + if (!res) return true; + if (errno != ENOENT && errno != ENOTDIR) + throw SysError("getting status of %1%", path); + return false; +} + +bool pathAccessible(const Path & path) +{ + try { + return pathExists(path); + } catch (SysError & e) { + // swallow EPERM + if (e.errNo == EPERM) return false; + throw; + } +} + + +Path readLink(const Path & path) +{ + checkInterrupt(); + std::vector buf; + for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { + buf.resize(bufSize); + ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); + if (rlSize == -1) + if (errno == EINVAL) + throw Error("'%1%' is not a symlink", path); + else + throw SysError("reading symbolic link '%1%'", path); + else if (rlSize < bufSize) + return std::string(buf.data(), rlSize); + } +} + + +bool isLink(const Path & path) +{ + struct stat st = lstat(path); + return S_ISLNK(st.st_mode); +} + + +DirEntries readDirectory(DIR *dir, const Path & path) +{ + DirEntries entries; + entries.reserve(64); + + struct dirent * dirent; + while (errno = 0, dirent = readdir(dir)) { /* sic */ + checkInterrupt(); + std::string name = dirent->d_name; + if (name == "." || name == "..") continue; + entries.emplace_back(name, dirent->d_ino, +#ifdef HAVE_STRUCT_DIRENT_D_TYPE + dirent->d_type +#else + DT_UNKNOWN +#endif + ); + } + if (errno) throw SysError("reading directory '%1%'", path); + + return entries; +} + +DirEntries readDirectory(const Path & path) +{ + AutoCloseDir dir(opendir(path.c_str())); + if (!dir) throw SysError("opening directory '%1%'", path); + + return readDirectory(dir.get(), path); +} + + +unsigned char getFileType(const Path & path) +{ + struct stat st = lstat(path); + if (S_ISDIR(st.st_mode)) return DT_DIR; + if (S_ISLNK(st.st_mode)) return DT_LNK; + if (S_ISREG(st.st_mode)) return DT_REG; + return DT_UNKNOWN; +} + + +std::string readFile(const Path & path) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%1%'", path); + return readFile(fd.get()); +} + + +void readFile(const Path & path, Sink & sink) +{ + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + if (!fd) + throw SysError("opening file '%s'", path); + drainFD(fd.get(), sink); +} + + +void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) + throw SysError("opening file '%1%'", path); + try { + writeFull(fd.get(), s); + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + + +void writeFile(const Path & path, Source & source, mode_t mode, bool sync) +{ + AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); + if (!fd) + throw SysError("opening file '%1%'", path); + + std::vector buf(64 * 1024); + + try { + while (true) { + try { + auto n = source.read(buf.data(), buf.size()); + writeFull(fd.get(), {buf.data(), n}); + } catch (EndOfFile &) { break; } + } + } catch (Error & e) { + e.addTrace({}, "writing file '%1%'", path); + throw; + } + if (sync) + fd.fsync(); + // Explicitly close to make sure exceptions are propagated. + fd.close(); + if (sync) + syncParent(path); +} + +void syncParent(const Path & path) +{ + AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0); + if (!fd) + throw SysError("opening file '%1%'", path); + fd.fsync(); +} + + +static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) +{ + checkInterrupt(); + + std::string name(baseNameOf(path)); + + struct stat st; + if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { + if (errno == ENOENT) return; + throw SysError("getting status of '%1%'", path); + } + + if (!S_ISDIR(st.st_mode)) { + /* We are about to delete a file. Will it likely free space? */ + + switch (st.st_nlink) { + /* Yes: last link. */ + case 1: + bytesFreed += st.st_size; + break; + /* Maybe: yes, if 'auto-optimise-store' or manual optimisation + was performed. Instead of checking for real let's assume + it's an optimised file and space will be freed. + + In worst case we will double count on freed space for files + with exactly two hardlinks for unoptimised packages. + */ + case 2: + bytesFreed += st.st_size; + break; + /* No: 3+ links. */ + default: + break; + } + } + + if (S_ISDIR(st.st_mode)) { + /* Make the directory accessible. */ + const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; + if ((st.st_mode & PERM_MASK) != PERM_MASK) { + if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) + throw SysError("chmod '%1%'", path); + } + + int fd = openat(parentfd, path.c_str(), O_RDONLY); + if (fd == -1) + throw SysError("opening directory '%1%'", path); + AutoCloseDir dir(fdopendir(fd)); + if (!dir) + throw SysError("opening directory '%1%'", path); + for (auto & i : readDirectory(dir.get(), path)) + _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); + } + + int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; + if (unlinkat(parentfd, name.c_str(), flags) == -1) { + if (errno == ENOENT) return; + throw SysError("cannot unlink '%1%'", path); + } +} + +static void _deletePath(const Path & path, uint64_t & bytesFreed) +{ + Path dir = dirOf(path); + if (dir == "") + dir = "/"; + + AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; + if (!dirfd) { + if (errno == ENOENT) return; + throw SysError("opening directory '%1%'", path); + } + + _deletePath(dirfd.get(), path, bytesFreed); +} + + +void deletePath(const Path & path) +{ + uint64_t dummy; + deletePath(path, dummy); +} + + +Paths createDirs(const Path & path) +{ + Paths created; + if (path == "/") return created; + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { + created = createDirs(dirOf(path)); + if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) + throw SysError("creating directory '%1%'", path); + st = lstat(path); + created.push_back(path); + } + + if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) + throw SysError("statting symlink '%1%'", path); + + if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); + + return created; +} + + +void deletePath(const Path & path, uint64_t & bytesFreed) +{ + //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); + bytesFreed = 0; + _deletePath(path, bytesFreed); +} + + +////////////////////////////////////////////////////////////////////// + +AutoDelete::AutoDelete() : del{false} {} + +AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p) +{ + del = true; + this->recursive = recursive; +} + +AutoDelete::~AutoDelete() +{ + try { + if (del) { + if (recursive) + deletePath(path); + else { + if (remove(path.c_str()) == -1) + throw SysError("cannot unlink '%1%'", path); + } + } + } catch (...) { + ignoreException(); + } +} + +void AutoDelete::cancel() +{ + del = false; +} + +void AutoDelete::reset(const Path & p, bool recursive) { + path = p; + this->recursive = recursive; + del = true; +} + +////////////////////////////////////////////////////////////////////// + +////////////////////////////////////////////////////////////////////// + +static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, + std::atomic & counter) +{ + tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); + if (includePid) + return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++); + else + return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++); +} + +Path createTempDir(const Path & tmpRoot, const Path & prefix, + bool includePid, bool useGlobalCounter, mode_t mode) +{ + static std::atomic globalCounter = 0; + std::atomic localCounter = 0; + auto & counter(useGlobalCounter ? globalCounter : localCounter); + + while (1) { + checkInterrupt(); + Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); + if (mkdir(tmpDir.c_str(), mode) == 0) { +#if __FreeBSD__ + /* Explicitly set the group of the directory. This is to + work around around problems caused by BSD's group + ownership semantics (directories inherit the group of + the parent). For instance, the group of /tmp on + FreeBSD is "wheel", so all directories created in /tmp + will be owned by "wheel"; but if the user is not in + "wheel", then "tar" will fail to unpack archives that + have the setgid bit set on directories. */ + if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) + throw SysError("setting group of directory '%1%'", tmpDir); +#endif + return tmpDir; + } + if (errno != EEXIST) + throw SysError("creating directory '%1%'", tmpDir); + } +} + + +std::pair createTempFile(const Path & prefix) +{ + Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); + // Strictly speaking, this is UB, but who cares... + // FIXME: use O_TMPFILE. + AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); + if (!fd) + throw SysError("creating temporary file '%s'", tmpl); + closeOnExec(fd.get()); + return {std::move(fd), tmpl}; +} + +void createSymlink(const Path & target, const Path & link) +{ + if (symlink(target.c_str(), link.c_str())) + throw SysError("creating symlink from '%1%' to '%2%'", link, target); +} + +void replaceSymlink(const Path & target, const Path & link) +{ + for (unsigned int n = 0; true; n++) { + Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + + try { + createSymlink(target, tmp); + } catch (SysError & e) { + if (e.errNo == EEXIST) continue; + throw; + } + + renameFile(tmp, link); + + break; + } +} + +void setWriteTime(const fs::path & p, const struct stat & st) +{ + struct timeval times[2]; + times[0] = { + .tv_sec = st.st_atime, + .tv_usec = 0, + }; + times[1] = { + .tv_sec = st.st_mtime, + .tv_usec = 0, + }; + if (lutimes(p.c_str(), times) != 0) + throw SysError("changing modification time of '%s'", p); +} + +void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) +{ + // TODO: Rewrite the `is_*` to use `symlink_status()` + auto statOfFrom = lstat(from.path().c_str()); + auto fromStatus = from.symlink_status(); + + // Mark the directory as writable so that we can delete its children + if (andDelete && fs::is_directory(fromStatus)) { + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + } + + + if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { + fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); + } else if (fs::is_directory(fromStatus)) { + fs::create_directory(to); + for (auto & entry : fs::directory_iterator(from.path())) { + copy(entry, to / entry.path().filename(), andDelete); + } + } else { + throw Error("file '%s' has an unsupported type", from.path()); + } + + setWriteTime(to, statOfFrom); + if (andDelete) { + if (!fs::is_symlink(fromStatus)) + fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); + fs::remove(from.path()); + } +} + +void renameFile(const Path & oldName, const Path & newName) +{ + fs::rename(oldName, newName); +} + +void moveFile(const Path & oldName, const Path & newName) +{ + try { + renameFile(oldName, newName); + } catch (fs::filesystem_error & e) { + auto oldPath = fs::path(oldName); + auto newPath = fs::path(newName); + // For the move to be as atomic as possible, copy to a temporary + // directory + fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); + Finally removeTemp = [&]() { fs::remove(temp); }; + auto tempCopyTarget = temp / "copy-target"; + if (e.code().value() == EXDEV) { + fs::remove(newPath); + warn("Can’t rename %s as %s, copying instead", oldName, newName); + copy(fs::directory_entry(oldPath), tempCopyTarget, true); + renameFile(tempCopyTarget, newPath); + } + } +} + +////////////////////////////////////////////////////////////////////// + +} diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh new file mode 100644 index 000000000..4637507b3 --- /dev/null +++ b/src/libutil/file-system.hh @@ -0,0 +1,238 @@ +#pragma once +/** + * @file + * + * Utiltities for working with the file sytem and file paths. + */ + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "file-descriptor.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +#ifndef HAVE_STRUCT_DIRENT_D_TYPE +#define DT_UNKNOWN 0 +#define DT_REG 1 +#define DT_LNK 2 +#define DT_DIR 3 +#endif + +namespace nix { + +struct Sink; +struct Source; + +/** + * @return An absolutized path, resolving paths relative to the + * specified directory, or the current directory otherwise. The path + * is also canonicalised. + */ +Path absPath(Path path, + std::optional dir = {}, + bool resolveSymlinks = false); + +/** + * Canonicalise a path by removing all `.` or `..` components and + * double or trailing slashes. Optionally resolves all symlink + * components such that each component of the resulting path is *not* + * a symbolic link. + */ +Path canonPath(PathView path, bool resolveSymlinks = false); + +/** + * @return The directory part of the given canonical path, i.e., + * everything before the final `/`. If the path is the root or an + * immediate child thereof (e.g., `/foo`), this means `/` + * is returned. + */ +Path dirOf(const PathView path); + +/** + * @return the base name of the given canonical path, i.e., everything + * following the final `/` (trailing slashes are removed). + */ +std::string_view baseNameOf(std::string_view path); + +/** + * Check whether 'path' is a descendant of 'dir'. Both paths must be + * canonicalized. + */ +bool isInDir(std::string_view path, std::string_view dir); + +/** + * Check whether 'path' is equal to 'dir' or a descendant of + * 'dir'. Both paths must be canonicalized. + */ +bool isDirOrInDir(std::string_view path, std::string_view dir); + +/** + * Get status of `path`. + */ +struct stat stat(const Path & path); +struct stat lstat(const Path & path); + +/** + * @return true iff the given path exists. + */ +bool pathExists(const Path & path); + +/** + * A version of pathExists that returns false on a permission error. + * Useful for inferring default paths across directories that might not + * be readable. + * @return true iff the given path can be accessed and exists + */ +bool pathAccessible(const Path & path); + +/** + * Read the contents (target) of a symbolic link. The result is not + * in any way canonicalised. + */ +Path readLink(const Path & path); + +bool isLink(const Path & path); + +/** + * Read the contents of a directory. The entries `.` and `..` are + * removed. + */ +struct DirEntry +{ + std::string name; + ino_t ino; + /** + * one of DT_* + */ + unsigned char type; + DirEntry(std::string name, ino_t ino, unsigned char type) + : name(std::move(name)), ino(ino), type(type) { } +}; + +typedef std::vector DirEntries; + +DirEntries readDirectory(const Path & path); + +unsigned char getFileType(const Path & path); + +/** + * Read the contents of a file into a string. + */ +std::string readFile(const Path & path); +void readFile(const Path & path, Sink & sink); + +/** + * Write a string to a file. + */ +void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); + +void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); + +/** + * Flush a file's parent directory to disk + */ +void syncParent(const Path & path); + +/** + * Delete a path; i.e., in the case of a directory, it is deleted + * recursively. It's not an error if the path does not exist. The + * second variant returns the number of bytes and blocks freed. + */ +void deletePath(const Path & path); + +void deletePath(const Path & path, uint64_t & bytesFreed); + +/** + * Create a directory and all its parents, if necessary. Returns the + * list of created directories, in order of creation. + */ +Paths createDirs(const Path & path); +inline Paths createDirs(PathView path) +{ + return createDirs(Path(path)); +} + +/** + * Create a symlink. + */ +void createSymlink(const Path & target, const Path & link); + +/** + * Atomically create or replace a symlink. + */ +void replaceSymlink(const Path & target, const Path & link); + +void renameFile(const Path & src, const Path & dst); + +/** + * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` + * are on a different filesystem. + * + * Beware that this might not be atomic because of the copy that happens behind + * the scenes + */ +void moveFile(const Path & src, const Path & dst); + + +/** + * Automatic cleanup of resources. + */ +class AutoDelete +{ + Path path; + bool del; + bool recursive; +public: + AutoDelete(); + AutoDelete(const Path & p, bool recursive = true); + ~AutoDelete(); + void cancel(); + void reset(const Path & p, bool recursive = true); + operator Path() const { return path; } + operator PathView() const { return path; } +}; + + +struct DIRDeleter +{ + void operator()(DIR * dir) const { + closedir(dir); + } +}; + +typedef std::unique_ptr AutoCloseDir; + + +/** + * Create a temporary directory. + */ +Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", + bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); + +/** + * Create a temporary file, returning a file handle and its path. + */ +std::pair createTempFile(const Path & prefix = "nix"); + + +/** + * Used in various places. + */ +typedef std::function PathFilter; + +extern PathFilter defaultPathFilter; + +} diff --git a/src/libutil/filesystem.cc b/src/libutil/filesystem.cc deleted file mode 100644 index 11cc0c0e7..000000000 --- a/src/libutil/filesystem.cc +++ /dev/null @@ -1,162 +0,0 @@ -#include -#include -#include - -#include "finally.hh" -#include "util.hh" -#include "types.hh" - -namespace fs = std::filesystem; - -namespace nix { - -static Path tempName(Path tmpRoot, const Path & prefix, bool includePid, - std::atomic & counter) -{ - tmpRoot = canonPath(tmpRoot.empty() ? getEnv("TMPDIR").value_or("/tmp") : tmpRoot, true); - if (includePid) - return fmt("%1%/%2%-%3%-%4%", tmpRoot, prefix, getpid(), counter++); - else - return fmt("%1%/%2%-%3%", tmpRoot, prefix, counter++); -} - -Path createTempDir(const Path & tmpRoot, const Path & prefix, - bool includePid, bool useGlobalCounter, mode_t mode) -{ - static std::atomic globalCounter = 0; - std::atomic localCounter = 0; - auto & counter(useGlobalCounter ? globalCounter : localCounter); - - while (1) { - checkInterrupt(); - Path tmpDir = tempName(tmpRoot, prefix, includePid, counter); - if (mkdir(tmpDir.c_str(), mode) == 0) { -#if __FreeBSD__ - /* Explicitly set the group of the directory. This is to - work around around problems caused by BSD's group - ownership semantics (directories inherit the group of - the parent). For instance, the group of /tmp on - FreeBSD is "wheel", so all directories created in /tmp - will be owned by "wheel"; but if the user is not in - "wheel", then "tar" will fail to unpack archives that - have the setgid bit set on directories. */ - if (chown(tmpDir.c_str(), (uid_t) -1, getegid()) != 0) - throw SysError("setting group of directory '%1%'", tmpDir); -#endif - return tmpDir; - } - if (errno != EEXIST) - throw SysError("creating directory '%1%'", tmpDir); - } -} - - -std::pair createTempFile(const Path & prefix) -{ - Path tmpl(getEnv("TMPDIR").value_or("/tmp") + "/" + prefix + ".XXXXXX"); - // Strictly speaking, this is UB, but who cares... - // FIXME: use O_TMPFILE. - AutoCloseFD fd(mkstemp((char *) tmpl.c_str())); - if (!fd) - throw SysError("creating temporary file '%s'", tmpl); - closeOnExec(fd.get()); - return {std::move(fd), tmpl}; -} - -void createSymlink(const Path & target, const Path & link) -{ - if (symlink(target.c_str(), link.c_str())) - throw SysError("creating symlink from '%1%' to '%2%'", link, target); -} - -void replaceSymlink(const Path & target, const Path & link) -{ - for (unsigned int n = 0; true; n++) { - Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); - - try { - createSymlink(target, tmp); - } catch (SysError & e) { - if (e.errNo == EEXIST) continue; - throw; - } - - renameFile(tmp, link); - - break; - } -} - -void setWriteTime(const fs::path & p, const struct stat & st) -{ - struct timeval times[2]; - times[0] = { - .tv_sec = st.st_atime, - .tv_usec = 0, - }; - times[1] = { - .tv_sec = st.st_mtime, - .tv_usec = 0, - }; - if (lutimes(p.c_str(), times) != 0) - throw SysError("changing modification time of '%s'", p); -} - -void copy(const fs::directory_entry & from, const fs::path & to, bool andDelete) -{ - // TODO: Rewrite the `is_*` to use `symlink_status()` - auto statOfFrom = lstat(from.path().c_str()); - auto fromStatus = from.symlink_status(); - - // Mark the directory as writable so that we can delete its children - if (andDelete && fs::is_directory(fromStatus)) { - fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - } - - - if (fs::is_symlink(fromStatus) || fs::is_regular_file(fromStatus)) { - fs::copy(from.path(), to, fs::copy_options::copy_symlinks | fs::copy_options::overwrite_existing); - } else if (fs::is_directory(fromStatus)) { - fs::create_directory(to); - for (auto & entry : fs::directory_iterator(from.path())) { - copy(entry, to / entry.path().filename(), andDelete); - } - } else { - throw Error("file '%s' has an unsupported type", from.path()); - } - - setWriteTime(to, statOfFrom); - if (andDelete) { - if (!fs::is_symlink(fromStatus)) - fs::permissions(from.path(), fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); - fs::remove(from.path()); - } -} - -void renameFile(const Path & oldName, const Path & newName) -{ - fs::rename(oldName, newName); -} - -void moveFile(const Path & oldName, const Path & newName) -{ - try { - renameFile(oldName, newName); - } catch (fs::filesystem_error & e) { - auto oldPath = fs::path(oldName); - auto newPath = fs::path(newName); - // For the move to be as atomic as possible, copy to a temporary - // directory - fs::path temp = createTempDir(newPath.parent_path(), "rename-tmp"); - Finally removeTemp = [&]() { fs::remove(temp); }; - auto tempCopyTarget = temp / "copy-target"; - if (e.code().value() == EXDEV) { - fs::remove(newPath); - warn("Can’t rename %s as %s, copying instead", oldName, newName); - copy(fs::directory_entry(oldPath), tempCopyTarget, true); - renameFile(tempCopyTarget, newPath); - } - } -} - -} diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index 727255b45..ac72e47fb 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -44,6 +44,11 @@ inline std::string fmt(const std::string & s) return s; } +inline std::string fmt(std::string_view s) +{ + return std::string(s); +} + inline std::string fmt(const char * s) { return s; diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index a08a723a4..925e6f05d 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -5,6 +5,54 @@ namespace nix { +void copyRecursive( + SourceAccessor & accessor, const CanonPath & from, + ParseSink & sink, const Path & to) +{ + auto stat = accessor.lstat(from); + + switch (stat.type) { + case SourceAccessor::tSymlink: + { + sink.createSymlink(to, accessor.readLink(from)); + } + + case SourceAccessor::tRegular: + { + sink.createRegularFile(to); + if (stat.isExecutable) + sink.isExecutable(); + LambdaSink sink2 { + [&](auto d) { + sink.receiveContents(d); + } + }; + accessor.readFile(from, sink2, [&](uint64_t size) { + sink.preallocateContents(size); + }); + break; + } + + case SourceAccessor::tDirectory: + { + sink.createDirectory(to); + for (auto & [name, _] : accessor.readDirectory(from)) { + copyRecursive( + accessor, from + name, + sink, to + "/" + name); + break; + } + } + + case SourceAccessor::tMisc: + throw Error("file '%1%' has an unsupported type", from); + + default: + abort(); + } +} + + struct RestoreSinkSettings : Config { Setting preallocateContents{this, false, "preallocate-contents", diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index 6837e2fc4..bf54b7301 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -3,6 +3,8 @@ #include "types.hh" #include "serialise.hh" +#include "source-accessor.hh" +#include "file-system.hh" namespace nix { @@ -11,32 +13,93 @@ namespace nix { */ struct ParseSink { - virtual void createDirectory(const Path & path) { }; + virtual void createDirectory(const Path & path) = 0; - virtual void createRegularFile(const Path & path) { }; - virtual void closeRegularFile() { }; - virtual void isExecutable() { }; + virtual void createRegularFile(const Path & path) = 0; + virtual void receiveContents(std::string_view data) = 0; + virtual void isExecutable() = 0; + virtual void closeRegularFile() = 0; + + virtual void createSymlink(const Path & path, const std::string & target) = 0; + + /** + * An optimization. By default, do nothing. + */ virtual void preallocateContents(uint64_t size) { }; - virtual void receiveContents(std::string_view data) { }; - - virtual void createSymlink(const Path & path, const std::string & target) { }; }; +/** + * Recusively copy file system objects from the source into the sink. + */ +void copyRecursive( + SourceAccessor & accessor, const CanonPath & sourcePath, + ParseSink & sink, const Path & destPath); + +/** + * Ignore everything and do nothing + */ +struct NullParseSink : ParseSink +{ + void createDirectory(const Path & path) override { } + void receiveContents(std::string_view data) override { } + void createSymlink(const Path & path, const std::string & target) override { } + void createRegularFile(const Path & path) override { } + void closeRegularFile() override { } + void isExecutable() override { } +}; + +/** + * Write files at the given path + */ struct RestoreSink : ParseSink { Path dstPath; - AutoCloseFD fd; - void createDirectory(const Path & path) override; void createRegularFile(const Path & path) override; - void closeRegularFile() override; - void isExecutable() override; - void preallocateContents(uint64_t size) override; void receiveContents(std::string_view data) override; + void isExecutable() override; + void closeRegularFile() override; void createSymlink(const Path & path, const std::string & target) override; + + void preallocateContents(uint64_t size) override; + +private: + AutoCloseFD fd; +}; + +/** + * Restore a single file at the top level, passing along + * `receiveContents` to the underlying `Sink`. For anything but a single + * file, set `regular = true` so the caller can fail accordingly. + */ +struct RegularFileSink : ParseSink +{ + bool regular = true; + Sink & sink; + + RegularFileSink(Sink & sink) : sink(sink) { } + + void createDirectory(const Path & path) override + { + regular = false; + } + + void receiveContents(std::string_view data) override + { + sink(data); + } + + void createSymlink(const Path & path, const std::string & target) override + { + regular = false; + } + + void createRegularFile(const Path & path) override { } + void closeRegularFile() override { } + void isExecutable() override { } }; } diff --git a/src/libutil/git.cc b/src/libutil/git.cc index f35c2fdb7..296b75628 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -1,9 +1,263 @@ -#include "git.hh" - +#include +#include +#include +#include #include +#include // for strcasecmp + +#include "signals.hh" +#include "config.hh" +#include "hash.hh" +#include "posix-source-accessor.hh" + +#include "git.hh" +#include "serialise.hh" + +namespace nix::git { + +using namespace nix; +using namespace std::string_literals; + +std::optional decodeMode(RawMode m) { + switch (m) { + case (RawMode) Mode::Directory: + case (RawMode) Mode::Executable: + case (RawMode) Mode::Regular: + case (RawMode) Mode::Symlink: + return (Mode) m; + default: + return std::nullopt; + } +} + + +static std::string getStringUntil(Source & source, char byte) +{ + std::string s; + char n[1]; + source(std::string_view { n, 1 }); + while (*n != byte) { + s += *n; + source(std::string_view { n, 1 }); + } + return s; +} + + +static std::string getString(Source & source, int n) +{ + std::string v; + v.resize(n); + source(v); + return v; +} + + +void parse( + ParseSink & sink, + const Path & sinkPath, + Source & source, + std::function hook, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + + auto type = getString(source, 5); + + if (type == "blob ") { + sink.createRegularFile(sinkPath); + + unsigned long long size = std::stoi(getStringUntil(source, 0)); + + sink.preallocateContents(size); + + unsigned long long left = size; + std::string buf; + buf.reserve(65536); + + while (left) { + checkInterrupt(); + buf.resize(std::min((unsigned long long)buf.capacity(), left)); + source(buf); + sink.receiveContents(buf); + left -= buf.size(); + } + } else if (type == "tree ") { + unsigned long long size = std::stoi(getStringUntil(source, 0)); + unsigned long long left = size; + + sink.createDirectory(sinkPath); + + while (left) { + std::string perms = getStringUntil(source, ' '); + left -= perms.size(); + left -= 1; + + RawMode rawMode = std::stoi(perms, 0, 8); + auto modeOpt = decodeMode(rawMode); + if (!modeOpt) + throw Error("Unknown Git permission: %o", perms); + auto mode = std::move(*modeOpt); + + std::string name = getStringUntil(source, '\0'); + left -= name.size(); + left -= 1; + + std::string hashs = getString(source, 20); + left -= 20; + + Hash hash(HashAlgorithm::SHA1); + std::copy(hashs.begin(), hashs.end(), hash.hash); + + hook(name, TreeEntry { + .mode = mode, + .hash = hash, + }); + + if (mode == Mode::Executable) + sink.isExecutable(); + } + } else throw Error("input doesn't look like a Git object"); +} + + +std::optional convertMode(SourceAccessor::Type type) +{ + switch (type) { + case SourceAccessor::tSymlink: return Mode::Symlink; + case SourceAccessor::tRegular: return Mode::Regular; + case SourceAccessor::tDirectory: return Mode::Directory; + case SourceAccessor::tMisc: return std::nullopt; + default: abort(); + } +} + + +void restore(ParseSink & sink, Source & source, std::function hook) +{ + parse(sink, "", source, [&](Path name, TreeEntry entry) { + auto [accessor, from] = hook(entry.hash); + auto stat = accessor->lstat(from); + auto gotOpt = convertMode(stat.type); + if (!gotOpt) + throw Error("file '%s' (git hash %s) has an unsupported type", + from, + entry.hash.to_string(HashFormat::Base16, false)); + auto & got = *gotOpt; + if (got != entry.mode) + throw Error("git mode of file '%s' (git hash %s) is %o but expected %o", + from, + entry.hash.to_string(HashFormat::Base16, false), + (RawMode) got, + (RawMode) entry.mode); + copyRecursive( + *accessor, from, + sink, name); + }); +} + + +void dumpBlobPrefix( + uint64_t size, Sink & sink, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + auto s = fmt("blob %d\0"s, std::to_string(size)); + sink(s); +} + + +void dumpTree(const Tree & entries, Sink & sink, + const ExperimentalFeatureSettings & xpSettings) +{ + xpSettings.require(Xp::GitHashing); + + std::string v1; + + for (auto & [name, entry] : entries) { + auto name2 = name; + if (entry.mode == Mode::Directory) { + assert(name2.back() == '/'); + name2.pop_back(); + } + v1 += fmt("%o %s\0"s, static_cast(entry.mode), name2); + std::copy(entry.hash.hash, entry.hash.hash + entry.hash.hashSize, std::back_inserter(v1)); + } + + { + auto s = fmt("tree %d\0"s, v1.size()); + sink(s); + } + + sink(v1); +} + + +Mode dump( + SourceAccessor & accessor, const CanonPath & path, + Sink & sink, + std::function hook, + PathFilter & filter, + const ExperimentalFeatureSettings & xpSettings) +{ + auto st = accessor.lstat(path); + + switch (st.type) { + case SourceAccessor::tRegular: + { + accessor.readFile(path, sink, [&](uint64_t size) { + dumpBlobPrefix(size, sink, xpSettings); + }); + return st.isExecutable + ? Mode::Executable + : Mode::Regular; + } + + case SourceAccessor::tDirectory: + { + Tree entries; + for (auto & [name, _] : accessor.readDirectory(path)) { + auto child = path + name; + if (!filter(child.abs())) continue; + + auto entry = hook(child); + + auto name2 = name; + if (entry.mode == Mode::Directory) + name2 += "/"; + + entries.insert_or_assign(std::move(name2), std::move(entry)); + } + dumpTree(entries, sink, xpSettings); + return Mode::Directory; + } + + case SourceAccessor::tSymlink: + case SourceAccessor::tMisc: + default: + throw Error("file '%1%' has an unsupported type", path); + } +} + + +TreeEntry dumpHash( + HashAlgorithm ha, + SourceAccessor & accessor, const CanonPath & path, PathFilter & filter) +{ + std::function hook; + hook = [&](const CanonPath & path) -> TreeEntry { + auto hashSink = HashSink(ha); + auto mode = dump(accessor, path, hashSink, hook, filter); + auto hash = hashSink.finish().first; + return { + .mode = mode, + .hash = hash, + }; + }; + + return hook(path); +} -namespace nix { -namespace git { std::optional parseLsRemoteLine(std::string_view line) { @@ -22,4 +276,3 @@ std::optional parseLsRemoteLine(std::string_view line) } } -} diff --git a/src/libutil/git.hh b/src/libutil/git.hh index bf2b9a286..b24b25dd3 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -5,9 +5,127 @@ #include #include -namespace nix { +#include "types.hh" +#include "serialise.hh" +#include "hash.hh" +#include "source-accessor.hh" +#include "fs-sink.hh" -namespace git { +namespace nix::git { + +using RawMode = uint32_t; + +enum struct Mode : RawMode { + Directory = 0040000, + Executable = 0100755, + Regular = 0100644, + Symlink = 0120000, +}; + +std::optional decodeMode(RawMode m); + +/** + * An anonymous Git tree object entry (no name part). + */ +struct TreeEntry +{ + Mode mode; + Hash hash; + + GENERATE_CMP(TreeEntry, me->mode, me->hash); +}; + +/** + * A Git tree object, fully decoded and stored in memory. + * + * Directory names must end in a `/` for sake of sorting. See + * https://github.com/mirage/irmin/issues/352 + */ +using Tree = std::map; + +/** + * Callback for processing a child hash with `parse` + * + * The function should + * + * 1. Obtain the file system objects denoted by `gitHash` + * + * 2. Ensure they match `mode` + * + * 3. Feed them into the same sink `parse` was called with + * + * Implementations may seek to memoize resources (bandwidth, storage, + * etc.) for the same Git hash. + */ +using SinkHook = void(const Path & name, TreeEntry entry); + +void parse( + ParseSink & sink, const Path & sinkPath, + Source & source, + std::function hook, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Assists with writing a `SinkHook` step (2). + */ +std::optional convertMode(SourceAccessor::Type type); + +/** + * Simplified version of `SinkHook` for `restore`. + * + * Given a `Hash`, return a `SourceAccessor` and `CanonPath` pointing to + * the file system object with that path. + */ +using RestoreHook = std::pair(Hash); + +/** + * Wrapper around `parse` and `RestoreSink` + */ +void restore(ParseSink & sink, Source & source, std::function hook); + +/** + * Dumps a single file to a sink + * + * @param xpSettings for testing purposes + */ +void dumpBlobPrefix( + uint64_t size, Sink & sink, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Dumps a representation of a git tree to a sink + */ +void dumpTree( + const Tree & entries, Sink & sink, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Callback for processing a child with `dump` + * + * The function should return the Git hash and mode of the file at the + * given path in the accessor passed to `dump`. + * + * Note that if the child is a directory, its child in must also be so + * processed in order to compute this information. + */ +using DumpHook = TreeEntry(const CanonPath & path); + +Mode dump( + SourceAccessor & accessor, const CanonPath & path, + Sink & sink, + std::function hook, + PathFilter & filter = defaultPathFilter, + const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + +/** + * Recursively dumps path, hashing as we go. + * + * A smaller wrapper around `dump`. + */ +TreeEntry dumpHash( + HashAlgorithm ha, + SourceAccessor & accessor, const CanonPath & path, + PathFilter & filter = defaultPathFilter); /** * A line from the output of `git ls-remote --symref`. @@ -16,15 +134,17 @@ namespace git { * * - Symbolic references of the form * - * ref: {target} {reference} - * - * where {target} is itself a reference and {reference} is optional + * ``` + * ref: {target} {reference} + * ``` + * where {target} is itself a reference and {reference} is optional * * - Object references of the form * - * {target} {reference} - * - * where {target} is a commit id and {reference} is mandatory + * ``` + * {target} {reference} + * ``` + * where {target} is a commit id and {reference} is mandatory */ struct LsRemoteRefLine { enum struct Kind { @@ -36,8 +156,9 @@ struct LsRemoteRefLine { std::optional reference; }; +/** + * Parse an `LsRemoteRefLine` + */ std::optional parseLsRemoteLine(std::string_view line); } - -} diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index e297c245b..30456ae5c 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -9,7 +9,6 @@ #include "hash.hh" #include "archive.hh" #include "split.hh" -#include "util.hh" #include #include @@ -17,23 +16,24 @@ namespace nix { -static size_t regularHashSize(HashType type) { +static size_t regularHashSize(HashAlgorithm type) { switch (type) { - case htMD5: return md5HashSize; - case htSHA1: return sha1HashSize; - case htSHA256: return sha256HashSize; - case htSHA512: return sha512HashSize; + case HashAlgorithm::MD5: return md5HashSize; + case HashAlgorithm::SHA1: return sha1HashSize; + case HashAlgorithm::SHA256: return sha256HashSize; + case HashAlgorithm::SHA512: return sha512HashSize; } abort(); } -std::set hashTypes = { "md5", "sha1", "sha256", "sha512" }; +const std::set hashAlgorithms = {"md5", "sha1", "sha256", "sha512" }; +const std::set hashFormats = {"base64", "nix32", "base16", "sri" }; -Hash::Hash(HashType type) : type(type) +Hash::Hash(HashAlgorithm algo) : algo(algo) { - hashSize = regularHashSize(type); + hashSize = regularHashSize(algo); assert(hashSize <= maxHashSize); memset(hash, 0, maxHashSize); } @@ -82,7 +82,7 @@ static std::string printHash16(const Hash & hash) // omitted: E O U T -const std::string base32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; +const std::string nix32Chars = "0123456789abcdfghijklmnpqrsvwxyz"; static std::string printHash32(const Hash & hash) @@ -101,7 +101,7 @@ static std::string printHash32(const Hash & hash) unsigned char c = (hash.hash[i] >> j) | (i >= hash.hashSize - 1 ? 0 : hash.hash[i + 1] << (8 - j)); - s.push_back(base32Chars[c & 0x1f]); + s.push_back(nix32Chars[c & 0x1f]); } return s; @@ -110,23 +110,23 @@ static std::string printHash32(const Hash & hash) std::string printHash16or32(const Hash & hash) { - assert(hash.type); - return hash.to_string(hash.type == htMD5 ? HashFormat::Base16 : HashFormat::Base32, false); + assert(static_cast(hash.algo)); + return hash.to_string(hash.algo == HashAlgorithm::MD5 ? HashFormat::Base16 : HashFormat::Nix32, false); } -std::string Hash::to_string(HashFormat hashFormat, bool includeType) const +std::string Hash::to_string(HashFormat hashFormat, bool includeAlgo) const { std::string s; - if (hashFormat == HashFormat::SRI || includeType) { - s += printHashType(type); + if (hashFormat == HashFormat::SRI || includeAlgo) { + s += printHashAlgo(algo); s += hashFormat == HashFormat::SRI ? '-' : ':'; } switch (hashFormat) { case HashFormat::Base16: s += printHash16(*this); break; - case HashFormat::Base32: + case HashFormat::Nix32: s += printHash32(*this); break; case HashFormat::Base64: @@ -137,7 +137,7 @@ std::string Hash::to_string(HashFormat hashFormat, bool includeType) const return s; } -Hash Hash::dummy(htSHA256); +Hash Hash::dummy(HashAlgorithm::SHA256); Hash Hash::parseSRI(std::string_view original) { auto rest = original; @@ -146,18 +146,18 @@ Hash Hash::parseSRI(std::string_view original) { auto hashRaw = splitPrefixTo(rest, '-'); if (!hashRaw) throw BadHash("hash '%s' is not SRI", original); - HashType parsedType = parseHashType(*hashRaw); + HashAlgorithm parsedType = parseHashAlgo(*hashRaw); return Hash(rest, parsedType, true); } // Mutates the string to eliminate the prefixes when found -static std::pair, bool> getParsedTypeAndSRI(std::string_view & rest) +static std::pair, bool> getParsedTypeAndSRI(std::string_view & rest) { bool isSRI = false; // Parse the hash type before the separator, if there was one. - std::optional optParsedType; + std::optional optParsedType; { auto hashRaw = splitPrefixTo(rest, ':'); @@ -167,7 +167,7 @@ static std::pair, bool> getParsedTypeAndSRI(std::string_ isSRI = true; } if (hashRaw) - optParsedType = parseHashType(*hashRaw); + optParsedType = parseHashAlgo(*hashRaw); } return {optParsedType, isSRI}; @@ -186,29 +186,29 @@ Hash Hash::parseAnyPrefixed(std::string_view original) return Hash(rest, *optParsedType, isSRI); } -Hash Hash::parseAny(std::string_view original, std::optional optType) +Hash Hash::parseAny(std::string_view original, std::optional optAlgo) { auto rest = original; auto [optParsedType, isSRI] = getParsedTypeAndSRI(rest); // Either the string or user must provide the type, if they both do they // must agree. - if (!optParsedType && !optType) + if (!optParsedType && !optAlgo) throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context", rest); - else if (optParsedType && optType && *optParsedType != *optType) - throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType)); + else if (optParsedType && optAlgo && *optParsedType != *optAlgo) + throw BadHash("hash '%s' should have type '%s'", original, printHashAlgo(*optAlgo)); - HashType hashType = optParsedType ? *optParsedType : *optType; - return Hash(rest, hashType, isSRI); + HashAlgorithm hashAlgo = optParsedType ? *optParsedType : *optAlgo; + return Hash(rest, hashAlgo, isSRI); } -Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashType type) +Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo) { - return Hash(s, type, false); + return Hash(s, algo, false); } -Hash::Hash(std::string_view rest, HashType type, bool isSRI) - : Hash(type) +Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI) + : Hash(algo) { if (!isSRI && rest.size() == base16Len()) { @@ -231,8 +231,8 @@ Hash::Hash(std::string_view rest, HashType type, bool isSRI) for (unsigned int n = 0; n < rest.size(); ++n) { char c = rest[rest.size() - n - 1]; unsigned char digit; - for (digit = 0; digit < base32Chars.size(); ++digit) /* !!! slow */ - if (base32Chars[digit] == c) break; + for (digit = 0; digit < nix32Chars.size(); ++digit) /* !!! slow */ + if (nix32Chars[digit] == c) break; if (digit >= 32) throw BadHash("invalid base-32 hash '%s'", rest); unsigned int b = n * 5; @@ -258,19 +258,19 @@ Hash::Hash(std::string_view rest, HashType type, bool isSRI) } else - throw BadHash("hash '%s' has wrong length for hash type '%s'", rest, printHashType(this->type)); + throw BadHash("hash '%s' has wrong length for hash algorithm '%s'", rest, printHashAlgo(this->algo)); } -Hash newHashAllowEmpty(std::string_view hashStr, std::optional ht) +Hash newHashAllowEmpty(std::string_view hashStr, std::optional ha) { if (hashStr.empty()) { - if (!ht) + if (!ha) throw BadHash("empty hash requires explicit hash type"); - Hash h(*ht); + Hash h(*ha); warn("found empty hash, assuming '%s'", h.to_string(HashFormat::SRI, true)); return h; } else - return Hash::parseAny(hashStr, ht); + return Hash::parseAny(hashStr, ha); } @@ -283,58 +283,58 @@ union Ctx }; -static void start(HashType ht, Ctx & ctx) +static void start(HashAlgorithm ha, Ctx & ctx) { - if (ht == htMD5) MD5_Init(&ctx.md5); - else if (ht == htSHA1) SHA1_Init(&ctx.sha1); - else if (ht == htSHA256) SHA256_Init(&ctx.sha256); - else if (ht == htSHA512) SHA512_Init(&ctx.sha512); + if (ha == HashAlgorithm::MD5) MD5_Init(&ctx.md5); + else if (ha == HashAlgorithm::SHA1) SHA1_Init(&ctx.sha1); + else if (ha == HashAlgorithm::SHA256) SHA256_Init(&ctx.sha256); + else if (ha == HashAlgorithm::SHA512) SHA512_Init(&ctx.sha512); } -static void update(HashType ht, Ctx & ctx, - std::string_view data) +static void update(HashAlgorithm ha, Ctx & ctx, + std::string_view data) { - if (ht == htMD5) MD5_Update(&ctx.md5, data.data(), data.size()); - else if (ht == htSHA1) SHA1_Update(&ctx.sha1, data.data(), data.size()); - else if (ht == htSHA256) SHA256_Update(&ctx.sha256, data.data(), data.size()); - else if (ht == htSHA512) SHA512_Update(&ctx.sha512, data.data(), data.size()); + if (ha == HashAlgorithm::MD5) MD5_Update(&ctx.md5, data.data(), data.size()); + else if (ha == HashAlgorithm::SHA1) SHA1_Update(&ctx.sha1, data.data(), data.size()); + else if (ha == HashAlgorithm::SHA256) SHA256_Update(&ctx.sha256, data.data(), data.size()); + else if (ha == HashAlgorithm::SHA512) SHA512_Update(&ctx.sha512, data.data(), data.size()); } -static void finish(HashType ht, Ctx & ctx, unsigned char * hash) +static void finish(HashAlgorithm ha, Ctx & ctx, unsigned char * hash) { - if (ht == htMD5) MD5_Final(hash, &ctx.md5); - else if (ht == htSHA1) SHA1_Final(hash, &ctx.sha1); - else if (ht == htSHA256) SHA256_Final(hash, &ctx.sha256); - else if (ht == htSHA512) SHA512_Final(hash, &ctx.sha512); + if (ha == HashAlgorithm::MD5) MD5_Final(hash, &ctx.md5); + else if (ha == HashAlgorithm::SHA1) SHA1_Final(hash, &ctx.sha1); + else if (ha == HashAlgorithm::SHA256) SHA256_Final(hash, &ctx.sha256); + else if (ha == HashAlgorithm::SHA512) SHA512_Final(hash, &ctx.sha512); } -Hash hashString(HashType ht, std::string_view s) +Hash hashString(HashAlgorithm ha, std::string_view s) { Ctx ctx; - Hash hash(ht); - start(ht, ctx); - update(ht, ctx, s); - finish(ht, ctx, hash.hash); + Hash hash(ha); + start(ha, ctx); + update(ha, ctx, s); + finish(ha, ctx, hash.hash); return hash; } -Hash hashFile(HashType ht, const Path & path) +Hash hashFile(HashAlgorithm ha, const Path & path) { - HashSink sink(ht); + HashSink sink(ha); readFile(path, sink); return sink.finish().first; } -HashSink::HashSink(HashType ht) : ht(ht) +HashSink::HashSink(HashAlgorithm ha) : ha(ha) { ctx = new Ctx; bytes = 0; - start(ht, *ctx); + start(ha, *ctx); } HashSink::~HashSink() @@ -346,14 +346,14 @@ HashSink::~HashSink() void HashSink::writeUnbuffered(std::string_view data) { bytes += data.size(); - update(ht, *ctx, data); + update(ha, *ctx, data); } HashResult HashSink::finish() { flush(); - Hash hash(ht); - nix::finish(ht, *ctx, hash.hash); + Hash hash(ha); + nix::finish(ha, *ctx, hash.hash); return HashResult(hash, bytes); } @@ -361,16 +361,16 @@ HashResult HashSink::currentHash() { flush(); Ctx ctx2 = *ctx; - Hash hash(ht); - nix::finish(ht, ctx2, hash.hash); + Hash hash(ha); + nix::finish(ha, ctx2, hash.hash); return HashResult(hash, bytes); } HashResult hashPath( - HashType ht, const Path & path, PathFilter & filter) + HashAlgorithm ha, const Path & path, PathFilter & filter) { - HashSink sink(ht); + HashSink sink(ha); dumpPath(path, sink, filter); return sink.finish(); } @@ -378,7 +378,7 @@ HashResult hashPath( Hash compressHash(const Hash & hash, unsigned int newSize) { - Hash h(hash.type); + Hash h(hash.algo); h.hashSize = newSize; for (unsigned int i = 0; i < hash.hashSize; ++i) h.hash[i % newSize] ^= hash.hash[i]; @@ -389,7 +389,11 @@ Hash compressHash(const Hash & hash, unsigned int newSize) std::optional parseHashFormatOpt(std::string_view hashFormatName) { if (hashFormatName == "base16") return HashFormat::Base16; - if (hashFormatName == "base32") return HashFormat::Base32; + if (hashFormatName == "nix32") return HashFormat::Nix32; + if (hashFormatName == "base32") { + warn(R"("base32" is a deprecated alias for hash format "nix32".)"); + return HashFormat::Nix32; + } if (hashFormatName == "base64") return HashFormat::Base64; if (hashFormatName == "sri") return HashFormat::SRI; return std::nullopt; @@ -408,8 +412,8 @@ std::string_view printHashFormat(HashFormat HashFormat) switch (HashFormat) { case HashFormat::Base64: return "base64"; - case HashFormat::Base32: - return "base32"; + case HashFormat::Nix32: + return "nix32"; case HashFormat::Base16: return "base16"; case HashFormat::SRI: @@ -421,31 +425,31 @@ std::string_view printHashFormat(HashFormat HashFormat) } } -std::optional parseHashTypeOpt(std::string_view s) +std::optional parseHashAlgoOpt(std::string_view s) { - if (s == "md5") return htMD5; - if (s == "sha1") return htSHA1; - if (s == "sha256") return htSHA256; - if (s == "sha512") return htSHA512; + if (s == "md5") return HashAlgorithm::MD5; + if (s == "sha1") return HashAlgorithm::SHA1; + if (s == "sha256") return HashAlgorithm::SHA256; + if (s == "sha512") return HashAlgorithm::SHA512; return std::nullopt; } -HashType parseHashType(std::string_view s) +HashAlgorithm parseHashAlgo(std::string_view s) { - auto opt_h = parseHashTypeOpt(s); + auto opt_h = parseHashAlgoOpt(s); if (opt_h) return *opt_h; else throw UsageError("unknown hash algorithm '%1%', expect 'md5', 'sha1', 'sha256', or 'sha512'", s); } -std::string_view printHashType(HashType ht) +std::string_view printHashAlgo(HashAlgorithm ha) { - switch (ht) { - case htMD5: return "md5"; - case htSHA1: return "sha1"; - case htSHA256: return "sha256"; - case htSHA512: return "sha512"; + switch (ha) { + case HashAlgorithm::MD5: return "md5"; + case HashAlgorithm::SHA1: return "sha1"; + case HashAlgorithm::SHA256: return "sha256"; + case HashAlgorithm::SHA512: return "sha512"; default: // illegal hash type enum value internally, as opposed to external input // which should be validated with nice error message. diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index cab3e6eca..7bed9e2bd 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "serialise.hh" +#include "file-system.hh" namespace nix { @@ -11,7 +12,7 @@ namespace nix { MakeError(BadHash, Error); -enum HashType : char { htMD5 = 42, htSHA1, htSHA256, htSHA512 }; +enum struct HashAlgorithm : char { MD5 = 42, SHA1, SHA256, SHA512 }; const int md5HashSize = 16; @@ -19,9 +20,9 @@ const int sha1HashSize = 20; const int sha256HashSize = 32; const int sha512HashSize = 64; -extern std::set hashTypes; +extern const std::set hashAlgorithms; -extern const std::string base32Chars; +extern const std::string nix32Chars; /** * @brief Enumeration representing the hash formats. @@ -30,8 +31,8 @@ enum struct HashFormat : int { /// @brief Base 64 encoding. /// @see [IETF RFC 4648, section 4](https://datatracker.ietf.org/doc/html/rfc4648#section-4). Base64, - /// @brief Nix-specific base-32 encoding. @see base32Chars - Base32, + /// @brief Nix-specific base-32 encoding. @see nix32Chars + Nix32, /// @brief Lowercase hexadecimal encoding. @see base16Chars Base16, /// @brief ":", format of the SRI integrity attribute. @@ -39,6 +40,7 @@ enum struct HashFormat : int { SRI }; +extern const std::set hashFormats; struct Hash { @@ -46,12 +48,12 @@ struct Hash size_t hashSize = 0; uint8_t hash[maxHashSize] = {}; - HashType type; + HashAlgorithm algo; /** * Create a zero-filled hash object. */ - Hash(HashType type); + explicit Hash(HashAlgorithm algo); /** * Parse the hash from a string representation in the format @@ -60,7 +62,7 @@ struct Hash * is not present, then the hash type must be specified in the * string. */ - static Hash parseAny(std::string_view s, std::optional type); + static Hash parseAny(std::string_view s, std::optional optAlgo); /** * Parse a hash from a string representation like the above, except the @@ -72,7 +74,7 @@ struct Hash * Parse a plain hash that musst not have any prefix indicating the type. * The type is passed in to disambiguate. */ - static Hash parseNonSRIUnprefixed(std::string_view s, HashType type); + static Hash parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo); static Hash parseSRI(std::string_view original); @@ -81,7 +83,7 @@ private: * The type must be provided, the string view must not include * prefix. `isSRI` helps disambigate the various base-* encodings. */ - Hash(std::string_view s, HashType type, bool isSRI); + Hash(std::string_view s, HashAlgorithm algo, bool isSRI); public: /** @@ -102,7 +104,7 @@ public: /** * Returns the length of a base-16 representation of this hash. */ - size_t base16Len() const + [[nodiscard]] size_t base16Len() const { return hashSize * 2; } @@ -110,7 +112,7 @@ public: /** * Returns the length of a base-32 representation of this hash. */ - size_t base32Len() const + [[nodiscard]] size_t base32Len() const { return (hashSize * 8 - 1) / 5 + 1; } @@ -118,24 +120,24 @@ public: /** * Returns the length of a base-64 representation of this hash. */ - size_t base64Len() const + [[nodiscard]] size_t base64Len() const { return ((4 * hashSize / 3) + 3) & ~3; } /** * Return a string representation of the hash, in base-16, base-32 - * or base-64. By default, this is prefixed by the hash type + * or base-64. By default, this is prefixed by the hash algo * (e.g. "sha256:"). */ - std::string to_string(HashFormat hashFormat, bool includeType) const; + [[nodiscard]] std::string to_string(HashFormat hashFormat, bool includeAlgo) const; - std::string gitRev() const + [[nodiscard]] std::string gitRev() const { return to_string(HashFormat::Base16, false); } - std::string gitShortRev() const + [[nodiscard]] std::string gitShortRev() const { return std::string(to_string(HashFormat::Base16, false), 0, 7); } @@ -146,7 +148,7 @@ public: /** * Helper that defaults empty hashes to the 0 hash. */ -Hash newHashAllowEmpty(std::string_view hashStr, std::optional ht); +Hash newHashAllowEmpty(std::string_view hashStr, std::optional ha); /** * Print a hash in base-16 if it's MD5, or base-32 otherwise. @@ -156,14 +158,14 @@ std::string printHash16or32(const Hash & hash); /** * Compute the hash of the given string. */ -Hash hashString(HashType ht, std::string_view s); +Hash hashString(HashAlgorithm ha, std::string_view s); /** * Compute the hash of the given file, hashing its contents directly. * * (Metadata, such as the executable permission bit, is ignored.) */ -Hash hashFile(HashType ht, const Path & path); +Hash hashFile(HashAlgorithm ha, const Path & path); /** * Compute the hash of the given path, serializing as a Nix Archive and @@ -172,8 +174,8 @@ Hash hashFile(HashType ht, const Path & path); * The hash is defined as (essentially) hashString(ht, dumpPath(path)). */ typedef std::pair HashResult; -HashResult hashPath(HashType ht, const Path & path, - PathFilter & filter = defaultPathFilter); +HashResult hashPath(HashAlgorithm ha, const Path & path, + PathFilter & filter = defaultPathFilter); /** * Compress a hash to the specified number of bytes by cyclically @@ -199,17 +201,17 @@ std::string_view printHashFormat(HashFormat hashFormat); /** * Parse a string representing a hash type. */ -HashType parseHashType(std::string_view s); +HashAlgorithm parseHashAlgo(std::string_view s); /** * Will return nothing on parse error */ -std::optional parseHashTypeOpt(std::string_view s); +std::optional parseHashAlgoOpt(std::string_view s); /** * And the reverse. */ -std::string_view printHashType(HashType ht); +std::string_view printHashAlgo(HashAlgorithm ha); union Ctx; @@ -222,12 +224,12 @@ struct AbstractHashSink : virtual Sink class HashSink : public BufferedSink, public AbstractHashSink { private: - HashType ht; + HashAlgorithm ha; Ctx * ctx; uint64_t bytes; public: - HashSink(HashType ht); + HashSink(HashAlgorithm ha); HashSink(const HashSink & h); ~HashSink(); void writeUnbuffered(std::string_view data) override; diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index 77c63595c..06dd80cf7 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -78,20 +78,29 @@ namespace nlohmann { */ template struct adl_serializer> { - static std::optional from_json(const json & json) { + /** + * @brief Convert a JSON type to an `optional` treating + * `null` as `std::nullopt`. + */ + static void from_json(const json & json, std::optional & t) { static_assert( nix::json_avoids_null::value, "null is already in use for underlying type's JSON"); - return json.is_null() + t = json.is_null() ? std::nullopt - : std::optional { adl_serializer::from_json(json) }; + : std::make_optional(json.template get()); } - static void to_json(json & json, std::optional t) { + + /** + * @brief Convert an optional type to a JSON type treating `std::nullopt` + * as `null`. + */ + static void to_json(json & json, const std::optional & t) { static_assert( nix::json_avoids_null::value, "null is already in use for underlying type's JSON"); if (t) - adl_serializer::to_json(json, *t); + json = *t; else json = nullptr; } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 9d7a141b3..60b0865bf 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -1,4 +1,7 @@ #include "logging.hh" +#include "file-descriptor.hh" +#include "environment-variables.hh" +#include "terminal.hh" #include "util.hh" #include "config.hh" diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 5aa6bee95..183f2d8e1 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -23,6 +23,7 @@ typedef enum { actQueryPathInfo = 109, actPostBuildHook = 110, actBuildWaiting = 111, + actFetchTree = 112, } ActivityType; typedef enum { @@ -34,6 +35,7 @@ typedef enum { resProgress = 105, resSetExpected = 106, resPostBuildLogLine = 107, + resFetchStatus = 108, } ResultType; typedef uint64_t ActivityId; diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc new file mode 100644 index 000000000..78a4dd298 --- /dev/null +++ b/src/libutil/memory-source-accessor.cc @@ -0,0 +1,180 @@ +#include "memory-source-accessor.hh" + +namespace nix { + +MemorySourceAccessor::File * +MemorySourceAccessor::open(const CanonPath & path, std::optional create) +{ + File * cur = &root; + + bool newF = false; + + for (std::string_view name : path) + { + auto * curDirP = std::get_if(&cur->raw); + if (!curDirP) + return nullptr; + auto & curDir = *curDirP; + + auto i = curDir.contents.find(name); + if (i == curDir.contents.end()) { + if (!create) + return nullptr; + else { + newF = true; + i = curDir.contents.insert(i, { + std::string { name }, + File::Directory {}, + }); + } + } + cur = &i->second; + } + + if (newF && create) *cur = std::move(*create); + + return cur; +} + +std::string MemorySourceAccessor::readFile(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * r = std::get_if(&f->raw)) + return r->contents; + else + throw Error("file '%s' is not a regular file", path); +} + +bool MemorySourceAccessor::pathExists(const CanonPath & path) +{ + return open(path, std::nullopt); +} + +MemorySourceAccessor::Stat MemorySourceAccessor::File::lstat() const +{ + return std::visit(overloaded { + [](const Regular & r) { + return Stat { + .type = tRegular, + .fileSize = r.contents.size(), + .isExecutable = r.executable, + }; + }, + [](const Directory &) { + return Stat { + .type = tDirectory, + }; + }, + [](const Symlink &) { + return Stat { + .type = tSymlink, + }; + }, + }, this->raw); +} + +std::optional +MemorySourceAccessor::maybeLstat(const CanonPath & path) +{ + const auto * f = open(path, std::nullopt); + return f ? std::optional { f->lstat() } : std::nullopt; +} + +MemorySourceAccessor::DirEntries MemorySourceAccessor::readDirectory(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * d = std::get_if(&f->raw)) { + DirEntries res; + for (auto & [name, file] : d->contents) + res.insert_or_assign(name, file.lstat().type); + return res; + } else + throw Error("file '%s' is not a directory", path); + return {}; +} + +std::string MemorySourceAccessor::readLink(const CanonPath & path) +{ + auto * f = open(path, std::nullopt); + if (!f) + throw Error("file '%s' does not exist", path); + if (auto * s = std::get_if(&f->raw)) + return s->target; + else + throw Error("file '%s' is not a symbolic link", path); +} + +CanonPath MemorySourceAccessor::addFile(CanonPath path, std::string && contents) +{ + auto * f = open(path, File { File::Regular {} }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (auto * r = std::get_if(&f->raw)) + r->contents = std::move(contents); + else + throw Error("file '%s' is not a regular file", path); + + return path; +} + + +using File = MemorySourceAccessor::File; + +void MemorySink::createDirectory(const Path & path) +{ + auto * f = dst.open(CanonPath{path}, File { File::Directory { } }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + + if (!std::holds_alternative(f->raw)) + throw Error("file '%s' is not a directory", path); +}; + +void MemorySink::createRegularFile(const Path & path) +{ + auto * f = dst.open(CanonPath{path}, File { File::Regular {} }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (!(r = std::get_if(&f->raw))) + throw Error("file '%s' is not a regular file", path); +} + +void MemorySink::closeRegularFile() +{ + r = nullptr; +} + +void MemorySink::isExecutable() +{ + assert(r); + r->executable = true; +} + +void MemorySink::preallocateContents(uint64_t len) +{ + assert(r); + r->contents.reserve(len); +} + +void MemorySink::receiveContents(std::string_view data) +{ + assert(r); + r->contents += data; +} + +void MemorySink::createSymlink(const Path & path, const std::string & target) +{ + auto * f = dst.open(CanonPath{path}, File { File::Symlink { } }); + if (!f) + throw Error("file '%s' cannot be made because some parent file is not a directory", path); + if (auto * s = std::get_if(&f->raw)) + s->target = target; + else + throw Error("file '%s' is not a symbolic link", path); +} + +} diff --git a/src/libutil/memory-source-accessor.hh b/src/libutil/memory-source-accessor.hh new file mode 100644 index 000000000..b908f3713 --- /dev/null +++ b/src/libutil/memory-source-accessor.hh @@ -0,0 +1,99 @@ +#include "source-accessor.hh" +#include "fs-sink.hh" +#include "variant-wrapper.hh" + +namespace nix { + +/** + * An source accessor for an in-memory file system. + */ +struct MemorySourceAccessor : virtual SourceAccessor +{ + /** + * In addition to being part of the implementation of + * `MemorySourceAccessor`, this has a side benefit of nicely + * defining what a "file system object" is in Nix. + */ + struct File { + struct Regular { + bool executable = false; + std::string contents; + + GENERATE_CMP(Regular, me->executable, me->contents); + }; + + struct Directory { + using Name = std::string; + + std::map> contents; + + GENERATE_CMP(Directory, me->contents); + }; + + struct Symlink { + std::string target; + + GENERATE_CMP(Symlink, me->target); + }; + + using Raw = std::variant; + Raw raw; + + MAKE_WRAPPER_CONSTRUCTOR(File); + + GENERATE_CMP(File, me->raw); + + Stat lstat() const; + }; + + File root { File::Directory {} }; + + GENERATE_CMP(MemorySourceAccessor, me->root); + + std::string readFile(const CanonPath & path) override; + bool pathExists(const CanonPath & path) override; + std::optional maybeLstat(const CanonPath & path) override; + DirEntries readDirectory(const CanonPath & path) override; + std::string readLink(const CanonPath & path) override; + + /** + * @param create If present, create this file and any parent directories + * that are needed. + * + * Return null if + * + * - `create = false`: File does not exist. + * + * - `create = true`: some parent file was not a dir, so couldn't + * look/create inside. + */ + File * open(const CanonPath & path, std::optional create); + + CanonPath addFile(CanonPath path, std::string && contents); +}; + +/** + * Write to a `MemorySourceAccessor` at the given path + */ +struct MemorySink : ParseSink +{ + MemorySourceAccessor & dst; + + MemorySink(MemorySourceAccessor & dst) : dst(dst) { } + + void createDirectory(const Path & path) override; + + void createRegularFile(const Path & path) override; + void receiveContents(std::string_view data) override; + void isExecutable() override; + void closeRegularFile() override; + + void createSymlink(const Path & path, const std::string & target) override; + + void preallocateContents(uint64_t size) override; + +private: + MemorySourceAccessor::File::Regular * r; +}; + +} diff --git a/src/libutil/monitor-fd.hh b/src/libutil/monitor-fd.hh index 86d0115fc..228fb13f8 100644 --- a/src/libutil/monitor-fd.hh +++ b/src/libutil/monitor-fd.hh @@ -10,6 +10,8 @@ #include #include +#include "signals.hh" + namespace nix { diff --git a/src/libutil/namespaces.cc b/src/libutil/namespaces.cc index f66accb10..a789b321e 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/namespaces.cc @@ -1,13 +1,22 @@ -#if __linux__ - -#include "namespaces.hh" +#include "current-process.hh" #include "util.hh" #include "finally.hh" +#include "file-system.hh" +#include "processes.hh" +#include "signals.hh" + +#if __linux__ +# include +# include +# include "cgroup.hh" +#endif #include namespace nix { +#if __linux__ + bool userNamespacesSupported() { static auto res = [&]() -> bool @@ -92,6 +101,60 @@ bool mountAndPidNamespacesSupported() return res; } +#endif + + +////////////////////////////////////////////////////////////////////// + +#if __linux__ +static AutoCloseFD fdSavedMountNamespace; +static AutoCloseFD fdSavedRoot; +#endif + +void saveMountNamespace() +{ +#if __linux__ + static std::once_flag done; + std::call_once(done, []() { + fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); + if (!fdSavedMountNamespace) + throw SysError("saving parent mount namespace"); + + fdSavedRoot = open("/proc/self/root", O_RDONLY); + }); +#endif } +void restoreMountNamespace() +{ +#if __linux__ + try { + auto savedCwd = absPath("."); + + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + throw SysError("restoring parent mount namespace"); + + if (fdSavedRoot) { + if (fchdir(fdSavedRoot.get())) + throw SysError("chdir into saved root"); + if (chroot(".")) + throw SysError("chroot into saved root"); + } + + if (chdir(savedCwd.c_str()) == -1) + throw SysError("restoring cwd"); + } catch (Error & e) { + debug(e.msg()); + } #endif +} + +void unshareFilesystem() +{ +#ifdef __linux__ + if (unshare(CLONE_FS) != 0 && errno != EPERM) + throw SysError("unsharing filesystem state in download thread"); +#endif +} + +} diff --git a/src/libutil/namespaces.hh b/src/libutil/namespaces.hh index 0b7eeb66c..7e4e921a8 100644 --- a/src/libutil/namespaces.hh +++ b/src/libutil/namespaces.hh @@ -1,8 +1,31 @@ #pragma once ///@file +#include + +#include "types.hh" + namespace nix { +/** + * Save the current mount namespace. Ignored if called more than + * once. + */ +void saveMountNamespace(); + +/** + * Restore the mount namespace saved by saveMountNamespace(). Ignored + * if saveMountNamespace() was never called. + */ +void restoreMountNamespace(); + +/** + * Cause this thread to not share any FS attributes with the main + * thread, because this causes setns() in restoreMountNamespace() to + * fail. + */ +void unshareFilesystem(); + #if __linux__ bool userNamespacesSupported(); diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 48b4fe626..15ff76e59 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -1,4 +1,8 @@ #include "posix-source-accessor.hh" +#include "signals.hh" +#include "sync.hh" + +#include namespace nix { @@ -7,9 +11,9 @@ void PosixSourceAccessor::readFile( Sink & sink, std::function sizeCallback) { - // FIXME: add O_NOFOLLOW since symlinks should be resolved by the - // caller? - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); + assertNoSymlinks(path); + + AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC | O_NOFOLLOW); if (!fd) throw SysError("opening file '%1%'", path); @@ -41,25 +45,55 @@ void PosixSourceAccessor::readFile( bool PosixSourceAccessor::pathExists(const CanonPath & path) { + if (auto parent = path.parent()) assertNoSymlinks(*parent); return nix::pathExists(path.abs()); } -SourceAccessor::Stat PosixSourceAccessor::lstat(const CanonPath & path) +std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path) { - auto st = nix::lstat(path.abs()); - mtime = std::max(mtime, st.st_mtime); + static Sync>> _cache; + + { + auto cache(_cache.lock()); + auto i = cache->find(path); + if (i != cache->end()) return i->second; + } + + std::optional st{std::in_place}; + if (::lstat(path.c_str(), &*st)) { + if (errno == ENOENT || errno == ENOTDIR) + st.reset(); + else + throw SysError("getting status of '%s'", showPath(path)); + } + + auto cache(_cache.lock()); + if (cache->size() >= 16384) cache->clear(); + cache->emplace(path, st); + + return st; +} + +std::optional PosixSourceAccessor::maybeLstat(const CanonPath & path) +{ + if (auto parent = path.parent()) assertNoSymlinks(*parent); + auto st = cachedLstat(path); + if (!st) return std::nullopt; + mtime = std::max(mtime, st->st_mtime); return Stat { .type = - S_ISREG(st.st_mode) ? tRegular : - S_ISDIR(st.st_mode) ? tDirectory : - S_ISLNK(st.st_mode) ? tSymlink : + S_ISREG(st->st_mode) ? tRegular : + S_ISDIR(st->st_mode) ? tDirectory : + S_ISLNK(st->st_mode) ? tSymlink : tMisc, - .isExecutable = S_ISREG(st.st_mode) && st.st_mode & S_IXUSR + .fileSize = S_ISREG(st->st_mode) ? std::optional(st->st_size) : std::nullopt, + .isExecutable = S_ISREG(st->st_mode) && st->st_mode & S_IXUSR, }; } SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & path) { + assertNoSymlinks(path); DirEntries res; for (auto & entry : nix::readDirectory(path.abs())) { std::optional type; @@ -75,6 +109,7 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & std::string PosixSourceAccessor::readLink(const CanonPath & path) { + if (auto parent = path.parent()) assertNoSymlinks(*parent); return nix::readLink(path.abs()); } @@ -83,4 +118,14 @@ std::optional PosixSourceAccessor::getPhysicalPath(const CanonPath & return path; } +void PosixSourceAccessor::assertNoSymlinks(CanonPath path) +{ + while (!path.isRoot()) { + auto st = cachedLstat(path); + if (st && S_ISLNK(st->st_mode)) + throw Error("path '%s' is a symlink", showPath(path)); + path.pop(); + } +} + } diff --git a/src/libutil/posix-source-accessor.hh b/src/libutil/posix-source-accessor.hh index 608f96ee2..b2bd39805 100644 --- a/src/libutil/posix-source-accessor.hh +++ b/src/libutil/posix-source-accessor.hh @@ -7,7 +7,7 @@ namespace nix { /** * A source accessor that uses the Unix filesystem. */ -struct PosixSourceAccessor : SourceAccessor +struct PosixSourceAccessor : virtual SourceAccessor { /** * The most recent mtime seen by lstat(). This is a hack to @@ -22,13 +22,22 @@ struct PosixSourceAccessor : SourceAccessor bool pathExists(const CanonPath & path) override; - Stat lstat(const CanonPath & path) override; + std::optional maybeLstat(const CanonPath & path) override; DirEntries readDirectory(const CanonPath & path) override; std::string readLink(const CanonPath & path) override; std::optional getPhysicalPath(const CanonPath & path) override; + +private: + + /** + * Throw an error if `path` or any of its ancestors are symlinks. + */ + void assertNoSymlinks(CanonPath path); + + std::optional cachedLstat(const CanonPath & path); }; } diff --git a/src/libutil/processes.cc b/src/libutil/processes.cc new file mode 100644 index 000000000..91a0ea66f --- /dev/null +++ b/src/libutil/processes.cc @@ -0,0 +1,421 @@ +#include "current-process.hh" +#include "environment-variables.hh" +#include "signals.hh" +#include "processes.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#ifdef __APPLE__ +# include +#endif + +#ifdef __linux__ +# include +# include +#endif + + +namespace nix { + +Pid::Pid() +{ +} + + +Pid::Pid(pid_t pid) + : pid(pid) +{ +} + + +Pid::~Pid() +{ + if (pid != -1) kill(); +} + + +void Pid::operator =(pid_t pid) +{ + if (this->pid != -1 && this->pid != pid) kill(); + this->pid = pid; + killSignal = SIGKILL; // reset signal to default +} + + +Pid::operator pid_t() +{ + return pid; +} + + +int Pid::kill() +{ + assert(pid != -1); + + debug("killing process %1%", pid); + + /* Send the requested signal to the child. If it has its own + process group, send the signal to every process in the child + process group (which hopefully includes *all* its children). */ + if (::kill(separatePG ? -pid : pid, killSignal) != 0) { + /* On BSDs, killing a process group will return EPERM if all + processes in the group are zombies (or something like + that). So try to detect and ignore that situation. */ +#if __FreeBSD__ || __APPLE__ + if (errno != EPERM || ::kill(pid, 0) != 0) +#endif + logError(SysError("killing process %d", pid).info()); + } + + return wait(); +} + + +int Pid::wait() +{ + assert(pid != -1); + while (1) { + int status; + int res = waitpid(pid, &status, 0); + if (res == pid) { + pid = -1; + return status; + } + if (errno != EINTR) + throw SysError("cannot get exit status of PID %d", pid); + checkInterrupt(); + } +} + + +void Pid::setSeparatePG(bool separatePG) +{ + this->separatePG = separatePG; +} + + +void Pid::setKillSignal(int signal) +{ + this->killSignal = signal; +} + + +pid_t Pid::release() +{ + pid_t p = pid; + pid = -1; + return p; +} + + +void killUser(uid_t uid) +{ + debug("killing all processes running under uid '%1%'", uid); + + assert(uid != 0); /* just to be safe... */ + + /* The system call kill(-1, sig) sends the signal `sig' to all + users to which the current process can send signals. So we + fork a process, switch to uid, and send a mass kill. */ + + Pid pid = startProcess([&]() { + + if (setuid(uid) == -1) + throw SysError("setting uid"); + + while (true) { +#ifdef __APPLE__ + /* OSX's kill syscall takes a third parameter that, among + other things, determines if kill(-1, signo) affects the + calling process. In the OSX libc, it's set to true, + which means "follow POSIX", which we don't want here + */ + if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; +#else + if (kill(-1, SIGKILL) == 0) break; +#endif + if (errno == ESRCH || errno == EPERM) break; /* no more processes */ + if (errno != EINTR) + throw SysError("cannot kill processes for uid '%1%'", uid); + } + + _exit(0); + }); + + int status = pid.wait(); + if (status != 0) + throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); + + /* !!! We should really do some check to make sure that there are + no processes left running under `uid', but there is no portable + way to do so (I think). The most reliable way may be `ps -eo + uid | grep -q $uid'. */ +} + + +////////////////////////////////////////////////////////////////////// + + +/* Wrapper around vfork to prevent the child process from clobbering + the caller's stack frame in the parent. */ +static pid_t doFork(bool allowVfork, std::function fun) __attribute__((noinline)); +static pid_t doFork(bool allowVfork, std::function fun) +{ +#ifdef __linux__ + pid_t pid = allowVfork ? vfork() : fork(); +#else + pid_t pid = fork(); +#endif + if (pid != 0) return pid; + fun(); + abort(); +} + + +#if __linux__ +static int childEntry(void * arg) +{ + auto main = (std::function *) arg; + (*main)(); + return 1; +} +#endif + + +pid_t startProcess(std::function fun, const ProcessOptions & options) +{ + std::function wrapper = [&]() { + if (!options.allowVfork) + logger = makeSimpleLogger(); + try { +#if __linux__ + if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) + throw SysError("setting death signal"); +#endif + fun(); + } catch (std::exception & e) { + try { + std::cerr << options.errorPrefix << e.what() << "\n"; + } catch (...) { } + } catch (...) { } + if (options.runExitHandlers) + exit(1); + else + _exit(1); + }; + + pid_t pid = -1; + + if (options.cloneFlags) { + #ifdef __linux__ + // Not supported, since then we don't know when to free the stack. + assert(!(options.cloneFlags & CLONE_VM)); + + size_t stackSize = 1 * 1024 * 1024; + auto stack = (char *) mmap(0, stackSize, + PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); + if (stack == MAP_FAILED) throw SysError("allocating stack"); + + Finally freeStack([&]() { munmap(stack, stackSize); }); + + pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); + #else + throw Error("clone flags are only supported on Linux"); + #endif + } else + pid = doFork(options.allowVfork, wrapper); + + if (pid == -1) throw SysError("unable to fork"); + + return pid; +} + + +std::string runProgram(Path program, bool searchPath, const Strings & args, + const std::optional & input, bool isInteractive) +{ + auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive}); + + if (!statusOk(res.first)) + throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); + + return res.second; +} + +// Output = error code + "standard out" output stream +std::pair runProgram(RunOptions && options) +{ + StringSink sink; + options.standardOut = &sink; + + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(sink.s)}; +} + +void runProgram2(const RunOptions & options) +{ + checkInterrupt(); + + assert(!(options.standardIn && options.input)); + + std::unique_ptr source_; + Source * source = options.standardIn; + + if (options.input) { + source_ = std::make_unique(*options.input); + source = source_.get(); + } + + /* Create a pipe. */ + Pipe out, in; + if (options.standardOut) out.create(); + if (source) in.create(); + + ProcessOptions processOptions; + // vfork implies that the environment of the main process and the fork will + // be shared (technically this is undefined, but in practice that's the + // case), so we can't use it if we alter the environment + processOptions.allowVfork = !options.environment; + + std::optional>> resumeLoggerDefer; + if (options.isInteractive) { + logger->pause(); + resumeLoggerDefer.emplace( + []() { + logger->resume(); + } + ); + } + + /* Fork. */ + Pid pid = startProcess([&]() { + if (options.environment) + replaceEnv(*options.environment); + if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) + throw SysError("dupping stdout"); + if (options.mergeStderrToStdout) + if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) + throw SysError("cannot dup stdout into stderr"); + if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) + throw SysError("dupping stdin"); + + if (options.chdir && chdir((*options.chdir).c_str()) == -1) + throw SysError("chdir failed"); + if (options.gid && setgid(*options.gid) == -1) + throw SysError("setgid failed"); + /* Drop all other groups if we're setgid. */ + if (options.gid && setgroups(0, 0) == -1) + throw SysError("setgroups failed"); + if (options.uid && setuid(*options.uid) == -1) + throw SysError("setuid failed"); + + Strings args_(options.args); + args_.push_front(options.program); + + restoreProcessContext(); + + if (options.searchPath) + execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + // This allows you to refer to a program with a pathname relative + // to the PATH variable. + else + execv(options.program.c_str(), stringsToCharPtrs(args_).data()); + + throw SysError("executing '%1%'", options.program); + }, processOptions); + + out.writeSide.close(); + + std::thread writerThread; + + std::promise promise; + + Finally doJoin([&]() { + if (writerThread.joinable()) + writerThread.join(); + }); + + + if (source) { + in.readSide.close(); + writerThread = std::thread([&]() { + try { + std::vector buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + writeFull(in.writeSide.get(), {buf.data(), n}); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide.close(); + }); + } + + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); + + /* Wait for the child to finish. */ + int status = pid.wait(); + + /* Wait for the writer thread to finish. */ + if (source) promise.get_future().get(); + + if (status) + throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); +} + +////////////////////////////////////////////////////////////////////// + +std::string statusToString(int status) +{ + if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { + if (WIFEXITED(status)) + return fmt("failed with exit code %1%", WEXITSTATUS(status)); + else if (WIFSIGNALED(status)) { + int sig = WTERMSIG(status); +#if HAVE_STRSIGNAL + const char * description = strsignal(sig); + return fmt("failed due to signal %1% (%2%)", sig, description); +#else + return fmt("failed due to signal %1%", sig); +#endif + } + else + return "died abnormally"; + } else return "succeeded"; +} + + +bool statusOk(int status) +{ + return WIFEXITED(status) && WEXITSTATUS(status) == 0; +} + +} diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh new file mode 100644 index 000000000..978c37105 --- /dev/null +++ b/src/libutil/processes.hh @@ -0,0 +1,123 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "ansicolor.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace nix { + +struct Sink; +struct Source; + +class Pid +{ + pid_t pid = -1; + bool separatePG = false; + int killSignal = SIGKILL; +public: + Pid(); + Pid(pid_t pid); + ~Pid(); + void operator =(pid_t pid); + operator pid_t(); + int kill(); + int wait(); + + void setSeparatePG(bool separatePG); + void setKillSignal(int signal); + pid_t release(); +}; + + +/** + * Kill all processes running under the specified uid by sending them + * a SIGKILL. + */ +void killUser(uid_t uid); + + +/** + * Fork a process that runs the given function, and return the child + * pid to the caller. + */ +struct ProcessOptions +{ + std::string errorPrefix = ""; + bool dieWithParent = true; + bool runExitHandlers = false; + bool allowVfork = false; + /** + * use clone() with the specified flags (Linux only) + */ + int cloneFlags = 0; +}; + +pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); + + +/** + * Run a program and return its stdout in a string (i.e., like the + * shell backtick operator). + */ +std::string runProgram(Path program, bool searchPath = false, + const Strings & args = Strings(), + const std::optional & input = {}, bool isInteractive = false); + +struct RunOptions +{ + Path program; + bool searchPath = true; + Strings args; + std::optional uid; + std::optional gid; + std::optional chdir; + std::optional> environment; + std::optional input; + Source * standardIn = nullptr; + Sink * standardOut = nullptr; + bool mergeStderrToStdout = false; + bool isInteractive = false; +}; + +std::pair runProgram(RunOptions && options); + +void runProgram2(const RunOptions & options); + + +class ExecError : public Error +{ +public: + int status; + + template + ExecError(int status, const Args & ... args) + : Error(args...), status(status) + { } +}; + + +/** + * Convert the exit status of a child as returned by wait() into an + * error string. + */ +std::string statusToString(int status); + +bool statusOk(int status); + +} diff --git a/src/libutil/references.cc b/src/libutil/references.cc index 7f59b4c09..b30e62c7b 100644 --- a/src/libutil/references.cc +++ b/src/libutil/references.cc @@ -1,6 +1,5 @@ #include "references.hh" #include "hash.hh" -#include "util.hh" #include "archive.hh" #include @@ -24,8 +23,8 @@ static void search( static bool isBase32[256]; std::call_once(initialised, [](){ for (unsigned int i = 0; i < 256; ++i) isBase32[i] = false; - for (unsigned int i = 0; i < base32Chars.size(); ++i) - isBase32[(unsigned char) base32Chars[i]] = true; + for (unsigned int i = 0; i < nix32Chars.size(); ++i) + isBase32[(unsigned char) nix32Chars[i]] = true; }); for (size_t i = 0; i + refLength <= s.size(); ) { @@ -111,8 +110,8 @@ void RewritingSink::flush() prev.clear(); } -HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus) - : hashSink(ht) +HashModuloSink::HashModuloSink(HashAlgorithm ha, const std::string & modulus) + : hashSink(ha) , rewritingSink(modulus, std::string(modulus.size(), 0), hashSink) { } diff --git a/src/libutil/references.hh b/src/libutil/references.hh index f0baeffe1..8bc9f7ec9 100644 --- a/src/libutil/references.hh +++ b/src/libutil/references.hh @@ -46,7 +46,7 @@ struct HashModuloSink : AbstractHashSink HashSink hashSink; RewritingSink rewritingSink; - HashModuloSink(HashType ht, const std::string & modulus); + HashModuloSink(HashAlgorithm ha, const std::string & modulus); void operator () (std::string_view data) override; diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 3d5121a19..f465bd0de 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -1,5 +1,5 @@ #include "serialise.hh" -#include "util.hh" +#include "signals.hh" #include #include @@ -74,6 +74,10 @@ void Source::operator () (char * data, size_t len) } } +void Source::operator () (std::string_view data) +{ + (*this)((char *)data.data(), data.size()); +} void Source::drainInto(Sink & sink) { @@ -444,7 +448,7 @@ Error readError(Source & source) auto msg = readString(source); ErrorInfo info { .level = level, - .msg = hintformat(fmt("%s", msg)), + .msg = hintfmt(msg), }; auto havePos = readNum(source); assert(havePos == 0); @@ -453,7 +457,7 @@ Error readError(Source & source) havePos = readNum(source); assert(havePos == 0); info.traces.push_back(Trace { - .hint = hintformat(fmt("%s", readString(source))) + .hint = hintfmt(readString(source)) }); } return Error(std::move(info)); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 333c254ea..3f57ce88b 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -5,6 +5,7 @@ #include "types.hh" #include "util.hh" +#include "file-descriptor.hh" namespace boost::context { struct stack_context; } @@ -72,6 +73,7 @@ struct Source * an error if it is not going to be available. */ void operator () (char * data, size_t len); + void operator () (std::string_view data); /** * Store up to ‘len’ in the buffer pointed to by ‘data’, and diff --git a/src/libutil/signals.cc b/src/libutil/signals.cc new file mode 100644 index 000000000..4632aa319 --- /dev/null +++ b/src/libutil/signals.cc @@ -0,0 +1,188 @@ +#include "signals.hh" +#include "util.hh" +#include "error.hh" +#include "sync.hh" +#include "terminal.hh" + +#include + +namespace nix { + +std::atomic _isInterrupted = false; + +static thread_local bool interruptThrown = false; +thread_local std::function interruptCheck; + +void setInterruptThrown() +{ + interruptThrown = true; +} + +void _interrupted() +{ + /* Block user interrupts while an exception is being handled. + Throwing an exception while another exception is being handled + kills the program! */ + if (!interruptThrown && !std::uncaught_exceptions()) { + interruptThrown = true; + throw Interrupted("interrupted by the user"); + } +} + + +////////////////////////////////////////////////////////////////////// + + +/* We keep track of interrupt callbacks using integer tokens, so we can iterate + safely without having to lock the data structure while executing arbitrary + functions. + */ +struct InterruptCallbacks { + typedef int64_t Token; + + /* We use unique tokens so that we can't accidentally delete the wrong + handler because of an erroneous double delete. */ + Token nextToken = 0; + + /* Used as a list, see InterruptCallbacks comment. */ + std::map> callbacks; +}; + +static Sync _interruptCallbacks; + +static void signalHandlerThread(sigset_t set) +{ + while (true) { + int signal = 0; + sigwait(&set, &signal); + + if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) + triggerInterrupt(); + + else if (signal == SIGWINCH) { + updateWindowSize(); + } + } +} + +void triggerInterrupt() +{ + _isInterrupted = true; + + { + InterruptCallbacks::Token i = 0; + while (true) { + std::function callback; + { + auto interruptCallbacks(_interruptCallbacks.lock()); + auto lb = interruptCallbacks->callbacks.lower_bound(i); + if (lb == interruptCallbacks->callbacks.end()) + break; + + callback = lb->second; + i = lb->first + 1; + } + + try { + callback(); + } catch (...) { + ignoreException(); + } + } + } +} + + +static sigset_t savedSignalMask; +static bool savedSignalMaskIsSet = false; + +void setChildSignalMask(sigset_t * sigs) +{ + assert(sigs); // C style function, but think of sigs as a reference + +#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE + sigemptyset(&savedSignalMask); + // There's no "assign" or "copy" function, so we rely on (math) idempotence + // of the or operator: a or a = a. + sigorset(&savedSignalMask, sigs, sigs); +#else + // Without sigorset, our best bet is to assume that sigset_t is a type that + // can be assigned directly, such as is the case for a sigset_t defined as + // an integer type. + savedSignalMask = *sigs; +#endif + + savedSignalMaskIsSet = true; +} + +void saveSignalMask() { + if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) + throw SysError("querying signal mask"); + + savedSignalMaskIsSet = true; +} + +void startSignalHandlerThread() +{ + updateWindowSize(); + + saveSignalMask(); + + sigset_t set; + sigemptyset(&set); + sigaddset(&set, SIGINT); + sigaddset(&set, SIGTERM); + sigaddset(&set, SIGHUP); + sigaddset(&set, SIGPIPE); + sigaddset(&set, SIGWINCH); + if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) + throw SysError("blocking signals"); + + std::thread(signalHandlerThread, set).detach(); +} + +void restoreSignals() +{ + // If startSignalHandlerThread wasn't called, that means we're not running + // in a proper libmain process, but a process that presumably manages its + // own signal handlers. Such a process should call either + // - initNix(), to be a proper libmain process + // - startSignalHandlerThread(), to resemble libmain regarding signal + // handling only + // - saveSignalMask(), for processes that define their own signal handling + // thread + // TODO: Warn about this? Have a default signal mask? The latter depends on + // whether we should generally inherit signal masks from the caller. + // I don't know what the larger unix ecosystem expects from us here. + if (!savedSignalMaskIsSet) + return; + + if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) + throw SysError("restoring signals"); +} + + +/* RAII helper to automatically deregister a callback. */ +struct InterruptCallbackImpl : InterruptCallback +{ + InterruptCallbacks::Token token; + ~InterruptCallbackImpl() override + { + auto interruptCallbacks(_interruptCallbacks.lock()); + interruptCallbacks->callbacks.erase(token); + } +}; + +std::unique_ptr createInterruptCallback(std::function callback) +{ + auto interruptCallbacks(_interruptCallbacks.lock()); + auto token = interruptCallbacks->nextToken++; + interruptCallbacks->callbacks.emplace(token, callback); + + auto res = std::make_unique(); + res->token = token; + + return std::unique_ptr(res.release()); +} + +} diff --git a/src/libutil/signals.hh b/src/libutil/signals.hh new file mode 100644 index 000000000..7e8beff33 --- /dev/null +++ b/src/libutil/signals.hh @@ -0,0 +1,104 @@ +#pragma once +///@file + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "ansicolor.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace nix { + +/* User interruption. */ + +extern std::atomic _isInterrupted; + +extern thread_local std::function interruptCheck; + +void setInterruptThrown(); + +void _interrupted(); + +void inline checkInterrupt() +{ + if (_isInterrupted || (interruptCheck && interruptCheck())) + _interrupted(); +} + +MakeError(Interrupted, BaseError); + + +/** + * Start a thread that handles various signals. Also block those signals + * on the current thread (and thus any threads created by it). + * Saves the signal mask before changing the mask to block those signals. + * See saveSignalMask(). + */ +void startSignalHandlerThread(); + +/** + * Saves the signal mask, which is the signal mask that nix will restore + * before creating child processes. + * See setChildSignalMask() to set an arbitrary signal mask instead of the + * current mask. + */ +void saveSignalMask(); + +/** + * To use in a process that already called `startSignalHandlerThread()` + * or `saveSignalMask()` first. + */ +void restoreSignals(); + +/** + * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't + * necessarily match the current thread's mask. + * See saveSignalMask() to set the saved mask to the current mask. + */ +void setChildSignalMask(sigset_t *sigs); + +struct InterruptCallback +{ + virtual ~InterruptCallback() { }; +}; + +/** + * Register a function that gets called on SIGINT (in a non-signal + * context). + */ +std::unique_ptr createInterruptCallback( + std::function callback); + +void triggerInterrupt(); + +/** + * A RAII class that causes the current thread to receive SIGUSR1 when + * the signal handler thread receives SIGINT. That is, this allows + * SIGINT to be multiplexed to multiple threads. + */ +struct ReceiveInterrupts +{ + pthread_t target; + std::unique_ptr callback; + + ReceiveInterrupts() + : target(pthread_self()) + , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) + { } +}; + + +} diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index d168a9667..afbbbe1a9 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -7,9 +7,15 @@ static std::atomic nextNumber{0}; SourceAccessor::SourceAccessor() : number(++nextNumber) + , displayPrefix{"«unknown»"} { } +bool SourceAccessor::pathExists(const CanonPath & path) +{ + return maybeLstat(path).has_value(); +} + std::string SourceAccessor::readFile(const CanonPath & path) { StringSink sink; @@ -33,26 +39,32 @@ void SourceAccessor::readFile( } Hash SourceAccessor::hashPath( - const CanonPath & path, - PathFilter & filter, - HashType ht) + const CanonPath & path, + PathFilter & filter, + HashAlgorithm ha) { - HashSink sink(ht); + HashSink sink(ha); dumpPath(path, sink, filter); return sink.finish().first; } -std::optional SourceAccessor::maybeLstat(const CanonPath & path) +SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path) { - // FIXME: merge these into one operation. - if (!pathExists(path)) - return {}; - return lstat(path); + if (auto st = maybeLstat(path)) + return *st; + else + throw Error("path '%s' does not exist", showPath(path)); +} + +void SourceAccessor::setPathDisplay(std::string displayPrefix, std::string displaySuffix) +{ + this->displayPrefix = std::move(displayPrefix); + this->displaySuffix = std::move(displaySuffix); } std::string SourceAccessor::showPath(const CanonPath & path) { - return path.abs(); + return displayPrefix + path.abs() + displaySuffix; } } diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index fd823aa39..4f4ff09c1 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -17,6 +17,8 @@ struct SourceAccessor { const size_t number; + std::string displayPrefix, displaySuffix; + SourceAccessor(); virtual ~SourceAccessor() @@ -24,6 +26,13 @@ struct SourceAccessor /** * Return the contents of a file as a string. + * + * @note Unlike Unix, this method should *not* follow symlinks. Nix + * by default wants to manipulate symlinks explicitly, and not + * implictly follow them, as they are frequently untrusted user data + * and thus may point to arbitrary locations. Acting on the targets + * targets of symlinks should only occasionally be done, and only + * with care. */ virtual std::string readFile(const CanonPath & path); @@ -32,7 +41,10 @@ struct SourceAccessor * called with the size of the file before any data is written to * the sink. * - * Note: subclasses of `SourceAccessor` need to implement at least + * @note Like the other `readFile`, this method should *not* follow + * symlinks. + * + * @note subclasses of `SourceAccessor` need to implement at least * one of the `readFile()` variants. */ virtual void readFile( @@ -40,7 +52,7 @@ struct SourceAccessor Sink & sink, std::function sizeCallback = [](uint64_t size){}); - virtual bool pathExists(const CanonPath & path) = 0; + virtual bool pathExists(const CanonPath & path); enum Type { tRegular, tSymlink, tDirectory, @@ -57,18 +69,37 @@ struct SourceAccessor struct Stat { Type type = tMisc; - //uint64_t fileSize = 0; // regular files only - bool isExecutable = false; // regular files only + + /** + * For regular files only: the size of the file. Not all + * accessors return this since it may be too expensive to + * compute. + */ + std::optional fileSize; + + /** + * For regular files only: whether this is an executable. + */ + bool isExecutable = false; + + /** + * For regular files only: the position of the contents of this + * file in the NAR. Only returned by NAR accessors. + */ + std::optional narOffset; }; - virtual Stat lstat(const CanonPath & path) = 0; + Stat lstat(const CanonPath & path); - std::optional maybeLstat(const CanonPath & path); + virtual std::optional maybeLstat(const CanonPath & path) = 0; typedef std::optional DirEntry; typedef std::map DirEntries; + /** + * @note Like `readFile`, this method should *not* follow symlinks. + */ virtual DirEntries readDirectory(const CanonPath & path) = 0; virtual std::string readLink(const CanonPath & path) = 0; @@ -79,9 +110,9 @@ struct SourceAccessor PathFilter & filter = defaultPathFilter); Hash hashPath( - const CanonPath & path, - PathFilter & filter = defaultPathFilter, - HashType ht = htSHA256); + const CanonPath & path, + PathFilter & filter = defaultPathFilter, + HashAlgorithm ha = HashAlgorithm::SHA256); /** * Return a corresponding path in the root filesystem, if @@ -101,6 +132,8 @@ struct SourceAccessor return number < x.number; } + void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); + virtual std::string showPath(const CanonPath & path); }; diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index 9510a5f0c..e67e986fb 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -1,7 +1,9 @@ #include "suggestions.hh" #include "ansicolor.hh" -#include "util.hh" +#include "terminal.hh" + #include +#include namespace nix { diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 5060a8f24..187b3e948 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -3,6 +3,7 @@ #include "serialise.hh" #include "tarfile.hh" +#include "file-system.hh" namespace nix { @@ -52,6 +53,7 @@ TarArchive::TarArchive(Source & source, bool raw) : buffer(65536) archive_read_support_format_raw(archive); archive_read_support_format_empty(archive); } + archive_read_set_option(archive, NULL, "mac-ext", NULL); check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)"); } @@ -62,6 +64,7 @@ TarArchive::TarArchive(const Path & path) archive_read_support_filter_all(archive); archive_read_support_format_all(archive); + archive_read_set_option(archive, NULL, "mac-ext", NULL); check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); } diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc new file mode 100644 index 000000000..8febc8771 --- /dev/null +++ b/src/libutil/terminal.cc @@ -0,0 +1,108 @@ +#include "terminal.hh" +#include "environment-variables.hh" +#include "sync.hh" + +#include +#include + +namespace nix { + +bool shouldANSI() +{ + return isatty(STDERR_FILENO) + && getEnv("TERM").value_or("dumb") != "dumb" + && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value()); +} + +std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) +{ + std::string t, e; + size_t w = 0; + auto i = s.begin(); + + while (w < (size_t) width && i != s.end()) { + + if (*i == '\e') { + std::string e; + e += *i++; + char last = 0; + + if (i != s.end() && *i == '[') { + e += *i++; + // eat parameter bytes + while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; + // eat intermediate bytes + while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; + // eat final byte + if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; + } else { + if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; + } + + if (!filterAll && last == 'm') + t += e; + } + + else if (*i == '\t') { + i++; t += ' '; w++; + while (w < (size_t) width && w % 8) { + t += ' '; w++; + } + } + + else if (*i == '\r' || *i == '\a') + // do nothing for now + i++; + + else { + w++; + // Copy one UTF-8 character. + if ((*i & 0xe0) == 0xc0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } else if ((*i & 0xf0) == 0xe0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } else if ((*i & 0xf8) == 0xf0) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) { + t += *i++; + if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; + } + } + } else + t += *i++; + } + } + + return t; +} + + +////////////////////////////////////////////////////////////////////// + +static Sync> windowSize{{0, 0}}; + + +void updateWindowSize() +{ + struct winsize ws; + if (ioctl(2, TIOCGWINSZ, &ws) == 0) { + auto windowSize_(windowSize.lock()); + windowSize_->first = ws.ws_row; + windowSize_->second = ws.ws_col; + } +} + + +std::pair getWindowSize() +{ + return *windowSize.lock(); +} + +} diff --git a/src/libutil/terminal.hh b/src/libutil/terminal.hh new file mode 100644 index 000000000..9cb191308 --- /dev/null +++ b/src/libutil/terminal.hh @@ -0,0 +1,38 @@ +#pragma once +///@file + +#include "types.hh" + +namespace nix { +/** + * Determine whether ANSI escape sequences are appropriate for the + * present output. + */ +bool shouldANSI(); + +/** + * Truncate a string to 'width' printable characters. If 'filterAll' + * is true, all ANSI escape sequences are filtered out. Otherwise, + * some escape sequences (such as colour setting) are copied but not + * included in the character count. Also, tabs are expanded to + * spaces. + */ +std::string filterANSIEscapes(std::string_view s, + bool filterAll = false, + unsigned int width = std::numeric_limits::max()); + +/** + * Recalculate the window size, updating a global variable. Used in the + * `SIGWINCH` signal handler. + */ +void updateWindowSize(); + +/** + * @return the number of rows and columns of the terminal. + * + * The value is cached so this is quick. The cached result is computed + * by `updateWindowSize()`. + */ +std::pair getWindowSize(); + +} diff --git a/src/libutil/tests/git.cc b/src/libutil/tests/git.cc deleted file mode 100644 index 5b5715fc2..000000000 --- a/src/libutil/tests/git.cc +++ /dev/null @@ -1,33 +0,0 @@ -#include "git.hh" -#include - -namespace nix { - - TEST(GitLsRemote, parseSymrefLineWithReference) { - auto line = "ref: refs/head/main HEAD"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic); - ASSERT_EQ(res->target, "refs/head/main"); - ASSERT_EQ(res->reference, "HEAD"); - } - - TEST(GitLsRemote, parseSymrefLineWithNoReference) { - auto line = "ref: refs/head/main"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Symbolic); - ASSERT_EQ(res->target, "refs/head/main"); - ASSERT_EQ(res->reference, std::nullopt); - } - - TEST(GitLsRemote, parseObjectRefLine) { - auto line = "abc123 refs/head/main"; - auto res = git::parseLsRemoteLine(line); - ASSERT_TRUE(res.has_value()); - ASSERT_EQ(res->kind, git::LsRemoteRefLine::Kind::Object); - ASSERT_EQ(res->target, "abc123"); - ASSERT_EQ(res->reference, "refs/head/main"); - } -} - diff --git a/src/libutil/tests/local.mk b/src/libutil/tests/local.mk deleted file mode 100644 index 167915439..000000000 --- a/src/libutil/tests/local.mk +++ /dev/null @@ -1,29 +0,0 @@ -check: libutil-tests_RUN - -programs += libutil-tests - -libutil-tests-exe_NAME = libnixutil-tests - -libutil-tests-exe_DIR := $(d) - -libutil-tests-exe_INSTALL_DIR := - -libutil-tests-exe_LIBS = libutil-tests - -libutil-tests-exe_LDFLAGS := $(GTEST_LIBS) - -libraries += libutil-tests - -libutil-tests_NAME = libnixutil-tests - -libutil-tests_DIR := $(d) - -libutil-tests_INSTALL_DIR := - -libutil-tests_SOURCES := $(wildcard $(d)/*.cc) - -libutil-tests_CXXFLAGS += -I src/libutil - -libutil-tests_LIBS = libutil - -libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index dc4067f1b..c5e735617 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -1,4 +1,6 @@ #include "thread-pool.hh" +#include "signals.hh" +#include "util.hh" namespace nix { diff --git a/src/libutil/thread-pool.hh b/src/libutil/thread-pool.hh index 0e09fae97..02765badc 100644 --- a/src/libutil/thread-pool.hh +++ b/src/libutil/thread-pool.hh @@ -1,8 +1,8 @@ #pragma once ///@file +#include "error.hh" #include "sync.hh" -#include "util.hh" #include #include diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc new file mode 100644 index 000000000..8949461d2 --- /dev/null +++ b/src/libutil/unix-domain-socket.cc @@ -0,0 +1,100 @@ +#include "file-system.hh" +#include "processes.hh" +#include "unix-domain-socket.hh" + +#include +#include +#include + +namespace nix { + +AutoCloseFD createUnixDomainSocket() +{ + AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM + #ifdef SOCK_CLOEXEC + | SOCK_CLOEXEC + #endif + , 0); + if (!fdSocket) + throw SysError("cannot create Unix domain socket"); + closeOnExec(fdSocket.get()); + return fdSocket; +} + + +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) +{ + auto fdSocket = nix::createUnixDomainSocket(); + + bind(fdSocket.get(), path); + + if (chmod(path.c_str(), mode) == -1) + throw SysError("changing permissions on '%1%'", path); + + if (listen(fdSocket.get(), 100) == -1) + throw SysError("cannot listen on socket '%1%'", path); + + return fdSocket; +} + + +void bind(int fd, const std::string & path) +{ + unlink(path.c_str()); + + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pid pid = startProcess([&]() { + Path dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%s'", path); + _exit(0); + }); + int status = pid.wait(); + if (status != 0) + throw Error("cannot bind to socket '%s'", path); + } else { + memcpy(addr.sun_path, path.c_str(), path.size() + 1); + if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot bind to socket '%s'", path); + } +} + + +void connect(int fd, const std::string & path) +{ + struct sockaddr_un addr; + addr.sun_family = AF_UNIX; + + if (path.size() + 1 >= sizeof(addr.sun_path)) { + Pid pid = startProcess([&]() { + Path dir = dirOf(path); + if (chdir(dir.c_str()) == -1) + throw SysError("chdir to '%s' failed", dir); + std::string base(baseNameOf(path)); + if (base.size() + 1 >= sizeof(addr.sun_path)) + throw Error("socket path '%s' is too long", base); + memcpy(addr.sun_path, base.c_str(), base.size() + 1); + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to socket at '%s'", path); + _exit(0); + }); + int status = pid.wait(); + if (status != 0) + throw Error("cannot connect to socket at '%s'", path); + } else { + memcpy(addr.sun_path, path.c_str(), path.size() + 1); + if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) + throw SysError("cannot connect to socket at '%s'", path); + } +} + +} diff --git a/src/libutil/unix-domain-socket.hh b/src/libutil/unix-domain-socket.hh new file mode 100644 index 000000000..b78feb454 --- /dev/null +++ b/src/libutil/unix-domain-socket.hh @@ -0,0 +1,31 @@ +#pragma once +///@file + +#include "types.hh" +#include "file-descriptor.hh" + +#include + +namespace nix { + +/** + * Create a Unix domain socket. + */ +AutoCloseFD createUnixDomainSocket(); + +/** + * Create a Unix domain socket in listen mode. + */ +AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); + +/** + * Bind a Unix domain socket to a path. + */ +void bind(int fd, const std::string & path); + +/** + * Connect to a Unix domain socket. + */ +void connect(int fd, const std::string & path); + +} diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 9b438e6cd..57b64d607 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -2,6 +2,7 @@ #include "url-parts.hh" #include "util.hh" #include "split.hh" +#include "canon-path.hh" namespace nix { @@ -141,6 +142,13 @@ bool ParsedURL::operator ==(const ParsedURL & other) const && fragment == other.fragment; } +ParsedURL ParsedURL::canonicalise() +{ + ParsedURL res(*this); + res.path = CanonPath(res.path).abs(); + return res; +} + /** * Parse a URL scheme of the form '(applicationScheme\+)?transportScheme' * into a tuple '(applicationScheme, transportScheme)' diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 26c2dcc28..833f54678 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -19,6 +19,11 @@ struct ParsedURL std::string to_string() const; bool operator ==(const ParsedURL & other) const; + + /** + * Remove `.` and `..` path elements. + */ + ParsedURL canonicalise(); }; MakeError(BadURL, Error); diff --git a/src/libutil/users.cc b/src/libutil/users.cc new file mode 100644 index 000000000..95a641322 --- /dev/null +++ b/src/libutil/users.cc @@ -0,0 +1,116 @@ +#include "util.hh" +#include "users.hh" +#include "environment-variables.hh" +#include "file-system.hh" + +#include +#include +#include + +namespace nix { + +std::string getUserName() +{ + auto pw = getpwuid(geteuid()); + std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); + if (name.empty()) + throw Error("cannot figure out user name"); + return name; +} + +Path getHomeOf(uid_t userId) +{ + std::vector buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + return pw->pw_dir; +} + +Path getHome() +{ + static Path homeDir = []() + { + std::optional unownedUserHomeDir = {}; + auto homeDir = getEnv("HOME"); + if (homeDir) { + // Only use $HOME if doesn't exist or is owned by the current user. + struct stat st; + int result = stat(homeDir->c_str(), &st); + if (result != 0) { + if (errno != ENOENT) { + warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); + homeDir.reset(); + } + } else if (st.st_uid != geteuid()) { + unownedUserHomeDir.swap(homeDir); + } + } + if (!homeDir) { + homeDir = getHomeOf(geteuid()); + if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { + warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); + } + } + return *homeDir; + }(); + return homeDir; +} + + +Path getCacheDir() +{ + auto cacheDir = getEnv("XDG_CACHE_HOME"); + return cacheDir ? *cacheDir : getHome() + "/.cache"; +} + + +Path getConfigDir() +{ + auto configDir = getEnv("XDG_CONFIG_HOME"); + return configDir ? *configDir : getHome() + "/.config"; +} + +std::vector getConfigDirs() +{ + Path configHome = getConfigDir(); + auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); + std::vector result = tokenizeString>(configDirs, ":"); + result.insert(result.begin(), configHome); + return result; +} + + +Path getDataDir() +{ + auto dataDir = getEnv("XDG_DATA_HOME"); + return dataDir ? *dataDir : getHome() + "/.local/share"; +} + +Path getStateDir() +{ + auto stateDir = getEnv("XDG_STATE_HOME"); + return stateDir ? *stateDir : getHome() + "/.local/state"; +} + +Path createNixStateDir() +{ + Path dir = getStateDir() + "/nix"; + createDirs(dir); + return dir; +} + + +std::string expandTilde(std::string_view path) +{ + // TODO: expand ~user ? + auto tilde = path.substr(0, 2); + if (tilde == "~/" || tilde == "~") + return getHome() + std::string(path.substr(1)); + else + return std::string(path); +} + +} diff --git a/src/libutil/users.hh b/src/libutil/users.hh new file mode 100644 index 000000000..cecbb8bfb --- /dev/null +++ b/src/libutil/users.hh @@ -0,0 +1,58 @@ +#pragma once +///@file + +#include "types.hh" + +#include + +namespace nix { + +std::string getUserName(); + +/** + * @return the given user's home directory from /etc/passwd. + */ +Path getHomeOf(uid_t userId); + +/** + * @return $HOME or the user's home directory from /etc/passwd. + */ +Path getHome(); + +/** + * @return $XDG_CACHE_HOME or $HOME/.cache. + */ +Path getCacheDir(); + +/** + * @return $XDG_CONFIG_HOME or $HOME/.config. + */ +Path getConfigDir(); + +/** + * @return the directories to search for user configuration files + */ +std::vector getConfigDirs(); + +/** + * @return $XDG_DATA_HOME or $HOME/.local/share. + */ +Path getDataDir(); + +/** + * @return $XDG_STATE_HOME or $HOME/.local/state. + */ +Path getStateDir(); + +/** + * Create the Nix state directory and return the path to it. + */ +Path createNixStateDir(); + +/** + * Perform tilde expansion on a path, replacing tilde with the user's + * home directory. + */ +std::string expandTilde(std::string_view path); + +} diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 3b4c181e5..5bb3f374b 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,48 +1,11 @@ #include "util.hh" -#include "sync.hh" -#include "finally.hh" -#include "serialise.hh" -#include "cgroup.hh" +#include "fmt.hh" #include #include -#include -#include -#include -#include -#include -#include #include -#include -#include -#include - -#include #include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __APPLE__ -#include -#include -#endif - -#ifdef __linux__ -#include -#include -#include - -#include -#endif - - -extern char * * environ __attribute__((weak)); +#include namespace nix { @@ -67,1099 +30,8 @@ void initLibUtil() { assert(caught); } -std::optional getEnv(const std::string & key) -{ - char * value = getenv(key.c_str()); - if (!value) return {}; - return std::string(value); -} - -std::optional getEnvNonEmpty(const std::string & key) { - auto value = getEnv(key); - if (value == "") return {}; - return value; -} - -std::map getEnv() -{ - std::map env; - for (size_t i = 0; environ[i]; ++i) { - auto s = environ[i]; - auto eq = strchr(s, '='); - if (!eq) - // invalid env, just keep going - continue; - env.emplace(std::string(s, eq), std::string(eq + 1)); - } - return env; -} - - -void clearEnv() -{ - for (auto & name : getEnv()) - unsetenv(name.first.c_str()); -} - -void replaceEnv(const std::map & newEnv) -{ - clearEnv(); - for (auto & newEnvVar : newEnv) - setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); -} - - -Path absPath(Path path, std::optional dir, bool resolveSymlinks) -{ - if (path[0] != '/') { - if (!dir) { -#ifdef __GNU__ - /* GNU (aka. GNU/Hurd) doesn't have any limitation on path - lengths and doesn't define `PATH_MAX'. */ - char *buf = getcwd(NULL, 0); - if (buf == NULL) -#else - char buf[PATH_MAX]; - if (!getcwd(buf, sizeof(buf))) -#endif - throw SysError("cannot get cwd"); - path = concatStrings(buf, "/", path); -#ifdef __GNU__ - free(buf); -#endif - } else - path = concatStrings(*dir, "/", path); - } - return canonPath(path, resolveSymlinks); -} - - -Path canonPath(PathView path, bool resolveSymlinks) -{ - assert(path != ""); - - std::string s; - s.reserve(256); - - if (path[0] != '/') - throw Error("not an absolute path: '%1%'", path); - - std::string temp; - - /* Count the number of times we follow a symlink and stop at some - arbitrary (but high) limit to prevent infinite loops. */ - unsigned int followCount = 0, maxFollow = 1024; - - while (1) { - - /* Skip slashes. */ - while (!path.empty() && path[0] == '/') path.remove_prefix(1); - if (path.empty()) break; - - /* Ignore `.'. */ - if (path == "." || path.substr(0, 2) == "./") - path.remove_prefix(1); - - /* If `..', delete the last component. */ - else if (path == ".." || path.substr(0, 3) == "../") - { - if (!s.empty()) s.erase(s.rfind('/')); - path.remove_prefix(2); - } - - /* Normal component; copy it. */ - else { - s += '/'; - if (const auto slash = path.find('/'); slash == std::string::npos) { - s += path; - path = {}; - } else { - s += path.substr(0, slash); - path = path.substr(slash); - } - - /* If s points to a symlink, resolve it and continue from there */ - if (resolveSymlinks && isLink(s)) { - if (++followCount >= maxFollow) - throw Error("infinite symlink recursion in path '%1%'", path); - temp = concatStrings(readLink(s), path); - path = temp; - if (!temp.empty() && temp[0] == '/') { - s.clear(); /* restart for symlinks pointing to absolute path */ - } else { - s = dirOf(s); - if (s == "/") { // we don’t want trailing slashes here, which dirOf only produces if s = / - s.clear(); - } - } - } - } - } - - return s.empty() ? "/" : std::move(s); -} - - -Path dirOf(const PathView path) -{ - Path::size_type pos = path.rfind('/'); - if (pos == std::string::npos) - return "."; - return pos == 0 ? "/" : Path(path, 0, pos); -} - - -std::string_view baseNameOf(std::string_view path) -{ - if (path.empty()) - return ""; - - auto last = path.size() - 1; - if (path[last] == '/' && last > 0) - last -= 1; - - auto pos = path.rfind('/', last); - if (pos == std::string::npos) - pos = 0; - else - pos += 1; - - return path.substr(pos, last - pos + 1); -} - - -std::string expandTilde(std::string_view path) -{ - // TODO: expand ~user ? - auto tilde = path.substr(0, 2); - if (tilde == "~/" || tilde == "~") - return getHome() + std::string(path.substr(1)); - else - return std::string(path); -} - - -bool isInDir(std::string_view path, std::string_view dir) -{ - return path.substr(0, 1) == "/" - && path.substr(0, dir.size()) == dir - && path.size() >= dir.size() + 2 - && path[dir.size()] == '/'; -} - - -bool isDirOrInDir(std::string_view path, std::string_view dir) -{ - return path == dir || isInDir(path, dir); -} - - -struct stat stat(const Path & path) -{ - struct stat st; - if (stat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); - return st; -} - - -struct stat lstat(const Path & path) -{ - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError("getting status of '%1%'", path); - return st; -} - - -bool pathExists(const Path & path) -{ - int res; - struct stat st; - res = lstat(path.c_str(), &st); - if (!res) return true; - if (errno != ENOENT && errno != ENOTDIR) - throw SysError("getting status of %1%", path); - return false; -} - -bool pathAccessible(const Path & path) -{ - try { - return pathExists(path); - } catch (SysError & e) { - // swallow EPERM - if (e.errNo == EPERM) return false; - throw; - } -} - - -Path readLink(const Path & path) -{ - checkInterrupt(); - std::vector buf; - for (ssize_t bufSize = PATH_MAX/4; true; bufSize += bufSize/2) { - buf.resize(bufSize); - ssize_t rlSize = readlink(path.c_str(), buf.data(), bufSize); - if (rlSize == -1) - if (errno == EINVAL) - throw Error("'%1%' is not a symlink", path); - else - throw SysError("reading symbolic link '%1%'", path); - else if (rlSize < bufSize) - return std::string(buf.data(), rlSize); - } -} - - -bool isLink(const Path & path) -{ - struct stat st = lstat(path); - return S_ISLNK(st.st_mode); -} - - -DirEntries readDirectory(DIR *dir, const Path & path) -{ - DirEntries entries; - entries.reserve(64); - - struct dirent * dirent; - while (errno = 0, dirent = readdir(dir)) { /* sic */ - checkInterrupt(); - std::string name = dirent->d_name; - if (name == "." || name == "..") continue; - entries.emplace_back(name, dirent->d_ino, -#ifdef HAVE_STRUCT_DIRENT_D_TYPE - dirent->d_type -#else - DT_UNKNOWN -#endif - ); - } - if (errno) throw SysError("reading directory '%1%'", path); - - return entries; -} - -DirEntries readDirectory(const Path & path) -{ - AutoCloseDir dir(opendir(path.c_str())); - if (!dir) throw SysError("opening directory '%1%'", path); - - return readDirectory(dir.get(), path); -} - - -unsigned char getFileType(const Path & path) -{ - struct stat st = lstat(path); - if (S_ISDIR(st.st_mode)) return DT_DIR; - if (S_ISLNK(st.st_mode)) return DT_LNK; - if (S_ISREG(st.st_mode)) return DT_REG; - return DT_UNKNOWN; -} - - -std::string readFile(int fd) -{ - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("statting file"); - - return drainFD(fd, true, st.st_size); -} - - -std::string readFile(const Path & path) -{ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening file '%1%'", path); - return readFile(fd.get()); -} - - -void readFile(const Path & path, Sink & sink) -{ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening file '%s'", path); - drainFD(fd.get(), sink); -} - - -void writeFile(const Path & path, std::string_view s, mode_t mode, bool sync) -{ - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); - if (!fd) - throw SysError("opening file '%1%'", path); - try { - writeFull(fd.get(), s); - } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", path); - throw; - } - if (sync) - fd.fsync(); - // Explicitly close to make sure exceptions are propagated. - fd.close(); - if (sync) - syncParent(path); -} - - -void writeFile(const Path & path, Source & source, mode_t mode, bool sync) -{ - AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode); - if (!fd) - throw SysError("opening file '%1%'", path); - - std::vector buf(64 * 1024); - - try { - while (true) { - try { - auto n = source.read(buf.data(), buf.size()); - writeFull(fd.get(), {buf.data(), n}); - } catch (EndOfFile &) { break; } - } - } catch (Error & e) { - e.addTrace({}, "writing file '%1%'", path); - throw; - } - if (sync) - fd.fsync(); - // Explicitly close to make sure exceptions are propagated. - fd.close(); - if (sync) - syncParent(path); -} - -void syncParent(const Path & path) -{ - AutoCloseFD fd = open(dirOf(path).c_str(), O_RDONLY, 0); - if (!fd) - throw SysError("opening file '%1%'", path); - fd.fsync(); -} - -std::string readLine(int fd) -{ - std::string s; - while (1) { - checkInterrupt(); - char ch; - // FIXME: inefficient - ssize_t rd = read(fd, &ch, 1); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading a line"); - } else if (rd == 0) - throw EndOfFile("unexpected EOF reading a line"); - else { - if (ch == '\n') return s; - s += ch; - } - } -} - - -void writeLine(int fd, std::string s) -{ - s += '\n'; - writeFull(fd, s); -} - - -static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed) -{ - checkInterrupt(); - - std::string name(baseNameOf(path)); - - struct stat st; - if (fstatat(parentfd, name.c_str(), &st, AT_SYMLINK_NOFOLLOW) == -1) { - if (errno == ENOENT) return; - throw SysError("getting status of '%1%'", path); - } - - if (!S_ISDIR(st.st_mode)) { - /* We are about to delete a file. Will it likely free space? */ - - switch (st.st_nlink) { - /* Yes: last link. */ - case 1: - bytesFreed += st.st_size; - break; - /* Maybe: yes, if 'auto-optimise-store' or manual optimisation - was performed. Instead of checking for real let's assume - it's an optimised file and space will be freed. - - In worst case we will double count on freed space for files - with exactly two hardlinks for unoptimised packages. - */ - case 2: - bytesFreed += st.st_size; - break; - /* No: 3+ links. */ - default: - break; - } - } - - if (S_ISDIR(st.st_mode)) { - /* Make the directory accessible. */ - const auto PERM_MASK = S_IRUSR | S_IWUSR | S_IXUSR; - if ((st.st_mode & PERM_MASK) != PERM_MASK) { - if (fchmodat(parentfd, name.c_str(), st.st_mode | PERM_MASK, 0) == -1) - throw SysError("chmod '%1%'", path); - } - - int fd = openat(parentfd, path.c_str(), O_RDONLY); - if (fd == -1) - throw SysError("opening directory '%1%'", path); - AutoCloseDir dir(fdopendir(fd)); - if (!dir) - throw SysError("opening directory '%1%'", path); - for (auto & i : readDirectory(dir.get(), path)) - _deletePath(dirfd(dir.get()), path + "/" + i.name, bytesFreed); - } - - int flags = S_ISDIR(st.st_mode) ? AT_REMOVEDIR : 0; - if (unlinkat(parentfd, name.c_str(), flags) == -1) { - if (errno == ENOENT) return; - throw SysError("cannot unlink '%1%'", path); - } -} - -static void _deletePath(const Path & path, uint64_t & bytesFreed) -{ - Path dir = dirOf(path); - if (dir == "") - dir = "/"; - - AutoCloseFD dirfd{open(dir.c_str(), O_RDONLY)}; - if (!dirfd) { - if (errno == ENOENT) return; - throw SysError("opening directory '%1%'", path); - } - - _deletePath(dirfd.get(), path, bytesFreed); -} - - -void deletePath(const Path & path) -{ - uint64_t dummy; - deletePath(path, dummy); -} - - -void deletePath(const Path & path, uint64_t & bytesFreed) -{ - //Activity act(*logger, lvlDebug, "recursively deleting path '%1%'", path); - bytesFreed = 0; - _deletePath(path, bytesFreed); -} - - -std::string getUserName() -{ - auto pw = getpwuid(geteuid()); - std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); - if (name.empty()) - throw Error("cannot figure out user name"); - return name; -} - -Path getHomeOf(uid_t userId) -{ - std::vector buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - return pw->pw_dir; -} - -Path getHome() -{ - static Path homeDir = []() - { - std::optional unownedUserHomeDir = {}; - auto homeDir = getEnv("HOME"); - if (homeDir) { - // Only use $HOME if doesn't exist or is owned by the current user. - struct stat st; - int result = stat(homeDir->c_str(), &st); - if (result != 0) { - if (errno != ENOENT) { - warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); - homeDir.reset(); - } - } else if (st.st_uid != geteuid()) { - unownedUserHomeDir.swap(homeDir); - } - } - if (!homeDir) { - homeDir = getHomeOf(geteuid()); - if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { - warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); - } - } - return *homeDir; - }(); - return homeDir; -} - - -Path getCacheDir() -{ - auto cacheDir = getEnv("XDG_CACHE_HOME"); - return cacheDir ? *cacheDir : getHome() + "/.cache"; -} - - -Path getConfigDir() -{ - auto configDir = getEnv("XDG_CONFIG_HOME"); - return configDir ? *configDir : getHome() + "/.config"; -} - -std::vector getConfigDirs() -{ - Path configHome = getConfigDir(); - auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); - std::vector result = tokenizeString>(configDirs, ":"); - result.insert(result.begin(), configHome); - return result; -} - - -Path getDataDir() -{ - auto dataDir = getEnv("XDG_DATA_HOME"); - return dataDir ? *dataDir : getHome() + "/.local/share"; -} - -Path getStateDir() -{ - auto stateDir = getEnv("XDG_STATE_HOME"); - return stateDir ? *stateDir : getHome() + "/.local/state"; -} - -Path createNixStateDir() -{ - Path dir = getStateDir() + "/nix"; - createDirs(dir); - return dir; -} - - -std::optional getSelfExe() -{ - static auto cached = []() -> std::optional - { - #if __linux__ - return readLink("/proc/self/exe"); - #elif __APPLE__ - char buf[1024]; - uint32_t size = sizeof(buf); - if (_NSGetExecutablePath(buf, &size) == 0) - return buf; - else - return std::nullopt; - #else - return std::nullopt; - #endif - }(); - return cached; -} - - -Paths createDirs(const Path & path) -{ - Paths created; - if (path == "/") return created; - - struct stat st; - if (lstat(path.c_str(), &st) == -1) { - created = createDirs(dirOf(path)); - if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) - throw SysError("creating directory '%1%'", path); - st = lstat(path); - created.push_back(path); - } - - if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) - throw SysError("statting symlink '%1%'", path); - - if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); - - return created; -} - - -void readFull(int fd, char * buf, size_t count) -{ - while (count) { - checkInterrupt(); - ssize_t res = read(fd, buf, count); - if (res == -1) { - if (errno == EINTR) continue; - throw SysError("reading from file"); - } - if (res == 0) throw EndOfFile("unexpected end-of-file"); - count -= res; - buf += res; - } -} - - -void writeFull(int fd, std::string_view s, bool allowInterrupts) -{ - while (!s.empty()) { - if (allowInterrupts) checkInterrupt(); - ssize_t res = write(fd, s.data(), s.size()); - if (res == -1 && errno != EINTR) - throw SysError("writing to file"); - if (res > 0) - s.remove_prefix(res); - } -} - - -std::string drainFD(int fd, bool block, const size_t reserveSize) -{ - // the parser needs two extra bytes to append terminating characters, other users will - // not care very much about the extra memory. - StringSink sink(reserveSize + 2); - drainFD(fd, sink, block); - return std::move(sink.s); -} - - -void drainFD(int fd, Sink & sink, bool block) -{ - // silence GCC maybe-uninitialized warning in finally - int saved = 0; - - if (!block) { - saved = fcntl(fd, F_GETFL); - if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) - throw SysError("making file descriptor non-blocking"); - } - - Finally finally([&]() { - if (!block) { - if (fcntl(fd, F_SETFL, saved) == -1) - throw SysError("making file descriptor blocking"); - } - }); - - std::vector buf(64 * 1024); - while (1) { - checkInterrupt(); - ssize_t rd = read(fd, buf.data(), buf.size()); - if (rd == -1) { - if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) - break; - if (errno != EINTR) - throw SysError("reading from file"); - } - else if (rd == 0) break; - else sink({(char *) buf.data(), (size_t) rd}); - } -} - ////////////////////////////////////////////////////////////////////// -unsigned int getMaxCPU() -{ - #if __linux__ - try { - auto cgroupFS = getCgroupFS(); - if (!cgroupFS) return 0; - - auto cgroups = getCgroups("/proc/self/cgroup"); - auto cgroup = cgroups[""]; - if (cgroup == "") return 0; - - auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; - - auto cpuMax = readFile(cpuFile); - auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); - auto quota = cpuMaxParts[0]; - auto period = cpuMaxParts[1]; - if (quota != "max") - return std::ceil(std::stoi(quota) / std::stof(period)); - } catch (Error &) { ignoreException(lvlDebug); } - #endif - - return 0; -} - -////////////////////////////////////////////////////////////////////// - - -AutoDelete::AutoDelete() : del{false} {} - -AutoDelete::AutoDelete(const std::string & p, bool recursive) : path(p) -{ - del = true; - this->recursive = recursive; -} - -AutoDelete::~AutoDelete() -{ - try { - if (del) { - if (recursive) - deletePath(path); - else { - if (remove(path.c_str()) == -1) - throw SysError("cannot unlink '%1%'", path); - } - } - } catch (...) { - ignoreException(); - } -} - -void AutoDelete::cancel() -{ - del = false; -} - -void AutoDelete::reset(const Path & p, bool recursive) { - path = p; - this->recursive = recursive; - del = true; -} - - - -////////////////////////////////////////////////////////////////////// - - -AutoCloseFD::AutoCloseFD() : fd{-1} {} - - -AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} - - -AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} -{ - that.fd = -1; -} - - -AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) -{ - close(); - fd = that.fd; - that.fd = -1; - return *this; -} - - -AutoCloseFD::~AutoCloseFD() -{ - try { - close(); - } catch (...) { - ignoreException(); - } -} - - -int AutoCloseFD::get() const -{ - return fd; -} - - -void AutoCloseFD::close() -{ - if (fd != -1) { - if (::close(fd) == -1) - /* This should never happen. */ - throw SysError("closing file descriptor %1%", fd); - fd = -1; - } -} - -void AutoCloseFD::fsync() -{ - if (fd != -1) { - int result; -#if __APPLE__ - result = ::fcntl(fd, F_FULLFSYNC); -#else - result = ::fsync(fd); -#endif - if (result == -1) - throw SysError("fsync file descriptor %1%", fd); - } -} - - -AutoCloseFD::operator bool() const -{ - return fd != -1; -} - - -int AutoCloseFD::release() -{ - int oldFD = fd; - fd = -1; - return oldFD; -} - - -void Pipe::create() -{ - int fds[2]; -#if HAVE_PIPE2 - if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); -#else - if (pipe(fds) != 0) throw SysError("creating pipe"); - closeOnExec(fds[0]); - closeOnExec(fds[1]); -#endif - readSide = fds[0]; - writeSide = fds[1]; -} - - -void Pipe::close() -{ - readSide.close(); - writeSide.close(); -} - - -////////////////////////////////////////////////////////////////////// - - -Pid::Pid() -{ -} - - -Pid::Pid(pid_t pid) - : pid(pid) -{ -} - - -Pid::~Pid() -{ - if (pid != -1) kill(); -} - - -void Pid::operator =(pid_t pid) -{ - if (this->pid != -1 && this->pid != pid) kill(); - this->pid = pid; - killSignal = SIGKILL; // reset signal to default -} - - -Pid::operator pid_t() -{ - return pid; -} - - -int Pid::kill() -{ - assert(pid != -1); - - debug("killing process %1%", pid); - - /* Send the requested signal to the child. If it has its own - process group, send the signal to every process in the child - process group (which hopefully includes *all* its children). */ - if (::kill(separatePG ? -pid : pid, killSignal) != 0) { - /* On BSDs, killing a process group will return EPERM if all - processes in the group are zombies (or something like - that). So try to detect and ignore that situation. */ -#if __FreeBSD__ || __APPLE__ - if (errno != EPERM || ::kill(pid, 0) != 0) -#endif - logError(SysError("killing process %d", pid).info()); - } - - return wait(); -} - - -int Pid::wait() -{ - assert(pid != -1); - while (1) { - int status; - int res = waitpid(pid, &status, 0); - if (res == pid) { - pid = -1; - return status; - } - if (errno != EINTR) - throw SysError("cannot get exit status of PID %d", pid); - checkInterrupt(); - } -} - - -void Pid::setSeparatePG(bool separatePG) -{ - this->separatePG = separatePG; -} - - -void Pid::setKillSignal(int signal) -{ - this->killSignal = signal; -} - - -pid_t Pid::release() -{ - pid_t p = pid; - pid = -1; - return p; -} - - -void killUser(uid_t uid) -{ - debug("killing all processes running under uid '%1%'", uid); - - assert(uid != 0); /* just to be safe... */ - - /* The system call kill(-1, sig) sends the signal `sig' to all - users to which the current process can send signals. So we - fork a process, switch to uid, and send a mass kill. */ - - Pid pid = startProcess([&]() { - - if (setuid(uid) == -1) - throw SysError("setting uid"); - - while (true) { -#ifdef __APPLE__ - /* OSX's kill syscall takes a third parameter that, among - other things, determines if kill(-1, signo) affects the - calling process. In the OSX libc, it's set to true, - which means "follow POSIX", which we don't want here - */ - if (syscall(SYS_kill, -1, SIGKILL, false) == 0) break; -#else - if (kill(-1, SIGKILL) == 0) break; -#endif - if (errno == ESRCH || errno == EPERM) break; /* no more processes */ - if (errno != EINTR) - throw SysError("cannot kill processes for uid '%1%'", uid); - } - - _exit(0); - }); - - int status = pid.wait(); - if (status != 0) - throw Error("cannot kill processes for uid '%1%': %2%", uid, statusToString(status)); - - /* !!! We should really do some check to make sure that there are - no processes left running under `uid', but there is no portable - way to do so (I think). The most reliable way may be `ps -eo - uid | grep -q $uid'. */ -} - - -////////////////////////////////////////////////////////////////////// - - -/* Wrapper around vfork to prevent the child process from clobbering - the caller's stack frame in the parent. */ -static pid_t doFork(bool allowVfork, std::function fun) __attribute__((noinline)); -static pid_t doFork(bool allowVfork, std::function fun) -{ -#ifdef __linux__ - pid_t pid = allowVfork ? vfork() : fork(); -#else - pid_t pid = fork(); -#endif - if (pid != 0) return pid; - fun(); - abort(); -} - - -#if __linux__ -static int childEntry(void * arg) -{ - auto main = (std::function *) arg; - (*main)(); - return 1; -} -#endif - - -pid_t startProcess(std::function fun, const ProcessOptions & options) -{ - std::function wrapper = [&]() { - if (!options.allowVfork) - logger = makeSimpleLogger(); - try { -#if __linux__ - if (options.dieWithParent && prctl(PR_SET_PDEATHSIG, SIGKILL) == -1) - throw SysError("setting death signal"); -#endif - fun(); - } catch (std::exception & e) { - try { - std::cerr << options.errorPrefix << e.what() << "\n"; - } catch (...) { } - } catch (...) { } - if (options.runExitHandlers) - exit(1); - else - _exit(1); - }; - - pid_t pid = -1; - - if (options.cloneFlags) { - #ifdef __linux__ - // Not supported, since then we don't know when to free the stack. - assert(!(options.cloneFlags & CLONE_VM)); - - size_t stackSize = 1 * 1024 * 1024; - auto stack = (char *) mmap(0, stackSize, - PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0); - if (stack == MAP_FAILED) throw SysError("allocating stack"); - - Finally freeStack([&]() { munmap(stack, stackSize); }); - - pid = clone(childEntry, stack + stackSize, options.cloneFlags | SIGCHLD, &wrapper); - #else - throw Error("clone flags are only supported on Linux"); - #endif - } else - pid = doFork(options.allowVfork, wrapper); - - if (pid == -1) throw SysError("unable to fork"); - - return pid; -} - - std::vector stringsToCharPtrs(const Strings & ss) { std::vector res; @@ -1168,211 +40,6 @@ std::vector stringsToCharPtrs(const Strings & ss) return res; } -std::string runProgram(Path program, bool searchPath, const Strings & args, - const std::optional & input, bool isInteractive) -{ - auto res = runProgram(RunOptions {.program = program, .searchPath = searchPath, .args = args, .input = input, .isInteractive = isInteractive}); - - if (!statusOk(res.first)) - throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); - - return res.second; -} - -// Output = error code + "standard out" output stream -std::pair runProgram(RunOptions && options) -{ - StringSink sink; - options.standardOut = &sink; - - int status = 0; - - try { - runProgram2(options); - } catch (ExecError & e) { - status = e.status; - } - - return {status, std::move(sink.s)}; -} - -void runProgram2(const RunOptions & options) -{ - checkInterrupt(); - - assert(!(options.standardIn && options.input)); - - std::unique_ptr source_; - Source * source = options.standardIn; - - if (options.input) { - source_ = std::make_unique(*options.input); - source = source_.get(); - } - - /* Create a pipe. */ - Pipe out, in; - if (options.standardOut) out.create(); - if (source) in.create(); - - ProcessOptions processOptions; - // vfork implies that the environment of the main process and the fork will - // be shared (technically this is undefined, but in practice that's the - // case), so we can't use it if we alter the environment - processOptions.allowVfork = !options.environment; - - std::optional>> resumeLoggerDefer; - if (options.isInteractive) { - logger->pause(); - resumeLoggerDefer.emplace( - []() { - logger->resume(); - } - ); - } - - /* Fork. */ - Pid pid = startProcess([&]() { - if (options.environment) - replaceEnv(*options.environment); - if (options.standardOut && dup2(out.writeSide.get(), STDOUT_FILENO) == -1) - throw SysError("dupping stdout"); - if (options.mergeStderrToStdout) - if (dup2(STDOUT_FILENO, STDERR_FILENO) == -1) - throw SysError("cannot dup stdout into stderr"); - if (source && dup2(in.readSide.get(), STDIN_FILENO) == -1) - throw SysError("dupping stdin"); - - if (options.chdir && chdir((*options.chdir).c_str()) == -1) - throw SysError("chdir failed"); - if (options.gid && setgid(*options.gid) == -1) - throw SysError("setgid failed"); - /* Drop all other groups if we're setgid. */ - if (options.gid && setgroups(0, 0) == -1) - throw SysError("setgroups failed"); - if (options.uid && setuid(*options.uid) == -1) - throw SysError("setuid failed"); - - Strings args_(options.args); - args_.push_front(options.program); - - restoreProcessContext(); - - if (options.searchPath) - execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); - // This allows you to refer to a program with a pathname relative - // to the PATH variable. - else - execv(options.program.c_str(), stringsToCharPtrs(args_).data()); - - throw SysError("executing '%1%'", options.program); - }, processOptions); - - out.writeSide.close(); - - std::thread writerThread; - - std::promise promise; - - Finally doJoin([&]() { - if (writerThread.joinable()) - writerThread.join(); - }); - - - if (source) { - in.readSide.close(); - writerThread = std::thread([&]() { - try { - std::vector buf(8 * 1024); - while (true) { - size_t n; - try { - n = source->read(buf.data(), buf.size()); - } catch (EndOfFile &) { - break; - } - writeFull(in.writeSide.get(), {buf.data(), n}); - } - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - in.writeSide.close(); - }); - } - - if (options.standardOut) - drainFD(out.readSide.get(), *options.standardOut); - - /* Wait for the child to finish. */ - int status = pid.wait(); - - /* Wait for the writer thread to finish. */ - if (source) promise.get_future().get(); - - if (status) - throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); -} - - -void closeMostFDs(const std::set & exceptions) -{ -#if __linux__ - try { - for (auto & s : readDirectory("/proc/self/fd")) { - auto fd = std::stoi(s.name); - if (!exceptions.count(fd)) { - debug("closing leaked FD %d", fd); - close(fd); - } - } - return; - } catch (SysError &) { - } -#endif - - int maxFD = 0; - maxFD = sysconf(_SC_OPEN_MAX); - for (int fd = 0; fd < maxFD; ++fd) - if (!exceptions.count(fd)) - close(fd); /* ignore result */ -} - - -void closeOnExec(int fd) -{ - int prev; - if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || - fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) - throw SysError("setting close-on-exec flag"); -} - - -////////////////////////////////////////////////////////////////////// - - -std::atomic _isInterrupted = false; - -static thread_local bool interruptThrown = false; -thread_local std::function interruptCheck; - -void setInterruptThrown() -{ - interruptThrown = true; -} - -void _interrupted() -{ - /* Block user interrupts while an exception is being handled. - Throwing an exception while another exception is being handled - kills the program! */ - if (!interruptThrown && !std::uncaught_exceptions()) { - interruptThrown = true; - throw Interrupted("interrupted by the user"); - } -} - ////////////////////////////////////////////////////////////////////// @@ -1438,32 +105,6 @@ std::string rewriteStrings(std::string s, const StringMap & rewrites) } -std::string statusToString(int status) -{ - if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { - if (WIFEXITED(status)) - return fmt("failed with exit code %1%", WEXITSTATUS(status)); - else if (WIFSIGNALED(status)) { - int sig = WTERMSIG(status); -#if HAVE_STRSIGNAL - const char * description = strsignal(sig); - return fmt("failed due to signal %1% (%2%)", sig, description); -#else - return fmt("failed due to signal %1%", sig); -#endif - } - else - return "died abnormally"; - } else return "succeeded"; -} - - -bool statusOk(int status) -{ - return WIFEXITED(status) && WEXITSTATUS(status) == 0; -} - - bool hasPrefix(std::string_view s, std::string_view prefix) { return s.compare(0, prefix.size(), prefix) == 0; @@ -1511,82 +152,6 @@ void ignoreException(Verbosity lvl) } catch (...) { } } -bool shouldANSI() -{ - return isatty(STDERR_FILENO) - && getEnv("TERM").value_or("dumb") != "dumb" - && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value()); -} - -std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) -{ - std::string t, e; - size_t w = 0; - auto i = s.begin(); - - while (w < (size_t) width && i != s.end()) { - - if (*i == '\e') { - std::string e; - e += *i++; - char last = 0; - - if (i != s.end() && *i == '[') { - e += *i++; - // eat parameter bytes - while (i != s.end() && *i >= 0x30 && *i <= 0x3f) e += *i++; - // eat intermediate bytes - while (i != s.end() && *i >= 0x20 && *i <= 0x2f) e += *i++; - // eat final byte - if (i != s.end() && *i >= 0x40 && *i <= 0x7e) e += last = *i++; - } else { - if (i != s.end() && *i >= 0x40 && *i <= 0x5f) e += *i++; - } - - if (!filterAll && last == 'm') - t += e; - } - - else if (*i == '\t') { - i++; t += ' '; w++; - while (w < (size_t) width && w % 8) { - t += ' '; w++; - } - } - - else if (*i == '\r' || *i == '\a') - // do nothing for now - i++; - - else { - w++; - // Copy one UTF-8 character. - if ((*i & 0xe0) == 0xc0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } else if ((*i & 0xf0) == 0xe0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } else if ((*i & 0xf8) == 0xf0) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) { - t += *i++; - if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++; - } - } - } else - t += *i++; - } - } - - return t; -} - constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; @@ -1703,386 +268,9 @@ std::pair getLine(std::string_view s) } -////////////////////////////////////////////////////////////////////// - -static Sync> windowSize{{0, 0}}; - - -static void updateWindowSize() -{ - struct winsize ws; - if (ioctl(2, TIOCGWINSZ, &ws) == 0) { - auto windowSize_(windowSize.lock()); - windowSize_->first = ws.ws_row; - windowSize_->second = ws.ws_col; - } -} - - -std::pair getWindowSize() -{ - return *windowSize.lock(); -} - - -/* We keep track of interrupt callbacks using integer tokens, so we can iterate - safely without having to lock the data structure while executing arbitrary - functions. - */ -struct InterruptCallbacks { - typedef int64_t Token; - - /* We use unique tokens so that we can't accidentally delete the wrong - handler because of an erroneous double delete. */ - Token nextToken = 0; - - /* Used as a list, see InterruptCallbacks comment. */ - std::map> callbacks; -}; - -static Sync _interruptCallbacks; - -static void signalHandlerThread(sigset_t set) -{ - while (true) { - int signal = 0; - sigwait(&set, &signal); - - if (signal == SIGINT || signal == SIGTERM || signal == SIGHUP) - triggerInterrupt(); - - else if (signal == SIGWINCH) { - updateWindowSize(); - } - } -} - -void triggerInterrupt() -{ - _isInterrupted = true; - - { - InterruptCallbacks::Token i = 0; - while (true) { - std::function callback; - { - auto interruptCallbacks(_interruptCallbacks.lock()); - auto lb = interruptCallbacks->callbacks.lower_bound(i); - if (lb == interruptCallbacks->callbacks.end()) - break; - - callback = lb->second; - i = lb->first + 1; - } - - try { - callback(); - } catch (...) { - ignoreException(); - } - } - } -} - -static sigset_t savedSignalMask; -static bool savedSignalMaskIsSet = false; - -void setChildSignalMask(sigset_t * sigs) -{ - assert(sigs); // C style function, but think of sigs as a reference - -#if _POSIX_C_SOURCE >= 1 || _XOPEN_SOURCE || _POSIX_SOURCE - sigemptyset(&savedSignalMask); - // There's no "assign" or "copy" function, so we rely on (math) idempotence - // of the or operator: a or a = a. - sigorset(&savedSignalMask, sigs, sigs); -#else - // Without sigorset, our best bet is to assume that sigset_t is a type that - // can be assigned directly, such as is the case for a sigset_t defined as - // an integer type. - savedSignalMask = *sigs; -#endif - - savedSignalMaskIsSet = true; -} - -void saveSignalMask() { - if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) - throw SysError("querying signal mask"); - - savedSignalMaskIsSet = true; -} - -void startSignalHandlerThread() -{ - updateWindowSize(); - - saveSignalMask(); - - sigset_t set; - sigemptyset(&set); - sigaddset(&set, SIGINT); - sigaddset(&set, SIGTERM); - sigaddset(&set, SIGHUP); - sigaddset(&set, SIGPIPE); - sigaddset(&set, SIGWINCH); - if (pthread_sigmask(SIG_BLOCK, &set, nullptr)) - throw SysError("blocking signals"); - - std::thread(signalHandlerThread, set).detach(); -} - -static void restoreSignals() -{ - // If startSignalHandlerThread wasn't called, that means we're not running - // in a proper libmain process, but a process that presumably manages its - // own signal handlers. Such a process should call either - // - initNix(), to be a proper libmain process - // - startSignalHandlerThread(), to resemble libmain regarding signal - // handling only - // - saveSignalMask(), for processes that define their own signal handling - // thread - // TODO: Warn about this? Have a default signal mask? The latter depends on - // whether we should generally inherit signal masks from the caller. - // I don't know what the larger unix ecosystem expects from us here. - if (!savedSignalMaskIsSet) - return; - - if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr)) - throw SysError("restoring signals"); -} - -#if __linux__ -rlim_t savedStackSize = 0; -#endif - -void setStackSize(size_t stackSize) -{ - #if __linux__ - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { - savedStackSize = limit.rlim_cur; - limit.rlim_cur = stackSize; - setrlimit(RLIMIT_STACK, &limit); - } - #endif -} - -#if __linux__ -static AutoCloseFD fdSavedMountNamespace; -static AutoCloseFD fdSavedRoot; -#endif - -void saveMountNamespace() -{ -#if __linux__ - static std::once_flag done; - std::call_once(done, []() { - fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); - if (!fdSavedMountNamespace) - throw SysError("saving parent mount namespace"); - - fdSavedRoot = open("/proc/self/root", O_RDONLY); - }); -#endif -} - -void restoreMountNamespace() -{ -#if __linux__ - try { - auto savedCwd = absPath("."); - - if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) - throw SysError("restoring parent mount namespace"); - - if (fdSavedRoot) { - if (fchdir(fdSavedRoot.get())) - throw SysError("chdir into saved root"); - if (chroot(".")) - throw SysError("chroot into saved root"); - } - - if (chdir(savedCwd.c_str()) == -1) - throw SysError("restoring cwd"); - } catch (Error & e) { - debug(e.msg()); - } -#endif -} - -void unshareFilesystem() -{ -#ifdef __linux__ - if (unshare(CLONE_FS) != 0 && errno != EPERM) - throw SysError("unsharing filesystem state in download thread"); -#endif -} - -void restoreProcessContext(bool restoreMounts) -{ - restoreSignals(); - if (restoreMounts) { - restoreMountNamespace(); - } - - #if __linux__ - if (savedStackSize) { - struct rlimit limit; - if (getrlimit(RLIMIT_STACK, &limit) == 0) { - limit.rlim_cur = savedStackSize; - setrlimit(RLIMIT_STACK, &limit); - } - } - #endif -} - -/* RAII helper to automatically deregister a callback. */ -struct InterruptCallbackImpl : InterruptCallback -{ - InterruptCallbacks::Token token; - ~InterruptCallbackImpl() override - { - auto interruptCallbacks(_interruptCallbacks.lock()); - interruptCallbacks->callbacks.erase(token); - } -}; - -std::unique_ptr createInterruptCallback(std::function callback) -{ - auto interruptCallbacks(_interruptCallbacks.lock()); - auto token = interruptCallbacks->nextToken++; - interruptCallbacks->callbacks.emplace(token, callback); - - auto res = std::make_unique(); - res->token = token; - - return std::unique_ptr(res.release()); -} - - -AutoCloseFD createUnixDomainSocket() -{ - AutoCloseFD fdSocket = socket(PF_UNIX, SOCK_STREAM - #ifdef SOCK_CLOEXEC - | SOCK_CLOEXEC - #endif - , 0); - if (!fdSocket) - throw SysError("cannot create Unix domain socket"); - closeOnExec(fdSocket.get()); - return fdSocket; -} - - -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode) -{ - auto fdSocket = nix::createUnixDomainSocket(); - - bind(fdSocket.get(), path); - - if (chmod(path.c_str(), mode) == -1) - throw SysError("changing permissions on '%1%'", path); - - if (listen(fdSocket.get(), 100) == -1) - throw SysError("cannot listen on socket '%1%'", path); - - return fdSocket; -} - - -void bind(int fd, const std::string & path) -{ - unlink(path.c_str()); - - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - - if (path.size() + 1 >= sizeof(addr.sun_path)) { - Pid pid = startProcess([&]() { - Path dir = dirOf(path); - if (chdir(dir.c_str()) == -1) - throw SysError("chdir to '%s' failed", dir); - std::string base(baseNameOf(path)); - if (base.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%s' is too long", base); - memcpy(addr.sun_path, base.c_str(), base.size() + 1); - if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot bind to socket '%s'", path); - _exit(0); - }); - int status = pid.wait(); - if (status != 0) - throw Error("cannot bind to socket '%s'", path); - } else { - memcpy(addr.sun_path, path.c_str(), path.size() + 1); - if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot bind to socket '%s'", path); - } -} - - -void connect(int fd, const std::string & path) -{ - struct sockaddr_un addr; - addr.sun_family = AF_UNIX; - - if (path.size() + 1 >= sizeof(addr.sun_path)) { - Pid pid = startProcess([&]() { - Path dir = dirOf(path); - if (chdir(dir.c_str()) == -1) - throw SysError("chdir to '%s' failed", dir); - std::string base(baseNameOf(path)); - if (base.size() + 1 >= sizeof(addr.sun_path)) - throw Error("socket path '%s' is too long", base); - memcpy(addr.sun_path, base.c_str(), base.size() + 1); - if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to socket at '%s'", path); - _exit(0); - }); - int status = pid.wait(); - if (status != 0) - throw Error("cannot connect to socket at '%s'", path); - } else { - memcpy(addr.sun_path, path.c_str(), path.size() + 1); - if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1) - throw SysError("cannot connect to socket at '%s'", path); - } -} - - std::string showBytes(uint64_t bytes) { return fmt("%.2f MiB", bytes / (1024.0 * 1024.0)); } - -// FIXME: move to libstore/build -void commonChildInit() -{ - logger = makeSimpleLogger(); - - const static std::string pathNullDevice = "/dev/null"; - restoreProcessContext(false); - - /* Put the child in a separate session (and thus a separate - process group) so that it has no controlling terminal (meaning - that e.g. ssh cannot open /dev/tty) and it doesn't receive - terminal signals. */ - if (setsid() == -1) - throw SysError("creating a new session"); - - /* Dup stderr to stdout. */ - if (dup2(STDERR_FILENO, STDOUT_FILENO) == -1) - throw SysError("cannot dup stderr into stdout"); - - /* Reroute stdin to /dev/null. */ - int fdDevNull = open(pathNullDevice.c_str(), O_RDWR); - if (fdDevNull == -1) - throw SysError("cannot open '%1%'", pathNullDevice); - if (dup2(fdDevNull, STDIN_FILENO) == -1) - throw SysError("cannot dup null device into stdin"); - close(fdDevNull); -} - } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index b302d6f45..27faa4d6d 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -4,491 +4,18 @@ #include "types.hh" #include "error.hh" #include "logging.hh" -#include "ansicolor.hh" - -#include -#include -#include -#include -#include #include -#include #include #include #include #include -#ifndef HAVE_STRUCT_DIRENT_D_TYPE -#define DT_UNKNOWN 0 -#define DT_REG 1 -#define DT_LNK 2 -#define DT_DIR 3 -#endif - namespace nix { -struct Sink; -struct Source; - void initLibUtil(); -/** - * The system for which Nix is compiled. - */ -extern const std::string nativeSystem; - - -/** - * @return an environment variable. - */ -std::optional getEnv(const std::string & key); - -/** - * @return a non empty environment variable. Returns nullopt if the env - * variable is set to "" - */ -std::optional getEnvNonEmpty(const std::string & key); - -/** - * Get the entire environment. - */ -std::map getEnv(); - -/** - * Clear the environment. - */ -void clearEnv(); - -/** - * @return An absolutized path, resolving paths relative to the - * specified directory, or the current directory otherwise. The path - * is also canonicalised. - */ -Path absPath(Path path, - std::optional dir = {}, - bool resolveSymlinks = false); - -/** - * Canonicalise a path by removing all `.` or `..` components and - * double or trailing slashes. Optionally resolves all symlink - * components such that each component of the resulting path is *not* - * a symbolic link. - */ -Path canonPath(PathView path, bool resolveSymlinks = false); - -/** - * @return The directory part of the given canonical path, i.e., - * everything before the final `/`. If the path is the root or an - * immediate child thereof (e.g., `/foo`), this means `/` - * is returned. - */ -Path dirOf(const PathView path); - -/** - * @return the base name of the given canonical path, i.e., everything - * following the final `/` (trailing slashes are removed). - */ -std::string_view baseNameOf(std::string_view path); - -/** - * Perform tilde expansion on a path. - */ -std::string expandTilde(std::string_view path); - -/** - * Check whether 'path' is a descendant of 'dir'. Both paths must be - * canonicalized. - */ -bool isInDir(std::string_view path, std::string_view dir); - -/** - * Check whether 'path' is equal to 'dir' or a descendant of - * 'dir'. Both paths must be canonicalized. - */ -bool isDirOrInDir(std::string_view path, std::string_view dir); - -/** - * Get status of `path`. - */ -struct stat stat(const Path & path); -struct stat lstat(const Path & path); - -/** - * @return true iff the given path exists. - */ -bool pathExists(const Path & path); - -/** - * A version of pathExists that returns false on a permission error. - * Useful for inferring default paths across directories that might not - * be readable. - * @return true iff the given path can be accessed and exists - */ -bool pathAccessible(const Path & path); - -/** - * Read the contents (target) of a symbolic link. The result is not - * in any way canonicalised. - */ -Path readLink(const Path & path); - -bool isLink(const Path & path); - -/** - * Read the contents of a directory. The entries `.` and `..` are - * removed. - */ -struct DirEntry -{ - std::string name; - ino_t ino; - /** - * one of DT_* - */ - unsigned char type; - DirEntry(std::string name, ino_t ino, unsigned char type) - : name(std::move(name)), ino(ino), type(type) { } -}; - -typedef std::vector DirEntries; - -DirEntries readDirectory(const Path & path); - -unsigned char getFileType(const Path & path); - -/** - * Read the contents of a file into a string. - */ -std::string readFile(int fd); -std::string readFile(const Path & path); -void readFile(const Path & path, Sink & sink); - -/** - * Write a string to a file. - */ -void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); - -void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); - -/** - * Flush a file's parent directory to disk - */ -void syncParent(const Path & path); - -/** - * Read a line from a file descriptor. - */ -std::string readLine(int fd); - -/** - * Write a line to a file descriptor. - */ -void writeLine(int fd, std::string s); - -/** - * Delete a path; i.e., in the case of a directory, it is deleted - * recursively. It's not an error if the path does not exist. The - * second variant returns the number of bytes and blocks freed. - */ -void deletePath(const Path & path); - -void deletePath(const Path & path, uint64_t & bytesFreed); - -std::string getUserName(); - -/** - * @return the given user's home directory from /etc/passwd. - */ -Path getHomeOf(uid_t userId); - -/** - * @return $HOME or the user's home directory from /etc/passwd. - */ -Path getHome(); - -/** - * @return $XDG_CACHE_HOME or $HOME/.cache. - */ -Path getCacheDir(); - -/** - * @return $XDG_CONFIG_HOME or $HOME/.config. - */ -Path getConfigDir(); - -/** - * @return the directories to search for user configuration files - */ -std::vector getConfigDirs(); - -/** - * @return $XDG_DATA_HOME or $HOME/.local/share. - */ -Path getDataDir(); - -/** - * @return the path of the current executable. - */ -std::optional getSelfExe(); - -/** - * @return $XDG_STATE_HOME or $HOME/.local/state. - */ -Path getStateDir(); - -/** - * Create the Nix state directory and return the path to it. - */ -Path createNixStateDir(); - -/** - * Create a directory and all its parents, if necessary. Returns the - * list of created directories, in order of creation. - */ -Paths createDirs(const Path & path); -inline Paths createDirs(PathView path) -{ - return createDirs(Path(path)); -} - -/** - * Create a symlink. - */ -void createSymlink(const Path & target, const Path & link); - -/** - * Atomically create or replace a symlink. - */ -void replaceSymlink(const Path & target, const Path & link); - -void renameFile(const Path & src, const Path & dst); - -/** - * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` - * are on a different filesystem. - * - * Beware that this might not be atomic because of the copy that happens behind - * the scenes - */ -void moveFile(const Path & src, const Path & dst); - - -/** - * Wrappers arount read()/write() that read/write exactly the - * requested number of bytes. - */ -void readFull(int fd, char * buf, size_t count); -void writeFull(int fd, std::string_view s, bool allowInterrupts = true); - -MakeError(EndOfFile, Error); - - -/** - * Read a file descriptor until EOF occurs. - */ -std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); - -void drainFD(int fd, Sink & sink, bool block = true); - -/** - * If cgroups are active, attempt to calculate the number of CPUs available. - * If cgroups are unavailable or if cpu.max is set to "max", return 0. - */ -unsigned int getMaxCPU(); - -/** - * Automatic cleanup of resources. - */ - - -class AutoDelete -{ - Path path; - bool del; - bool recursive; -public: - AutoDelete(); - AutoDelete(const Path & p, bool recursive = true); - ~AutoDelete(); - void cancel(); - void reset(const Path & p, bool recursive = true); - operator Path() const { return path; } - operator PathView() const { return path; } -}; - - -class AutoCloseFD -{ - int fd; -public: - AutoCloseFD(); - AutoCloseFD(int fd); - AutoCloseFD(const AutoCloseFD & fd) = delete; - AutoCloseFD(AutoCloseFD&& fd); - ~AutoCloseFD(); - AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; - AutoCloseFD& operator =(AutoCloseFD&& fd); - int get() const; - explicit operator bool() const; - int release(); - void close(); - void fsync(); -}; - - -/** - * Create a temporary directory. - */ -Path createTempDir(const Path & tmpRoot = "", const Path & prefix = "nix", - bool includePid = true, bool useGlobalCounter = true, mode_t mode = 0755); - -/** - * Create a temporary file, returning a file handle and its path. - */ -std::pair createTempFile(const Path & prefix = "nix"); - - -class Pipe -{ -public: - AutoCloseFD readSide, writeSide; - void create(); - void close(); -}; - - -struct DIRDeleter -{ - void operator()(DIR * dir) const { - closedir(dir); - } -}; - -typedef std::unique_ptr AutoCloseDir; - - -class Pid -{ - pid_t pid = -1; - bool separatePG = false; - int killSignal = SIGKILL; -public: - Pid(); - Pid(pid_t pid); - ~Pid(); - void operator =(pid_t pid); - operator pid_t(); - int kill(); - int wait(); - - void setSeparatePG(bool separatePG); - void setKillSignal(int signal); - pid_t release(); -}; - - -/** - * Kill all processes running under the specified uid by sending them - * a SIGKILL. - */ -void killUser(uid_t uid); - - -/** - * Fork a process that runs the given function, and return the child - * pid to the caller. - */ -struct ProcessOptions -{ - std::string errorPrefix = ""; - bool dieWithParent = true; - bool runExitHandlers = false; - bool allowVfork = false; - /** - * use clone() with the specified flags (Linux only) - */ - int cloneFlags = 0; -}; - -pid_t startProcess(std::function fun, const ProcessOptions & options = ProcessOptions()); - - -/** - * Run a program and return its stdout in a string (i.e., like the - * shell backtick operator). - */ -std::string runProgram(Path program, bool searchPath = false, - const Strings & args = Strings(), - const std::optional & input = {}, bool isInteractive = false); - -struct RunOptions -{ - Path program; - bool searchPath = true; - Strings args; - std::optional uid; - std::optional gid; - std::optional chdir; - std::optional> environment; - std::optional input; - Source * standardIn = nullptr; - Sink * standardOut = nullptr; - bool mergeStderrToStdout = false; - bool isInteractive = false; -}; - -std::pair runProgram(RunOptions && options); - -void runProgram2(const RunOptions & options); - - -/** - * Change the stack size. - */ -void setStackSize(size_t stackSize); - - -/** - * Restore the original inherited Unix process context (such as signal - * masks, stack size). - - * See startSignalHandlerThread(), saveSignalMask(). - */ -void restoreProcessContext(bool restoreMounts = true); - -/** - * Save the current mount namespace. Ignored if called more than - * once. - */ -void saveMountNamespace(); - -/** - * Restore the mount namespace saved by saveMountNamespace(). Ignored - * if saveMountNamespace() was never called. - */ -void restoreMountNamespace(); - -/** - * Cause this thread to not share any FS attributes with the main - * thread, because this causes setns() in restoreMountNamespace() to - * fail. - */ -void unshareFilesystem(); - - -class ExecError : public Error -{ -public: - int status; - - template - ExecError(int status, const Args & ... args) - : Error(args...), status(status) - { } -}; - /** * Convert a list of strings to a null-terminated vector of `char * *`s. The result must not be accessed beyond the lifetime of the @@ -496,36 +23,6 @@ public: */ std::vector stringsToCharPtrs(const Strings & ss); -/** - * Close all file descriptors except those listed in the given set. - * Good practice in child processes. - */ -void closeMostFDs(const std::set & exceptions); - -/** - * Set the close-on-exec flag for the given file descriptor. - */ -void closeOnExec(int fd); - - -/* User interruption. */ - -extern std::atomic _isInterrupted; - -extern thread_local std::function interruptCheck; - -void setInterruptThrown(); - -void _interrupted(); - -void inline checkInterrupt() -{ - if (_isInterrupted || (interruptCheck && interruptCheck())) - _interrupted(); -} - -MakeError(Interrupted, BaseError); - MakeError(FormatError, Error); @@ -601,15 +98,6 @@ std::string replaceStrings( std::string rewriteStrings(std::string s, const StringMap & rewrites); -/** - * Convert the exit status of a child as returned by wait() into an - * error string. - */ -std::string statusToString(int status); - -bool statusOk(int status); - - /** * Parse a string into an integer. */ @@ -701,10 +189,8 @@ std::string toLower(const std::string & s); std::string shellEscape(const std::string_view s); -/** - * Exception handling in destructors: print an error message, then - * ignore the exception. - */ +/* Exception handling in destructors: print an error message, then + ignore the exception. */ void ignoreException(Verbosity lvl = lvlError); @@ -717,23 +203,6 @@ constexpr char treeLast[] = "└───"; constexpr char treeLine[] = "│ "; constexpr char treeNull[] = " "; -/** - * Determine whether ANSI escape sequences are appropriate for the - * present output. - */ -bool shouldANSI(); - -/** - * Truncate a string to 'width' printable characters. If 'filterAll' - * is true, all ANSI escape sequences are filtered out. Otherwise, - * some escape sequences (such as colour setting) are copied but not - * included in the character count. Also, tabs are expanded to - * spaces. - */ -std::string filterANSIEscapes(std::string_view s, - bool filterAll = false, - unsigned int width = std::numeric_limits::max()); - /** * Base64 encoding/decoding. @@ -821,61 +290,6 @@ template class Callback; -/** - * Start a thread that handles various signals. Also block those signals - * on the current thread (and thus any threads created by it). - * Saves the signal mask before changing the mask to block those signals. - * See saveSignalMask(). - */ -void startSignalHandlerThread(); - -/** - * Saves the signal mask, which is the signal mask that nix will restore - * before creating child processes. - * See setChildSignalMask() to set an arbitrary signal mask instead of the - * current mask. - */ -void saveSignalMask(); - -/** - * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't - * necessarily match the current thread's mask. - * See saveSignalMask() to set the saved mask to the current mask. - */ -void setChildSignalMask(sigset_t *sigs); - -struct InterruptCallback -{ - virtual ~InterruptCallback() { }; -}; - -/** - * Register a function that gets called on SIGINT (in a non-signal - * context). - */ -std::unique_ptr createInterruptCallback( - std::function callback); - -void triggerInterrupt(); - -/** - * A RAII class that causes the current thread to receive SIGUSR1 when - * the signal handler thread receives SIGINT. That is, this allows - * SIGINT to be multiplexed to multiple threads. - */ -struct ReceiveInterrupts -{ - pthread_t target; - std::unique_ptr callback; - - ReceiveInterrupts() - : target(pthread_self()) - , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) - { } -}; - - - /** * A RAII helper that increments a counter on construction and * decrements it on destruction. @@ -890,45 +304,6 @@ struct MaintainCount }; -/** - * @return the number of rows and columns of the terminal. - */ -std::pair getWindowSize(); - - -/** - * Used in various places. - */ -typedef std::function PathFilter; - -extern PathFilter defaultPathFilter; - -/** - * Common initialisation performed in child processes. - */ -void commonChildInit(); - -/** - * Create a Unix domain socket. - */ -AutoCloseFD createUnixDomainSocket(); - -/** - * Create a Unix domain socket in listen mode. - */ -AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode); - -/** - * Bind a Unix domain socket to a path. - */ -void bind(int fd, const std::string & path); - -/** - * Connect to a Unix domain socket. - */ -void connect(int fd, const std::string & path); - - /** * A Rust/Python-like enumerate() iterator adapter. * diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index e62c4f6b1..01da028d8 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -9,12 +9,13 @@ #include +#include "current-process.hh" #include "parsed-derivations.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "globals.hh" +#include "realisation.hh" #include "derivations.hh" -#include "util.hh" #include "shared.hh" #include "path-with-outputs.hh" #include "eval.hh" @@ -310,8 +311,11 @@ static void main_nix_build(int argc, char * * argv) else /* If we're in a #! script, interpret filenames relative to the script. */ - exprs.push_back(state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, - inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i))))); + exprs.push_back( + state->parseExprFromFile( + resolveExprPath( + lookupFileArg(*state, + inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))); } } @@ -449,7 +453,7 @@ static void main_nix_build(int argc, char * * argv) } } for (const auto & src : drv.inputSrcs) { - pathsToBuild.push_back(DerivedPath::Opaque{src}); + pathsToBuild.emplace_back(DerivedPath::Opaque{src}); pathsToCopy.insert(src); } diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc old mode 100755 new mode 100644 index 4504441fa..79db78236 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -5,7 +5,7 @@ #include "store-api.hh" #include "legacy.hh" #include "eval-settings.hh" // for defexpr -#include "util.hh" +#include "users.hh" #include "tarball.hh" #include diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 70af53b28..bb3f1bc6a 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -1,3 +1,5 @@ +#include "file-system.hh" +#include "signals.hh" #include "store-api.hh" #include "store-cast.hh" #include "gc-store.hh" diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc old mode 100755 new mode 100644 index 7f2bb93b6..b64af758f --- a/src/nix-copy-closure/nix-copy-closure.cc +++ b/src/nix-copy-closure/nix-copy-closure.cc @@ -1,4 +1,5 @@ #include "shared.hh" +#include "realisation.hh" #include "store-api.hh" #include "legacy.hh" diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 01742daa8..e2bbd9775 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1,3 +1,4 @@ +#include "users.hh" #include "attr-path.hh" #include "common-eval-args.hh" #include "derivations.hh" @@ -11,7 +12,6 @@ #include "store-api.hh" #include "local-fs-store.hh" #include "user-env.hh" -#include "util.hh" #include "value-to-json.hh" #include "xml-writer.hh" #include "legacy.hh" @@ -97,7 +97,7 @@ static bool isNixExpr(const SourcePath & path, struct InputAccessor::Stat & st) { return st.type == InputAccessor::tRegular - || (st.type == InputAccessor::tDirectory && (path + "default.nix").pathExists()); + || (st.type == InputAccessor::tDirectory && (path + "default.nix").resolveSymlinks().pathExists()); } @@ -116,11 +116,11 @@ static void getAllExprs(EvalState & state, are implemented using profiles). */ if (i == "manifest.nix") continue; - SourcePath path2 = path + i; + auto path2 = (path + i).resolveSymlinks(); InputAccessor::Stat st; try { - st = path2.resolveSymlinks().lstat(); + st = path2.lstat(); } catch (Error &) { continue; // ignore dangling symlinks in ~/.nix-defexpr } @@ -172,7 +172,7 @@ static void loadSourceExpr(EvalState & state, const SourcePath & path, Value & v directory). */ else if (st.type == InputAccessor::tDirectory) { auto attrs = state.buildBindings(maxAttrs); - attrs.alloc("_combineChannels").mkList(0); + state.mkList(attrs.alloc("_combineChannels"), 0); StringSet seen; getAllExprs(state, path, seen, attrs); v.mkAttrs(attrs); @@ -481,12 +481,12 @@ static void printMissing(EvalState & state, DrvInfos & elems) std::vector targets; for (auto & i : elems) if (auto drvPath = i.queryDrvPath()) - targets.push_back(DerivedPath::Built{ + targets.emplace_back(DerivedPath::Built{ .drvPath = makeConstantStorePathRef(*drvPath), .outputs = OutputsSpec::All { }, }); else - targets.push_back(DerivedPath::Opaque{ + targets.emplace_back(DerivedPath::Opaque{ .path = i.queryOutPath(), }); @@ -922,7 +922,7 @@ static VersionDiff compareVersionAgainstSet( } -static void queryJSON(Globals & globals, std::vector & elems, bool printOutPath, bool printMeta) +static void queryJSON(Globals & globals, std::vector & elems, bool printOutPath, bool printDrvPath, bool printMeta) { using nlohmann::json; json topObj = json::object(); @@ -953,6 +953,11 @@ static void queryJSON(Globals & globals, std::vector & elems, bool prin } } + if (printDrvPath) { + auto drvPath = i.queryDrvPath(); + if (drvPath) pkgObj["drvPath"] = globals.state->store->printStorePath(*drvPath); + } + if (printMeta) { json &metaObj = pkgObj["meta"]; metaObj = json::object(); @@ -1079,7 +1084,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) /* Print the desired columns, or XML output. */ if (jsonOutput) { - queryJSON(globals, elems, printOutPath, printMeta); + queryJSON(globals, elems, printOutPath, printDrvPath, printMeta); cout << '\n'; return; } diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index d12d70f33..34f6bd005 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -1,5 +1,4 @@ #include "user-env.hh" -#include "util.hh" #include "derivations.hh" #include "store-api.hh" #include "path-with-outputs.hh" @@ -22,7 +21,7 @@ DrvInfos queryInstalled(EvalState & state, const Path & userEnv) auto manifestFile = userEnv + "/manifest.nix"; if (pathExists(manifestFile)) { Value v; - state.evalFile(state.rootPath(CanonPath(manifestFile)), v); + state.evalFile(state.rootPath(CanonPath(manifestFile)).resolveSymlinks(), v); Bindings & bindings(*state.allocBindings(0)); getDerivations(state, v, "", bindings, elems, false); } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index d40196497..86b9be17d 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -6,7 +6,6 @@ #include "attr-path.hh" #include "value-to-xml.hh" #include "value-to-json.hh" -#include "util.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "common-eval-args.hh" @@ -184,7 +183,7 @@ static int main_nix_instantiate(int argc, char * * argv) for (auto & i : files) { Expr * e = fromArgs ? state->parseExprFromString(i, state->rootPath(CanonPath::fromCwd())) - : state->parseExprFromFile(resolveExprPath(state->checkSourcePath(lookupFileArg(*state, i)))); + : state->parseExprFromFile(resolveExprPath(lookupFileArg(*state, i))); processExpr(*state, attrPaths, parseOnly, strict, autoArgs, evalOnly, outputKind, xmlOutputSourceLocation, e); } diff --git a/src/nix-store/dotgraph.cc b/src/nix-store/dotgraph.cc index 577cadceb..2c530999b 100644 --- a/src/nix-store/dotgraph.cc +++ b/src/nix-store/dotgraph.cc @@ -1,5 +1,4 @@ #include "dotgraph.hh" -#include "util.hh" #include "store-api.hh" #include diff --git a/src/nix-store/graphml.cc b/src/nix-store/graphml.cc index 439557658..3e789a2d8 100644 --- a/src/nix-store/graphml.cc +++ b/src/nix-store/graphml.cc @@ -1,5 +1,4 @@ #include "graphml.hh" -#include "util.hh" #include "store-api.hh" #include "derivations.hh" diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index e4dd94585..d361dc0ac 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -11,10 +11,10 @@ #include "serve-protocol.hh" #include "serve-protocol-impl.hh" #include "shared.hh" -#include "util.hh" #include "graphml.hh" #include "legacy.hh" #include "path-with-outputs.hh" +#include "posix-fs-canonicalise.hh" #include #include @@ -193,7 +193,7 @@ static void opAddFixed(Strings opFlags, Strings opArgs) if (opArgs.empty()) throw UsageError("first argument must be hash algorithm"); - HashType hashAlgo = parseHashType(opArgs.front()); + HashAlgorithm hashAlgo = parseHashAlgo(opArgs.front()); opArgs.pop_front(); for (auto & i : opArgs) @@ -214,7 +214,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) throw UsageError("'--print-fixed-path' requires three arguments"); Strings::iterator i = opArgs.begin(); - HashType hashAlgo = parseHashType(*i++); + HashAlgorithm hashAlgo = parseHashAlgo(*i++); std::string hash = *i++; std::string name = *i++; @@ -405,8 +405,8 @@ static void opQuery(Strings opFlags, Strings opArgs) for (auto & j : maybeUseOutputs(store->followLinksToStorePath(i), useOutput, forceRealise)) { auto info = store->queryPathInfo(j); if (query == qHash) { - assert(info->narHash.type == htSHA256); - cout << fmt("%s\n", info->narHash.to_string(HashFormat::Base32, true)); + assert(info->narHash.algo == HashAlgorithm::SHA256); + cout << fmt("%s\n", info->narHash.to_string(HashFormat::Nix32, true)); } else if (query == qSize) cout << fmt("%d\n", info->narSize); } @@ -541,7 +541,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) if (canonicalise) canonicalisePathMetaData(store->printStorePath(info->path), {}); if (!hashGiven) { - HashResult hash = hashPath(htSHA256, store->printStorePath(info->path)); + HashResult hash = hashPath(HashAlgorithm::SHA256, store->printStorePath(info->path)); info->narHash = hash.first; info->narSize = hash.second; } @@ -763,14 +763,14 @@ static void opVerifyPath(Strings opFlags, Strings opArgs) auto path = store->followLinksToStorePath(i); printMsg(lvlTalkative, "checking path '%s'...", store->printStorePath(path)); auto info = store->queryPathInfo(path); - HashSink sink(info->narHash.type); + HashSink sink(info->narHash.algo); store->narFromPath(path, sink); auto current = sink.finish(); if (current.first != info->narHash) { printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(path), - info->narHash.to_string(HashFormat::Base32, true), - current.first.to_string(HashFormat::Base32, true)); + info->narHash.to_string(HashFormat::Nix32, true), + current.first.to_string(HashFormat::Nix32, true)); status = 1; } } @@ -835,27 +835,33 @@ static void opServe(Strings opFlags, Strings opArgs) verbosity = lvlError; settings.keepLog = false; settings.useSubstitutes = false; - settings.maxSilentTime = readInt(in); - settings.buildTimeout = readInt(in); + + auto options = ServeProto::Serialise::read(*store, rconn); + + // Only certain feilds get initialized based on the protocol + // version. This is why not all the code below is unconditional. + // See how the serialization logic in + // `ServeProto::Serialise` matches + // these conditions. + settings.maxSilentTime = options.maxSilentTime; + settings.buildTimeout = options.buildTimeout; if (GET_PROTOCOL_MINOR(clientVersion) >= 2) - settings.maxLogSize = readNum(in); + settings.maxLogSize = options.maxLogSize; if (GET_PROTOCOL_MINOR(clientVersion) >= 3) { - auto nrRepeats = readInt(in); - if (nrRepeats != 0) { + if (options.nrRepeats != 0) { throw Error("client requested repeating builds, but this is not currently implemented"); } - // Ignore 'enforceDeterminism'. It used to be true by - // default, but also only never had any effect when - // `nrRepeats == 0`. We have already asserted that - // `nrRepeats` in fact is 0, so we can safely ignore this - // without doing something other than what the client - // asked for. - readInt(in); - + // Ignore 'options.enforceDeterminism'. + // + // It used to be true by default, but also only never had + // any effect when `nrRepeats == 0`. We have already + // checked that `nrRepeats` in fact is 0, so we can safely + // ignore this without doing something other than what the + // client asked for. settings.runDiffHook = true; } if (GET_PROTOCOL_MINOR(clientVersion) >= 7) { - settings.keepFailed = (bool) readInt(in); + settings.keepFailed = options.keepFailed; } }; @@ -891,16 +897,8 @@ static void opServe(Strings opFlags, Strings opArgs) for (auto & i : paths) { try { auto info = store->queryPathInfo(i); - out << store->printStorePath(info->path) - << (info->deriver ? store->printStorePath(*info->deriver) : ""); - ServeProto::write(*store, wconn, info->references); - // !!! Maybe we want compression? - out << info->narSize // downloadSize - << info->narSize; - if (GET_PROTOCOL_MINOR(clientVersion) >= 4) - out << info->narHash.to_string(HashFormat::Base32, true) - << renderContentAddress(info->ca) - << info->sigs; + out << store->printStorePath(info->path); + ServeProto::write(*store, wconn, static_cast(*info)); } catch (InvalidPath &) { } } @@ -979,7 +977,7 @@ static void opServe(Strings opFlags, Strings opArgs) auto deriver = readString(in); ValidPathInfo info { store->parseStorePath(path), - Hash::parseAny(readString(in), htSHA256), + Hash::parseAny(readString(in), HashAlgorithm::SHA256), }; if (deriver != "") info.deriver = store->parseStorePath(deriver); diff --git a/src/nix/add-file.md b/src/nix/add-file.md deleted file mode 100644 index ed237a035..000000000 --- a/src/nix/add-file.md +++ /dev/null @@ -1,28 +0,0 @@ -R""( - -# Description - -Copy the regular file *path* to the Nix store, and print the resulting -store path on standard output. - -> **Warning** -> -> The resulting store path is not registered as a garbage -> collector root, so it could be deleted before you have a -> chance to register it. - -# Examples - -Add a regular file to the store: - -```console -# echo foo > bar - -# nix store add-file ./bar -/nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar - -# cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar -foo -``` - -)"" diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 39e5cc99d..02de796b5 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -5,11 +5,22 @@ using namespace nix; +static FileIngestionMethod parseIngestionMethod(std::string_view input) +{ + if (input == "flat") { + return FileIngestionMethod::Flat; + } else if (input == "nar") { + return FileIngestionMethod::Recursive; + } else { + throw UsageError("Unknown hash mode '%s', expect `flat` or `nar`"); + } +} + struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; - FileIngestionMethod ingestionMethod; + FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive; CmdAddToStore() { @@ -23,6 +34,23 @@ struct CmdAddToStore : MixDryRun, StoreCommand .labels = {"name"}, .handler = {&namePart}, }); + + addFlag({ + .longName = "mode", + .shortName = 'n', + .description = R"( + How to compute the hash of the input. + One of: + + - `nar` (the default): Serialises the input as an archive (following the [_Nix Archive Format_](https://edolstra.github.io/pubs/phd-thesis.pdf#page=101)) and passes that to the hash function. + + - `flat`: Assumes that the input is a single file and directly passes it to the hash function; + )", + .labels = {"hash-mode"}, + .handler = {[this](std::string s) { + this->ingestionMethod = parseIngestionMethod(s); + }}, + }); } void run(ref store) override @@ -32,11 +60,11 @@ struct CmdAddToStore : MixDryRun, StoreCommand StringSink sink; dumpPath(path, sink); - auto narHash = hashString(htSHA256, sink.s); + auto narHash = hashString(HashAlgorithm::SHA256, sink.s); Hash hash = narHash; if (ingestionMethod == FileIngestionMethod::Flat) { - HashSink hsink(htSHA256); + HashSink hsink(HashAlgorithm::SHA256); readFile(path, hsink); hash = hsink.finish().first; } @@ -62,6 +90,22 @@ struct CmdAddToStore : MixDryRun, StoreCommand } }; +struct CmdAdd : CmdAddToStore +{ + + std::string description() override + { + return "Add a file or directory to the Nix store"; + } + + std::string doc() override + { + return + #include "add.md" + ; + } +}; + struct CmdAddFile : CmdAddToStore { CmdAddFile() @@ -71,36 +115,18 @@ struct CmdAddFile : CmdAddToStore std::string description() override { - return "add a regular file to the Nix store"; - } - - std::string doc() override - { - return - #include "add-file.md" - ; + return "Deprecated. Use [`nix store add --mode flat`](@docroot@/command-ref/new-cli/nix3-store-add.md) instead."; } }; struct CmdAddPath : CmdAddToStore { - CmdAddPath() - { - ingestionMethod = FileIngestionMethod::Recursive; - } - std::string description() override { - return "add a path to the Nix store"; - } - - std::string doc() override - { - return - #include "add-path.md" - ; + return "Deprecated alias to [`nix store add`](@docroot@/command-ref/new-cli/nix3-store-add.md)."; } }; static auto rCmdAddFile = registerCommand2({"store", "add-file"}); static auto rCmdAddPath = registerCommand2({"store", "add-path"}); +static auto rCmdAdd = registerCommand2({"store", "add"}); diff --git a/src/nix/add-path.md b/src/nix/add.md similarity index 94% rename from src/nix/add-path.md rename to src/nix/add.md index 87473611d..d38cd21d8 100644 --- a/src/nix/add-path.md +++ b/src/nix/add.md @@ -19,7 +19,7 @@ Add a directory to the store: # mkdir dir # echo foo > dir/bar -# nix store add-path ./dir +# nix store add ./dir /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir # cat /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir/bar diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 504e35c81..54cc6a17f 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -4,7 +4,6 @@ #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "fs-accessor.hh" #include "eval-inline.hh" using namespace nix; diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 60aa66ce0..4df086d4f 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "store-api.hh" -#include "fs-accessor.hh" #include "nar-accessor.hh" +#include "progress-bar.hh" using namespace nix; @@ -9,15 +9,13 @@ struct MixCat : virtual Args { std::string path; - void cat(ref accessor) + void cat(ref accessor) { - auto st = accessor->stat(path); - if (st.type == FSAccessor::Type::tMissing) - throw Error("path '%1%' does not exist", path); - if (st.type != FSAccessor::Type::tRegular) + auto st = accessor->lstat(CanonPath(path)); + if (st.type != SourceAccessor::Type::tRegular) throw Error("path '%1%' is not a regular file", path); - - writeFull(STDOUT_FILENO, accessor->readFile(path)); + stopProgressBar(); + writeFull(STDOUT_FILENO, accessor->readFile(CanonPath(path))); } }; diff --git a/src/nix/doctor.cc b/src/nix/config-check.cc similarity index 97% rename from src/nix/doctor.cc rename to src/nix/config-check.cc index 1aa6831d3..410feca2f 100644 --- a/src/nix/doctor.cc +++ b/src/nix/config-check.cc @@ -6,7 +6,6 @@ #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" -#include "util.hh" #include "worker-protocol.hh" using namespace nix; @@ -39,7 +38,7 @@ void checkInfo(const std::string & msg) { } -struct CmdDoctor : StoreCommand +struct CmdConfigCheck : StoreCommand { bool success = true; @@ -153,4 +152,4 @@ struct CmdDoctor : StoreCommand } }; -static auto rCmdDoctor = registerCommand("doctor"); +static auto rCmdConfigCheck = registerCommand2({ "config", "check" }); diff --git a/src/nix/show-config.cc b/src/nix/config.cc similarity index 73% rename from src/nix/show-config.cc rename to src/nix/config.cc index 3530584f9..52706afcf 100644 --- a/src/nix/show-config.cc +++ b/src/nix/config.cc @@ -7,11 +7,24 @@ using namespace nix; -struct CmdShowConfig : Command, MixJSON +struct CmdConfig : NixMultiCommand +{ + CmdConfig() : NixMultiCommand("config", RegisterCommand::getCommandsFor({"config"})) + { } + + std::string description() override + { + return "manipulate the Nix configuration"; + } + + Category category() override { return catUtility; } +}; + +struct CmdConfigShow : Command, MixJSON { std::optional name; - CmdShowConfig() { + CmdConfigShow() { expectArgs({ .label = {"name"}, .optional = true, @@ -56,4 +69,5 @@ struct CmdShowConfig : Command, MixJSON } }; -static auto rShowConfig = registerCommand("show-config"); +static auto rCmdConfig = registerCommand("config"); +static auto rShowConfig = registerCommand2({"config", "show"}); diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc index af428018a..4dada8e0e 100644 --- a/src/nix/daemon.cc +++ b/src/nix/daemon.cc @@ -1,11 +1,12 @@ ///@file +#include "signals.hh" +#include "unix-domain-socket.hh" #include "command.hh" #include "shared.hh" #include "local-store.hh" #include "remote-store.hh" #include "remote-store-connection.hh" -#include "util.hh" #include "serialise.hh" #include "archive.hh" #include "globals.hh" @@ -442,16 +443,23 @@ static void processStdioConnection(ref store, TrustedFlag trustClient) * * @param forceTrustClientOpt See `daemonLoop()` and the parameter with * the same name over there for details. + * + * @param procesOps Whether to force processing ops even if the next + * store also is a remote store and could process it directly. */ -static void runDaemon(bool stdio, std::optional forceTrustClientOpt) +static void runDaemon(bool stdio, std::optional forceTrustClientOpt, bool processOps) { if (stdio) { auto store = openUncachedStore(); + std::shared_ptr remoteStore; + // If --force-untrusted is passed, we cannot forward the connection and // must process it ourselves (before delegating to the next store) to // force untrusting the client. - if (auto remoteStore = store.dynamic_pointer_cast(); remoteStore && (!forceTrustClientOpt || *forceTrustClientOpt != NotTrusted)) + processOps |= !forceTrustClientOpt || *forceTrustClientOpt != NotTrusted; + + if (!processOps && (remoteStore = store.dynamic_pointer_cast())) forwardStdioConnection(*remoteStore); else // `Trusted` is passed in the auto (no override case) because we @@ -467,6 +475,7 @@ static int main_nix_daemon(int argc, char * * argv) { auto stdio = false; std::optional isTrustedOpt = std::nullopt; + auto processOps = false; parseCmdLine(argc, argv, [&](Strings::iterator & arg, const Strings::iterator & end) { if (*arg == "--daemon") @@ -486,11 +495,14 @@ static int main_nix_daemon(int argc, char * * argv) } else if (*arg == "--default-trust") { experimentalFeatureSettings.require(Xp::DaemonTrustOverride); isTrustedOpt = std::nullopt; + } else if (*arg == "--process-ops") { + experimentalFeatureSettings.require(Xp::MountedSSHStore); + processOps = true; } else return false; return true; }); - runDaemon(stdio, isTrustedOpt); + runDaemon(stdio, isTrustedOpt, processOps); return 0; } @@ -502,6 +514,7 @@ struct CmdDaemon : StoreCommand { bool stdio = false; std::optional isTrustedOpt = std::nullopt; + bool processOps = false; CmdDaemon() { @@ -537,6 +550,19 @@ struct CmdDaemon : StoreCommand }}, .experimentalFeature = Xp::DaemonTrustOverride, }); + + addFlag({ + .longName = "process-ops", + .description = R"( + Forces the daemon to process received commands itself rather than forwarding the commands straight to the remote store. + + This is useful for the `mounted-ssh://` store where some actions need to be performed on the remote end but as connected user, and not as the user of the underlying daemon on the remote end. + )", + .handler = {[&]() { + processOps = true; + }}, + .experimentalFeature = Xp::MountedSSHStore, + }); } std::string description() override @@ -555,7 +581,7 @@ struct CmdDaemon : StoreCommand void run(ref store) override { - runDaemon(stdio, isTrustedOpt); + runDaemon(stdio, isTrustedOpt, processOps); } }; diff --git a/src/nix/derivation.cc b/src/nix/derivation.cc index cd3975a4f..59a78d378 100644 --- a/src/nix/derivation.cc +++ b/src/nix/derivation.cc @@ -2,9 +2,9 @@ using namespace nix; -struct CmdDerivation : virtual NixMultiCommand +struct CmdDerivation : NixMultiCommand { - CmdDerivation() : MultiCommand(RegisterCommand::getCommandsFor({"derivation"})) + CmdDerivation() : NixMultiCommand("derivation", RegisterCommand::getCommandsFor({"derivation"})) { } std::string description() override @@ -13,13 +13,6 @@ struct CmdDerivation : virtual NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix derivation' requires a sub-command."); - command->second->run(); - } }; static auto rCmdDerivation = registerCommand("derivation"); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index b080a3939..606b044b0 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -8,7 +8,6 @@ #include "derivations.hh" #include "progress-bar.hh" #include "run.hh" -#include "util.hh" #include #include @@ -663,7 +662,7 @@ struct CmdDevelop : Common, MixEnvironment } } - runProgramInStore(store, shell, args, buildEnvironment.getSystem()); + runProgramInStore(store, UseSearchPath::Use, shell, args, buildEnvironment.getSystem()); } }; diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index c4edc894b..0850d4c1c 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -61,4 +61,12 @@ struct CmdDumpPath2 : Command } }; -static auto rDumpPath2 = registerCommand2({"nar", "dump-path"}); +struct CmdNarDumpPath : CmdDumpPath2 { + void run() override { + warn("'nix nar dump-path' is a deprecated alias for 'nix nar pack'"); + CmdDumpPath2::run(); + } +}; + +static auto rCmdNarPack = registerCommand2({"nar", "pack"}); +static auto rCmdNarDumpPath = registerCommand2({"nar", "dump-path"}); diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 66629fab0..9cbab230b 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -1,3 +1,4 @@ +#include "current-process.hh" #include "command-installable-value.hh" #include "shared.hh" #include "eval.hh" diff --git a/src/nix/flake-init.md b/src/nix/flake-init.md index fc1f4f805..ea274bf29 100644 --- a/src/nix/flake-init.md +++ b/src/nix/flake-init.md @@ -30,7 +30,7 @@ using `-t`. # Template definitions A flake can declare templates through its `templates` output -attribute. A template has two attributes: +attribute. A template has the following attributes: * `description`: A one-line description of the template, in CommonMark syntax. diff --git a/src/nix/flake-lock.md b/src/nix/flake-lock.md index 2af0ad81e..6d10258e3 100644 --- a/src/nix/flake-lock.md +++ b/src/nix/flake-lock.md @@ -2,37 +2,39 @@ R""( # Examples -* Update the `nixpkgs` and `nix` inputs of the flake in the current - directory: +* Create the lock file for the flake in the current directory: ```console - # nix flake lock --update-input nixpkgs --update-input nix - * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' - * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' + # nix flake lock + warning: creating lock file '/home/myself/repos/testflake/flake.lock': + • Added input 'nix': + 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28) + • Added input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) ``` +* Add missing inputs to the lock file for a flake in a different directory: + + ```console + # nix flake lock ~/repos/another + warning: updating lock file '/home/myself/repos/another/flake.lock': + • Added input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + ``` + + > **Note** + > + > When trying to refer to a flake in a subdirectory, write `./another` + > instead of `another`. + > Otherwise Nix will try to look up the flake in the registry. + # Description -This command updates the lock file of a flake (`flake.lock`) so that -it contains a lock for every flake input specified in -`flake.nix`. Existing lock file entries are not updated unless -required by a flag such as `--update-input`. +This command adds inputs to the lock file of a flake (`flake.lock`) +so that it contains a lock for every flake input specified in +`flake.nix`. Existing lock file entries are not updated. -Note that every command that operates on a flake will also update the -lock file if needed, and supports the same flags. Therefore, - -```console -# nix flake lock --update-input nixpkgs -# nix build -``` - -is equivalent to: - -```console -# nix build --update-input nixpkgs -``` - -Thus, this command is only useful if you want to update the lock file -separately from any other action such as building. +If you want to update existing lock entries, use +[`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) )"" diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md index 8c6042d94..63df3b12a 100644 --- a/src/nix/flake-update.md +++ b/src/nix/flake-update.md @@ -2,33 +2,57 @@ R""( # Examples -* Recreate the lock file (i.e. update all inputs) and commit the new - lock file: +* Update all inputs (i.e. recreate the lock file from scratch): ```console - # nix flake update --commit-lock-file - * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' - * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' - … - warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad' + # nix flake update + warning: updating lock file '/home/myself/repos/testflake/flake.lock': + • Updated input 'nix': + 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' (2023-06-28) + → 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' (2023-07-11) + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) ``` +* Update only a single input: + + ```console + # nix flake update nixpkgs + warning: updating lock file '/home/myself/repos/testflake/flake.lock': + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) + ``` + +* Update only a single input of a flake in a different directory: + + ```console + # nix flake update nixpkgs --flake ~/repos/another + warning: updating lock file '/home/myself/repos/another/flake.lock': + • Updated input 'nixpkgs': + 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' (2023-06-30) + → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) + ``` + + > **Note** + > + > When trying to refer to a flake in a subdirectory, write `./another` + > instead of `another`. + > Otherwise Nix will try to look up the flake in the registry. + # Description -This command recreates the lock file of a flake (`flake.lock`), thus -updating the lock for every unlocked input (like `nixpkgs`) to its -current version. This is equivalent to passing `--recreate-lock-file` -to any command that operates on a flake. That is, +This command updates the inputs in a lock file (`flake.lock`). +**By default, all inputs are updated**. If the lock file doesn't exist +yet, it will be created. If inputs are not in the lock file yet, they will be added. -```console -# nix flake update -# nix build -``` +Unlike other `nix flake` commands, `nix flake update` takes a list of names of inputs +to update as its positional arguments and operates on the flake in the current directory. +You can pass a different flake-url with `--flake` to override that default. -is equivalent to: - -```console -# nix build --recreate-lock-file -``` +The related command [`nix flake lock`](@docroot@/command-ref/new-cli/nix3-flake-lock.md) +also creates lock files and adds missing inputs, but is safer as it +will never update inputs already in the lock file. )"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 0116eff2e..2b6e56283 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -15,6 +15,7 @@ #include "registry.hh" #include "eval-cache.hh" #include "markdown.hh" +#include "users.hh" #include #include @@ -24,8 +25,10 @@ using namespace nix; using namespace nix::flake; using json = nlohmann::json; +struct CmdFlakeUpdate; class FlakeCommand : virtual Args, public MixFlakeOptions { +protected: std::string flakeUrl = "."; public: @@ -63,6 +66,8 @@ public: struct CmdFlakeUpdate : FlakeCommand { +public: + std::string description() override { return "update flake lock file"; @@ -70,9 +75,37 @@ struct CmdFlakeUpdate : FlakeCommand CmdFlakeUpdate() { + expectedArgs.clear(); + addFlag({ + .longName="flake", + .description="The flake to operate on. Default is the current directory.", + .labels={"flake-url"}, + .handler={&flakeUrl}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeRef(completions, getStore(), prefix); + }} + }); + expectArgs({ + .label="inputs", + .optional=true, + .handler={[&](std::string inputToUpdate){ + InputPath inputPath; + try { + inputPath = flake::parseInputPath(inputToUpdate); + } catch (Error & e) { + warn("Invalid flake input '%s'. To update a specific flake, use 'nix flake update --flake %s' instead.", inputToUpdate, inputToUpdate); + throw e; + } + if (lockFlags.inputUpdates.contains(inputPath)) + warn("Input '%s' was specified multiple times. You may have done this by accident."); + lockFlags.inputUpdates.insert(inputPath); + }}, + .completer = {[&](AddCompletions & completions, size_t, std::string_view prefix) { + completeFlakeInputPath(completions, getEvalState(), getFlakeRefsForCompletion(), prefix); + }} + }); + /* Remove flags that don't make sense. */ - removeFlag("recreate-lock-file"); - removeFlag("update-input"); removeFlag("no-update-lock-file"); removeFlag("no-write-lock-file"); } @@ -87,8 +120,9 @@ struct CmdFlakeUpdate : FlakeCommand void run(nix::ref store) override { settings.tarballTtl = 0; + auto updateAll = lockFlags.inputUpdates.empty(); - lockFlags.recreateLockFile = true; + lockFlags.recreateLockFile = updateAll; lockFlags.writeLockFile = true; lockFlags.applyNixConfig = true; @@ -1365,7 +1399,9 @@ struct CmdFlakePrefetch : FlakeCommand, MixJSON struct CmdFlake : NixMultiCommand { CmdFlake() - : MultiCommand({ + : NixMultiCommand( + "flake", + { {"update", []() { return make_ref(); }}, {"lock", []() { return make_ref(); }}, {"metadata", []() { return make_ref(); }}, @@ -1395,10 +1431,8 @@ struct CmdFlake : NixMultiCommand void run() override { - if (!command) - throw UsageError("'nix flake' requires a sub-command."); experimentalFeatureSettings.require(Xp::Flakes); - command->second->run(); + NixMultiCommand::run(); } }; diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc index c85eacded..059904150 100644 --- a/src/nix/fmt.cc +++ b/src/nix/fmt.cc @@ -49,7 +49,7 @@ struct CmdFmt : SourceExprCommand { } } - runProgramInStore(store, app.program, programArgs); + runProgramInStore(store, UseSearchPath::DontUse, app.program, programArgs); }; }; diff --git a/src/nix/hash.cc b/src/nix/hash.cc index d6595dcca..0bba3b7d2 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -8,16 +8,21 @@ using namespace nix; +/** + * Base for `nix hash file` (deprecated), `nix hash path` and `nix-hash` (legacy). + * + * Deprecation Issue: https://github.com/NixOS/nix/issues/8876 + */ struct CmdHashBase : Command { FileIngestionMethod mode; HashFormat hashFormat = HashFormat::SRI; bool truncate = false; - HashType ht = htSHA256; + HashAlgorithm ha = HashAlgorithm::SHA256; std::vector paths; std::optional modulus; - CmdHashBase(FileIngestionMethod mode) : mode(mode) + explicit CmdHashBase(FileIngestionMethod mode) : mode(mode) { addFlag({ .longName = "sri", @@ -34,7 +39,7 @@ struct CmdHashBase : Command addFlag({ .longName = "base32", .description = "Print the hash in base-32 (Nix-specific) format.", - .handler = {&hashFormat, HashFormat::Base32}, + .handler = {&hashFormat, HashFormat::Nix32}, }); addFlag({ @@ -43,7 +48,7 @@ struct CmdHashBase : Command .handler = {&hashFormat, HashFormat::Base16}, }); - addFlag(Flag::mkHashTypeFlag("type", &ht)); + addFlag(Flag::mkHashAlgoFlag("type", &ha)); #if 0 addFlag({ @@ -79,9 +84,9 @@ struct CmdHashBase : Command std::unique_ptr hashSink; if (modulus) - hashSink = std::make_unique(ht, *modulus); + hashSink = std::make_unique(ha, *modulus); else - hashSink = std::make_unique(ht); + hashSink = std::make_unique(ha); switch (mode) { case FileIngestionMethod::Flat: @@ -102,39 +107,90 @@ struct CmdHashBase : Command struct CmdToBase : Command { HashFormat hashFormat; - std::optional ht; + std::optional ht; std::vector args; CmdToBase(HashFormat hashFormat) : hashFormat(hashFormat) { - addFlag(Flag::mkHashTypeOptFlag("type", &ht)); + addFlag(Flag::mkHashAlgoOptFlag("type", &ht)); expectArgs("strings", &args); } std::string description() override { - return fmt("convert a hash to %s representation", + return fmt("convert a hash to %s representation (deprecated, use `nix hash convert` instead)", hashFormat == HashFormat::Base16 ? "base-16" : - hashFormat == HashFormat::Base32 ? "base-32" : + hashFormat == HashFormat::Nix32 ? "base-32" : hashFormat == HashFormat::Base64 ? "base-64" : "SRI"); } void run() override { + warn("The old format conversion sub commands of `nix hash` where deprecated in favor of `nix hash convert`."); for (auto s : args) logger->cout(Hash::parseAny(s, ht).to_string(hashFormat, hashFormat == HashFormat::SRI)); } }; +/** + * `nix hash convert` + */ +struct CmdHashConvert : Command +{ + std::optional from; + HashFormat to; + std::optional algo; + std::vector hashStrings; + + CmdHashConvert(): to(HashFormat::SRI) { + addFlag(Args::Flag::mkHashFormatOptFlag("from", &from)); + addFlag(Args::Flag::mkHashFormatFlagWithDefault("to", &to)); + addFlag(Args::Flag::mkHashAlgoOptFlag("algo", &algo)); + expectArgs({ + .label = "hashes", + .handler = {&hashStrings}, + }); + } + + std::string description() override + { + std::string descr( "convert between different hash formats. Choose from: "); + auto iter = hashFormats.begin(); + assert(iter != hashFormats.end()); + descr += *iter++; + while (iter != hashFormats.end()) { + descr += ", " + *iter++; + } + + return descr; + } + + Category category() override { return catUtility; } + + void run() override { + for (const auto& s: hashStrings) { + Hash h = Hash::parseAny(s, algo); + if (from && h.to_string(*from, from == HashFormat::SRI) != s) { + auto from_as_string = printHashFormat(*from); + throw BadHash("input hash '%s' does not have the expected format '--from %s'", s, from_as_string); + } + logger->cout(h.to_string(to, to == HashFormat::SRI)); + } + } +}; + struct CmdHash : NixMultiCommand { CmdHash() - : MultiCommand({ + : NixMultiCommand( + "hash", + { + {"convert", []() { return make_ref();}}, {"file", []() { return make_ref(FileIngestionMethod::Flat);; }}, {"path", []() { return make_ref(FileIngestionMethod::Recursive); }}, {"to-base16", []() { return make_ref(HashFormat::Base16); }}, - {"to-base32", []() { return make_ref(HashFormat::Base32); }}, + {"to-base32", []() { return make_ref(HashFormat::Nix32); }}, {"to-base64", []() { return make_ref(HashFormat::Base64); }}, {"to-sri", []() { return make_ref(HashFormat::SRI); }}, }) @@ -146,13 +202,6 @@ struct CmdHash : NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix hash' requires a sub-command."); - command->second->run(); - } }; static auto rCmdHash = registerCommand("hash"); @@ -160,7 +209,10 @@ static auto rCmdHash = registerCommand("hash"); /* Legacy nix-hash command. */ static int compatNixHash(int argc, char * * argv) { - std::optional ht; + // Wait until `nix hash convert` is not hidden behind experimental flags anymore. + // warn("`nix-hash` has been deprecated in favor of `nix hash convert`."); + + std::optional ha; bool flat = false; HashFormat hashFormat = HashFormat::Base16; bool truncate = false; @@ -174,13 +226,13 @@ static int compatNixHash(int argc, char * * argv) printVersion("nix-hash"); else if (*arg == "--flat") flat = true; else if (*arg == "--base16") hashFormat = HashFormat::Base16; - else if (*arg == "--base32") hashFormat = HashFormat::Base32; + else if (*arg == "--base32") hashFormat = HashFormat::Nix32; else if (*arg == "--base64") hashFormat = HashFormat::Base64; else if (*arg == "--sri") hashFormat = HashFormat::SRI; else if (*arg == "--truncate") truncate = true; else if (*arg == "--type") { std::string s = getArg(*arg, arg, end); - ht = parseHashType(s); + ha = parseHashAlgo(s); } else if (*arg == "--to-base16") { op = opTo; @@ -188,7 +240,7 @@ static int compatNixHash(int argc, char * * argv) } else if (*arg == "--to-base32") { op = opTo; - hashFormat = HashFormat::Base32; + hashFormat = HashFormat::Nix32; } else if (*arg == "--to-base64") { op = opTo; @@ -207,8 +259,8 @@ static int compatNixHash(int argc, char * * argv) if (op == opHash) { CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); - if (!ht.has_value()) ht = htMD5; - cmd.ht = ht.value(); + if (!ha.has_value()) ha = HashAlgorithm::MD5; + cmd.ha = ha.value(); cmd.hashFormat = hashFormat; cmd.truncate = truncate; cmd.paths = ss; @@ -218,7 +270,7 @@ static int compatNixHash(int argc, char * * argv) else { CmdToBase cmd(hashFormat); cmd.args = ss; - if (ht.has_value()) cmd.ht = ht; + if (ha.has_value()) cmd.ht = ha; cmd.run(); } diff --git a/src/nix/local.mk b/src/nix/local.mk index 57f8259c4..a21aa705f 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -31,10 +31,23 @@ src/nix/develop.cc: src/nix/get-env.sh.gen.hh src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh -src/nix/main.cc: doc/manual/generate-manpage.nix.gen.hh doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh doc/manual/generate-store-info.nix.gen.hh +src/nix/main.cc: \ + doc/manual/generate-manpage.nix.gen.hh \ + doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh \ + doc/manual/generate-store-info.nix.gen.hh \ + src/nix/generated-doc/help-stores.md -src/nix/doc/files/%.md: doc/manual/src/command-ref/files/%.md +src/nix/generated-doc/files/%.md: doc/manual/src/command-ref/files/%.md @mkdir -p $$(dirname $@) @cp $< $@ -src/nix/profile.cc: src/nix/profile.md src/nix/doc/files/profiles.md.gen.hh +src/nix/profile.cc: src/nix/profile.md src/nix/generated-doc/files/profiles.md.gen.hh + +src/nix/generated-doc/help-stores.md: doc/manual/src/store/types/index.md.in + @mkdir -p $$(dirname $@) + @echo 'R"(' >> $@.tmp + @echo >> $@.tmp + @cat $^ >> $@.tmp + @echo >> $@.tmp + @echo ')"' >> $@.tmp + @mv $@.tmp $@ diff --git a/src/nix/ls.cc b/src/nix/ls.cc index c990a303c..231456c9c 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -1,6 +1,5 @@ #include "command.hh" #include "store-api.hh" -#include "fs-accessor.hh" #include "nar-accessor.hh" #include "common-args.hh" #include @@ -39,61 +38,58 @@ struct MixLs : virtual Args, MixJSON }); } - void listText(ref accessor) + void listText(ref accessor) { - std::function doPath; + std::function doPath; - auto showFile = [&](const Path & curPath, const std::string & relPath) { + auto showFile = [&](const CanonPath & curPath, std::string_view relPath) { if (verbose) { - auto st = accessor->stat(curPath); + auto st = accessor->lstat(curPath); std::string tp = - st.type == FSAccessor::Type::tRegular ? + st.type == SourceAccessor::Type::tRegular ? (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : - st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : + st.type == SourceAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); - if (st.type == FSAccessor::Type::tSymlink) + auto line = fmt("%s %20d %s", tp, st.fileSize.value_or(0), relPath); + if (st.type == SourceAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); logger->cout(line); - if (recursive && st.type == FSAccessor::Type::tDirectory) + if (recursive && st.type == SourceAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } else { logger->cout(relPath); if (recursive) { - auto st = accessor->stat(curPath); - if (st.type == FSAccessor::Type::tDirectory) + auto st = accessor->lstat(curPath); + if (st.type == SourceAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } } }; - doPath = [&](const FSAccessor::Stat & st, const Path & curPath, - const std::string & relPath, bool showDirectory) + doPath = [&](const SourceAccessor::Stat & st, const CanonPath & curPath, + std::string_view relPath, bool showDirectory) { - if (st.type == FSAccessor::Type::tDirectory && !showDirectory) { + if (st.type == SourceAccessor::Type::tDirectory && !showDirectory) { auto names = accessor->readDirectory(curPath); - for (auto & name : names) - showFile(curPath + "/" + name, relPath + "/" + name); + for (auto & [name, type] : names) + showFile(curPath + name, relPath + "/" + name); } else showFile(curPath, relPath); }; - auto st = accessor->stat(path); - if (st.type == FSAccessor::Type::tMissing) - throw Error("path '%1%' does not exist", path); - doPath(st, path, - st.type == FSAccessor::Type::tDirectory ? "." : std::string(baseNameOf(path)), + auto path2 = CanonPath(path); + auto st = accessor->lstat(path2); + doPath(st, path2, + st.type == SourceAccessor::Type::tDirectory ? "." : path2.baseName().value_or(""), showDirectory); } - void list(ref accessor) + void list(ref accessor) { - if (path == "/") path = ""; - if (json) { if (showDirectory) throw UsageError("'--directory' is useless with '--json'"); - logger->cout("%s", listNar(accessor, path, recursive)); + logger->cout("%s", listNar(accessor, CanonPath(path), recursive)); } else listText(accessor); } diff --git a/src/nix/main.cc b/src/nix/main.cc index ffba10099..39c04069b 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -1,6 +1,8 @@ #include #include "args/root.hh" +#include "current-process.hh" +#include "namespaces.hh" #include "command.hh" #include "common-args.hh" #include "eval.hh" @@ -20,6 +22,7 @@ #include #include #include +#include #include @@ -64,7 +67,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs bool helpRequested = false; bool showVersion = false; - NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix") + NixArgs() : MultiCommand("", RegisterCommand::getCommandsFor({})), MixCommonArgs("nix") { categories.clear(); categories[catHelp] = "Help commands"; @@ -131,10 +134,12 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs {"ping-store", {"store", "ping"}}, {"sign-paths", {"store", "sign"}}, {"show-derivation", {"derivation", "show"}}, + {"show-config", {"config", "show"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, {"verify", {"store", "verify"}}, + {"doctor", {"config", "check"}}, }; bool aliasUsed = false; @@ -188,6 +193,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs j["experimentalFeature"] = storeConfig->experimentalFeature(); } res["stores"] = std::move(stores); + res["fetchers"] = fetchers::dumpRegisterInputSchemeInfo(); return res.dump(); } @@ -292,7 +298,7 @@ struct CmdHelpStores : Command std::string doc() override { return - #include "help-stores.md" + #include "generated-doc/help-stores.md" ; } @@ -367,6 +373,7 @@ void mainWrapped(int argc, char * * argv) Xp::Flakes, Xp::FetchClosure, Xp::DynamicDerivations, + Xp::FetchTree, }; evalSettings.pureEval = false; EvalState state({}, openStore("dummy://")); @@ -425,7 +432,9 @@ void mainWrapped(int argc, char * * argv) }); try { - args.parseCmdline(argvToStrings(argc, argv)); + auto isNixCommand = std::regex_search(programName, std::regex("nix$")); + auto allowShebang = isNixCommand && argc > 1; + args.parseCmdline(argvToStrings(argc, argv),allowShebang); } catch (UsageError &) { if (!args.helpRequested && !args.completions) throw; } diff --git a/src/nix/nar-dump-path.md b/src/nix/nar-dump-path.md index 26191ad25..de82202de 100644 --- a/src/nix/nar-dump-path.md +++ b/src/nix/nar-dump-path.md @@ -5,7 +5,7 @@ R""( * To serialise directory `foo` as a NAR: ```console - # nix nar dump-path ./foo > foo.nar + # nix nar pack ./foo > foo.nar ``` # Description diff --git a/src/nix/nar.cc b/src/nix/nar.cc index 9815410cf..8ad4f92a7 100644 --- a/src/nix/nar.cc +++ b/src/nix/nar.cc @@ -4,7 +4,7 @@ using namespace nix; struct CmdNar : NixMultiCommand { - CmdNar() : MultiCommand(RegisterCommand::getCommandsFor({"nar"})) + CmdNar() : NixMultiCommand("nar", RegisterCommand::getCommandsFor({"nar"})) { } std::string description() override @@ -20,13 +20,6 @@ struct CmdNar : NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix nar' requires a sub-command."); - command->second->run(); - } }; static auto rCmdNar = registerCommand("nar"); diff --git a/src/nix/nix.md b/src/nix/nix.md index 6e7e8a649..749456014 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -235,7 +235,74 @@ operate are determined as follows: # Nix stores -Most `nix` subcommands operate on a *Nix store*. These are documented -in [`nix help-stores`](./nix3-help-stores.md). +Most `nix` subcommands operate on a *Nix store*. +The various store types are documented in the +[Store Types](@docroot@/store/types/index.md) +section of the manual. + +The same information is also available from the [`nix help-stores`](./nix3-help-stores.md) command. + +# Shebang interpreter + +The `nix` command can be used as a `#!` interpreter. +Arguments to Nix can be passed on subsequent lines in the script. + +Verbatim strings may be passed in double backtick (```` `` ````) quotes. +Sequences of _n_ backticks of 3 or longer are parsed as _n-1_ literal backticks. +A single space before the closing ```` `` ```` is ignored if present. + +`--file` and `--expr` resolve relative paths based on the script location. + +Examples: + +``` +#!/usr/bin/env nix +#! nix shell --file ```` hello cowsay --command bash + +hello | cowsay +``` + +or with **flakes**: + +``` +#!/usr/bin/env nix +#! nix shell nixpkgs#bash nixpkgs#hello nixpkgs#cowsay --command bash + +hello | cowsay +``` + +or with an **expression**: + +```bash +#! /usr/bin/env nix +#! nix shell --impure --expr `` +#! nix with (import (builtins.getFlake "nixpkgs") {}); +#! nix terraform.withPlugins (plugins: [ plugins.openstack ]) +#! nix `` +#! nix --command bash + +terraform "$@" +``` + +or with cascading interpreters. Note that the `#! nix` lines don't need to follow after the first line, to accomodate other interpreters. + +``` +#!/usr/bin/env nix +//! ```cargo +//! [dependencies] +//! time = "0.1.25" +//! ``` +/* +#!nix shell nixpkgs#rustc nixpkgs#rust-script nixpkgs#cargo --command rust-script +*/ +fn main() { + for argument in std::env::args().skip(1) { + println!("{}", argument); + }; + println!("{}", std::env::var("HOME").expect("")); + println!("{}", time::now().rfc822z()); +} +// vim: ft=rust +``` )"" diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index c16864d30..5f10cfb61 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -2,6 +2,7 @@ #include "shared.hh" #include "store-api.hh" #include "common-args.hh" +#include "nar-info.hh" #include #include @@ -9,6 +10,75 @@ #include using namespace nix; +using nlohmann::json; + +/** + * @return the total size of a set of store objects (specified by path), + * that is, the sum of the size of the NAR serialisation of each object + * in the set. + */ +static uint64_t getStoreObjectsTotalSize(Store & store, const StorePathSet & closure) +{ + uint64_t totalNarSize = 0; + for (auto & p : closure) { + totalNarSize += store.queryPathInfo(p)->narSize; + } + return totalNarSize; +} + + +/** + * Write a JSON representation of store object metadata, such as the + * hash and the references. + * + * @param showClosureSize If true, the closure size of each path is + * included. + */ +static json pathInfoToJSON( + Store & store, + const StorePathSet & storePaths, + bool showClosureSize) +{ + json::object_t jsonAllObjects = json::object(); + + for (auto & storePath : storePaths) { + json jsonObject; + + try { + auto info = store.queryPathInfo(storePath); + + jsonObject = info->toJSON(store, true, HashFormat::SRI); + + if (showClosureSize) { + StorePathSet closure; + store.computeFSClosure(storePath, closure, false, false); + + jsonObject["closureSize"] = getStoreObjectsTotalSize(store, closure); + + if (dynamic_cast(&*info)) { + uint64_t totalDownloadSize = 0; + for (auto & p : closure) { + auto depInfo = store.queryPathInfo(p); + if (auto * depNarInfo = dynamic_cast(&*depInfo)) + totalDownloadSize += depNarInfo->fileSize; + else + throw Error("Missing .narinfo for dep %s of %s", + store.printStorePath(p), + store.printStorePath(storePath)); + } + jsonObject["closureDownloadSize"] = totalDownloadSize; + } + } + + } catch (InvalidPath &) { + jsonObject = nullptr; + } + + jsonAllObjects[store.printStorePath(storePath)] = std::move(jsonObject); + } + return jsonAllObjects; +} + struct CmdPathInfo : StorePathsCommand, MixJSON { @@ -87,10 +157,11 @@ struct CmdPathInfo : StorePathsCommand, MixJSON pathLen = std::max(pathLen, store->printStorePath(storePath).size()); if (json) { - std::cout << store->pathInfoToJSON( + std::cout << pathInfoToJSON( + *store, // FIXME: preserve order? StorePathSet(storePaths.begin(), storePaths.end()), - true, showClosureSize, HashFormat::SRI, AllowInvalid).dump(); + showClosureSize).dump(); } else { @@ -107,8 +178,11 @@ struct CmdPathInfo : StorePathsCommand, MixJSON if (showSize) printSize(info->narSize); - if (showClosureSize) - printSize(store->getClosureSize(info->path).first); + if (showClosureSize) { + StorePathSet closure; + store->computeFSClosure(storePath, closure, false, false); + printSize(getStoreObjectsTotalSize(*store, closure)); + } if (showSigs) { std::cout << '\t'; diff --git a/src/nix/path-info.md b/src/nix/path-info.md index 2dda866d0..4594854eb 100644 --- a/src/nix/path-info.md +++ b/src/nix/path-info.md @@ -43,7 +43,7 @@ R""( command): ```console - # nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path' + # nix path-info --json --all | jq -r 'to_entries | sort_by(.value.registrationTime) | .[-11:-1][] | .key' ``` * Show the size of the entire Nix store: @@ -58,13 +58,13 @@ R""( ```console # nix path-info --json --all --closure-size \ - | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])' + | jq 'map_values(.closureSize | select(. < 1e9)) | to_entries | sort_by(.value)' [ …, - [ - "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65", - 5887562256 - ] + { + .key = "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65", + .value = 5887562256, + } ] ``` diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 3ed7946a8..bbfeb8aa4 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -46,13 +46,13 @@ std::string resolveMirrorUrl(EvalState & state, const std::string & url) } std::tuple prefetchFile( - ref store, - std::string_view url, - std::optional name, - HashType hashType, - std::optional expectedHash, - bool unpack, - bool executable) + ref store, + std::string_view url, + std::optional name, + HashAlgorithm hashAlgo, + std::optional expectedHash, + bool unpack, + bool executable) { auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; @@ -69,7 +69,7 @@ std::tuple prefetchFile( /* If an expected hash is given, the file may already exist in the store. */ if (expectedHash) { - hashType = expectedHash->type; + hashAlgo = expectedHash->algo; storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { .method = ingestionMethod, .hash = *expectedHash, @@ -122,7 +122,7 @@ std::tuple prefetchFile( Activity act(*logger, lvlChatty, actUnknown, fmt("adding '%s' to the store", url)); - auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); + auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashAlgo, expectedHash); storePath = info.path; assert(info.ca); hash = info.ca->hash; @@ -134,7 +134,7 @@ std::tuple prefetchFile( static int main_nix_prefetch_url(int argc, char * * argv) { { - HashType ht = htSHA256; + HashAlgorithm ha = HashAlgorithm::SHA256; std::vector args; bool printPath = getEnv("PRINT_PATH") == "1"; bool fromExpr = false; @@ -155,7 +155,7 @@ static int main_nix_prefetch_url(int argc, char * * argv) printVersion("nix-prefetch-url"); else if (*arg == "--type") { auto s = getArg(*arg, arg, end); - ht = parseHashType(s); + ha = parseHashAlgo(s); } else if (*arg == "--print-path") printPath = true; @@ -233,10 +233,10 @@ static int main_nix_prefetch_url(int argc, char * * argv) std::optional expectedHash; if (args.size() == 2) - expectedHash = Hash::parseAny(args[1], ht); + expectedHash = Hash::parseAny(args[1], ha); auto [storePath, hash] = prefetchFile( - store, resolveMirrorUrl(*state, url), name, ht, expectedHash, unpack, executable); + store, resolveMirrorUrl(*state, url), name, ha, expectedHash, unpack, executable); stopProgressBar(); @@ -258,7 +258,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON std::string url; bool executable = false; std::optional name; - HashType hashType = htSHA256; + HashAlgorithm hashAlgo = HashAlgorithm::SHA256; std::optional expectedHash; CmdStorePrefetchFile() @@ -275,11 +275,11 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON .description = "The expected hash of the file.", .labels = {"hash"}, .handler = {[&](std::string s) { - expectedHash = Hash::parseAny(s, hashType); + expectedHash = Hash::parseAny(s, hashAlgo); }} }); - addFlag(Flag::mkHashTypeFlag("hash-type", &hashType)); + addFlag(Flag::mkHashAlgoFlag("hash-type", &hashAlgo)); addFlag({ .longName = "executable", @@ -305,7 +305,7 @@ struct CmdStorePrefetchFile : StoreCommand, MixJSON } void run(ref store) override { - auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, false, executable); + auto [storePath, hash] = prefetchFile(store, url, name, hashAlgo, expectedHash, false, executable); if (json) { auto res = nlohmann::json::object(); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 476ddcd60..9d9492da9 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -216,7 +216,7 @@ struct ProfileManifest StringSink sink; dumpPath(tempDir, sink); - auto narHash = hashString(htSHA256, sink.s); + auto narHash = hashString(HashAlgorithm::SHA256, sink.s); ValidPathInfo info { *store, @@ -825,7 +825,9 @@ struct CmdProfileWipeHistory : virtual StoreCommand, MixDefaultProfile, MixDryRu struct CmdProfile : NixMultiCommand { CmdProfile() - : MultiCommand({ + : NixMultiCommand( + "profile", + { {"install", []() { return make_ref(); }}, {"remove", []() { return make_ref(); }}, {"upgrade", []() { return make_ref(); }}, @@ -848,13 +850,6 @@ struct CmdProfile : NixMultiCommand #include "profile.md" ; } - - void run() override - { - if (!command) - throw UsageError("'nix profile' requires a sub-command."); - command->second->run(); - } }; static auto rCmdProfile = registerCommand("profile"); diff --git a/src/nix/profile.md b/src/nix/profile.md index bd13f906f..9b2f86f4a 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -11,7 +11,7 @@ them to be rolled back easily. )"" -#include "doc/files/profiles.md.gen.hh" +#include "generated-doc/files/profiles.md.gen.hh" R""( diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc index e19e93219..e1f231222 100644 --- a/src/nix/realisation.cc +++ b/src/nix/realisation.cc @@ -5,9 +5,9 @@ using namespace nix; -struct CmdRealisation : virtual NixMultiCommand +struct CmdRealisation : NixMultiCommand { - CmdRealisation() : MultiCommand(RegisterCommand::getCommandsFor({"realisation"})) + CmdRealisation() : NixMultiCommand("realisation", RegisterCommand::getCommandsFor({"realisation"})) { } std::string description() override @@ -16,13 +16,6 @@ struct CmdRealisation : virtual NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix realisation' requires a sub-command."); - command->second->run(); - } }; static auto rCmdRealisation = registerCommand("realisation"); diff --git a/src/nix/registry.cc b/src/nix/registry.cc index f509ccae8..0346ec1e0 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -196,10 +196,12 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand } }; -struct CmdRegistry : virtual NixMultiCommand +struct CmdRegistry : NixMultiCommand { CmdRegistry() - : MultiCommand({ + : NixMultiCommand( + "registry", + { {"list", []() { return make_ref(); }}, {"add", []() { return make_ref(); }}, {"remove", []() { return make_ref(); }}, @@ -221,14 +223,6 @@ struct CmdRegistry : virtual NixMultiCommand } Category category() override { return catSecondary; } - - void run() override - { - experimentalFeatureSettings.require(Xp::Flakes); - if (!command) - throw UsageError("'nix registry' requires a sub-command."); - command->second->run(); - } }; static auto rCmdRegistry = registerCommand("registry"); diff --git a/src/nix/run.cc b/src/nix/run.cc index 1baf299ab..efc0c56a1 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -1,3 +1,4 @@ +#include "current-process.hh" #include "run.hh" #include "command-installable-value.hh" #include "common-args.hh" @@ -6,7 +7,7 @@ #include "derivations.hh" #include "local-store.hh" #include "finally.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "progress-bar.hh" #include "eval.hh" #include "build/personality.hh" @@ -24,6 +25,7 @@ std::string chrootHelperName = "__run_in_chroot"; namespace nix { void runProgramInStore(ref store, + UseSearchPath useSearchPath, const std::string & program, const Strings & args, std::optional system) @@ -57,7 +59,10 @@ void runProgramInStore(ref store, if (system) setPersonality(*system); - execvp(program.c_str(), stringsToCharPtrs(args).data()); + if (useSearchPath == UseSearchPath::Use) + execvp(program.c_str(), stringsToCharPtrs(args).data()); + else + execv(program.c_str(), stringsToCharPtrs(args).data()); throw SysError("unable to execute '%s'", program); } @@ -119,9 +124,9 @@ struct CmdShell : InstallablesCommand, MixEnvironment if (true) unixPath.push_front(store->printStorePath(path) + "/bin"); - auto propPath = store->printStorePath(path) + "/nix-support/propagated-user-env-packages"; - if (accessor->stat(propPath).type == FSAccessor::tRegular) { - for (auto & p : tokenizeString(readFile(propPath))) + auto propPath = CanonPath(store->printStorePath(path)) + "nix-support" + "propagated-user-env-packages"; + if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) { + for (auto & p : tokenizeString(accessor->readFile(propPath))) todo.push(store->parseStorePath(p)); } } @@ -131,7 +136,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment Strings args; for (auto & arg : command) args.push_back(arg); - runProgramInStore(store, *command.begin(), args); + runProgramInStore(store, UseSearchPath::Use, *command.begin(), args); } }; @@ -193,7 +198,7 @@ struct CmdRun : InstallableValueCommand Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); - runProgramInStore(store, app.program, allArgs); + runProgramInStore(store, UseSearchPath::DontUse, app.program, allArgs); } }; diff --git a/src/nix/run.hh b/src/nix/run.hh index 97ddef19b..a55917b06 100644 --- a/src/nix/run.hh +++ b/src/nix/run.hh @@ -5,7 +5,13 @@ namespace nix { +enum struct UseSearchPath { + Use, + DontUse +}; + void runProgramInStore(ref store, + UseSearchPath useSearchPath, const std::string & program, const Strings & args, std::optional system = std::nullopt); diff --git a/src/nix/shell.md b/src/nix/shell.md index f36919575..7c315fb3f 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -51,4 +51,120 @@ R""( provides the specified [*installables*](./nix.md#installable). If no command is specified, it starts the default shell of your user account specified by `$SHELL`. +# Use as a `#!`-interpreter + +You can use `nix` as a script interpreter to allow scripts written +in arbitrary languages to obtain their own dependencies via Nix. This is +done by starting the script with the following lines: + +```bash +#! /usr/bin/env nix +#! nix shell installables --command real-interpreter +``` + +where *real-interpreter* is the “real” script interpreter that will be +invoked by `nix shell` after it has obtained the dependencies and +initialised the environment, and *installables* are the attribute names of +the dependencies in Nixpkgs. + +The lines starting with `#! nix` specify options (see above). Note that you +cannot write `#! /usr/bin/env nix shell -i ...` because many operating systems +only allow one argument in `#!` lines. + +For example, here is a Python script that depends on Python and the +`prettytable` package: + +```python +#! /usr/bin/env nix +#! nix shell github:tomberek/-#python3With.prettytable --command python + +import prettytable + +# Print a simple table. +t = prettytable.PrettyTable(["N", "N^2"]) +for n in range(1, 10): t.add_row([n, n * n]) +print t +``` + +Similarly, the following is a Perl script that specifies that it +requires Perl and the `HTML::TokeParser::Simple` and `LWP` packages: + +```perl +#! /usr/bin/env nix +#! nix shell github:tomberek/-#perlWith.HTMLTokeParserSimple.LWP --command perl -x + +use HTML::TokeParser::Simple; + +# Fetch nixos.org and print all hrefs. +my $p = HTML::TokeParser::Simple->new(url => 'http://nixos.org/'); + +while (my $token = $p->get_tag("a")) { + my $href = $token->get_attr("href"); + print "$href\n" if $href; +} +``` + +Sometimes you need to pass a simple Nix expression to customize a +package like Terraform: + +```bash +#! /usr/bin/env nix +#! nix shell --impure --expr `` +#! nix with (import (builtins.getFlake ''nixpkgs'') {}); +#! nix terraform.withPlugins (plugins: [ plugins.openstack ]) +#! nix `` +#! nix --command bash + +terraform "$@" +``` + +> **Note** +> +> You must use double backticks (```` `` ````) when passing a simple Nix expression +> in a nix shell shebang. + +Finally, using the merging of multiple nix shell shebangs the following +Haskell script uses a specific branch of Nixpkgs/NixOS (the 21.11 stable +branch): + +```haskell +#!/usr/bin/env nix +#!nix shell --override-input nixpkgs github:NixOS/nixpkgs/nixos-21.11 +#!nix github:tomberek/-#haskellWith.download-curl.tagsoup --command runghc + +import Network.Curl.Download +import Text.HTML.TagSoup +import Data.Either +import Data.ByteString.Char8 (unpack) + +-- Fetch nixos.org and print all hrefs. +main = do + resp <- openURI "https://nixos.org/" + let tags = filter (isTagOpenName "a") $ parseTags $ unpack $ fromRight undefined resp + let tags' = map (fromAttrib "href") tags + mapM_ putStrLn $ filter (/= "") tags' +``` + +If you want to be even more precise, you can specify a specific revision +of Nixpkgs: + + #!nix shell --override-input nixpkgs github:NixOS/nixpkgs/eabc38219184cc3e04a974fe31857d8e0eac098d + +You can also use a Nix expression to build your own dependencies. For example, +the Python example could have been written as: + +```python +#! /usr/bin/env nix +#! nix shell --impure --file deps.nix -i python +``` + +where the file `deps.nix` in the same directory as the `#!`-script +contains: + +```nix +with import {}; +python3.withPackages (ps: with ps; [ prettytable ]) +``` + + )"" diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 45cd2e1a6..a57a407e6 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -1,7 +1,9 @@ +#include "signals.hh" #include "command.hh" #include "shared.hh" #include "store-api.hh" #include "thread-pool.hh" +#include "progress-bar.hh" #include @@ -173,6 +175,7 @@ struct CmdKeyGenerateSecret : Command if (!keyName) throw UsageError("required argument '--key-name' is missing"); + stopProgressBar(); writeFull(STDOUT_FILENO, SecretKey::generate(*keyName).to_string()); } }; @@ -194,6 +197,7 @@ struct CmdKeyConvertSecretToPublic : Command void run() override { SecretKey secretKey(drainFD(STDIN_FILENO)); + stopProgressBar(); writeFull(STDOUT_FILENO, secretKey.toPublicKey().to_string()); } }; @@ -201,7 +205,9 @@ struct CmdKeyConvertSecretToPublic : Command struct CmdKey : NixMultiCommand { CmdKey() - : MultiCommand({ + : NixMultiCommand( + "key", + { {"generate-secret", []() { return make_ref(); }}, {"convert-secret-to-public", []() { return make_ref(); }}, }) @@ -214,13 +220,6 @@ struct CmdKey : NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix key' requires a sub-command."); - command->second->run(); - } }; static auto rCmdKey = registerCommand("key"); diff --git a/src/nix/store.cc b/src/nix/store.cc index 2879e03b3..79b41e096 100644 --- a/src/nix/store.cc +++ b/src/nix/store.cc @@ -2,9 +2,9 @@ using namespace nix; -struct CmdStore : virtual NixMultiCommand +struct CmdStore : NixMultiCommand { - CmdStore() : MultiCommand(RegisterCommand::getCommandsFor({"store"})) + CmdStore() : NixMultiCommand("store", RegisterCommand::getCommandsFor({"store"})) { } std::string description() override @@ -13,13 +13,6 @@ struct CmdStore : virtual NixMultiCommand } Category category() override { return catUtility; } - - void run() override - { - if (!command) - throw UsageError("'nix store' requires a sub-command."); - command->second->run(); - } }; static auto rCmdStore = registerCommand("store"); diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index d238456db..4c7a74e16 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -1,3 +1,4 @@ +#include "processes.hh" #include "command.hh" #include "common-args.hh" #include "store-api.hh" @@ -13,7 +14,6 @@ using namespace nix; struct CmdUpgradeNix : MixDryRun, StoreCommand { Path profileDir; - std::string storePathsUrl = "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix"; CmdUpgradeNix() { @@ -29,7 +29,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand .longName = "nix-store-paths-url", .description = "The URL of the file that contains the store paths of the latest Nix release.", .labels = {"url"}, - .handler = {&storePathsUrl} + .handler = {&(std::string&) settings.upgradeNixStorePathUrl} }); } @@ -43,7 +43,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand std::string description() override { - return "upgrade Nix to the stable version declared in Nixpkgs"; + return "upgrade Nix to the latest stable version"; } std::string doc() override @@ -144,7 +144,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); // FIXME: use nixos.org? - auto req = FileTransferRequest(storePathsUrl); + auto req = FileTransferRequest((std::string&) settings.upgradeNixStorePathUrl); auto res = getFileTransfer()->download(req); auto state = std::make_unique(SearchPath{}, store); diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md index cce88c397..3a3bf61b9 100644 --- a/src/nix/upgrade-nix.md +++ b/src/nix/upgrade-nix.md @@ -16,8 +16,10 @@ R""( # Description -This command upgrades Nix to the stable version declared in Nixpkgs. -This stable version is defined in [nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix) +This command upgrades Nix to the stable version. + +By default, the latest stable version is defined by Nixpkgs, in +[nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix) and updated manually. It may not always be the latest tagged release. By default, it locates the directory containing the `nix` binary in the `$PATH` diff --git a/src/nix/verify.cc b/src/nix/verify.cc index adaa33c0c..f0234f7be 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -4,6 +4,7 @@ #include "sync.hh" #include "thread-pool.hh" #include "references.hh" +#include "signals.hh" #include @@ -97,7 +98,7 @@ struct CmdVerify : StorePathsCommand if (!noContents) { - auto hashSink = HashSink(info->narHash.type); + auto hashSink = HashSink(info->narHash.algo); store->narFromPath(info->path, hashSink); @@ -108,8 +109,8 @@ struct CmdVerify : StorePathsCommand act2.result(resCorruptedPath, store->printStorePath(info->path)); printError("path '%s' was modified! expected hash '%s', got '%s'", store->printStorePath(info->path), - info->narHash.to_string(HashFormat::Base32, true), - hash.first.to_string(HashFormat::Base32, true)); + info->narHash.to_string(HashFormat::Nix32, true), + hash.first.to_string(HashFormat::Nix32, true)); } } diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 055cf6d0d..aecf65922 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -1,7 +1,7 @@ #include "command.hh" #include "store-api.hh" #include "progress-bar.hh" -#include "fs-accessor.hh" +#include "source-accessor.hh" #include "shared.hh" #include @@ -175,7 +175,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions struct BailOut { }; printNode = [&](Node & node, const std::string & firstPad, const std::string & tailPad) { - auto pathS = store->printStorePath(node.path); + CanonPath pathS(store->printStorePath(node.path)); assert(node.dist != inf); if (precise) { @@ -183,7 +183,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions firstPad, node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "→ " : "", - pathS); + pathS.abs()); } if (node.path == dependencyPath && !all @@ -210,24 +210,25 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions contain the reference. */ std::map hits; - std::function visitPath; + std::function visitPath; - visitPath = [&](const Path & p) { - auto st = accessor->stat(p); + visitPath = [&](const CanonPath & p) { + auto st = accessor->maybeLstat(p); + assert(st); - auto p2 = p == pathS ? "/" : std::string(p, pathS.size() + 1); + auto p2 = p == pathS ? "/" : p.abs().substr(pathS.abs().size() + 1); auto getColour = [&](const std::string & hash) { return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; }; - if (st.type == FSAccessor::Type::tDirectory) { + if (st->type == SourceAccessor::Type::tDirectory) { auto names = accessor->readDirectory(p); - for (auto & name : names) - visitPath(p + "/" + name); + for (auto & [name, type] : names) + visitPath(p + name); } - else if (st.type == FSAccessor::Type::tRegular) { + else if (st->type == SourceAccessor::Type::tRegular) { auto contents = accessor->readFile(p); for (auto & hash : hashes) { @@ -245,7 +246,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions } } - else if (st.type == FSAccessor::Type::tSymlink) { + else if (st->type == SourceAccessor::Type::tSymlink) { auto target = accessor->readLink(p); for (auto & hash : hashes) { diff --git a/tests/functional/add.sh b/tests/functional/add.sh index 5c3eed793..d0fedcb25 100644 --- a/tests/functional/add.sh +++ b/tests/functional/add.sh @@ -26,3 +26,20 @@ hash2=$(nix-hash --type sha256 --base32 ./dummy) echo $hash2 test "$hash1" = "sha256:$hash2" + +#### New style commands + +clearStore + +( + path1=$(nix store add ./dummy) + path2=$(nix store add --mode nar ./dummy) + path3=$(nix store add-path ./dummy) + [[ "$path1" == "$path2" ]] + [[ "$path1" == "$path3" ]] +) +( + path1=$(nix store add --mode flat ./dummy) + path2=$(nix store add-file ./dummy) + [[ "$path1" == "$path2" ]] +) diff --git a/tests/functional/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh index fad1def59..3d4a4b097 100644 --- a/tests/functional/build-remote-trustless-should-fail-0.sh +++ b/tests/functional/build-remote-trustless-should-fail-0.sh @@ -4,6 +4,7 @@ enableFeatures "daemon-trust-override" restartDaemon +requireSandboxSupport [[ $busybox =~ busybox ]] || skipTest "no busybox" unset NIX_STORE_DIR diff --git a/tests/functional/build-remote-with-mounted-ssh-ng.sh b/tests/functional/build-remote-with-mounted-ssh-ng.sh new file mode 100644 index 000000000..443acb6ca --- /dev/null +++ b/tests/functional/build-remote-with-mounted-ssh-ng.sh @@ -0,0 +1,22 @@ +source common.sh + +requireSandboxSupport +[[ $busybox =~ busybox ]] || skipTest "no busybox" + +enableFeatures mounted-ssh-store + +nix build -Lvf simple.nix \ + --arg busybox $busybox \ + --out-link $TEST_ROOT/result-from-remote \ + --store mounted-ssh-ng://localhost + +nix build -Lvf simple.nix \ + --arg busybox $busybox \ + --out-link $TEST_ROOT/result-from-remote-new-cli \ + --store 'mounted-ssh-ng://localhost?remote-program=nix daemon' + +# This verifies that the out link was actually created and valid. The ability +# to create out links (permanent gc roots) is the distinguishing feature of +# the mounted-ssh-ng store. +cat $TEST_ROOT/result-from-remote/hello | grepQuiet 'Hello World!' +cat $TEST_ROOT/result-from-remote-new-cli/hello | grepQuiet 'Hello World!' diff --git a/tests/functional/common/vars-and-functions.sh.in b/tests/functional/common/vars-and-functions.sh.in index 967d6be54..848988af9 100644 --- a/tests/functional/common/vars-and-functions.sh.in +++ b/tests/functional/common/vars-and-functions.sh.in @@ -4,7 +4,7 @@ if [[ -z "${COMMON_VARS_AND_FUNCTIONS_SH_SOURCED-}" ]]; then COMMON_VARS_AND_FUNCTIONS_SH_SOURCED=1 -export PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) ' +set +x export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/${TEST_NAME:-default/tests\/functional//} export NIX_STORE_DIR diff --git a/tests/functional/completions.sh b/tests/functional/completions.sh index 7c1e4b287..d3d5bbd48 100644 --- a/tests/functional/completions.sh +++ b/tests/functional/completions.sh @@ -44,15 +44,18 @@ EOF # Input override completion [[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '')" == $'normal\na\t' ]] [[ "$(NIX_GET_COMPLETIONS=5 nix flake show ./foo --override-input '')" == $'normal\na\t' ]] +cd ./foo +[[ "$(NIX_GET_COMPLETIONS=3 nix flake update '')" == $'normal\na\t' ]] +cd .. +[[ "$(NIX_GET_COMPLETIONS=5 nix flake update --flake './foo' '')" == $'normal\na\t' ]] ## With multiple input flakes [[ "$(NIX_GET_COMPLETIONS=5 nix build ./foo ./bar --override-input '')" == $'normal\na\t\nb\t' ]] ## With tilde expansion [[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix build '~/foo' --override-input '')" == $'normal\na\t' ]] -[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake show '~/foo' --update-input '')" == $'normal\na\t' ]] -[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=4 nix run '~/foo' --update-input '')" == $'normal\na\t' ]] +[[ "$(HOME=$PWD NIX_GET_COMPLETIONS=5 nix flake update --flake '~/foo' '')" == $'normal\na\t' ]] ## Out of order -[[ "$(NIX_GET_COMPLETIONS=3 nix build --update-input '' ./foo)" == $'normal\na\t' ]] -[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --update-input '' ./bar)" == $'normal\na\t\nb\t' ]] +[[ "$(NIX_GET_COMPLETIONS=3 nix build --override-input '' '' ./foo)" == $'normal\na\t' ]] +[[ "$(NIX_GET_COMPLETIONS=4 nix build ./foo --override-input '' '' ./bar)" == $'normal\na\t\nb\t' ]] # Cli flag completion NIX_GET_COMPLETIONS=2 nix build --log-form | grep -- "--log-format" diff --git a/tests/functional/config.sh b/tests/functional/config.sh index 723f575ed..324fe95bd 100644 --- a/tests/functional/config.sh +++ b/tests/functional/config.sh @@ -40,19 +40,20 @@ files=$(nix-build --verbose --version | grep "User config" | cut -d ':' -f2- | x # Test that it's possible to load the config from a custom location here=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")") export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf -var=$(nix show-config | grep '^substituters =' | cut -d '=' -f 2 | xargs) +var=$(nix config show | grep '^substituters =' | cut -d '=' -f 2 | xargs) [[ $var == https://example.com ]] # Test that it's possible to load config from the environment -prev=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) +prev=$(nix config show | grep '^cores' | cut -d '=' -f 2 | xargs) export NIX_CONFIG="cores = 4242"$'\n'"experimental-features = nix-command flakes" -exp_cores=$(nix show-config | grep '^cores' | cut -d '=' -f 2 | xargs) -exp_features=$(nix show-config | grep '^experimental-features' | cut -d '=' -f 2 | xargs) +exp_cores=$(nix config show | grep '^cores' | cut -d '=' -f 2 | xargs) +exp_features=$(nix config show | grep '^experimental-features' | cut -d '=' -f 2 | xargs) [[ $prev != $exp_cores ]] [[ $exp_cores == "4242" ]] -[[ $exp_features == "flakes nix-command" ]] +# flakes implies fetch-tree +[[ $exp_features == "fetch-tree flakes nix-command" ]] # Test that it's possible to retrieve a single setting's value -val=$(nix show-config | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) -val2=$(nix show-config warn-dirty) +val=$(nix config show | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) +val2=$(nix config show warn-dirty) [[ $val == $val2 ]] diff --git a/tests/functional/experimental-features.sh b/tests/functional/experimental-features.sh index 607bf0a8e..9ee4a53d4 100644 --- a/tests/functional/experimental-features.sh +++ b/tests/functional/experimental-features.sh @@ -31,7 +31,7 @@ source common.sh NIX_CONFIG=' experimental-features = nix-command accept-flake-config = true -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "false" $TEST_ROOT/stdout grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr @@ -39,7 +39,7 @@ grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature ' NIX_CONFIG=' accept-flake-config = true experimental-features = nix-command -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "false" $TEST_ROOT/stdout grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr @@ -47,7 +47,7 @@ grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature ' NIX_CONFIG=' experimental-features = nix-command flakes accept-flake-config = true -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "true" $TEST_ROOT/stdout grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr @@ -55,7 +55,7 @@ grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr NIX_CONFIG=' accept-flake-config = true experimental-features = nix-command flakes -' nix show-config accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr +' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr grepQuiet "true" $TEST_ROOT/stdout grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index fc89f2040..4985c7764 100644 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -51,9 +51,7 @@ git -C $repo add differentbranch git -C $repo commit -m 'Test2' git -C $repo checkout master devrev=$(git -C $repo rev-parse devtest) -out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }" 2>&1) || status=$? -[[ $status == 1 ]] -[[ $out =~ 'Cannot find Git revision' ]] +nix eval --impure --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }" [[ $(nix eval --raw --expr "builtins.readFile (builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; allRefs = true; } + \"/differentbranch\")") = 'different file' ]] @@ -185,11 +183,7 @@ path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = # Nuke the cache rm -rf $TEST_HOME/.cache/nix -# Try again, but without 'git' on PATH. This should fail. -NIX=$(command -v nix) -(! PATH= $NIX eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath" ) - -# Try again, with 'git' available. This should work. +# Try again. This should work. path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") [[ $path3 = $path5 ]] @@ -241,6 +235,7 @@ rm -rf $repo/.git # should succeed for a repo without commits git init $repo +git -C $repo add hello # need to add at least one file to cause the root of the repo to be visible path10=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath") # should succeed for a path with a space diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh index df81232e5..369cdc5db 100644 --- a/tests/functional/fetchGitSubmodules.sh +++ b/tests/functional/fetchGitSubmodules.sh @@ -118,11 +118,3 @@ cloneRepo=$TEST_ROOT/a/b/gitSubmodulesClone # NB /a/b to make the relative path git clone $rootRepo $cloneRepo pathIndirect=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath") [[ $pathIndirect = $pathWithRelative ]] - -# Test that if the clone has the submodule already, we're not fetching -# it again. -git -C $cloneRepo submodule update --init -rm $TEST_HOME/.cache/nix/fetcher-cache* -rm -rf $subRepo -pathSubmoduleGone=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath") -[[ $pathSubmoduleGone = $pathWithRelative ]] diff --git a/tests/functional/fetchGitVerification.sh b/tests/functional/fetchGitVerification.sh new file mode 100644 index 000000000..b80e061b5 --- /dev/null +++ b/tests/functional/fetchGitVerification.sh @@ -0,0 +1,82 @@ +source common.sh + +requireGit +[[ $(type -p ssh-keygen) ]] || skipTest "ssh-keygen not installed" # require ssh-keygen + +enableFeatures "verified-fetches" + +clearStore + +repo="$TEST_ROOT/git" + +# generate signing keys +keysDir=$TEST_ROOT/.ssh +mkdir -p "$keysDir" +ssh-keygen -f "$keysDir/testkey1" -t ed25519 -P "" -C "test key 1" +key1File="$keysDir/testkey1.pub" +publicKey1=$(awk '{print $2}' "$key1File") +ssh-keygen -f "$keysDir/testkey2" -t rsa -P "" -C "test key 2" +key2File="$keysDir/testkey2.pub" +publicKey2=$(awk '{print $2}' "$key2File") + +git init $repo +git -C $repo config user.email "foobar@example.com" +git -C $repo config user.name "Foobar" +git -C $repo config gpg.format ssh + +echo 'hello' > $repo/text +git -C $repo add text +git -C $repo -c "user.signingkey=$key1File" commit -S -m 'initial commit' + +out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = \"file://$repo\"; keytype = \"ssh-rsa\"; publicKey = \"$publicKey2\"; }" 2>&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'No principal matched.' ]] +[[ $(nix eval --impure --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; publicKey = \"$publicKey1\"; } + \"/text\")") = 'hello' ]] + +echo 'hello world' > $repo/text + +# Verification on a dirty repo should fail. +out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = \"file://$repo\"; keytype = \"ssh-rsa\"; publicKey = \"$publicKey2\"; }" 2>&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'dirty' ]] + +git -C $repo add text +git -C $repo -c "user.signingkey=$key2File" commit -S -m 'second commit' + +[[ $(nix eval --impure --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; publicKeys = [{key = \"$publicKey1\";} {type = \"ssh-rsa\"; key = \"$publicKey2\";}]; } + \"/text\")") = 'hello world' ]] + +# Flake input test +flakeDir="$TEST_ROOT/flake" +mkdir -p "$flakeDir" +cat > "$flakeDir/flake.nix" < "$flakeDir/flake.nix" <&1) || status=$? +[[ $status == 1 ]] +[[ $out =~ 'No principal matched.' ]] diff --git a/tests/functional/flakes/circular.sh b/tests/functional/flakes/circular.sh index 09cd02edf..d3bb8e8a3 100644 --- a/tests/functional/flakes/circular.sh +++ b/tests/functional/flakes/circular.sh @@ -42,7 +42,8 @@ git -C $flakeB commit -a -m 'Foo' sed -i $flakeB/flake.nix -e 's/456/789/' git -C $flakeB commit -a -m 'Foo' -[[ $(nix eval --update-input b $flakeA#foo) = 1912 ]] +nix flake update b --flake $flakeA +[[ $(nix eval $flakeA#foo) = 1912 ]] # Test list-inputs with circular dependencies nix flake metadata $flakeA diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index 8aed296e6..fc45cf7bf 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -11,6 +11,7 @@ writeSimpleFlake() { outputs = inputs: rec { packages.$system = rec { foo = import ./simple.nix; + fooScript = (import ./shell.nix {}).foo; default = foo; }; packages.someOtherSystem = rec { @@ -24,13 +25,13 @@ writeSimpleFlake() { } EOF - cp ../simple.nix ../simple.builder.sh ../config.nix $flakeDir/ + cp ../simple.nix ../shell.nix ../simple.builder.sh ../config.nix $flakeDir/ } createSimpleGitFlake() { local flakeDir="$1" writeSimpleFlake $flakeDir - git -C $flakeDir add flake.nix simple.nix simple.builder.sh config.nix + git -C $flakeDir add flake.nix simple.nix shell.nix simple.builder.sh config.nix git -C $flakeDir commit -m 'Initial' } diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index 21a4b52de..85a4d3389 100644 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -46,7 +46,16 @@ echo '"expression in root repo"' > $rootRepo/root.nix git -C $rootRepo add root.nix git -C $rootRepo commit -m "Add root.nix" +flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule + # Flake can live inside a submodule and can be accessed via ?dir=submodule -[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#sub ) = '"expression in submodule"' ]] +[[ $(nix eval --json $flakeref#sub ) = '"expression in submodule"' ]] + # The flake can access content outside of the submodule -[[ $(nix eval --json git+file://$rootRepo\?submodules=1\&dir=submodule#root ) = '"expression in root repo"' ]] +[[ $(nix eval --json $flakeref#root ) = '"expression in root repo"' ]] + +# Check that dirtying a submodule makes the entire thing dirty. +[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) != null ]] +echo '"foo"' > $rootRepo/submodule/sub.nix +[[ $(nix eval --json $flakeref#sub ) = '"foo"' ]] +[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) = null ]] diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 70de28628..ccf1699f9 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -66,9 +66,82 @@ cat > "$nonFlakeDir/README.md" < "$nonFlakeDir/shebang.sh" < $nonFlakeDir/shebang-comments.sh < $nonFlakeDir/shebang-reject.sh < $nonFlakeDir/shebang-inline-expr.sh <> $nonFlakeDir/shebang-inline-expr.sh <<"EOF" +#! nix --offline shell +#! nix --impure --expr `` +#! nix let flake = (builtins.getFlake (toString ../flake1)).packages; +#! nix fooScript = flake.${builtins.currentSystem}.fooScript; +#! nix /* just a comment !@#$%^&*()__+ # */ +#! nix in fooScript +#! nix `` +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-inline-expr.sh + +cat > $nonFlakeDir/fooScript.nix <<"EOF" +let flake = (builtins.getFlake (toString ../flake1)).packages; + fooScript = flake.${builtins.currentSystem}.fooScript; + in fooScript +EOF + +cat > $nonFlakeDir/shebang-file.sh <> $nonFlakeDir/shebang-file.sh <<"EOF" +#! nix --offline shell +#! nix --impure --file ./fooScript.nix +#! nix --no-write-lock-file --command bash +set -ex +foo +echo "$@" +EOF +chmod +x $nonFlakeDir/shebang-file.sh + # Construct a custom registry, additionally test the --registry flag nix registry add --registry "$registry" flake1 "git+file://$flake1Dir" nix registry add --registry "$registry" flake2 "git+file://$percentEncodedFlake2Dir" @@ -300,7 +373,7 @@ nix build -o "$TEST_ROOT/result" flake4#xyzzy nix flake lock "$flake3Dir" [[ -z $(git -C "$flake3Dir" diff master || echo failed) ]] -nix flake update "$flake3Dir" --override-flake flake2 nixpkgs +nix flake update --flake "$flake3Dir" --override-flake flake2 nixpkgs [[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]] # Make branch "removeXyzzy" where flake3 doesn't have xyzzy anymore @@ -437,7 +510,7 @@ cat > "$flake3Dir/flake.nix" <&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' diff --git a/tests/functional/flakes/follow-paths.sh b/tests/functional/flakes/follow-paths.sh index 8573b5511..1afd91bd2 100644 --- a/tests/functional/flakes/follow-paths.sh +++ b/tests/functional/flakes/follow-paths.sh @@ -77,7 +77,7 @@ git -C $flakeFollowsA add flake.nix flakeB/flake.nix \ nix flake metadata $flakeFollowsA -nix flake update $flakeFollowsA +nix flake update --flake $flakeFollowsA nix flake lock $flakeFollowsA @@ -228,7 +228,7 @@ git -C "$flakeFollowsOverloadA" add flake.nix flakeB/flake.nix \ flakeB/flakeC/flake.nix flakeB/flakeC/flakeD/flake.nix nix flake metadata "$flakeFollowsOverloadA" -nix flake update "$flakeFollowsOverloadA" +nix flake update --flake "$flakeFollowsOverloadA" nix flake lock "$flakeFollowsOverloadA" # Now test follow cycle detection @@ -260,3 +260,79 @@ EOF checkRes=$(nix flake lock "$flakeFollowCycle" 2>&1 && fail "nix flake lock should have failed." || true) echo $checkRes | grep -F "error: follow cycle detected: [baz -> foo -> bar -> baz]" + + +# Test transitive input url locking +# This tests the following lockfile issue: https://github.com/NixOS/nix/issues/9143 +# +# We construct the following graph, where p->q means p has input q. +# +# A -> B -> C +# +# And override B/C to flake D, first in A's flake.nix and then with --override-input. +# +# A -> B -> D +flakeFollowsCustomUrlA="$TEST_ROOT/follows/custom-url/flakeA" +flakeFollowsCustomUrlB="$TEST_ROOT/follows/custom-url/flakeA/flakeB" +flakeFollowsCustomUrlC="$TEST_ROOT/follows/custom-url/flakeA/flakeB/flakeC" +flakeFollowsCustomUrlD="$TEST_ROOT/follows/custom-url/flakeA/flakeB/flakeD" + + +createGitRepo "$flakeFollowsCustomUrlA" +mkdir -p "$flakeFollowsCustomUrlB" +mkdir -p "$flakeFollowsCustomUrlC" +mkdir -p "$flakeFollowsCustomUrlD" + +cat > "$flakeFollowsCustomUrlD/flake.nix" < "$flakeFollowsCustomUrlC/flake.nix" < "$flakeFollowsCustomUrlB/flake.nix" < "$flakeFollowsCustomUrlA/flake.nix" <&1 || echo "exit: $?") + [[ "$fail" == *"error: input hash"*"exit: 1" ]] + fail=$(nix hash convert --algo "$1" --from base16 "$3" 2>&1 || echo "exit: $?") + [[ "$fail" == *"error: input hash"*"exit: 1" ]] + fail=$(nix hash convert --algo "$1" --from nix32 "$4" 2>&1 || echo "exit: $?") + [[ "$fail" == *"error: input hash"*"exit: 1" ]] + } + try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" try3 sha256 "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" "1b8m03r63zqhnjf7l5wnldhh7c134ap5vpj0850ymkq1iyzicy5s" "ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=" try3 sha512 "204a8fc6dda82f0a0ced7beb8e08a41657c16ef468b228a8279be331a703c33596fd15c13b1b07f9aa1d3bea57789ca031ad85c7a71dd70354ec631238ca3445" "12k9jiq29iyqm03swfsgiw5mlqs173qazm3n7daz43infy12pyrcdf30fkk3qwv4yl2ick8yipc2mqnlh48xsvvxl60lbx8vp38yji0" "IEqPxt2oLwoM7XvrjgikFlfBbvRosiioJ5vjMacDwzWW/RXBOxsH+aodO+pXeJygMa2Fx6cd1wNU7GMSOMo0RQ==" diff --git a/tests/functional/init.sh b/tests/functional/init.sh index 2ec5fa1dd..97b1b0587 100755 --- a/tests/functional/init.sh +++ b/tests/functional/init.sh @@ -20,7 +20,7 @@ cat > "$NIX_CONF_DIR"/nix.conf < "$NIX_CONF_DIR"/nix.conf.extra < nix-instantiate --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuietInverse Hello nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grepQuietInverse Hello expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' | grepQuiet Hello +expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello %" (throw "Foo")' | grepQuiet 'Hello %' nix-instantiate --eval -E 'let x = builtins.trace { x = x; } true; in x' \ 2>&1 | grepQuiet -E 'trace: { x = «potential infinite recursion»; }' diff --git a/tests/functional/lang/eval-fail-attr-name-type.err.exp b/tests/functional/lang/eval-fail-attr-name-type.err.exp new file mode 100644 index 000000000..5f9a073dd --- /dev/null +++ b/tests/functional/lang/eval-fail-attr-name-type.err.exp @@ -0,0 +1,20 @@ +error: + … while evaluating the attribute 'puppy."${key}"' + + at /pwd/lang/eval-fail-attr-name-type.nix:3:5: + + 2| attrs = { + 3| puppy.doggy = {}; + | ^ + 4| }; + + … while evaluating an attribute name + + at /pwd/lang/eval-fail-attr-name-type.nix:7:17: + + 6| in + 7| attrs.puppy.${key} + | ^ + 8| + + error: value is an integer while a string was expected diff --git a/tests/functional/lang/eval-fail-attr-name-type.nix b/tests/functional/lang/eval-fail-attr-name-type.nix new file mode 100644 index 000000000..a0e76004a --- /dev/null +++ b/tests/functional/lang/eval-fail-attr-name-type.nix @@ -0,0 +1,7 @@ +let + attrs = { + puppy.doggy = {}; + }; + key = 1; +in + attrs.puppy.${key} diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp index dea119ae8..a287067cd 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-2.err.exp @@ -1 +1 @@ -error: getting status of '/pwd/lang/fnord': No such file or directory +error: path '/pwd/lang/fnord' does not exist diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp new file mode 100644 index 000000000..07843a480 --- /dev/null +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp @@ -0,0 +1,11 @@ +error: + … while evaluating a path segment + + at /pwd/lang/eval-fail-bad-string-interpolation-4.nix:9:3: + + 8| # The error message should not be too long. + 9| ''${pkgs}'' + | ^ + 10| + + error: cannot coerce a set to a string diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix b/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix new file mode 100644 index 000000000..457b5f06a --- /dev/null +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.nix @@ -0,0 +1,9 @@ +let + # Basically a "billion laughs" attack, but toned down to simulated `pkgs`. + ha = x: y: { a = x y; b = x y; c = x y; d = x y; e = x y; f = x y; g = x y; h = x y; j = x y; }; + has = ha (ha (ha (ha (x: x)))) "ha"; + # A large structure that has already been evaluated. + pkgs = builtins.deepSeq has has; +in +# The error message should not be too long. +''${pkgs}'' diff --git a/tests/functional/lang/eval-fail-call-primop.err.exp b/tests/functional/lang/eval-fail-call-primop.err.exp new file mode 100644 index 000000000..19b407c47 --- /dev/null +++ b/tests/functional/lang/eval-fail-call-primop.err.exp @@ -0,0 +1,12 @@ +error: + … while calling the 'length' builtin + + at /pwd/lang/eval-fail-call-primop.nix:1:1: + + 1| builtins.length 1 + | ^ + 2| + + … while evaluating the first argument passed to builtins.length + + error: value is an integer while a list was expected diff --git a/tests/functional/lang/eval-fail-call-primop.nix b/tests/functional/lang/eval-fail-call-primop.nix new file mode 100644 index 000000000..972eb72c7 --- /dev/null +++ b/tests/functional/lang/eval-fail-call-primop.nix @@ -0,0 +1 @@ +builtins.length 1 diff --git a/tests/functional/lang/eval-fail-nonexist-path.err.exp b/tests/functional/lang/eval-fail-nonexist-path.err.exp index dea119ae8..a287067cd 100644 --- a/tests/functional/lang/eval-fail-nonexist-path.err.exp +++ b/tests/functional/lang/eval-fail-nonexist-path.err.exp @@ -1 +1 @@ -error: getting status of '/pwd/lang/fnord': No such file or directory +error: path '/pwd/lang/fnord' does not exist diff --git a/tests/functional/lang/eval-fail-not-throws.err.exp b/tests/functional/lang/eval-fail-not-throws.err.exp new file mode 100644 index 000000000..b290afb0a --- /dev/null +++ b/tests/functional/lang/eval-fail-not-throws.err.exp @@ -0,0 +1,18 @@ +error: + … in the argument of the not operator + + at /pwd/lang/eval-fail-not-throws.nix:1:4: + + 1| ! (throw "uh oh!") + | ^ + 2| + + … while calling the 'throw' builtin + + at /pwd/lang/eval-fail-not-throws.nix:1:4: + + 1| ! (throw "uh oh!") + | ^ + 2| + + error: uh oh! diff --git a/tests/functional/lang/eval-fail-not-throws.nix b/tests/functional/lang/eval-fail-not-throws.nix new file mode 100644 index 000000000..a74ce4ebe --- /dev/null +++ b/tests/functional/lang/eval-fail-not-throws.nix @@ -0,0 +1 @@ +! (throw "uh oh!") diff --git a/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp new file mode 100644 index 000000000..811d01b03 --- /dev/null +++ b/tests/functional/lang/eval-fail-using-set-as-attr-name.err.exp @@ -0,0 +1,11 @@ +error: + … while evaluating an attribute name + + at /pwd/lang/eval-fail-using-set-as-attr-name.nix:5:10: + + 4| in + 5| attr.${key} + | ^ + 6| + + error: value is a set while a string was expected diff --git a/tests/functional/lang/eval-fail-using-set-as-attr-name.nix b/tests/functional/lang/eval-fail-using-set-as-attr-name.nix new file mode 100644 index 000000000..48e071a41 --- /dev/null +++ b/tests/functional/lang/eval-fail-using-set-as-attr-name.nix @@ -0,0 +1,5 @@ +let + attr = {foo = "bar";}; + key = {}; +in + attr.${key} diff --git a/tests/functional/lang/eval-okay-convertHash.err.exp b/tests/functional/lang/eval-okay-convertHash.err.exp new file mode 100644 index 000000000..41d746725 --- /dev/null +++ b/tests/functional/lang/eval-okay-convertHash.err.exp @@ -0,0 +1,108 @@ +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". +warning: "base32" is a deprecated alias for hash format "nix32". diff --git a/tests/functional/lang/eval-okay-convertHash.exp b/tests/functional/lang/eval-okay-convertHash.exp index 60e0a3c49..16b0240e5 100644 --- a/tests/functional/lang/eval-okay-convertHash.exp +++ b/tests/functional/lang/eval-okay-convertHash.exp @@ -1 +1 @@ -{ hashesBase16 = [ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]; hashesBase32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesBase64 = [ "1B2M2Y8AsgTpgAmY7PhCfg==" "bGnufyEcZAQZ1TZswHauRg==" "uzQ4+6vUYOptvSfRU+IjOw==" "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "zVToVowbN88eW62wd5vL84IhIYk=" "bRLhCx0zHa0hDkf9JdTyYIArfnc=" "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; hashesSRI = [ "md5-1B2M2Y8AsgTpgAmY7PhCfg==" "md5-bGnufyEcZAQZ1TZswHauRg==" "md5-uzQ4+6vUYOptvSfRU+IjOw==" "sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "sha1-zVToVowbN88eW62wd5vL84IhIYk=" "sha1-bRLhCx0zHa0hDkf9JdTyYIArfnc=" "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "sha256-kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "sha256-rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "sha512-nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "sha512-IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; } +{ hashesBase16 = [ "d41d8cd98f00b204e9800998ecf8427e" "6c69ee7f211c640419d5366cc076ae46" "bb3438fbabd460ea6dbd27d153e2233b" "da39a3ee5e6b4b0d3255bfef95601890afd80709" "cd54e8568c1b37cf1e5badb0779bcbf382212189" "6d12e10b1d331dad210e47fd25d4f260802b7e77" "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" "900a4469df00ccbfd0c145c6d1e4b7953dd0afafadd7534e3a4019e8d38fc663" "ad0387b3bd8652f730ca46d25f9c170af0fd589f42e7f23f5a9e6412d97d7e56" "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e" "9d0886f8c6b389398a16257bc79780fab9831c7fc11c8ab07fa732cb7b348feade382f92617c9c5305fefba0af02ab5fd39a587d330997ff5bd0db19f7666653" "21644b72aa259e5a588cd3afbafb1d4310f4889680f6c83b9d531596a5a284f34dbebff409d23bcc86aee6bad10c891606f075c6f4755cb536da27db5693f3a7" ]; hashesBase32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesBase64 = [ "1B2M2Y8AsgTpgAmY7PhCfg==" "bGnufyEcZAQZ1TZswHauRg==" "uzQ4+6vUYOptvSfRU+IjOw==" "2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "zVToVowbN88eW62wd5vL84IhIYk=" "bRLhCx0zHa0hDkf9JdTyYIArfnc=" "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; hashesNix32 = [ "3y8bwfr609h3lh9ch0izcqq7fl" "26mrvc0v1nslch8r0w45zywsbc" "1v4gi57l97pmnylq6lmgxkhd5v" "143xibwh31h9bvxzalr0sjvbbvpa6ffs" "i4hj30pkrfdpgc5dbcgcydqviibfhm6d" "fxz2p030yba2bza71qhss79k3l5y24kd" "0mdqa9w1p6cmli6976v4wi0sw9r4p5prkj7lzfd1877wk11c9c73" "0qy6iz9yh6a079757mxdmypx0gcmnzjd3ij5q78bzk00vxll82lh" "0mkygpci4r4yb8zz5rs2kxcgvw0a2yf5zlj6r8qgfll6pnrqf0xd" "0zdl9zrg8r3i9c1g90lgg9ip5ijzv3yhz91i0zzn3r8ap9ws784gkp9dk9j3aglhgf1amqb0pj21mh7h1nxcl18akqvvf7ggqsy30yg" "19ncrpp37dx0nzzjw4k6zaqkb9mzaq2myhgpzh5aff7qqcj5wwdxslg6ixwncm7gyq8l761gwf87fgsh2bwfyr52s53k2dkqvw8c24x" "2kz74snvckxldmmbisz9ikmy031d28cs6xfdbl6rhxx42glpyz4vww4lajrc5akklxwixl0js4g84233pxvmbykiic5m7i5m9r4nr11" ]; hashesSRI = [ "md5-1B2M2Y8AsgTpgAmY7PhCfg==" "md5-bGnufyEcZAQZ1TZswHauRg==" "md5-uzQ4+6vUYOptvSfRU+IjOw==" "sha1-2jmj7l5rSw0yVb/vlWAYkK/YBwk=" "sha1-zVToVowbN88eW62wd5vL84IhIYk=" "sha1-bRLhCx0zHa0hDkf9JdTyYIArfnc=" "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=" "sha256-kApEad8AzL/QwUXG0eS3lT3Qr6+t11NOOkAZ6NOPxmM=" "sha256-rQOHs72GUvcwykbSX5wXCvD9WJ9C5/I/Wp5kEtl9flY=" "sha512-z4PhNX7vuL3xVChQ1m2AB9Yg5AULVxXcg/SpIdNs6c5H0NE8XYXysP+DGNKHfuwvY7kxvUdBeoGlODJ6+SfaPg==" "sha512-nQiG+MaziTmKFiV7x5eA+rmDHH/BHIqwf6cyy3s0j+reOC+SYXycUwX++6CvAqtf05pYfTMJl/9b0NsZ92ZmUw==" "sha512-IWRLcqolnlpYjNOvuvsdQxD0iJaA9sg7nVMVlqWihPNNvr/0CdI7zIau5rrRDIkWBvB1xvR1XLU22ifbVpPzpw==" ]; } diff --git a/tests/functional/lang/eval-okay-convertHash.nix b/tests/functional/lang/eval-okay-convertHash.nix index cf4909aaf..a0191ee8d 100644 --- a/tests/functional/lang/eval-okay-convertHash.nix +++ b/tests/functional/lang/eval-okay-convertHash.nix @@ -5,12 +5,14 @@ let map2' = f: fsts: snds: map2 f { inherit fsts snds; }; getOutputHashes = hashes: { hashesBase16 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base16";}) hashAlgos hashes; + hashesNix32 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";}) hashAlgos hashes; hashesBase32 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";}) hashAlgos hashes; hashesBase64 = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base64";}) hashAlgos hashes; hashesSRI = map2' (hashAlgo: hash: builtins.convertHash { inherit hash hashAlgo; toHashFormat = "sri" ;}) hashAlgos hashes; }; getOutputHashesColon = hashes: { hashesBase16 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base16";}) hashAlgos hashes; + hashesNix32 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "nix32";}) hashAlgos hashes; hashesBase32 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base32";}) hashAlgos hashes; hashesBase64 = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "base64";}) hashAlgos hashes; hashesSRI = map2' (hashAlgo: hashBody: builtins.convertHash { hash = hashAlgo + ":" + hashBody; toHashFormat = "sri" ;}) hashAlgos hashes; diff --git a/tests/functional/lang/eval-okay-symlink-resolution.exp b/tests/functional/lang/eval-okay-symlink-resolution.exp new file mode 100644 index 000000000..8b8441b91 --- /dev/null +++ b/tests/functional/lang/eval-okay-symlink-resolution.exp @@ -0,0 +1 @@ +"test" diff --git a/tests/functional/lang/eval-okay-symlink-resolution.nix b/tests/functional/lang/eval-okay-symlink-resolution.nix new file mode 100644 index 000000000..ffb1818bd --- /dev/null +++ b/tests/functional/lang/eval-okay-symlink-resolution.nix @@ -0,0 +1 @@ +import symlink-resolution/foo/overlays/overlay.nix diff --git a/tests/functional/lang/symlink-resolution/foo/lib/default.nix b/tests/functional/lang/symlink-resolution/foo/lib/default.nix new file mode 100644 index 000000000..8b8441b91 --- /dev/null +++ b/tests/functional/lang/symlink-resolution/foo/lib/default.nix @@ -0,0 +1 @@ +"test" diff --git a/tests/functional/lang/symlink-resolution/foo/overlays b/tests/functional/lang/symlink-resolution/foo/overlays new file mode 120000 index 000000000..0d44a21c5 --- /dev/null +++ b/tests/functional/lang/symlink-resolution/foo/overlays @@ -0,0 +1 @@ +../overlays \ No newline at end of file diff --git a/tests/functional/lang/symlink-resolution/overlays/overlay.nix b/tests/functional/lang/symlink-resolution/overlays/overlay.nix new file mode 100644 index 000000000..b0368308e --- /dev/null +++ b/tests/functional/lang/symlink-resolution/overlays/overlay.nix @@ -0,0 +1 @@ +import ../lib diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 3679349f8..10b399d75 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -55,6 +55,7 @@ nix_tests = \ secure-drv-outputs.sh \ restricted.sh \ fetchGitSubmodules.sh \ + fetchGitVerification.sh \ flakes/search-root.sh \ readfile-context.sh \ nix-channel.sh \ @@ -68,6 +69,7 @@ nix_tests = \ build-remote-trustless-should-pass-2.sh \ build-remote-trustless-should-pass-3.sh \ build-remote-trustless-should-fail-0.sh \ + build-remote-with-mounted-ssh-ng.sh \ nar-access.sh \ pure-eval.sh \ eval.sh \ @@ -119,6 +121,7 @@ nix_tests = \ flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ + path-info.sh \ toString-path.sh \ read-only-store.sh \ nested-sandboxing.sh \ @@ -137,9 +140,9 @@ ifeq ($(ENABLE_BUILD), yes) endif $(d)/test-libstoreconsumer.sh.test $(d)/test-libstoreconsumer.sh.test-debug: \ - $(d)/test-libstoreconsumer/test-libstoreconsumer + $(buildprefix)$(d)/test-libstoreconsumer/test-libstoreconsumer $(d)/plugins.sh.test $(d)/plugins.sh.test-debug: \ - $(d)/plugins/libplugintest.$(SO_EXT) + $(buildprefix)$(d)/plugins/libplugintest.$(SO_EXT) install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh index d487d58d2..87981e7d9 100644 --- a/tests/functional/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -25,6 +25,11 @@ diff -u baz.cat-nar $storePath/foo/baz nix store cat $storePath/foo/baz > baz.cat-nar diff -u baz.cat-nar $storePath/foo/baz +# Check that 'nix store cat' fails on invalid store paths. +invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo" +cp -r $storePath $invalidPath +expect 1 nix store cat $invalidPath/foo/baz + # Test --json. diff -u \ <(nix nar ls --json $narFile / | jq -S) \ @@ -46,7 +51,7 @@ diff -u \ <(echo '{"type":"regular","size":0}' | jq -S) # Test missing files. -expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist in NAR' +expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist' expect 1 nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist' # Test failure to dump. diff --git a/tests/functional/path-info.sh b/tests/functional/path-info.sh new file mode 100644 index 000000000..763935eb7 --- /dev/null +++ b/tests/functional/path-info.sh @@ -0,0 +1,23 @@ +source common.sh + +echo foo > $TEST_ROOT/foo +foo=$(nix store add-file $TEST_ROOT/foo) + +echo bar > $TEST_ROOT/bar +bar=$(nix store add-file $TEST_ROOT/bar) + +echo baz > $TEST_ROOT/baz +baz=$(nix store add-file $TEST_ROOT/baz) +nix-store --delete "$baz" + +diff --unified --color=always \ + <(nix path-info --json "$foo" "$bar" "$baz" | + jq --sort-keys 'map_values(.narHash)') \ + <(jq --sort-keys <<-EOF + { + "$foo": "sha256-QvtAMbUl/uvi+LCObmqOhvNOapHdA2raiI4xG5zI5pA=", + "$bar": "sha256-9fhYGu9fqxcQC2Kc81qh2RMo1QcLBUBo8U+pPn+jthQ=", + "$baz": null + } +EOF + ) diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index 197ae7a10..3de26eb36 100644 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -14,8 +14,8 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I sr (! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel') nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src -(! nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ') -nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in ' -I src=. +expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' | grepQuiet "forbidden in restricted mode" +nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' -I src=. p=$(nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)") cmp $p restricted.sh @@ -39,6 +39,18 @@ nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT - [[ $(nix eval --raw --impure --restrict-eval -I . --expr 'builtins.readFile "${import ./simple.nix}/hello"') == 'Hello World!' ]] +# Check that we can't follow a symlink outside of the allowed paths. +mkdir -p $TEST_ROOT/tunnel.d $TEST_ROOT/foo2 +ln -sfn .. $TEST_ROOT/tunnel.d/tunnel +echo foo > $TEST_ROOT/bar + +expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readFile " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" + +expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" + +# Reading the parents of allowed paths should show only the ancestors of the allowed paths. +[[ $(nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d) == '{ "tunnel.d" = "directory"; }' ]] + # Check whether we can leak symlink information through directory traversal. traverseDir="$(pwd)/restricted-traverse-me" ln -sfn "$(pwd)/restricted-secret" "$(pwd)/restricted-innocent" diff --git a/tests/functional/tarball.sh b/tests/functional/tarball.sh index 6e621a28c..e59ee400e 100644 --- a/tests/functional/tarball.sh +++ b/tests/functional/tarball.sh @@ -18,7 +18,7 @@ test_tarball() { local compressor="$2" tarball=$TEST_ROOT/tarball.tar$ext - (cd $TEST_ROOT && tar cf - tarball) | $compressor > $tarball + (cd $TEST_ROOT && GNUTAR_REPRODUCIBLE= tar --mtime=$tarroot/default.nix --owner=0 --group=0 --numeric-owner --sort=name -c -f - tarball) | $compressor > $tarball nix-env -f file://$tarball -qa --out-path | grepQuiet dependencies diff --git a/tests/functional/user-envs.sh b/tests/functional/user-envs.sh index d1260ba04..dcd6b1b97 100644 --- a/tests/functional/user-envs.sh +++ b/tests/functional/user-envs.sh @@ -26,6 +26,7 @@ nix-env -f ./user-envs.nix -qa --json --out-path | jq -e '.[] | select(.name == .outputName == "out", (.outputs.out | test("'$NIX_STORE_DIR'.*-0\\.1")) ] | all' +nix-env -f ./user-envs.nix -qa --json --drv-path | jq -e '.[] | select(.name == "bar-0.1") | (.drvPath | test("'$NIX_STORE_DIR'.*-0\\.1\\.drv"))' # Query descriptions. nix-env -f ./user-envs.nix -qa '*' --description | grepQuiet silly diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index b391d7ef2..4459aa664 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -21,6 +21,8 @@ in remoteBuilds = runNixOSTestFor "x86_64-linux" ./remote-builds.nix; + remoteBuildsSshNg = runNixOSTestFor "x86_64-linux" ./remote-builds-ssh-ng.nix; + nix-copy-closure = runNixOSTestFor "x86_64-linux" ./nix-copy-closure.nix; nix-copy = runNixOSTestFor "x86_64-linux" ./nix-copy.nix; diff --git a/tests/nixos/remote-builds-ssh-ng.nix b/tests/nixos/remote-builds-ssh-ng.nix new file mode 100644 index 000000000..b59dde9bf --- /dev/null +++ b/tests/nixos/remote-builds-ssh-ng.nix @@ -0,0 +1,108 @@ +{ config, lib, hostPkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + # Trivial Nix expression to build remotely. + expr = config: nr: pkgs.writeText "expr.nix" + '' + let utils = builtins.storePath ${config.system.build.extraUtils}; in + derivation { + name = "hello-${toString nr}"; + system = "i686-linux"; + PATH = "''${utils}/bin"; + builder = "''${utils}/bin/sh"; + args = [ "-c" "${ + lib.concatStringsSep "; " [ + ''if [[ -n $NIX_LOG_FD ]]'' + ''then echo '@nix {\"action\":\"setPhase\",\"phase\":\"buildPhase\"}' >&''$NIX_LOG_FD'' + "fi" + "echo Hello" + "mkdir $out" + "cat /proc/sys/kernel/hostname > $out/host" + ] + }" ]; + outputs = [ "out" ]; + } + ''; +in + +{ + name = "remote-builds-ssh-ng"; + + nodes = + { builder = + { config, pkgs, ... }: + { services.openssh.enable = true; + virtualisation.writableStore = true; + nix.settings.sandbox = true; + nix.settings.substituters = lib.mkForce [ ]; + }; + + client = + { config, lib, pkgs, ... }: + { nix.settings.max-jobs = 0; # force remote building + nix.distributedBuilds = true; + nix.buildMachines = + [ { hostName = "builder"; + sshUser = "root"; + sshKey = "/root/.ssh/id_ed25519"; + system = "i686-linux"; + maxJobs = 1; + protocol = "ssh-ng"; + } + ]; + virtualisation.writableStore = true; + virtualisation.additionalPaths = [ config.system.build.extraUtils ]; + nix.settings.substituters = lib.mkForce [ ]; + programs.ssh.extraConfig = "ConnectTimeout 30"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import subprocess + + start_all() + + # Create an SSH key on the client. + subprocess.run([ + "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + client.succeed("mkdir -p -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + # Install the SSH key on the builder. + client.wait_for_unit("network.target") + builder.succeed("mkdir -p -m 700 /root/.ssh") + builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + builder.wait_for_unit("sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") + + # Perform a build + out = client.succeed("nix-build ${expr nodes.client.config 1} 2> build-output") + + # Verify that the build was done on the builder + builder.succeed(f"test -e {out.strip()}") + + # Print the build log, prefix the log lines to avoid nix intercepting lines starting with @nix + buildOutput = client.succeed("sed -e 's/^/build-output:/' build-output") + print(buildOutput) + + # Make sure that we get the expected build output + client.succeed("grep -qF Hello build-output") + + # We don't want phase reporting in the build output + client.fail("grep -qF '@nix' build-output") + + # Get the log file + client.succeed(f"nix-store --read-log {out.strip()} > log-output") + # Prefix the log lines to avoid nix intercepting lines starting with @nix + logOutput = client.succeed("sed -e 's/^/log-file:/' log-output") + print(logOutput) + + # Check that we get phase reporting in the log file + client.succeed("grep -q '@nix {\"action\":\"setPhase\",\"phase\":\"buildPhase\"}' log-output") + ''; +} diff --git a/tests/unit/libexpr-support/local.mk b/tests/unit/libexpr-support/local.mk new file mode 100644 index 000000000..12a76206a --- /dev/null +++ b/tests/unit/libexpr-support/local.mk @@ -0,0 +1,23 @@ +libraries += libexpr-test-support + +libexpr-test-support_NAME = libnixexpr-test-support + +libexpr-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-test-support_INSTALL_DIR := $(checklibdir) +else + libexpr-test-support_INSTALL_DIR := +endif + +libexpr-test-support_SOURCES := \ + $(wildcard $(d)/tests/*.cc) \ + $(wildcard $(d)/tests/value/*.cc) + +libexpr-test-support_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) + +libexpr-test-support_LIBS = \ + libstore-test-support libutil-test-support \ + libexpr libstore libutil + +libexpr-test-support_LDFLAGS := -pthread -lrapidcheck diff --git a/src/libexpr/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh similarity index 100% rename from src/libexpr/tests/libexpr.hh rename to tests/unit/libexpr-support/tests/libexpr.hh diff --git a/tests/unit/libexpr-support/tests/value/context.cc b/tests/unit/libexpr-support/tests/value/context.cc new file mode 100644 index 000000000..8658bdaef --- /dev/null +++ b/tests/unit/libexpr-support/tests/value/context.cc @@ -0,0 +1,30 @@ +#include + +#include "tests/path.hh" +#include "tests/value/context.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + return gen::just(NixStringContextElem::DrvDeep { + .drvPath = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + case 2: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +} diff --git a/src/libexpr/tests/value/context.hh b/tests/unit/libexpr-support/tests/value/context.hh similarity index 95% rename from src/libexpr/tests/value/context.hh rename to tests/unit/libexpr-support/tests/value/context.hh index c0bc97ba3..8c68c78bb 100644 --- a/src/libexpr/tests/value/context.hh +++ b/tests/unit/libexpr-support/tests/value/context.hh @@ -3,7 +3,7 @@ #include -#include +#include "value/context.hh" namespace rc { using namespace nix; diff --git a/src/libexpr/tests/derived-path.cc b/tests/unit/libexpr/derived-path.cc similarity index 100% rename from src/libexpr/tests/derived-path.cc rename to tests/unit/libexpr/derived-path.cc diff --git a/src/libexpr/tests/error_traces.cc b/tests/unit/libexpr/error_traces.cc similarity index 99% rename from src/libexpr/tests/error_traces.cc rename to tests/unit/libexpr/error_traces.cc index 139366bcd..81498f65a 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/tests/unit/libexpr/error_traces.cc @@ -906,12 +906,12 @@ namespace nix { ASSERT_TRACE2("concatMap (x: 1) [ \"foo\" ] # TODO", TypeError, hintfmt("value is %s while a list was expected", "an integer"), - hintfmt("while evaluating the return value of the function passed to buitlins.concatMap")); + hintfmt("while evaluating the return value of the function passed to builtins.concatMap")); ASSERT_TRACE2("concatMap (x: \"foo\") [ 1 2 ] # TODO", TypeError, hintfmt("value is %s while a list was expected", "a string"), - hintfmt("while evaluating the return value of the function passed to buitlins.concatMap")); + hintfmt("while evaluating the return value of the function passed to builtins.concatMap")); } diff --git a/src/libexpr/tests/flakeref.cc b/tests/unit/libexpr/flakeref.cc similarity index 100% rename from src/libexpr/tests/flakeref.cc rename to tests/unit/libexpr/flakeref.cc diff --git a/src/libexpr/tests/json.cc b/tests/unit/libexpr/json.cc similarity index 100% rename from src/libexpr/tests/json.cc rename to tests/unit/libexpr/json.cc diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk new file mode 100644 index 000000000..5743880d7 --- /dev/null +++ b/tests/unit/libexpr/local.mk @@ -0,0 +1,36 @@ +check: libexpr-tests_RUN + +programs += libexpr-tests + +libexpr-tests_NAME := libnixexpr-tests + +libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libexpr-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libexpr-tests_INSTALL_DIR := $(checkbindir) +else + libexpr-tests_INSTALL_DIR := +endif + +libexpr-tests_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) + +libexpr-tests_EXTRA_INCLUDES = \ + -I tests/unit/libexpr-support \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + -I src/libexpr \ + -I src/libfetchers \ + -I src/libstore \ + -I src/libutil + +libexpr-tests_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) + +libexpr-tests_LIBS = \ + libexpr-test-support libstore-test-support libutils-test-support \ + libexpr libfetchers libstore libutil + +libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock diff --git a/src/libexpr/tests/primops.cc b/tests/unit/libexpr/primops.cc similarity index 98% rename from src/libexpr/tests/primops.cc rename to tests/unit/libexpr/primops.cc index d820b860e..7485fa0d0 100644 --- a/src/libexpr/tests/primops.cc +++ b/tests/unit/libexpr/primops.cc @@ -814,6 +814,14 @@ namespace nix { ASSERT_THAT(*v.listElems()[0], IsStringEq("FOO")); } + TEST_F(PrimOpTest, match5) { + // The regex "\\{}" is valid and matches the string "{}". + // Caused a regression before when trying to switch from std::regex to boost::regex. + // See https://github.com/NixOS/nix/pull/7762#issuecomment-1834303659 + auto v = eval("builtins.match \"\\\\{}\" \"{}\""); + ASSERT_THAT(v, IsListOfSize(0)); + } + TEST_F(PrimOpTest, attrNames) { auto v = eval("builtins.attrNames { x = 1; y = 2; z = 3; a = 2; }"); ASSERT_THAT(v, IsListOfSize(4)); diff --git a/src/libexpr/tests/search-path.cc b/tests/unit/libexpr/search-path.cc similarity index 100% rename from src/libexpr/tests/search-path.cc rename to tests/unit/libexpr/search-path.cc diff --git a/src/libexpr/tests/trivial.cc b/tests/unit/libexpr/trivial.cc similarity index 100% rename from src/libexpr/tests/trivial.cc rename to tests/unit/libexpr/trivial.cc diff --git a/src/libexpr/tests/value/context.cc b/tests/unit/libexpr/value/context.cc similarity index 83% rename from src/libexpr/tests/value/context.cc rename to tests/unit/libexpr/value/context.cc index 92d4889ab..761286dbd 100644 --- a/src/libexpr/tests/value/context.cc +++ b/tests/unit/libexpr/value/context.cc @@ -117,36 +117,6 @@ TEST(NixStringContextElemTest, built_built_xp) { NixStringContextElem::parse("!foo!bar!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"), MissingExperimentalFeature); } -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - return gen::just(NixStringContextElem::DrvDeep { - .drvPath = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - case 2: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -} - -namespace nix { - #ifndef COVERAGE RC_GTEST_PROP( diff --git a/src/libexpr/tests/value/print.cc b/tests/unit/libexpr/value/print.cc similarity index 99% rename from src/libexpr/tests/value/print.cc rename to tests/unit/libexpr/value/print.cc index 5e96e12ec..a4f6fc014 100644 --- a/src/libexpr/tests/value/print.cc +++ b/tests/unit/libexpr/value/print.cc @@ -114,7 +114,8 @@ TEST_F(ValuePrintingTests, vLambda) TEST_F(ValuePrintingTests, vPrimOp) { Value vPrimOp; - vPrimOp.mkPrimOp(nullptr); + PrimOp primOp{}; + vPrimOp.mkPrimOp(&primOp); test(vPrimOp, ""); } diff --git a/tests/unit/libstore-support/local.mk b/tests/unit/libstore-support/local.mk new file mode 100644 index 000000000..ff075c96a --- /dev/null +++ b/tests/unit/libstore-support/local.mk @@ -0,0 +1,21 @@ +libraries += libstore-test-support + +libstore-test-support_NAME = libnixstore-test-support + +libstore-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-test-support_INSTALL_DIR := $(checklibdir) +else + libstore-test-support_INSTALL_DIR := +endif + +libstore-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) + +libstore-test-support_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) + +libstore-test-support_LIBS = \ + libutil-test-support \ + libstore libutil + +libstore-test-support_LDFLAGS := -pthread -lrapidcheck diff --git a/tests/unit/libstore-support/tests/derived-path.cc b/tests/unit/libstore-support/tests/derived-path.cc new file mode 100644 index 000000000..091706dba --- /dev/null +++ b/tests/unit/libstore-support/tests/derived-path.cc @@ -0,0 +1,57 @@ +#include + +#include + +#include "tests/derived-path.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Opaque { + .path = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(SingleDerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .output = (*gen::arbitrary()).name, + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(DerivedPath::Built { + .drvPath = make_ref(*gen::arbitrary()), + .outputs = *gen::arbitrary(), + }); +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just(*gen::arbitrary()); + case 1: + return gen::just(*gen::arbitrary()); + default: + assert(false); + } +} + +} diff --git a/src/libstore/tests/derived-path.hh b/tests/unit/libstore-support/tests/derived-path.hh similarity index 100% rename from src/libstore/tests/derived-path.hh rename to tests/unit/libstore-support/tests/derived-path.hh diff --git a/src/libstore/tests/libstore.hh b/tests/unit/libstore-support/tests/libstore.hh similarity index 86% rename from src/libstore/tests/libstore.hh rename to tests/unit/libstore-support/tests/libstore.hh index ef93457b5..78b162b95 100644 --- a/src/libstore/tests/libstore.hh +++ b/tests/unit/libstore-support/tests/libstore.hh @@ -8,7 +8,7 @@ namespace nix { -class LibStoreTest : public ::testing::Test { +class LibStoreTest : public virtual ::testing::Test { public: static void SetUpTestSuite() { initLibStore(); diff --git a/tests/unit/libstore-support/tests/outputs-spec.cc b/tests/unit/libstore-support/tests/outputs-spec.cc new file mode 100644 index 000000000..e9d602203 --- /dev/null +++ b/tests/unit/libstore-support/tests/outputs-spec.cc @@ -0,0 +1,24 @@ +#include "tests/outputs-spec.hh" + +#include + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + switch (*gen::inRange(0, std::variant_size_v)) { + case 0: + return gen::just((OutputsSpec) OutputsSpec::All { }); + case 1: + return gen::just((OutputsSpec) OutputsSpec::Names { + *gen::nonEmpty(gen::container(gen::map( + gen::arbitrary(), + [](StorePathName n) { return n.name; }))), + }); + default: + assert(false); + } +} + +} diff --git a/src/libstore/tests/outputs-spec.hh b/tests/unit/libstore-support/tests/outputs-spec.hh similarity index 89% rename from src/libstore/tests/outputs-spec.hh rename to tests/unit/libstore-support/tests/outputs-spec.hh index ded331b33..f5bf9042d 100644 --- a/src/libstore/tests/outputs-spec.hh +++ b/tests/unit/libstore-support/tests/outputs-spec.hh @@ -5,7 +5,7 @@ #include -#include +#include "tests/path.hh" namespace rc { using namespace nix; diff --git a/tests/unit/libstore-support/tests/path.cc b/tests/unit/libstore-support/tests/path.cc new file mode 100644 index 000000000..e5f169e94 --- /dev/null +++ b/tests/unit/libstore-support/tests/path.cc @@ -0,0 +1,82 @@ +#include + +#include + +#include "path-regex.hh" +#include "store-api.hh" + +#include "tests/hash.hh" +#include "tests/path.hh" + +namespace nix { + +void showValue(const StorePath & p, std::ostream & os) +{ + os << p.to_string(); +} + +} + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + auto len = *gen::inRange( + 1, + StorePath::MaxPathLen - StorePath::HashLen); + + std::string pre; + pre.reserve(len); + + for (size_t c = 0; c < len; ++c) { + switch (auto i = *gen::inRange(0, 10 + 2 * 26 + 6)) { + case 0 ... 9: + pre += '0' + i; + case 10 ... 35: + pre += 'A' + (i - 10); + break; + case 36 ... 61: + pre += 'a' + (i - 36); + break; + case 62: + pre += '+'; + break; + case 63: + pre += '-'; + break; + case 64: + // names aren't permitted to start with a period, + // so just fall through to the next case here + if (c != 0) { + pre += '.'; + break; + } + case 65: + pre += '_'; + break; + case 66: + pre += '?'; + break; + case 67: + pre += '='; + break; + default: + assert(false); + } + } + + return gen::just(StorePathName { + .name = std::move(pre), + }); +} + +Gen Arbitrary::arbitrary() +{ + return gen::just(StorePath { + *gen::arbitrary(), + (*gen::arbitrary()).name, + }); +} + +} // namespace rc diff --git a/src/libstore/tests/path.hh b/tests/unit/libstore-support/tests/path.hh similarity index 82% rename from src/libstore/tests/path.hh rename to tests/unit/libstore-support/tests/path.hh index 21cb62310..4751b3373 100644 --- a/src/libstore/tests/path.hh +++ b/tests/unit/libstore-support/tests/path.hh @@ -11,6 +11,9 @@ struct StorePathName { std::string name; }; +// For rapidcheck +void showValue(const StorePath & p, std::ostream & os); + } namespace rc { diff --git a/tests/unit/libstore-support/tests/protocol.hh b/tests/unit/libstore-support/tests/protocol.hh new file mode 100644 index 000000000..3c9e52c11 --- /dev/null +++ b/tests/unit/libstore-support/tests/protocol.hh @@ -0,0 +1,75 @@ +#pragma once +///@file + +#include +#include + +#include "tests/libstore.hh" +#include "tests/characterization.hh" + +namespace nix { + +template +class ProtoTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/" + protocolDir; + + Path goldenMaster(std::string_view testStem) const override { + return unitTestData + "/" + testStem + ".bin"; + } +}; + +template +class VersionedProtoTest : public ProtoTest +{ +public: + /** + * Golden test for `T` reading + */ + template + void readProtoTest(PathView testStem, typename Proto::Version version, T expected) + { + CharacterizationTest::readTest(testStem, [&](const auto & encoded) { + T got = ({ + StringSource from { encoded }; + Proto::template Serialise::read( + *LibStoreTest::store, + typename Proto::ReadConn { + .from = from, + .version = version, + }); + }); + + ASSERT_EQ(got, expected); + }); + } + + /** + * Golden test for `T` write + */ + template + void writeProtoTest(PathView testStem, typename Proto::Version version, const T & decoded) + { + CharacterizationTest::writeTest(testStem, [&]() { + StringSink to; + Proto::template Serialise::write( + *LibStoreTest::store, + typename Proto::WriteConn { + .to = to, + .version = version, + }, + decoded); + return std::move(to.s); + }); + } +}; + +#define VERSIONED_CHARACTERIZATION_TEST(FIXTURE, NAME, STEM, VERSION, VALUE) \ + TEST_F(FIXTURE, NAME ## _read) { \ + readProtoTest(STEM, VERSION, VALUE); \ + } \ + TEST_F(FIXTURE, NAME ## _write) { \ + writeProtoTest(STEM, VERSION, VALUE); \ + } + +} diff --git a/src/libstore/tests/common-protocol.cc b/tests/unit/libstore/common-protocol.cc similarity index 78% rename from src/libstore/tests/common-protocol.cc rename to tests/unit/libstore/common-protocol.cc index 61c2cb70c..d23805fc3 100644 --- a/src/libstore/tests/common-protocol.cc +++ b/tests/unit/libstore/common-protocol.cc @@ -20,61 +20,43 @@ public: * Golden test for `T` reading */ template - void readTest(PathView testStem, T value) + void readProtoTest(PathView testStem, const T & expected) { - if (testAccept()) - { - GTEST_SKIP() << "Cannot read golden master because another test is also updating it"; - } - else - { - auto expected = readFile(goldenMaster(testStem)); - + CharacterizationTest::readTest(testStem, [&](const auto & encoded) { T got = ({ - StringSource from { expected }; + StringSource from { encoded }; CommonProto::Serialise::read( *store, CommonProto::ReadConn { .from = from }); }); - ASSERT_EQ(got, value); - } + ASSERT_EQ(got, expected); + }); } /** * Golden test for `T` write */ template - void writeTest(PathView testStem, const T & value) + void writeProtoTest(PathView testStem, const T & decoded) { - auto file = goldenMaster(testStem); - - StringSink to; - CommonProto::write( - *store, - CommonProto::WriteConn { .to = to }, - value); - - if (testAccept()) - { - createDirs(dirOf(file)); - writeFile(file, to.s); - GTEST_SKIP() << "Updating golden master"; - } - else - { - auto expected = readFile(file); - ASSERT_EQ(to.s, expected); - } + CharacterizationTest::writeTest(testStem, [&]() -> std::string { + StringSink to; + CommonProto::Serialise::write( + *store, + CommonProto::WriteConn { .to = to }, + decoded); + return to.s; + }); } }; #define CHARACTERIZATION_TEST(NAME, STEM, VALUE) \ TEST_F(CommonProtoTest, NAME ## _read) { \ - readTest(STEM, VALUE); \ + readProtoTest(STEM, VALUE); \ } \ TEST_F(CommonProtoTest, NAME ## _write) { \ - writeTest(STEM, VALUE); \ + writeProtoTest(STEM, VALUE); \ } CHARACTERIZATION_TEST( @@ -102,15 +84,15 @@ CHARACTERIZATION_TEST( (std::tuple { ContentAddress { .method = TextIngestionMethod {}, - .hash = hashString(HashType::htSHA256, "Derive(...)"), + .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -197,7 +179,7 @@ CHARACTERIZATION_TEST( std::optional { ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, })) diff --git a/unit-test-data/libstore/common-protocol/content-address.bin b/tests/unit/libstore/data/common-protocol/content-address.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/content-address.bin rename to tests/unit/libstore/data/common-protocol/content-address.bin diff --git a/unit-test-data/libstore/common-protocol/drv-output.bin b/tests/unit/libstore/data/common-protocol/drv-output.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/drv-output.bin rename to tests/unit/libstore/data/common-protocol/drv-output.bin diff --git a/unit-test-data/libstore/common-protocol/optional-content-address.bin b/tests/unit/libstore/data/common-protocol/optional-content-address.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/optional-content-address.bin rename to tests/unit/libstore/data/common-protocol/optional-content-address.bin diff --git a/unit-test-data/libstore/common-protocol/optional-store-path.bin b/tests/unit/libstore/data/common-protocol/optional-store-path.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/optional-store-path.bin rename to tests/unit/libstore/data/common-protocol/optional-store-path.bin diff --git a/unit-test-data/libstore/common-protocol/realisation.bin b/tests/unit/libstore/data/common-protocol/realisation.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/realisation.bin rename to tests/unit/libstore/data/common-protocol/realisation.bin diff --git a/unit-test-data/libstore/common-protocol/set.bin b/tests/unit/libstore/data/common-protocol/set.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/set.bin rename to tests/unit/libstore/data/common-protocol/set.bin diff --git a/unit-test-data/libstore/common-protocol/store-path.bin b/tests/unit/libstore/data/common-protocol/store-path.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/store-path.bin rename to tests/unit/libstore/data/common-protocol/store-path.bin diff --git a/unit-test-data/libstore/common-protocol/string.bin b/tests/unit/libstore/data/common-protocol/string.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/string.bin rename to tests/unit/libstore/data/common-protocol/string.bin diff --git a/unit-test-data/libstore/common-protocol/vector.bin b/tests/unit/libstore/data/common-protocol/vector.bin similarity index 100% rename from unit-test-data/libstore/common-protocol/vector.bin rename to tests/unit/libstore/data/common-protocol/vector.bin diff --git a/tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv b/tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv new file mode 100644 index 000000000..3cd1ded02 --- /dev/null +++ b/tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv @@ -0,0 +1 @@ +Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/bad-version.drv b/tests/unit/libstore/data/derivation/bad-version.drv new file mode 100644 index 000000000..bbf75c114 --- /dev/null +++ b/tests/unit/libstore/data/derivation/bad-version.drv @@ -0,0 +1 @@ +DrvWithVersion("invalid-version",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/dynDerivationDeps.drv b/tests/unit/libstore/data/derivation/dynDerivationDeps.drv new file mode 100644 index 000000000..cfffe48ec --- /dev/null +++ b/tests/unit/libstore/data/derivation/dynDerivationDeps.drv @@ -0,0 +1 @@ +DrvWithVersion("xp-dyn-drv",[],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",(["cat","dog"],[("cat",["kitten"]),("goose",["gosling"])]))],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/dynDerivationDeps.json b/tests/unit/libstore/data/derivation/dynDerivationDeps.json new file mode 100644 index 000000000..9dbeb1f15 --- /dev/null +++ b/tests/unit/libstore/data/derivation/dynDerivationDeps.json @@ -0,0 +1,38 @@ +{ + "args": [ + "bar", + "baz" + ], + "builder": "foo", + "env": { + "BIG_BAD": "WOLF" + }, + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": { + "cat": { + "dynamicOutputs": {}, + "outputs": [ + "kitten" + ] + }, + "goose": { + "dynamicOutputs": {}, + "outputs": [ + "gosling" + ] + } + }, + "outputs": [ + "cat", + "dog" + ] + } + }, + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "name": "dyn-dep-derivation", + "outputs": {}, + "system": "wasm-sel4" +} diff --git a/tests/unit/libstore/data/derivation/output-caFixedFlat.json b/tests/unit/libstore/data/derivation/output-caFixedFlat.json new file mode 100644 index 000000000..fe000ea36 --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-caFixedFlat.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "sha256", + "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" +} diff --git a/tests/unit/libstore/data/derivation/output-caFixedNAR.json b/tests/unit/libstore/data/derivation/output-caFixedNAR.json new file mode 100644 index 000000000..1afd60223 --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-caFixedNAR.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "r:sha256", + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" +} diff --git a/tests/unit/libstore/data/derivation/output-caFixedText.json b/tests/unit/libstore/data/derivation/output-caFixedText.json new file mode 100644 index 000000000..0b2cc8bbc --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-caFixedText.json @@ -0,0 +1,5 @@ +{ + "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", + "hashAlgo": "text:sha256", + "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" +} diff --git a/tests/unit/libstore/data/derivation/output-caFloating.json b/tests/unit/libstore/data/derivation/output-caFloating.json new file mode 100644 index 000000000..9115de851 --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-caFloating.json @@ -0,0 +1,3 @@ +{ + "hashAlgo": "r:sha256" +} diff --git a/tests/unit/libstore/data/derivation/output-deferred.json b/tests/unit/libstore/data/derivation/output-deferred.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-deferred.json @@ -0,0 +1 @@ +{} diff --git a/tests/unit/libstore/data/derivation/output-impure.json b/tests/unit/libstore/data/derivation/output-impure.json new file mode 100644 index 000000000..62b61cdca --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-impure.json @@ -0,0 +1,4 @@ +{ + "hashAlgo": "r:sha256", + "impure": true +} diff --git a/tests/unit/libstore/data/derivation/output-inputAddressed.json b/tests/unit/libstore/data/derivation/output-inputAddressed.json new file mode 100644 index 000000000..86c7f3a05 --- /dev/null +++ b/tests/unit/libstore/data/derivation/output-inputAddressed.json @@ -0,0 +1,3 @@ +{ + "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" +} diff --git a/tests/unit/libstore/data/derivation/simple.drv b/tests/unit/libstore/data/derivation/simple.drv new file mode 100644 index 000000000..bda74ad25 --- /dev/null +++ b/tests/unit/libstore/data/derivation/simple.drv @@ -0,0 +1 @@ +Derive([],[("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv",["cat","dog"])],["/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"],"wasm-sel4","foo",["bar","baz"],[("BIG_BAD","WOLF")]) \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/simple.json b/tests/unit/libstore/data/derivation/simple.json new file mode 100644 index 000000000..20d0f8933 --- /dev/null +++ b/tests/unit/libstore/data/derivation/simple.json @@ -0,0 +1,25 @@ +{ + "args": [ + "bar", + "baz" + ], + "builder": "foo", + "env": { + "BIG_BAD": "WOLF" + }, + "inputDrvs": { + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "dynamicOutputs": {}, + "outputs": [ + "cat", + "dog" + ] + } + }, + "inputSrcs": [ + "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + ], + "name": "simple-derivation", + "outputs": {}, + "system": "wasm-sel4" +} diff --git a/tests/unit/libstore/data/nar-info/impure.json b/tests/unit/libstore/data/nar-info/impure.json new file mode 100644 index 000000000..bb9791a6a --- /dev/null +++ b/tests/unit/libstore/data/nar-info/impure.json @@ -0,0 +1,20 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "compression": "xz", + "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "downloadHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "downloadSize": 4029176, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + "asdf", + "qwer" + ], + "ultimate": true, + "url": "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz" +} diff --git a/tests/unit/libstore/data/nar-info/pure.json b/tests/unit/libstore/data/nar-info/pure.json new file mode 100644 index 000000000..955baec31 --- /dev/null +++ b/tests/unit/libstore/data/nar-info/pure.json @@ -0,0 +1,9 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ] +} diff --git a/tests/unit/libstore/data/path-info/impure.json b/tests/unit/libstore/data/path-info/impure.json new file mode 100644 index 000000000..0c452cc49 --- /dev/null +++ b/tests/unit/libstore/data/path-info/impure.json @@ -0,0 +1,16 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "deriver": "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ], + "registrationTime": 23423, + "signatures": [ + "asdf", + "qwer" + ], + "ultimate": true +} diff --git a/tests/unit/libstore/data/path-info/pure.json b/tests/unit/libstore/data/path-info/pure.json new file mode 100644 index 000000000..955baec31 --- /dev/null +++ b/tests/unit/libstore/data/path-info/pure.json @@ -0,0 +1,9 @@ +{ + "ca": "fixed:r:sha256:1lr187v6dck1rjh2j6svpikcfz53wyl3qrlcbb405zlh13x0khhh", + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 34878, + "references": [ + "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + "/nix/store/n5wkd9frr45pa74if5gpz9j7mifg27fh-foo" + ] +} diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.1.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.1.bin new file mode 100644 index 000000000..61e1d9728 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/build-options-2.1.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.2.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.2.bin new file mode 100644 index 000000000..045c2ff2b Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/build-options-2.2.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.3.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.3.bin new file mode 100644 index 000000000..5c5345883 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/build-options-2.3.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.7.bin b/tests/unit/libstore/data/serve-protocol/build-options-2.7.bin new file mode 100644 index 000000000..1bc7b02db Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/build-options-2.7.bin differ diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.2.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.2.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/build-result-2.2.bin rename to tests/unit/libstore/data/serve-protocol/build-result-2.2.bin diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.3.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.3.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/build-result-2.3.bin rename to tests/unit/libstore/data/serve-protocol/build-result-2.3.bin diff --git a/unit-test-data/libstore/serve-protocol/build-result-2.6.bin b/tests/unit/libstore/data/serve-protocol/build-result-2.6.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/build-result-2.6.bin rename to tests/unit/libstore/data/serve-protocol/build-result-2.6.bin diff --git a/unit-test-data/libstore/serve-protocol/content-address.bin b/tests/unit/libstore/data/serve-protocol/content-address.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/content-address.bin rename to tests/unit/libstore/data/serve-protocol/content-address.bin diff --git a/unit-test-data/libstore/serve-protocol/drv-output.bin b/tests/unit/libstore/data/serve-protocol/drv-output.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/drv-output.bin rename to tests/unit/libstore/data/serve-protocol/drv-output.bin diff --git a/unit-test-data/libstore/serve-protocol/optional-content-address.bin b/tests/unit/libstore/data/serve-protocol/optional-content-address.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/optional-content-address.bin rename to tests/unit/libstore/data/serve-protocol/optional-content-address.bin diff --git a/unit-test-data/libstore/serve-protocol/optional-store-path.bin b/tests/unit/libstore/data/serve-protocol/optional-store-path.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/optional-store-path.bin rename to tests/unit/libstore/data/serve-protocol/optional-store-path.bin diff --git a/unit-test-data/libstore/serve-protocol/realisation.bin b/tests/unit/libstore/data/serve-protocol/realisation.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/realisation.bin rename to tests/unit/libstore/data/serve-protocol/realisation.bin diff --git a/unit-test-data/libstore/serve-protocol/set.bin b/tests/unit/libstore/data/serve-protocol/set.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/set.bin rename to tests/unit/libstore/data/serve-protocol/set.bin diff --git a/unit-test-data/libstore/serve-protocol/store-path.bin b/tests/unit/libstore/data/serve-protocol/store-path.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/store-path.bin rename to tests/unit/libstore/data/serve-protocol/store-path.bin diff --git a/unit-test-data/libstore/serve-protocol/string.bin b/tests/unit/libstore/data/serve-protocol/string.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/string.bin rename to tests/unit/libstore/data/serve-protocol/string.bin diff --git a/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin b/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin new file mode 100644 index 000000000..8056ec055 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin differ diff --git a/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.4.bin b/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.4.bin new file mode 100644 index 000000000..521b5c423 Binary files /dev/null and b/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.4.bin differ diff --git a/unit-test-data/libstore/serve-protocol/vector.bin b/tests/unit/libstore/data/serve-protocol/vector.bin similarity index 100% rename from unit-test-data/libstore/serve-protocol/vector.bin rename to tests/unit/libstore/data/serve-protocol/vector.bin diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.27.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.27.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/build-result-1.27.bin rename to tests/unit/libstore/data/worker-protocol/build-result-1.27.bin diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.28.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.28.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/build-result-1.28.bin rename to tests/unit/libstore/data/worker-protocol/build-result-1.28.bin diff --git a/unit-test-data/libstore/worker-protocol/build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/build-result-1.29.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/build-result-1.29.bin rename to tests/unit/libstore/data/worker-protocol/build-result-1.29.bin diff --git a/unit-test-data/libstore/worker-protocol/content-address.bin b/tests/unit/libstore/data/worker-protocol/content-address.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/content-address.bin rename to tests/unit/libstore/data/worker-protocol/content-address.bin diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.29.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/derived-path-1.29.bin rename to tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin diff --git a/unit-test-data/libstore/worker-protocol/derived-path-1.30.bin b/tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/derived-path-1.30.bin rename to tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin diff --git a/unit-test-data/libstore/worker-protocol/drv-output.bin b/tests/unit/libstore/data/worker-protocol/drv-output.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/drv-output.bin rename to tests/unit/libstore/data/worker-protocol/drv-output.bin diff --git a/unit-test-data/libstore/worker-protocol/keyed-build-result-1.29.bin b/tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/keyed-build-result-1.29.bin rename to tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin diff --git a/unit-test-data/libstore/worker-protocol/optional-content-address.bin b/tests/unit/libstore/data/worker-protocol/optional-content-address.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/optional-content-address.bin rename to tests/unit/libstore/data/worker-protocol/optional-content-address.bin diff --git a/unit-test-data/libstore/worker-protocol/optional-store-path.bin b/tests/unit/libstore/data/worker-protocol/optional-store-path.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/optional-store-path.bin rename to tests/unit/libstore/data/worker-protocol/optional-store-path.bin diff --git a/unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin b/tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/optional-trusted-flag.bin rename to tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin diff --git a/unit-test-data/libstore/worker-protocol/realisation.bin b/tests/unit/libstore/data/worker-protocol/realisation.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/realisation.bin rename to tests/unit/libstore/data/worker-protocol/realisation.bin diff --git a/unit-test-data/libstore/worker-protocol/set.bin b/tests/unit/libstore/data/worker-protocol/set.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/set.bin rename to tests/unit/libstore/data/worker-protocol/set.bin diff --git a/unit-test-data/libstore/worker-protocol/store-path.bin b/tests/unit/libstore/data/worker-protocol/store-path.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/store-path.bin rename to tests/unit/libstore/data/worker-protocol/store-path.bin diff --git a/unit-test-data/libstore/worker-protocol/string.bin b/tests/unit/libstore/data/worker-protocol/string.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/string.bin rename to tests/unit/libstore/data/worker-protocol/string.bin diff --git a/unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/unkeyed-valid-path-info-1.15.bin rename to tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin diff --git a/unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/valid-path-info-1.15.bin rename to tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin diff --git a/unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin b/tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/valid-path-info-1.16.bin rename to tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin diff --git a/unit-test-data/libstore/worker-protocol/vector.bin b/tests/unit/libstore/data/worker-protocol/vector.bin similarity index 100% rename from unit-test-data/libstore/worker-protocol/vector.bin rename to tests/unit/libstore/data/worker-protocol/vector.bin diff --git a/tests/unit/libstore/derivation.cc b/tests/unit/libstore/derivation.cc new file mode 100644 index 000000000..7a4b1403a --- /dev/null +++ b/tests/unit/libstore/derivation.cc @@ -0,0 +1,298 @@ +#include +#include + +#include "experimental-features.hh" +#include "derivations.hh" + +#include "tests/libstore.hh" +#include "tests/characterization.hh" + +namespace nix { + +using nlohmann::json; + +class DerivationTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/derivation"; + +public: + Path goldenMaster(std::string_view testStem) const override { + return unitTestData + "/" + testStem; + } + + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; +}; + +class CaDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "ca-derivations"); + } +}; + +class DynDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "dynamic-derivations ca-derivations"); + } +}; + +class ImpureDerivationTest : public DerivationTest +{ + void SetUp() override + { + mockXpSettings.set("experimental-features", "impure-derivations"); + } +}; + +TEST_F(DerivationTest, BadATerm_version) { + ASSERT_THROW( + parseDerivation( + *store, + readFile(goldenMaster("bad-version.drv")), + "whatever", + mockXpSettings), + FormatError); +} + +TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) { + ASSERT_THROW( + parseDerivation( + *store, + readFile(goldenMaster("bad-old-version-dyn-deps.drv")), + "dyn-dep-derivation", + mockXpSettings), + FormatError); +} + +#define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME) \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _from_json) { \ + readTest("output-" #NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + DerivationOutput got = DerivationOutput::fromJSON( \ + *store, \ + DRV_NAME, \ + OUTPUT_NAME, \ + encoded, \ + mockXpSettings); \ + DerivationOutput expected { VAL }; \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(FIXTURE, DerivationOutput_ ## NAME ## _to_json) { \ + writeTest("output-" #NAME ".json", [&]() -> json { \ + return DerivationOutput { (VAL) }.toJSON( \ + *store, \ + (DRV_NAME), \ + (OUTPUT_NAME)); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +TEST_JSON(DerivationTest, inputAddressed, + (DerivationOutput::InputAddressed { + .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"), + }), + "drv-name", "output-name") + +TEST_JSON(DerivationTest, caFixedFlat, + (DerivationOutput::CAFixed { + .ca = { + .method = FileIngestionMethod::Flat, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationTest, caFixedNAR, + (DerivationOutput::CAFixed { + .ca = { + .method = FileIngestionMethod::Recursive, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }), + "drv-name", "output-name") + +TEST_JSON(DynDerivationTest, caFixedText, + (DerivationOutput::CAFixed { + .ca = { + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }), + "drv-name", "output-name") + +TEST_JSON(CaDerivationTest, caFloating, + (DerivationOutput::CAFloating { + .method = FileIngestionMethod::Recursive, + .hashAlgo = HashAlgorithm::SHA256, + }), + "drv-name", "output-name") + +TEST_JSON(DerivationTest, deferred, + DerivationOutput::Deferred { }, + "drv-name", "output-name") + +TEST_JSON(ImpureDerivationTest, impure, + (DerivationOutput::Impure { + .method = FileIngestionMethod::Recursive, + .hashAlgo = HashAlgorithm::SHA256, + }), + "drv-name", "output-name") + +#undef TEST_JSON + +#define TEST_JSON(FIXTURE, NAME, VAL) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_json) { \ + readTest(#NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + Derivation expected { VAL }; \ + Derivation got = Derivation::fromJSON( \ + *store, \ + encoded, \ + mockXpSettings); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_json) { \ + writeTest(#NAME ".json", [&]() -> json { \ + return Derivation { VAL }.toJSON(*store); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +#define TEST_ATERM(FIXTURE, NAME, VAL, DRV_NAME) \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _from_aterm) { \ + readTest(#NAME ".drv", [&](auto encoded) { \ + Derivation expected { VAL }; \ + auto got = parseDerivation( \ + *store, \ + std::move(encoded), \ + DRV_NAME, \ + mockXpSettings); \ + ASSERT_EQ(got.toJSON(*store), expected.toJSON(*store)) ; \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(FIXTURE, Derivation_ ## NAME ## _to_aterm) { \ + writeTest(#NAME ".drv", [&]() -> std::string { \ + return (VAL).unparse(*store, false); \ + }); \ + } + +Derivation makeSimpleDrv(const Store & store) { + Derivation drv; + drv.name = "simple-derivation"; + drv.inputSrcs = { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + .map = { + { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + { + .value = { + "cat", + "dog", + }, + }, + }, + }, + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = { + { + "BIG_BAD", + "WOLF", + }, + }; + return drv; +} + +TEST_JSON(DerivationTest, simple, makeSimpleDrv(*store)) + +TEST_ATERM(DerivationTest, simple, + makeSimpleDrv(*store), + "simple-derivation") + +Derivation makeDynDepDerivation(const Store & store) { + Derivation drv; + drv.name = "dyn-dep-derivation"; + drv.inputSrcs = { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + }; + drv.inputDrvs = { + .map = { + { + store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + DerivedPathMap::ChildNode { + .value = { + "cat", + "dog", + }, + .childMap = { + { + "cat", + DerivedPathMap::ChildNode { + .value = { + "kitten", + }, + }, + }, + { + "goose", + DerivedPathMap::ChildNode { + .value = { + "gosling", + }, + }, + }, + }, + }, + }, + }, + }; + drv.platform = "wasm-sel4"; + drv.builder = "foo"; + drv.args = { + "bar", + "baz", + }; + drv.env = { + { + "BIG_BAD", + "WOLF", + }, + }; + return drv; +} + +TEST_JSON(DynDerivationTest, dynDerivationDeps, makeDynDepDerivation(*store)) + +TEST_ATERM(DynDerivationTest, dynDerivationDeps, + makeDynDepDerivation(*store), + "dyn-dep-derivation") + +#undef TEST_JSON +#undef TEST_ATERM + +} diff --git a/src/libstore/tests/derived-path.cc b/tests/unit/libstore/derived-path.cc similarity index 66% rename from src/libstore/tests/derived-path.cc rename to tests/unit/libstore/derived-path.cc index 3fa3c0801..c62d79a78 100644 --- a/src/libstore/tests/derived-path.cc +++ b/tests/unit/libstore/derived-path.cc @@ -1,64 +1,11 @@ #include -#include #include #include #include "tests/derived-path.hh" #include "tests/libstore.hh" -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - return gen::just(DerivedPath::Opaque { - .path = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(SingleDerivedPath::Built { - .drvPath = make_ref(*gen::arbitrary()), - .output = (*gen::arbitrary()).name, - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(DerivedPath::Built { - .drvPath = make_ref(*gen::arbitrary()), - .outputs = *gen::arbitrary(), - }); -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just(*gen::arbitrary()); - case 1: - return gen::just(*gen::arbitrary()); - default: - assert(false); - } -} - -} - namespace nix { class DerivedPathTest : public LibStoreTest diff --git a/src/libstore/tests/downstream-placeholder.cc b/tests/unit/libstore/downstream-placeholder.cc similarity index 100% rename from src/libstore/tests/downstream-placeholder.cc rename to tests/unit/libstore/downstream-placeholder.cc diff --git a/tests/unit/libstore/local.mk b/tests/unit/libstore/local.mk new file mode 100644 index 000000000..63f6d011f --- /dev/null +++ b/tests/unit/libstore/local.mk @@ -0,0 +1,31 @@ +check: libstore-tests_RUN + +programs += libstore-tests + +libstore-tests_NAME = libnixstore-tests + +libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libstore-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libstore-tests_INSTALL_DIR := $(checkbindir) +else + libstore-tests_INSTALL_DIR := +endif + +libstore-tests_SOURCES := $(wildcard $(d)/*.cc) + +libstore-tests_EXTRA_INCLUDES = \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + -I src/libstore \ + -I src/libutil + +libstore-tests_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) + +libstore-tests_LIBS = \ + libstore-test-support libutil-test-support \ + libstore libutil + +libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/src/libstore/tests/machines.cc b/tests/unit/libstore/machines.cc similarity index 97% rename from src/libstore/tests/machines.cc rename to tests/unit/libstore/machines.cc index f51052b14..5b66e5a5b 100644 --- a/src/libstore/tests/machines.cc +++ b/tests/unit/libstore/machines.cc @@ -1,5 +1,7 @@ #include "machines.hh" #include "globals.hh" +#include "file-system.hh" +#include "util.hh" #include @@ -137,7 +139,7 @@ TEST(machines, getMachinesWithIncorrectFormat) { } TEST(machines, getMachinesWithCorrectFileReference) { - auto path = absPath("src/libstore/tests/test-data/machines.valid"); + auto path = absPath("tests/unit/libstore/test-data/machines.valid"); ASSERT_TRUE(pathExists(path)); settings.builders = std::string("@") + path; @@ -164,6 +166,6 @@ TEST(machines, getMachinesWithIncorrectFileReference) { } TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) { - settings.builders = std::string("@") + absPath("src/libstore/tests/test-data/machines.bad_format"); + settings.builders = std::string("@") + absPath("tests/unit/libstore/test-data/machines.bad_format"); EXPECT_THROW(getMachines(), FormatError); } diff --git a/src/libstore/tests/nar-info-disk-cache.cc b/tests/unit/libstore/nar-info-disk-cache.cc similarity index 100% rename from src/libstore/tests/nar-info-disk-cache.cc rename to tests/unit/libstore/nar-info-disk-cache.cc diff --git a/tests/unit/libstore/nar-info.cc b/tests/unit/libstore/nar-info.cc new file mode 100644 index 000000000..bd10602e7 --- /dev/null +++ b/tests/unit/libstore/nar-info.cc @@ -0,0 +1,86 @@ +#include +#include + +#include "path-info.hh" +#include "nar-info.hh" + +#include "tests/characterization.hh" +#include "tests/libstore.hh" + +namespace nix { + +using nlohmann::json; + +class NarInfoTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/nar-info"; + + Path goldenMaster(PathView testStem) const override { + return unitTestData + "/" + testStem + ".json"; + } +}; + +static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { + NarInfo info = ValidPathInfo { + store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashAlgorithm::SHA256, "(...)"), + + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.narSize = 34878; + if (includeImpureInfo) { + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.registrationTime = 23423; + info.ultimate = true; + info.sigs = { "asdf", "qwer" }; + + info.url = "nar/1w1fff338fvdw53sqgamddn1b2xgds473pv6y13gizdbqjv4i5p3.nar.xz"; + info.compression = "xz"; + info.fileHash = Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="); + info.fileSize = 4029176; + } + return info; +} + +#define JSON_TEST(STEM, PURE) \ + TEST_F(NarInfoTest, NarInfo_ ## STEM ## _from_json) { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + auto expected = makeNarInfo(*store, PURE); \ + NarInfo got = NarInfo::fromJSON( \ + *store, \ + expected.path, \ + encoded); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(NarInfoTest, NarInfo_ ## STEM ## _to_json) { \ + writeTest(#STEM, [&]() -> json { \ + return makeNarInfo(*store, PURE) \ + .toJSON(*store, PURE, HashFormat::SRI); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +JSON_TEST(pure, false) +JSON_TEST(impure, true) + +} diff --git a/src/libstore/tests/outputs-spec.cc b/tests/unit/libstore/outputs-spec.cc similarity index 92% rename from src/libstore/tests/outputs-spec.cc rename to tests/unit/libstore/outputs-spec.cc index 952945185..456196be1 100644 --- a/src/libstore/tests/outputs-spec.cc +++ b/tests/unit/libstore/outputs-spec.cc @@ -1,4 +1,4 @@ -#include "outputs-spec.hh" +#include "tests/outputs-spec.hh" #include #include @@ -199,31 +199,6 @@ TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Expl #undef TEST_JSON -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - switch (*gen::inRange(0, std::variant_size_v)) { - case 0: - return gen::just((OutputsSpec) OutputsSpec::All { }); - case 1: - return gen::just((OutputsSpec) OutputsSpec::Names { - *gen::nonEmpty(gen::container(gen::map( - gen::arbitrary(), - [](StorePathName n) { return n.name; }))), - }); - default: - assert(false); - } -} - -} - -namespace nix { - #ifndef COVERAGE RC_GTEST_PROP( diff --git a/tests/unit/libstore/path-info.cc b/tests/unit/libstore/path-info.cc new file mode 100644 index 000000000..80d6fcfed --- /dev/null +++ b/tests/unit/libstore/path-info.cc @@ -0,0 +1,79 @@ +#include +#include + +#include "path-info.hh" + +#include "tests/characterization.hh" +#include "tests/libstore.hh" + +namespace nix { + +using nlohmann::json; + +class PathInfoTest : public CharacterizationTest, public LibStoreTest +{ + Path unitTestData = getUnitTestData() + "/path-info"; + + Path goldenMaster(PathView testStem) const override { + return unitTestData + "/" + testStem + ".json"; + } +}; + +static UnkeyedValidPathInfo makePathInfo(const Store & store, bool includeImpureInfo) { + UnkeyedValidPathInfo info = ValidPathInfo { + store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashAlgorithm::SHA256, "(...)"), + + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.narSize = 34878; + if (includeImpureInfo) { + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.registrationTime = 23423; + info.ultimate = true; + info.sigs = { "asdf", "qwer" }; + } + return info; +} + +#define JSON_TEST(STEM, PURE) \ + TEST_F(PathInfoTest, PathInfo_ ## STEM ## _from_json) { \ + readTest(#STEM, [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + UnkeyedValidPathInfo got = UnkeyedValidPathInfo::fromJSON( \ + *store, \ + encoded); \ + auto expected = makePathInfo(*store, PURE); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(PathInfoTest, PathInfo_ ## STEM ## _to_json) { \ + writeTest(#STEM, [&]() -> json { \ + return makePathInfo(*store, PURE) \ + .toJSON(*store, PURE, HashFormat::SRI); \ + }, [](const auto & file) { \ + return json::parse(readFile(file)); \ + }, [](const auto & file, const auto & got) { \ + return writeFile(file, got.dump(2) + "\n"); \ + }); \ + } + +JSON_TEST(pure, false) +JSON_TEST(impure, true) + +} diff --git a/src/libstore/tests/path.cc b/tests/unit/libstore/path.cc similarity index 59% rename from src/libstore/tests/path.cc rename to tests/unit/libstore/path.cc index 5a84d646c..30631b5fd 100644 --- a/src/libstore/tests/path.cc +++ b/tests/unit/libstore/path.cc @@ -66,79 +66,6 @@ TEST_DO_PARSE(equals_sign, "foo=foo") #undef TEST_DO_PARSE -// For rapidcheck -void showValue(const StorePath & p, std::ostream & os) { - os << p.to_string(); -} - -} - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - auto len = *gen::inRange( - 1, - StorePath::MaxPathLen - std::string_view { HASH_PART }.size()); - - std::string pre; - pre.reserve(len); - - for (size_t c = 0; c < len; ++c) { - switch (auto i = *gen::inRange(0, 10 + 2 * 26 + 6)) { - case 0 ... 9: - pre += '0' + i; - case 10 ... 35: - pre += 'A' + (i - 10); - break; - case 36 ... 61: - pre += 'a' + (i - 36); - break; - case 62: - pre += '+'; - break; - case 63: - pre += '-'; - break; - case 64: - // names aren't permitted to start with a period, - // so just fall through to the next case here - if (c != 0) { - pre += '.'; - break; - } - case 65: - pre += '_'; - break; - case 66: - pre += '?'; - break; - case 67: - pre += '='; - break; - default: - assert(false); - } - } - - return gen::just(StorePathName { - .name = std::move(pre), - }); -} - -Gen Arbitrary::arbitrary() -{ - return gen::just(StorePath { - *gen::arbitrary(), - (*gen::arbitrary()).name, - }); -} - -} // namespace rc - -namespace nix { - #ifndef COVERAGE RC_GTEST_FIXTURE_PROP( diff --git a/src/libstore/tests/references.cc b/tests/unit/libstore/references.cc similarity index 100% rename from src/libstore/tests/references.cc rename to tests/unit/libstore/references.cc diff --git a/src/libstore/tests/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc similarity index 68% rename from src/libstore/tests/serve-protocol.cc rename to tests/unit/libstore/serve-protocol.cc index c8ac87a04..8f256d1e6 100644 --- a/src/libstore/tests/serve-protocol.cc +++ b/tests/unit/libstore/serve-protocol.cc @@ -53,15 +53,15 @@ VERSIONED_CHARACTERIZATION_TEST( (std::tuple { ContentAddress { .method = TextIngestionMethod {}, - .hash = hashString(HashType::htSHA256, "Derive(...)"), + .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -225,6 +225,131 @@ VERSIONED_CHARACTERIZATION_TEST( t; })) +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + unkeyedValidPathInfo_2_3, + "unkeyed-valid-path-info-2.3", + 2 << 8 | 3, + (std::tuple { + ({ + UnkeyedValidPathInfo info { Hash::dummy }; + info.narSize = 34878; + info; + }), + ({ + UnkeyedValidPathInfo info { Hash::dummy }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv", + }, + }; + info.narSize = 34878; + info; + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + unkeyedValidPathInfo_2_4, + "unkeyed-valid-path-info-2.4", + 2 << 8 | 4, + (std::tuple { + ({ + UnkeyedValidPathInfo info { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.references = { + StorePath { + "g1w7hyyyy1w7hy3qg1w7hy3qgqqqqy3q-foo.drv", + }, + }; + info.narSize = 34878; + info; + }), + ({ + ValidPathInfo info { + *LibStoreTest::store, + "foo", + FixedOutputInfo { + .method = FileIngestionMethod::Recursive, + .hash = hashString(HashAlgorithm::SHA256, "(...)"), + .references = { + .others = { + StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar", + }, + }, + .self = true, + }, + }, + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; + info.deriver = StorePath { + "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", + }; + info.narSize = 34878; + info.sigs = { + "fake-sig-1", + "fake-sig-2", + }, + static_cast(std::move(info)); + }), + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + build_options_2_1, + "build-options-2.1", + 2 << 8 | 1, + (ServeProto::BuildOptions { + .maxSilentTime = 5, + .buildTimeout = 6, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + build_options_2_2, + "build-options-2.2", + 2 << 8 | 2, + (ServeProto::BuildOptions { + .maxSilentTime = 5, + .buildTimeout = 6, + .maxLogSize = 7, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + build_options_2_3, + "build-options-2.3", + 2 << 8 | 3, + (ServeProto::BuildOptions { + .maxSilentTime = 5, + .buildTimeout = 6, + .maxLogSize = 7, + .nrRepeats = 8, + .enforceDeterminism = true, + })) + +VERSIONED_CHARACTERIZATION_TEST( + ServeProtoTest, + build_options_2_7, + "build-options-2.7", + 2 << 8 | 7, + (ServeProto::BuildOptions { + .maxSilentTime = 5, + .buildTimeout = 6, + .maxLogSize = 7, + .nrRepeats = 8, + .enforceDeterminism = false, + .keepFailed = true, + })) + VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, vector, @@ -271,7 +396,7 @@ VERSIONED_CHARACTERIZATION_TEST( std::optional { ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, })) diff --git a/src/libstore/tests/test-data/machines.bad_format b/tests/unit/libstore/test-data/machines.bad_format similarity index 100% rename from src/libstore/tests/test-data/machines.bad_format rename to tests/unit/libstore/test-data/machines.bad_format diff --git a/src/libstore/tests/test-data/machines.valid b/tests/unit/libstore/test-data/machines.valid similarity index 100% rename from src/libstore/tests/test-data/machines.valid rename to tests/unit/libstore/test-data/machines.valid diff --git a/src/libstore/tests/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc similarity index 98% rename from src/libstore/tests/worker-protocol.cc rename to tests/unit/libstore/worker-protocol.cc index ad5943c69..91f804f0c 100644 --- a/src/libstore/tests/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -55,15 +55,15 @@ VERSIONED_CHARACTERIZATION_TEST( (std::tuple { ContentAddress { .method = TextIngestionMethod {}, - .hash = hashString(HashType::htSHA256, "Derive(...)"), + .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -464,7 +464,7 @@ VERSIONED_CHARACTERIZATION_TEST( "foo", FixedOutputInfo { .method = FileIngestionMethod::Recursive, - .hash = hashString(HashType::htSHA256, "(...)"), + .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { .others = { StorePath { @@ -539,7 +539,7 @@ VERSIONED_CHARACTERIZATION_TEST( std::optional { ContentAddress { .method = FileIngestionMethod::Flat, - .hash = hashString(HashType::htSHA1, "blob blob..."), + .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, })) diff --git a/tests/unit/libutil-support/local.mk b/tests/unit/libutil-support/local.mk new file mode 100644 index 000000000..2ee2cdb6c --- /dev/null +++ b/tests/unit/libutil-support/local.mk @@ -0,0 +1,19 @@ +libraries += libutil-test-support + +libutil-test-support_NAME = libnixutil-test-support + +libutil-test-support_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-test-support_INSTALL_DIR := $(checklibdir) +else + libutil-test-support_INSTALL_DIR := +endif + +libutil-test-support_SOURCES := $(wildcard $(d)/tests/*.cc) + +libutil-test-support_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) + +libutil-test-support_LIBS = libutil + +libutil-test-support_LDFLAGS := -pthread -lrapidcheck diff --git a/tests/unit/libutil-support/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh new file mode 100644 index 000000000..9d6c850f0 --- /dev/null +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -0,0 +1,108 @@ +#pragma once +///@file + +#include + +#include "types.hh" +#include "environment-variables.hh" + +namespace nix { + +/** + * The path to the unit test data directory. See the contributing guide + * in the manual for further details. + */ +static Path getUnitTestData() { + return getEnv("_NIX_TEST_UNIT_DATA").value(); +} + +/** + * Whether we should update "golden masters" instead of running tests + * against them. See the contributing guide in the manual for further + * details. + */ +static bool testAccept() { + return getEnv("_NIX_TEST_ACCEPT") == "1"; +} + +/** + * Mixin class for writing characterization tests + */ +class CharacterizationTest : public virtual ::testing::Test +{ +protected: + /** + * While the "golden master" for this characterization test is + * located. It should not be shared with any other test. + */ + virtual Path goldenMaster(PathView testStem) const = 0; + +public: + /** + * Golden test for reading + * + * @param test hook that takes the contents of the file and does the + * actual work + */ + void readTest(PathView testStem, auto && test) + { + auto file = goldenMaster(testStem); + + if (testAccept()) + { + GTEST_SKIP() + << "Cannot read golden master " + << file + << "because another test is also updating it"; + } + else + { + test(readFile(file)); + } + } + + /** + * Golden test for writing + * + * @param test hook that produces contents of the file and does the + * actual work + */ + void writeTest( + PathView testStem, auto && test, auto && readFile2, auto && writeFile2) + { + auto file = goldenMaster(testStem); + + auto got = test(); + + if (testAccept()) + { + createDirs(dirOf(file)); + writeFile2(file, got); + GTEST_SKIP() + << "Updating golden master " + << file; + } + else + { + decltype(got) expected = readFile2(file); + ASSERT_EQ(got, expected); + } + } + + /** + * Specialize to `std::string` + */ + void writeTest(PathView testStem, auto && test) + { + writeTest( + testStem, test, + [](const Path & f) -> std::string { + return readFile(f); + }, + [](const Path & f, const std::string & c) { + return writeFile(f, c); + }); + } +}; + +} diff --git a/tests/unit/libutil-support/tests/hash.cc b/tests/unit/libutil-support/tests/hash.cc new file mode 100644 index 000000000..50889cd33 --- /dev/null +++ b/tests/unit/libutil-support/tests/hash.cc @@ -0,0 +1,20 @@ +#include + +#include + +#include "hash.hh" + +#include "tests/hash.hh" + +namespace rc { +using namespace nix; + +Gen Arbitrary::arbitrary() +{ + Hash hash(HashAlgorithm::SHA1); + for (size_t i = 0; i < hash.hashSize; ++i) + hash.hash[i] = *gen::arbitrary(); + return gen::just(hash); +} + +} diff --git a/src/libutil/tests/hash.hh b/tests/unit/libutil-support/tests/hash.hh similarity index 100% rename from src/libutil/tests/hash.hh rename to tests/unit/libutil-support/tests/hash.hh diff --git a/tests/unit/libutil/args.cc b/tests/unit/libutil/args.cc new file mode 100644 index 000000000..950224430 --- /dev/null +++ b/tests/unit/libutil/args.cc @@ -0,0 +1,168 @@ +#include "args.hh" +#include "fs-sink.hh" +#include + +#include +#include + +namespace nix { + + TEST(parseShebangContent, basic) { + std::list r = parseShebangContent("hi there"); + ASSERT_EQ(r.size(), 2); + auto i = r.begin(); + ASSERT_EQ(*i++, "hi"); + ASSERT_EQ(*i++, "there"); + } + + TEST(parseShebangContent, empty) { + std::list r = parseShebangContent(""); + ASSERT_EQ(r.size(), 0); + } + + TEST(parseShebangContent, doubleBacktick) { + std::list r = parseShebangContent("``\"ain't that nice\"``"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, "\"ain't that nice\""); + } + + TEST(parseShebangContent, doubleBacktickEmpty) { + std::list r = parseShebangContent("````"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, ""); + } + + TEST(parseShebangContent, doubleBacktickMarkdownInlineCode) { + std::list r = parseShebangContent("``# I'm markdown section about `coolFunction` ``"); + ASSERT_EQ(r.size(), 1); + auto i = r.begin(); + ASSERT_EQ(*i++, "# I'm markdown section about `coolFunction`"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockNaive) { + std::list r = parseShebangContent("``Example 1\n```nix\na: a\n``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n``nix\na: a\n``"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlockCorrect) { + std::list r = parseShebangContent("``Example 1\n````nix\na: a\n```` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```"); + } + + TEST(parseShebangContent, doubleBacktickMarkdownCodeBlock2) { + std::list r = parseShebangContent("``Example 1\n````nix\na: a\n````\nExample 2\n````nix\na: a\n```` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "Example 1\n```nix\na: a\n```\nExample 2\n```nix\na: a\n```"); + } + + TEST(parseShebangContent, singleBacktickInDoubleBacktickQuotes) { + std::list r = parseShebangContent("``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "`"); + } + + TEST(parseShebangContent, singleBacktickAndSpaceInDoubleBacktickQuotes) { + std::list r = parseShebangContent("``` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "` "); + } + + TEST(parseShebangContent, doubleBacktickInDoubleBacktickQuotes) { + std::list r = parseShebangContent("````` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 1); + ASSERT_EQ(*i++, "``"); + } + + TEST(parseShebangContent, increasingQuotes) { + std::list r = parseShebangContent("```` ``` `` ````` `` `````` ``"); + auto i = r.begin(); + ASSERT_EQ(r.size(), 4); + ASSERT_EQ(*i++, ""); + ASSERT_EQ(*i++, "`"); + ASSERT_EQ(*i++, "``"); + ASSERT_EQ(*i++, "```"); + } + + +#ifndef COVERAGE + +// quick and dirty +static inline std::string escape(std::string_view s_) { + + std::string_view s = s_; + std::string r = "``"; + + // make a guess to allocate ahead of time + r.reserve( + // plain chars + s.size() + // quotes + + 5 + // some "escape" backticks + + s.size() / 8); + + while (!s.empty()) { + if (s[0] == '`' && s.size() >= 2 && s[1] == '`') { + // escape it + r += "`"; + while (!s.empty() && s[0] == '`') { + r += "`"; + s = s.substr(1); + } + } else { + r += s[0]; + s = s.substr(1); + } + } + + if (!r.empty() + && ( + r[r.size() - 1] == '`' + || r[r.size() - 1] == ' ' + )) { + r += " "; + } + + r += "``"; + + return r; +}; + +RC_GTEST_PROP( + parseShebangContent, + prop_round_trip_single, + (const std::string & orig)) +{ + auto escaped = escape(orig); + // RC_LOG() << "escaped: <[[" << escaped << "]]>" << std::endl; + auto ss = parseShebangContent(escaped); + RC_ASSERT(ss.size() == 1); + RC_ASSERT(*ss.begin() == orig); +} + +RC_GTEST_PROP( + parseShebangContent, + prop_round_trip_two, + (const std::string & one, const std::string & two)) +{ + auto ss = parseShebangContent(escape(one) + " " + escape(two)); + RC_ASSERT(ss.size() == 2); + auto i = ss.begin(); + RC_ASSERT(*i++ == one); + RC_ASSERT(*i++ == two); +} + + +#endif + +} diff --git a/src/libutil/tests/canon-path.cc b/tests/unit/libutil/canon-path.cc similarity index 100% rename from src/libutil/tests/canon-path.cc rename to tests/unit/libutil/canon-path.cc diff --git a/src/libutil/tests/chunked-vector.cc b/tests/unit/libutil/chunked-vector.cc similarity index 100% rename from src/libutil/tests/chunked-vector.cc rename to tests/unit/libutil/chunked-vector.cc diff --git a/src/libutil/tests/closure.cc b/tests/unit/libutil/closure.cc similarity index 100% rename from src/libutil/tests/closure.cc rename to tests/unit/libutil/closure.cc diff --git a/src/libutil/tests/compression.cc b/tests/unit/libutil/compression.cc similarity index 100% rename from src/libutil/tests/compression.cc rename to tests/unit/libutil/compression.cc diff --git a/src/libutil/tests/config.cc b/tests/unit/libutil/config.cc similarity index 100% rename from src/libutil/tests/config.cc rename to tests/unit/libutil/config.cc diff --git a/tests/unit/libutil/data/git/check-data.sh b/tests/unit/libutil/data/git/check-data.sh new file mode 100644 index 000000000..b3f59c4f1 --- /dev/null +++ b/tests/unit/libutil/data/git/check-data.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +export TEST_ROOT=$(realpath ${TMPDIR:-/tmp}/nix-test)/git-hashing/check-data +mkdir -p $TEST_ROOT + +repo="$TEST_ROOT/scratch" +git init "$repo" + +git -C "$repo" config user.email "you@example.com" +git -C "$repo" config user.name "Your Name" + +# `-w` to write for tree test +freshlyAddedHash=$(git -C "$repo" hash-object -w -t blob --stdin < "./hello-world.bin") +encodingHash=$(sha1sum -b < "./hello-world-blob.bin" | head -c 40) + +# If the hashes match, then `hello-world-blob.bin` must be the encoding +# of `hello-world.bin`. +[[ "$encodingHash" == "$freshlyAddedHash" ]] + +# Create empty directory object for tree test +echo -n | git -C "$repo" hash-object -w -t tree --stdin + +# Relies on both child hashes already existing in the git store +freshlyAddedHash=$(git -C "$repo" mktree < "./tree.txt") +encodingHash=$(sha1sum -b < "./tree.bin" | head -c 40) + +# If the hashes match, then `tree.bin` must be the encoding of the +# directory denoted by `tree.txt` interpreted as git directory listing. +[[ "$encodingHash" == "$freshlyAddedHash" ]] diff --git a/tests/unit/libutil/data/git/hello-world-blob.bin b/tests/unit/libutil/data/git/hello-world-blob.bin new file mode 100644 index 000000000..255f5df55 Binary files /dev/null and b/tests/unit/libutil/data/git/hello-world-blob.bin differ diff --git a/tests/unit/libutil/data/git/hello-world.bin b/tests/unit/libutil/data/git/hello-world.bin new file mode 100644 index 000000000..63ddb3401 Binary files /dev/null and b/tests/unit/libutil/data/git/hello-world.bin differ diff --git a/tests/unit/libutil/data/git/tree.bin b/tests/unit/libutil/data/git/tree.bin new file mode 100644 index 000000000..5256ec140 Binary files /dev/null and b/tests/unit/libutil/data/git/tree.bin differ diff --git a/tests/unit/libutil/data/git/tree.txt b/tests/unit/libutil/data/git/tree.txt new file mode 100644 index 000000000..be3d02920 --- /dev/null +++ b/tests/unit/libutil/data/git/tree.txt @@ -0,0 +1,3 @@ +100644 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2 Foo +100755 blob 63ddb340119baf8492d2da53af47e8c7cfcd5eb2 bAr +040000 tree 4b825dc642cb6eb9a060e54bf8d69288fbee4904 baZ diff --git a/tests/unit/libutil/git.cc b/tests/unit/libutil/git.cc new file mode 100644 index 000000000..141a55816 --- /dev/null +++ b/tests/unit/libutil/git.cc @@ -0,0 +1,236 @@ +#include + +#include "git.hh" +#include "memory-source-accessor.hh" + +#include "tests/characterization.hh" + +namespace nix { + +using namespace git; + +class GitTest : public CharacterizationTest +{ + Path unitTestData = getUnitTestData() + "/git"; + +public: + + Path goldenMaster(std::string_view testStem) const override { + return unitTestData + "/" + testStem; + } + + /** + * We set these in tests rather than the regular globals so we don't have + * to worry about race conditions if the tests run concurrently. + */ + ExperimentalFeatureSettings mockXpSettings; + +private: + + void SetUp() override + { + mockXpSettings.set("experimental-features", "git-hashing"); + } +}; + +TEST(GitMode, gitMode_directory) { + Mode m = Mode::Directory; + RawMode r = 0040000; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_executable) { + Mode m = Mode::Executable; + RawMode r = 0100755; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_regular) { + Mode m = Mode::Regular; + RawMode r = 0100644; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST(GitMode, gitMode_symlink) { + Mode m = Mode::Symlink; + RawMode r = 0120000; + ASSERT_EQ(static_cast(m), r); + ASSERT_EQ(decodeMode(r), std::optional { m }); +}; + +TEST_F(GitTest, blob_read) { + readTest("hello-world-blob.bin", [&](const auto & encoded) { + StringSource in { encoded }; + StringSink out; + RegularFileSink out2 { out }; + parse(out2, "", in, [](auto &, auto) {}, mockXpSettings); + + auto expected = readFile(goldenMaster("hello-world.bin")); + + ASSERT_EQ(out.s, expected); + }); +} + +TEST_F(GitTest, blob_write) { + writeTest("hello-world-blob.bin", [&]() { + auto decoded = readFile(goldenMaster("hello-world.bin")); + StringSink s; + dumpBlobPrefix(decoded.size(), s, mockXpSettings); + s(decoded); + return s.s; + }); +} + +/** + * This data is for "shallow" tree tests. However, we use "real" hashes + * so that we can check our test data in a small shell script test test + * (`tests/unit/libutil/data/git/check-data.sh`). + */ +const static Tree tree = { + { + "Foo", + { + .mode = Mode::Regular, + // hello world with special chars from above + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1), + }, + }, + { + "bAr", + { + .mode = Mode::Executable, + // ditto + .hash = Hash::parseAny("63ddb340119baf8492d2da53af47e8c7cfcd5eb2", HashAlgorithm::SHA1), + }, + }, + { + "baZ/", + { + .mode = Mode::Directory, + // Empty directory hash + .hash = Hash::parseAny("4b825dc642cb6eb9a060e54bf8d69288fbee4904", HashAlgorithm::SHA1), + }, + }, +}; + +TEST_F(GitTest, tree_read) { + readTest("tree.bin", [&](const auto & encoded) { + StringSource in { encoded }; + NullParseSink out; + Tree got; + parse(out, "", in, [&](auto & name, auto entry) { + auto name2 = name; + if (entry.mode == Mode::Directory) + name2 += '/'; + got.insert_or_assign(name2, std::move(entry)); + }, mockXpSettings); + + ASSERT_EQ(got, tree); + }); +} + +TEST_F(GitTest, tree_write) { + writeTest("tree.bin", [&]() { + StringSink s; + dumpTree(tree, s, mockXpSettings); + return s.s; + }); +} + +TEST_F(GitTest, both_roundrip) { + using File = MemorySourceAccessor::File; + + MemorySourceAccessor files; + files.root = File::Directory { + .contents { + { + "foo", + File::Regular { + .contents = "hello\n\0\n\tworld!", + }, + }, + { + "bar", + File::Directory { + .contents = { + { + "baz", + File::Regular { + .executable = true, + .contents = "good day,\n\0\n\tworld!", + }, + }, + }, + }, + }, + }, + }; + + std::map cas; + + std::function dumpHook; + dumpHook = [&](const CanonPath & path) { + StringSink s; + HashSink hashSink { HashAlgorithm::SHA1 }; + TeeSink s2 { s, hashSink }; + auto mode = dump( + files, path, s2, dumpHook, + defaultPathFilter, mockXpSettings); + auto hash = hashSink.finish().first; + cas.insert_or_assign(hash, std::move(s.s)); + return TreeEntry { + .mode = mode, + .hash = hash, + }; + }; + + auto root = dumpHook(CanonPath::root); + + MemorySourceAccessor files2; + + MemorySink sinkFiles2 { files2 }; + + std::function mkSinkHook; + mkSinkHook = [&](const Path prefix, const Hash & hash) { + StringSource in { cas[hash] }; + parse(sinkFiles2, prefix, in, [&](const Path & name, const auto & entry) { + mkSinkHook(prefix + "/" + name, entry.hash); + }, mockXpSettings); + }; + + mkSinkHook("", root.hash); + + ASSERT_EQ(files, files2); +} + +TEST(GitLsRemote, parseSymrefLineWithReference) { + auto line = "ref: refs/head/main HEAD"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Symbolic); + ASSERT_EQ(res->target, "refs/head/main"); + ASSERT_EQ(res->reference, "HEAD"); +} + +TEST(GitLsRemote, parseSymrefLineWithNoReference) { + auto line = "ref: refs/head/main"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Symbolic); + ASSERT_EQ(res->target, "refs/head/main"); + ASSERT_EQ(res->reference, std::nullopt); +} + +TEST(GitLsRemote, parseObjectRefLine) { + auto line = "abc123 refs/head/main"; + auto res = parseLsRemoteLine(line); + ASSERT_TRUE(res.has_value()); + ASSERT_EQ(res->kind, LsRemoteRefLine::Kind::Object); + ASSERT_EQ(res->target, "abc123"); + ASSERT_EQ(res->reference, "refs/head/main"); +} + +} diff --git a/src/libutil/tests/hash.cc b/tests/unit/libutil/hash.cc similarity index 80% rename from src/libutil/tests/hash.cc rename to tests/unit/libutil/hash.cc index 9a5ebbb30..a88994d0b 100644 --- a/src/libutil/tests/hash.cc +++ b/tests/unit/libutil/hash.cc @@ -1,12 +1,8 @@ #include -#include #include -#include -#include - -#include "tests/hash.hh" +#include "hash.hh" namespace nix { @@ -17,28 +13,28 @@ namespace nix { TEST(hashString, testKnownMD5Hashes1) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s1 = ""; - auto hash = hashString(HashType::htMD5, s1); + auto hash = hashString(HashAlgorithm::MD5, s1); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:d41d8cd98f00b204e9800998ecf8427e"); } TEST(hashString, testKnownMD5Hashes2) { // values taken from: https://tools.ietf.org/html/rfc1321 auto s2 = "abc"; - auto hash = hashString(HashType::htMD5, s2); + auto hash = hashString(HashAlgorithm::MD5, s2); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "md5:900150983cd24fb0d6963f7d28e17f72"); } TEST(hashString, testKnownSHA1Hashes1) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abc"; - auto hash = hashString(HashType::htSHA1, s); + auto hash = hashString(HashAlgorithm::SHA1, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:a9993e364706816aba3e25717850c26c9cd0d89d"); } TEST(hashString, testKnownSHA1Hashes2) { // values taken from: https://tools.ietf.org/html/rfc3174 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; - auto hash = hashString(HashType::htSHA1, s); + auto hash = hashString(HashAlgorithm::SHA1, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true),"sha1:84983e441c3bd26ebaae4aa1f95129e5e54670f1"); } @@ -46,7 +42,7 @@ namespace nix { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; - auto hash = hashString(HashType::htSHA256, s); + auto hash = hashString(HashAlgorithm::SHA256, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); } @@ -54,7 +50,7 @@ namespace nix { TEST(hashString, testKnownSHA256Hashes2) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; - auto hash = hashString(HashType::htSHA256, s); + auto hash = hashString(HashAlgorithm::SHA256, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha256:248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1"); } @@ -62,18 +58,17 @@ namespace nix { TEST(hashString, testKnownSHA512Hashes1) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abc"; - auto hash = hashString(HashType::htSHA512, s); + auto hash = hashString(HashAlgorithm::SHA512, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:ddaf35a193617abacc417349ae20413112e6fa4e89a9" "7ea20a9eeee64b55d39a2192992a274fc1a836ba3c23a3feebbd" "454d4423643ce80e2a9ac94fa54ca49f"); } - TEST(hashString, testKnownSHA512Hashes2) { // values taken from: https://tools.ietf.org/html/rfc4634 auto s = "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"; - auto hash = hashString(HashType::htSHA512, s); + auto hash = hashString(HashAlgorithm::SHA512, s); ASSERT_EQ(hash.to_string(HashFormat::Base16, true), "sha512:8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa1" "7299aeadb6889018501d289e4900f7e4331b99dec4b5433a" @@ -85,7 +80,7 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(hashFormat, testRoundTripPrintParse) { - for (const HashFormat hashFormat: { HashFormat::Base64, HashFormat::Base32, HashFormat::Base16, HashFormat::SRI}) { + for (const HashFormat hashFormat: { HashFormat::Base64, HashFormat::Nix32, HashFormat::Base16, HashFormat::SRI}) { ASSERT_EQ(parseHashFormat(printHashFormat(hashFormat)), hashFormat); ASSERT_EQ(*parseHashFormatOpt(printHashFormat(hashFormat)), hashFormat); } @@ -95,16 +90,3 @@ namespace nix { ASSERT_EQ(parseHashFormatOpt("sha0042"), std::nullopt); } } - -namespace rc { -using namespace nix; - -Gen Arbitrary::arbitrary() -{ - Hash hash(htSHA1); - for (size_t i = 0; i < hash.hashSize; ++i) - hash.hash[i] = *gen::arbitrary(); - return gen::just(hash); -} - -} diff --git a/src/libutil/tests/hilite.cc b/tests/unit/libutil/hilite.cc similarity index 100% rename from src/libutil/tests/hilite.cc rename to tests/unit/libutil/hilite.cc diff --git a/tests/unit/libutil/json-utils.cc b/tests/unit/libutil/json-utils.cc new file mode 100644 index 000000000..f0ce15c93 --- /dev/null +++ b/tests/unit/libutil/json-utils.cc @@ -0,0 +1,58 @@ +#include +#include + +#include + +#include "json-utils.hh" + +namespace nix { + +/* Test `to_json` and `from_json` with `std::optional` types. + * We are specifically interested in whether we can _nest_ optionals in STL + * containers so we that we can leverage existing adl_serializer templates. */ + +TEST(to_json, optionalInt) { + std::optional val = std::make_optional(420); + ASSERT_EQ(nlohmann::json(val), nlohmann::json(420)); + val = std::nullopt; + ASSERT_EQ(nlohmann::json(val), nlohmann::json(nullptr)); +} + +TEST(to_json, vectorOfOptionalInts) { + std::vector> vals = { + std::make_optional(420), + std::nullopt, + }; + ASSERT_EQ(nlohmann::json(vals), nlohmann::json::parse("[420,null]")); +} + +TEST(to_json, optionalVectorOfInts) { + std::optional> val = std::make_optional(std::vector { + -420, + 420, + }); + ASSERT_EQ(nlohmann::json(val), nlohmann::json::parse("[-420,420]")); + val = std::nullopt; + ASSERT_EQ(nlohmann::json(val), nlohmann::json(nullptr)); +} + +TEST(from_json, optionalInt) { + nlohmann::json json = 420; + std::optional val = json; + ASSERT_TRUE(val.has_value()); + ASSERT_EQ(*val, 420); + json = nullptr; + json.get_to(val); + ASSERT_FALSE(val.has_value()); +} + +TEST(from_json, vectorOfOptionalInts) { + nlohmann::json json = { 420, nullptr }; + std::vector> vals = json; + ASSERT_EQ(vals.size(), 2); + ASSERT_TRUE(vals.at(0).has_value()); + ASSERT_EQ(*vals.at(0), 420); + ASSERT_FALSE(vals.at(1).has_value()); +} + +} /* namespace nix */ diff --git a/tests/unit/libutil/local.mk b/tests/unit/libutil/local.mk new file mode 100644 index 000000000..930efb90b --- /dev/null +++ b/tests/unit/libutil/local.mk @@ -0,0 +1,31 @@ +check: libutil-tests_RUN + +programs += libutil-tests + +libutil-tests_NAME = libnixutil-tests + +libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libutil-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libutil-tests_INSTALL_DIR := $(checkbindir) +else + libutil-tests_INSTALL_DIR := +endif + +libutil-tests_SOURCES := $(wildcard $(d)/*.cc) + +libutil-tests_EXTRA_INCLUDES = \ + -I tests/unit/libutil-support \ + -I src/libutil + +libutil-tests_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) + +libutil-tests_LIBS = libutil-test-support libutil + +libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) + +check: $(d)/data/git/check-data.sh.test + +$(eval $(call run-test,$(d)/data/git/check-data.sh)) diff --git a/src/libutil/tests/logging.cc b/tests/unit/libutil/logging.cc similarity index 99% rename from src/libutil/tests/logging.cc rename to tests/unit/libutil/logging.cc index 2ffdc2e9b..c6dfe63d3 100644 --- a/src/libutil/tests/logging.cc +++ b/tests/unit/libutil/logging.cc @@ -2,7 +2,6 @@ #include "logging.hh" #include "nixexpr.hh" -#include "util.hh" #include #include diff --git a/src/libutil/tests/lru-cache.cc b/tests/unit/libutil/lru-cache.cc similarity index 100% rename from src/libutil/tests/lru-cache.cc rename to tests/unit/libutil/lru-cache.cc diff --git a/src/libutil/tests/pool.cc b/tests/unit/libutil/pool.cc similarity index 100% rename from src/libutil/tests/pool.cc rename to tests/unit/libutil/pool.cc diff --git a/src/libutil/tests/references.cc b/tests/unit/libutil/references.cc similarity index 100% rename from src/libutil/tests/references.cc rename to tests/unit/libutil/references.cc diff --git a/src/libutil/tests/suggestions.cc b/tests/unit/libutil/suggestions.cc similarity index 100% rename from src/libutil/tests/suggestions.cc rename to tests/unit/libutil/suggestions.cc diff --git a/src/libutil/tests/tests.cc b/tests/unit/libutil/tests.cc similarity index 99% rename from src/libutil/tests/tests.cc rename to tests/unit/libutil/tests.cc index f3c1e8248..568f03f70 100644 --- a/src/libutil/tests/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -1,5 +1,8 @@ #include "util.hh" #include "types.hh" +#include "file-system.hh" +#include "processes.hh" +#include "terminal.hh" #include #include diff --git a/src/libutil/tests/url.cc b/tests/unit/libutil/url.cc similarity index 100% rename from src/libutil/tests/url.cc rename to tests/unit/libutil/url.cc diff --git a/src/libutil/tests/xml-writer.cc b/tests/unit/libutil/xml-writer.cc similarity index 100% rename from src/libutil/tests/xml-writer.cc rename to tests/unit/libutil/xml-writer.cc