diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md index be3f6af97..cf663e28d 100644 --- a/.github/ISSUE_TEMPLATE/missing_documentation.md +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -23,7 +23,7 @@ assignees: '' - [ ] checked [open documentation issues and pull requests] for possible duplicates [latest Nix manual]: https://nixos.org/manual/nix/unstable/ -[source]: https://github.com/NixOS/nix/tree/master/doc/manual/src +[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source [open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation ## Priorities diff --git a/.github/labeler.yml b/.github/labeler.yml index 0e6fd3e26..97ca9284d 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,7 +1,7 @@ "c api": - changed-files: - any-glob-to-any-file: "src/lib*-c/**/*" - - any-glob-to-any-file: "test/unit/**/nix_api_*" + - any-glob-to-any-file: "src/*test*/**/nix_api_*" - any-glob-to-any-file: "doc/external-api/**/*" "contributor-experience": @@ -9,7 +9,7 @@ - any-glob-to-any-file: "CONTRIBUTING.md" - any-glob-to-any-file: ".github/ISSUE_TEMPLATE/*" - any-glob-to-any-file: ".github/PULL_REQUEST_TEMPLATE.md" - - any-glob-to-any-file: "doc/manual/src/contributing/**" + - any-glob-to-any-file: "doc/manual/source/contributing/**" "documentation": - changed-files: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99e50d198..0e2e07da2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v29 + - uses: cachix/install-nix-action@v30 with: # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" @@ -89,7 +89,7 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v29 + - uses: cachix/install-nix-action@v30 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install - uses: cachix/cachix-action@v15 @@ -112,7 +112,7 @@ jobs: steps: - uses: actions/checkout@v4 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v29 + - uses: cachix/install-nix-action@v30 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -142,7 +142,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v29 + - uses: cachix/install-nix-action@v30 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV diff --git a/.gitignore b/.gitignore index a17b627f4..11a80ab5b 100644 --- a/.gitignore +++ b/.gitignore @@ -23,17 +23,17 @@ perl/Makefile.config /doc/manual/conf-file.json /doc/manual/language.json /doc/manual/xp-features.json -/doc/manual/src/SUMMARY.md -/doc/manual/src/SUMMARY-rl-next.md -/doc/manual/src/store/types/* -!/doc/manual/src/store/types/index.md.in -/doc/manual/src/command-ref/new-cli -/doc/manual/src/command-ref/conf-file.md -/doc/manual/src/command-ref/experimental-features-shortlist.md -/doc/manual/src/contributing/experimental-feature-descriptions.md -/doc/manual/src/language/builtins.md -/doc/manual/src/language/builtin-constants.md -/doc/manual/src/release-notes/rl-next.md +/doc/manual/source/SUMMARY.md +/doc/manual/source/SUMMARY-rl-next.md +/doc/manual/source/store/types/* +!/doc/manual/source/store/types/index.md.in +/doc/manual/source/command-ref/new-cli +/doc/manual/source/command-ref/conf-file.md +/doc/manual/source/command-ref/experimental-features-shortlist.md +/doc/manual/source/contributing/experimental-feature-descriptions.md +/doc/manual/source/language/builtins.md +/doc/manual/source/language/builtin-constants.md +/doc/manual/source/release-notes/rl-next.md # /scripts/ /scripts/nix-profile.sh @@ -49,22 +49,22 @@ perl/Makefile.config /src/libexpr/parser-tab.output /src/libexpr/nix.tbl /src/libexpr/tests -/tests/unit/libexpr/libnixexpr-tests +/src/libexpr-tests/libnixexpr-tests # /src/libfetchers -/tests/unit/libfetchers/libnixfetchers-tests +/src/libfetchers-tests/libnixfetchers-tests # /src/libflake -/tests/unit/libflake/libnixflake-tests +/src/libflake-tests/libnixflake-tests # /src/libstore/ *.gen.* /src/libstore/tests -/tests/unit/libstore/libnixstore-tests +/src/libstore-tests/libnixstore-tests # /src/libutil/ /src/libutil/tests -/tests/unit/libutil/libnixutil-tests +/src/libutil-tests/libnixutil-tests /src/nix/nix diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 56508df34..ad8678962 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,7 +79,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). - Functional tests – [`tests/functional/**.sh`](./tests/functional) - Unit tests – [`src/*/tests`](./src/) - Integration tests – [`tests/nixos/*`](./tests/nixos) - - [ ] User documentation in the [manual](./doc/manual/src) + - [ ] User documentation in the [manual](./doc/manual/source) - [ ] API documentation in header files - [ ] Code and comments are self-explanatory - [ ] Commit message explains **why** the change was made @@ -90,7 +90,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). ## Making changes to the Nix manual The Nix reference manual is hosted on https://nixos.org/manual/nix. -The underlying source files are located in [`doc/manual/src`](./doc/manual/src). +The underlying source files are located in [`doc/manual/source`](./doc/manual/source). For small changes you can [use GitHub to edit these files](https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files) For larger changes see the [Nix reference manual](https://nix.dev/manual/nix/development/development/contributing.html). diff --git a/HACKING.md b/HACKING.md index d3576d60d..02971da3e 120000 --- a/HACKING.md +++ b/HACKING.md @@ -1 +1 @@ -doc/manual/src/development/building.md \ No newline at end of file +doc/manual/source/development/building.md \ No newline at end of file diff --git a/Makefile b/Makefile index dbf510a3e..4d70be0e8 100644 --- a/Makefile +++ b/Makefile @@ -40,14 +40,14 @@ endif ifeq ($(ENABLE_UNIT_TESTS), yes) makefiles += \ - tests/unit/libutil/local.mk \ - tests/unit/libutil-support/local.mk \ - tests/unit/libstore/local.mk \ - tests/unit/libstore-support/local.mk \ - tests/unit/libfetchers/local.mk \ - tests/unit/libexpr/local.mk \ - tests/unit/libexpr-support/local.mk \ - tests/unit/libflake/local.mk + src/libutil-tests/local.mk \ + src/libutil-test-support/local.mk \ + src/libstore-tests/local.mk \ + src/libstore-test-support/local.mk \ + src/libfetchers-tests/local.mk \ + src/libexpr-tests/local.mk \ + src/libexpr-test-support/local.mk \ + src/libflake-tests/local.mk endif ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes) diff --git a/configure.ac b/configure.ac index 198198dea..4df5c80f0 100644 --- a/configure.ac +++ b/configure.ac @@ -62,12 +62,16 @@ AC_CHECK_TOOL([AR], [ar]) AC_SYS_LARGEFILE -# Solaris-specific stuff. +# OS-specific stuff. case "$host_os" in solaris*) # Solaris requires -lsocket -lnsl for network functions LDFLAGS="-lsocket -lnsl $LDFLAGS" ;; + darwin*) + # Need to link to libsandbox. + LDFLAGS="-lsandbox $LDFLAGS" + ;; esac @@ -89,9 +93,10 @@ AC_LANG_POP(C++) AC_CHECK_FUNCS([statvfs pipe2 close_range]) -# Check for lutimes, optionally used for changing the mtime of -# symlinks. -AC_CHECK_FUNCS([lutimes]) +# Check for lutimes and utimensat, optionally used for changing the +# mtime of symlinks. +AC_CHECK_DECLS([AT_SYMLINK_NOFOLLOW], [], [], [[#include ]]) +AC_CHECK_FUNCS([lutimes utimensat]) # Check whether the store optimiser can optimise symlinks. @@ -170,6 +175,10 @@ AS_IF( [test "$ENABLE_FUNCTIONAL_TESTS" == "yes" || test "$ENABLE_DOC_GEN" == "yes"], [NEED_PROG(jq, jq)]) +AS_IF( + [test "$ENABLE_DOC_GEN" == "yes"], + [NEED_PROG(man, man)]) + AS_IF([test "$ENABLE_BUILD" == "yes"],[ # Look for boost, a required dependency. diff --git a/doc/manual/.version b/doc/manual/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/doc/manual/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/doc/manual/book.toml b/doc/manual/book.toml index 73fb7e75e..213739174 100644 --- a/doc/manual/book.toml +++ b/doc/manual/book.toml @@ -1,5 +1,6 @@ [book] title = "Nix Reference Manual" +src = "source" [output.html] additional-css = ["custom.css"] @@ -7,9 +8,21 @@ additional-js = ["redirects.js"] edit-url-template = "https://github.com/NixOS/nix/tree/master/doc/manual/{path}" git-repository-url = "https://github.com/NixOS/nix" +# Handles replacing @docroot@ with a path to ./source relative to that markdown file, +# {{#include handlebars}}, and the @generated@ syntax used within these. it mostly +# but not entirely replaces the links preprocessor (which we cannot simply use due +# to @generated@ files living in a different directory to make meson happy). we do +# not want to disable the links preprocessor entirely though because that requires +# disabling *all* built-in preprocessors and selectively reenabling those we want. +[preprocessor.substitute] +command = "python3 ./substitute.py" +before = ["anchors", "links"] + [preprocessor.anchors] renderers = ["html"] -command = "jq --from-file doc/manual/anchors.jq" +command = "jq --from-file ./anchors.jq" + +[output.markdown] [output.linkcheck] # no Internet during the build (in the sandbox) diff --git a/doc/manual/generate-deps.py b/doc/manual/generate-deps.py new file mode 100755 index 000000000..297bd3939 --- /dev/null +++ b/doc/manual/generate-deps.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 + +import glob +import sys + +# meson expects makefile-style dependency declarations, i.e. +# +# target: dependency... +# +# meson seems to pass depfiles straight on to ninja even though +# it also parses the file itself (or at least has code to do so +# in its tree), so we must live by ninja's rules: only slashes, +# spaces and octothorpes can be escaped, anything else is taken +# literally. since the rules for these aren't even the same for +# all three we will just fail when we encounter any of them (if +# asserts are off for some reason the depfile will likely point +# to nonexistant paths, making everything phony and thus fine.) +for path in glob.glob(sys.argv[1] + '/**', recursive=True): + assert '\\' not in path + assert ' ' not in path + assert '#' not in path + print("ignored:", path) diff --git a/doc/manual/generate-store-types.nix b/doc/manual/generate-store-types.nix index 3b78a0e1b..46179abc5 100644 --- a/doc/manual/generate-store-types.nix +++ b/doc/manual/generate-store-types.nix @@ -21,7 +21,7 @@ let "index.md" = replaceStrings [ "@store-types@" ] [ index ] - (readFile ./src/store/types/index.md.in); + (readFile ./source/store/types/index.md.in); tableOfContents = let diff --git a/doc/manual/local.mk b/doc/manual/local.mk index fcc50f460..36cccc506 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -4,8 +4,8 @@ doc_nix = $(nix_PATH) MANUAL_SRCS := \ - $(call rwildcard, $(d)/src, *.md) \ - $(call rwildcard, $(d)/src, */*.md) + $(call rwildcard, $(d)/source, *.md) \ + $(call rwildcard, $(d)/source, */*.md) man-pages := $(foreach n, \ nix-env.1 nix-store.1 \ @@ -18,11 +18,11 @@ man-pages := $(foreach n, \ , $(d)/$(n)) # man pages for subcommands -# convert from `$(d)/src/command-ref/nix-{1}/{2}.md` to `$(d)/nix-{1}-{2}.1` +# convert from `$(d)/source/command-ref/nix-{1}/{2}.md` to `$(d)/nix-{1}-{2}.1` # FIXME: unify with how nix3-cli man pages are generated man-pages += $(foreach subcommand, \ - $(filter-out %opt-common.md %env-common.md, $(wildcard $(d)/src/command-ref/nix-*/*.md)), \ - $(d)/$(subst /,-,$(subst $(d)/src/command-ref/,,$(subst .md,.1,$(subcommand))))) + $(filter-out %opt-common.md %env-common.md, $(wildcard $(d)/source/command-ref/nix-*/*.md)), \ + $(d)/$(subst /,-,$(subst $(d)/source/command-ref/,,$(subst .md,.1,$(subcommand))))) clean-files += $(d)/*.1 $(d)/*.5 $(d)/*.8 @@ -49,11 +49,11 @@ define process-includes done < <(grep '{{#include' $(1)) endef -$(d)/nix-env-%.1: $(d)/src/command-ref/nix-env/%.md +$(d)/nix-env-%.1: $(d)/source/command-ref/nix-env/%.md @printf "Title: %s\n\n" "$(subst nix-env-,nix-env --,$$(basename "$@" .1))" > $^.tmp $(render-subcommand) -$(d)/nix-store-%.1: $(d)/src/command-ref/nix-store/%.md +$(d)/nix-store-%.1: $(d)/source/command-ref/nix-store/%.md @printf -- 'Title: %s\n\n' "$(subst nix-store-,nix-store --,$$(basename "$@" .1))" > $^.tmp $(render-subcommand) @@ -69,50 +69,50 @@ define render-subcommand endef -$(d)/%.1: $(d)/src/command-ref/%.md +$(d)/%.1: $(d)/source/command-ref/%.md @printf "Title: %s\n\n" "$$(basename $@ .1)" > $^.tmp @cat $^ >> $^.tmp @$(call process-includes,$^,$^.tmp) $(trace-gen) lowdown -sT man --nroff-nolinks -M section=1 $^.tmp -o $@ @rm $^.tmp -$(d)/%.8: $(d)/src/command-ref/%.md +$(d)/%.8: $(d)/source/command-ref/%.md @printf "Title: %s\n\n" "$$(basename $@ .8)" > $^.tmp @cat $^ >> $^.tmp $(trace-gen) lowdown -sT man --nroff-nolinks -M section=8 $^.tmp -o $@ @rm $^.tmp -$(d)/nix.conf.5: $(d)/src/command-ref/conf-file.md +$(d)/nix.conf.5: $(d)/source/command-ref/conf-file.md @printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp @cat $^ >> $^.tmp @$(call process-includes,$^,$^.tmp) $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md +$(d)/nix-profiles.5: $(d)/source/command-ref/files/profiles.md @printf "Title: %s\n\n" "$$(basename $@ .5)" > $^.tmp @cat $^ >> $^.tmp $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md +$(d)/source/SUMMARY.md: $(d)/source/SUMMARY.md.in $(d)/source/SUMMARY-rl-next.md $(d)/source/store/types $(d)/source/command-ref/new-cli $(d)/source/development/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) -$(d)/src/store/types: $(d)/nix.json $(d)/utils.nix $(d)/generate-store-info.nix $(d)/generate-store-types.nix $(d)/src/store/types/index.md.in $(doc_nix) +$(d)/source/store/types: $(d)/nix.json $(d)/utils.nix $(d)/generate-store-info.nix $(d)/generate-store-types.nix $(d)/source/store/types/index.md.in $(doc_nix) @# FIXME: build out of tree! @rm -rf $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-store-types.nix (builtins.fromJSON (builtins.readFile $<)).stores' @# do not destroy existing contents @mv $@.tmp/* $@/ -$(d)/src/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix) +$(d)/source/command-ref/new-cli: $(d)/nix.json $(d)/utils.nix $(d)/generate-manpage.nix $(d)/generate-settings.nix $(d)/generate-store-info.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-manpage.nix true (builtins.readFile $<)' @mv $@.tmp $@ -$(d)/src/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/src/command-ref/conf-file-prefix.md $(d)/src/command-ref/experimental-features-shortlist.md $(doc_nix) - @cat doc/manual/src/command-ref/conf-file-prefix.md > $@.tmp +$(d)/source/command-ref/conf-file.md: $(d)/conf-file.json $(d)/utils.nix $(d)/generate-settings.nix $(d)/source/command-ref/conf-file-prefix.md $(d)/source/command-ref/experimental-features-shortlist.md $(doc_nix) + @cat doc/manual/source/command-ref/conf-file-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-settings.nix { prefix = "conf"; } (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @mv $@.tmp $@ @@ -124,12 +124,12 @@ $(d)/conf-file.json: $(doc_nix) $(trace-gen) $(dummy-env) $(doc_nix) config show --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/development/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) +$(d)/source/development/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ -$(d)/src/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix) +$(d)/source/command-ref/experimental-features-shortlist.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features-shortlist.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features-shortlist.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ @@ -138,10 +138,10 @@ $(d)/xp-features.json: $(doc_nix) $(trace-gen) $(dummy-env) $(doc_nix) __dump-xp-features > $@.tmp @mv $@.tmp $@ -$(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(doc_nix) - @cat doc/manual/src/language/builtins-prefix.md > $@.tmp +$(d)/source/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/source/language/builtins-prefix.md $(doc_nix) + @cat doc/manual/source/language/builtins-prefix.md > $@.tmp $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; - @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp + @cat doc/manual/source/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ $(d)/language.json: $(doc_nix) @@ -149,7 +149,7 @@ $(d)/language.json: $(doc_nix) @mv $@.tmp $@ # Generate "Upcoming release" notes (or clear it and remove from menu) -$(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/* +$(d)/source/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/* @if type -p changelog-d > /dev/null; then \ echo " GEN " $@; \ changelog-d doc/manual/rl-next > $@; \ @@ -158,7 +158,7 @@ $(d)/src/release-notes/rl-next.md: $(d)/rl-next $(d)/rl-next/* true > $@; \ fi -$(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md +$(d)/source/SUMMARY-rl-next.md: $(d)/source/release-notes/rl-next.md $(trace-gen) true @if [ -s $< ]; then \ echo ' - [Upcoming release](release-notes/rl-next.md)' > $@; \ @@ -194,9 +194,9 @@ $(mandir)/man1/nix3-manpages: doc/manual/generated/man1/nix3-manpages @mkdir -p $(DESTDIR)$$(dirname $@) $(trace-install) install -m 0644 $$(dirname $<)/* $(DESTDIR)$$(dirname $@) -doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli +doc/manual/generated/man1/nix3-manpages: $(d)/source/command-ref/new-cli @mkdir -p $(DESTDIR)$$(dirname $@) - $(trace-gen) for i in doc/manual/src/command-ref/new-cli/*.md; do \ + $(trace-gen) for i in doc/manual/source/command-ref/new-cli/*.md; do \ name=$$(basename $$i .md); \ tmpFile=$$(mktemp); \ if [[ $$name = SUMMARY ]]; then continue; fi; \ @@ -211,7 +211,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli # `@docroot@` is to be preserved for documenting the mechanism # FIXME: maybe contributing guides should live right next to the code # instead of in the manual -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/source/SUMMARY.md $(d)/source/store/types $(d)/source/command-ref/new-cli $(d)/source/development/experimental-feature-descriptions.md $(d)/source/command-ref/conf-file.md $(d)/source/language/builtins.md $(d)/source/release-notes/rl-next.md $(d)/source/figures $(d)/source/favicon.png $(d)/source/favicon.svg $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ @@ -219,12 +219,17 @@ $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/ $(call process-includes,$$file,$$file); \ done; \ find "$$tmp" -name '*.md' ! -name 'documentation.md' | while read -r file; do \ - docroot="$$(realpath --relative-to="$$(dirname "$$file")" $$tmp/manual/src)"; \ + docroot="$$(realpath --relative-to="$$(dirname "$$file")" $$tmp/manual/source)"; \ sed -i "s,@docroot@,$$docroot,g" "$$file"; \ done; \ set -euo pipefail; \ - RUST_LOG=warn mdbook build "$$tmp/manual" -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \ - | { grep -Fv "because fragment resolution isn't implemented" || :; }; \ + ( \ + cd "$$tmp/manual"; \ + RUST_LOG=warn \ + MDBOOK_SUBSTITUTE_SEARCH=$(d)/source \ + mdbook build -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \ + | { grep -Fv "because fragment resolution isn't implemented" || :; } \ + ); \ rm -rf "$$tmp/manual" @rm -rf $(DESTDIR)$(docdir)/manual @mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual diff --git a/doc/manual/meson.build b/doc/manual/meson.build new file mode 100644 index 000000000..3630e2dc8 --- /dev/null +++ b/doc/manual/meson.build @@ -0,0 +1,353 @@ +project('nix-manual', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +nix = find_program('nix', native : true) + +mdbook = find_program('mdbook', native : true) +bash = find_program('bash', native : true) + +pymod = import('python') +python = pymod.find_installation('python3') + +nix_env_for_docs = { + 'HOME': '/dummy', + 'NIX_CONF_DIR': '/dummy', + 'NIX_SSL_CERT_FILE': '/dummy/no-ca-bundle.crt', + 'NIX_STATE_DIR': '/dummy', + 'NIX_CONFIG': 'cores = 0', +} + +nix_for_docs = [nix, '--experimental-features', 'nix-command'] +nix_eval_for_docs_common = nix_for_docs + [ + 'eval', + '-I', 'nix=' + meson.current_source_dir(), + '--store', 'dummy://', + '--impure', +] +nix_eval_for_docs = nix_eval_for_docs_common + '--raw' + +conf_file_json = custom_target( + command : nix_for_docs + ['config', 'show', '--json'], + capture : true, + output : 'conf-file.json', + env : nix_env_for_docs, +) + +language_json = custom_target( + command: [nix, '__dump-language'], + output : 'language.json', + capture : true, + env : nix_env_for_docs, +) + +nix3_cli_json = custom_target( + command : [nix, '__dump-cli'], + capture : true, + output : 'nix.json', + env : nix_env_for_docs, +) + +generate_manual_deps = files( + 'generate-deps.py', +) + +# Generates types +subdir('source/store') +# Generates builtins.md and builtin-constants.md. +subdir('source/language') +# Generates new-cli pages, experimental-features-shortlist.md, and conf-file.md. +subdir('source/command-ref') +# Generates experimental-feature-descriptions.md. +subdir('source/development') +# Generates rl-next-generated.md. +subdir('source/release-notes') +subdir('source') + +# Hacky way to figure out if `nix` is an `ExternalProgram` or +# `Exectuable`. Only the latter can occur in custom target input lists. +if nix.full_path().startswith(meson.build_root()) + nix_input = nix +else + nix_input = [] +endif + +manual = custom_target( + 'manual', + command : [ + bash, + '-euo', 'pipefail', + '-c', + ''' + @0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@ + @0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/source/SUMMARY.md.in > @2@/source/SUMMARY.md + rsync -r --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/ + (cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3 + rm -rf @2@/manual + mv @2@/html @2@/manual + find @2@/manual -iname meson.build -delete + '''.format( + python.full_path(), + mdbook.full_path(), + meson.current_build_dir(), + ), + ], + input : [ + generate_manual_deps, + 'substitute.py', + 'book.toml', + 'anchors.jq', + 'custom.css', + nix3_cli_files, + experimental_features_shortlist_md, + experimental_feature_descriptions_md, + types_dir, + conf_file_md, + builtins_md, + rl_next_generated, + summary_rl_next, + nix_input, + ], + output : [ + 'manual', + 'markdown', + ], + depfile : 'manual.d', + env : { + 'RUST_LOG': 'info', + 'MDBOOK_SUBSTITUTE_SEARCH': meson.current_build_dir() / 'source', + }, +) +manual_html = manual[0] +manual_md = manual[1] + +install_subdir( + manual_html.full_path(), + install_dir : get_option('datadir') / 'doc/nix', +) + +nix_nested_manpages = [ + [ 'nix-env', + [ + 'delete-generations', + 'install', + 'list-generations', + 'query', + 'rollback', + 'set-flag', + 'set', + 'switch-generation', + 'switch-profile', + 'uninstall', + 'upgrade', + ], + ], + [ 'nix-store', + [ + 'add-fixed', + 'add', + 'delete', + 'dump-db', + 'dump', + 'export', + 'gc', + 'generate-binary-cache-key', + 'import', + 'load-db', + 'optimise', + 'print-env', + 'query', + 'read-log', + 'realise', + 'repair-path', + 'restore', + 'serve', + 'verify', + 'verify-path', + ], + ], +] + +foreach command : nix_nested_manpages + foreach page : command[1] + title = command[0] + ' --' + page + section = '1' + custom_target( + command : [ + bash, + files('./render-manpage.sh'), + '--out-no-smarty', + title, + section, + '@INPUT0@/command-ref' / command[0] / (page + '.md'), + '@OUTPUT0@', + ], + input : [ + manual_md, + nix_input, + ], + output : command[0] + '-' + page + '.1', + install : true, + install_dir : get_option('mandir') / 'man1', + ) + endforeach +endforeach + +nix3_manpages = [ + 'nix3-build', + 'nix3-bundle', + 'nix3-config', + 'nix3-config-show', + 'nix3-copy', + 'nix3-daemon', + 'nix3-derivation-add', + 'nix3-derivation', + 'nix3-derivation-show', + 'nix3-develop', + #'nix3-doctor', + 'nix3-edit', + 'nix3-eval', + 'nix3-flake-archive', + 'nix3-flake-check', + 'nix3-flake-clone', + 'nix3-flake-info', + 'nix3-flake-init', + 'nix3-flake-lock', + 'nix3-flake', + 'nix3-flake-metadata', + 'nix3-flake-new', + 'nix3-flake-prefetch', + 'nix3-flake-show', + 'nix3-flake-update', + 'nix3-fmt', + 'nix3-hash-file', + 'nix3-hash', + 'nix3-hash-path', + 'nix3-hash-to-base16', + 'nix3-hash-to-base32', + 'nix3-hash-to-base64', + 'nix3-hash-to-sri', + 'nix3-help', + 'nix3-help-stores', + 'nix3-key-convert-secret-to-public', + 'nix3-key-generate-secret', + 'nix3-key', + 'nix3-log', + 'nix3-nar-cat', + 'nix3-nar-dump-path', + 'nix3-nar-ls', + 'nix3-nar', + 'nix3-path-info', + 'nix3-print-dev-env', + 'nix3-profile-diff-closures', + 'nix3-profile-history', + 'nix3-profile-install', + 'nix3-profile-list', + 'nix3-profile', + 'nix3-profile-remove', + 'nix3-profile-rollback', + 'nix3-profile-upgrade', + 'nix3-profile-wipe-history', + 'nix3-realisation-info', + 'nix3-realisation', + 'nix3-registry-add', + 'nix3-registry-list', + 'nix3-registry', + 'nix3-registry-pin', + 'nix3-registry-remove', + 'nix3-repl', + 'nix3-run', + 'nix3-search', + #'nix3-shell', + 'nix3-store-add-file', + 'nix3-store-add-path', + 'nix3-store-cat', + 'nix3-store-copy-log', + 'nix3-store-copy-sigs', + 'nix3-store-delete', + 'nix3-store-diff-closures', + 'nix3-store-dump-path', + 'nix3-store-gc', + 'nix3-store-ls', + 'nix3-store-make-content-addressed', + 'nix3-store', + 'nix3-store-optimise', + 'nix3-store-path-from-hash-part', + 'nix3-store-ping', + 'nix3-store-prefetch-file', + 'nix3-store-repair', + 'nix3-store-sign', + 'nix3-store-verify', + 'nix3-upgrade-nix', + 'nix3-why-depends', + 'nix', +] + +foreach page : nix3_manpages + section = '1' + custom_target( + command : [ + bash, + '@INPUT0@', + page, + section, + '@INPUT1@/command-ref/new-cli/@0@.md'.format(page), + '@OUTPUT@', + ], + input : [ + files('./render-manpage.sh'), + manual_md, + nix_input, + ], + output : page + '.1', + install : true, + install_dir : get_option('mandir') / 'man1', + ) +endforeach + +nix_manpages = [ + [ 'nix-env', 1 ], + [ 'nix-store', 1 ], + [ 'nix-build', 1 ], + [ 'nix-shell', 1 ], + [ 'nix-instantiate', 1 ], + [ 'nix-collect-garbage', 1 ], + [ 'nix-prefetch-url', 1 ], + [ 'nix-channel', 1 ], + [ 'nix-hash', 1 ], + [ 'nix-copy-closure', 1 ], + [ 'nix.conf', 5, conf_file_md.full_path() ], + [ 'nix-daemon', 8 ], + [ 'nix-profiles', 5, 'files/profiles.md' ], +] + +foreach entry : nix_manpages + title = entry[0] + # nix.conf.5 and nix-profiles.5 are based off of conf-file.md and files/profiles.md, + # rather than a stem identical to its mdbook source. + # Therefore we use an optional third element of this array to override the name pattern + md_file = entry.get(2, title + '.md') + section = entry[1].to_string() + md_file_resolved = join_paths('@INPUT1@/command-ref/', md_file) + custom_target( + command : [ + bash, + '@INPUT0@', + title, + section, + md_file_resolved, + '@OUTPUT@', + ], + input : [ + files('./render-manpage.sh'), + manual_md, + entry.get(3, []), + nix_input, + ], + output : '@0@.@1@'.format(entry[0], entry[1]), + install : true, + install_dir : get_option('mandir') / 'man@0@'.format(entry[1]), + ) +endforeach diff --git a/doc/manual/package.nix b/doc/manual/package.nix new file mode 100644 index 000000000..2e6fcede3 --- /dev/null +++ b/doc/manual/package.nix @@ -0,0 +1,71 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, lowdown +, mdbook +, mdbook-linkcheck +, jq +, python3 +, rsync +, nix-cli + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-manual"; + inherit version; + + workDir = ./.; + fileset = fileset.difference + (fileset.unions [ + ../../.version + # Too many different types of files to filter for now + ../../doc/manual + ./. + ]) + # Do a blacklist instead + ../../doc/manual/package.nix; + + # TODO the man pages should probably be separate + outputs = [ "out" "man" ]; + + # Hack for sake of the dev shell + passthru.externalNativeBuildInputs = [ + meson + ninja + (lib.getBin lowdown) + mdbook + mdbook-linkcheck + jq + python3 + rsync + ]; + + nativeBuildInputs = finalAttrs.passthru.externalNativeBuildInputs ++ [ + nix-cli + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''$out/nix-support + echo "doc manual ''$out/share/doc/nix/manual" >> ''$out/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index cb8cd18fa..dea141391 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -1,7 +1,7 @@ // redirect rules for URL fragments (client-side) to prevent link rot. // this must be done on the client side, as web servers do not see the fragment part of the URL. // it will only work with JavaScript enabled in the browser, but this is the best we can do here. -// see src/_redirects for path redirects (server-side) +// see source/_redirects for path redirects (server-side) // redirects are declared as follows: // each entry has as its key a path matching the requested URL path, relative to the mdBook document root. diff --git a/doc/manual/remove_before_wrapper.py b/doc/manual/remove_before_wrapper.py new file mode 100644 index 000000000..6da4c19b0 --- /dev/null +++ b/doc/manual/remove_before_wrapper.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import sys +import shutil +import typing as t + +def main(): + if len(sys.argv) < 4 or '--' not in sys.argv: + print("Usage: remove-before-wrapper -- ") + sys.exit(1) + + # Extract the parts + output: str = sys.argv[1] + nix_command_idx: int = sys.argv.index('--') + 1 + nix_command: t.List[str] = sys.argv[nix_command_idx:] + + output_temp: str = output + '.tmp' + + # Remove the output and temp output in case they exist + shutil.rmtree(output, ignore_errors=True) + shutil.rmtree(output_temp, ignore_errors=True) + + # Execute nix command with `--write-to` tempary output + nix_command_write_to = nix_command + ['--write-to', output_temp] + subprocess.run(nix_command_write_to, check=True) + + # Move the temporary output to the intended location + os.rename(output_temp, output) + +if __name__ == "__main__": + main() diff --git a/doc/manual/render-manpage.sh b/doc/manual/render-manpage.sh new file mode 100755 index 000000000..65a9c124e --- /dev/null +++ b/doc/manual/render-manpage.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -euo pipefail + +lowdown_args= + +if [ "$1" = --out-no-smarty ]; then + lowdown_args=--out-no-smarty + shift +fi + +[ "$#" = 4 ] || { + echo "wrong number of args passed" >&2 + exit 1 +} + +title="$1" +section="$2" +infile="$3" +outfile="$4" + +( + printf "Title: %s\n\n" "$title" + cat "$infile" +) | lowdown -sT man --nroff-nolinks $lowdown_args -M section="$section" -o "$outfile" diff --git a/doc/manual/rl-next/build-hook-default.md b/doc/manual/rl-next/build-hook-default.md index 0d5a130c0..f13537983 100644 --- a/doc/manual/rl-next/build-hook-default.md +++ b/doc/manual/rl-next/build-hook-default.md @@ -16,7 +16,7 @@ This has a small adverse affect on remote building --- the `build-remote` execut This means that other applications linking `libnixstore` that wish to use remote building must arrange for the `nix` command to be on the PATH (or manually overriding `build-hook`) in order for that to work. Long term we don't envision this being a downside, because we plan to [get rid of `build-remote` and the build hook setting entirely](https://github.com/NixOS/nix/issues/1221). -There is simply no need to add a second layer of remote-procedure-calling when we want to connect to a remote builder. +There should simply be no need to have an extra, intermediate layer of remote-procedure-calling when we want to connect to a remote builder. The build hook protocol did in principle support custom ways of remote building, but that can also be accomplished with a custom service for the ssh or daemon/ssh-ng protocols, or with a custom [store type](@docroot@/store/types/index.md) i.e. `Store` subclass. -The Perl bindings no longer expose `getBinDir` either, since they libraries those bindings wrap no longer know the location of installed binaries as described above. +The Perl bindings no longer expose `getBinDir` either, since the underlying C++ libraries those bindings wrap no longer know the location of installed binaries as described above. diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in similarity index 99% rename from doc/manual/src/SUMMARY.md.in rename to doc/manual/source/SUMMARY.md.in index eef7d189c..d7c312ae7 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -121,6 +121,7 @@ - [Development](development/index.md) - [Building](development/building.md) - [Testing](development/testing.md) + - [Debugging](development/debugging.md) - [Documentation](development/documentation.md) - [CLI guideline](development/cli-guideline.md) - [JSON guideline](development/json-guideline.md) diff --git a/doc/manual/src/_redirects b/doc/manual/source/_redirects similarity index 100% rename from doc/manual/src/_redirects rename to doc/manual/source/_redirects diff --git a/doc/manual/src/advanced-topics/cores-vs-jobs.md b/doc/manual/source/advanced-topics/cores-vs-jobs.md similarity index 100% rename from doc/manual/src/advanced-topics/cores-vs-jobs.md rename to doc/manual/source/advanced-topics/cores-vs-jobs.md diff --git a/doc/manual/src/advanced-topics/diff-hook.md b/doc/manual/source/advanced-topics/diff-hook.md similarity index 100% rename from doc/manual/src/advanced-topics/diff-hook.md rename to doc/manual/source/advanced-topics/diff-hook.md diff --git a/doc/manual/source/advanced-topics/distributed-builds.md b/doc/manual/source/advanced-topics/distributed-builds.md new file mode 100644 index 000000000..66e371888 --- /dev/null +++ b/doc/manual/source/advanced-topics/distributed-builds.md @@ -0,0 +1,108 @@ +# Remote Builds + +A local Nix installation can forward Nix builds to other machines, +this allows multiple builds to be performed in parallel. + +Remote builds also allow Nix to perform multi-platform builds in a +semi-transparent way. For example, if you perform a build for a +`x86_64-darwin` on an `i686-linux` machine, Nix can automatically +forward the build to a `x86_64-darwin` machine, if one is available. + +## Requirements + +For a local machine to forward a build to a remote machine, the remote machine must: + +- Have Nix installed +- Be running an SSH server, e.g. `sshd` +- Be accessible via SSH from the local machine over the network +- Have the local machine's public SSH key in `/etc/ssh/authorized_keys.d/` +- Have the username of the SSH user in the `trusted-users` setting in `nix.conf` + +## Testing + +To test connecting to a remote Nix instance (in this case `mac`), run: + +```console +nix store info --store ssh://username@mac +``` + +To specify an SSH identity file as part of the remote store URI add a +query paramater, e.g. + +```console +nix store info --store ssh://username@mac?ssh-key=/home/alice/my-key +``` + +Since builds should be non-interactive, the key should not have a +passphrase. Alternatively, you can load identities ahead of time into +`ssh-agent` or `gpg-agent`. + +In a multi-user installation (default), builds are executed by the Nix +Daemon. The Nix Daemon cannot prompt for a passphrase via the terminal +or `ssh-agent`, so the SSH key must not have a passphrase. + +In addition, the Nix Daemon's user (typically root) needs to have SSH +access to the remote builder. + +Access can be verified by running `sudo su`, and then validating SSH +access, e.g. by running `ssh mac`. SSH identity files for root users +are usually stored in `/root/.ssh/` (Linux) or `/var/root/.ssh` (MacOS). + +If you get the error + +```console +bash: nix: command not found +error: cannot connect to 'mac' +``` + +then you need to ensure that the `PATH` of non-interactive login shells +contains Nix. + +The [list of remote build machines](@docroot@/command-ref/conf-file.md#conf-builders) can be specified on the command line or in the Nix configuration file. +For example, the following command allows you to build a derivation for `x86_64-darwin` on a Linux machine: + +```console +uname +``` + +```console +Linux +``` + +```console +nix build --impure \ + --expr '(with import { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \ + --builders 'ssh://mac x86_64-darwin' +``` + +```console +[1/0/1 built, 0.0 MiB DL] building foo on ssh://mac +``` + +```console +cat ./result +``` + +```console +Darwin +``` + +It is possible to specify multiple build machines separated by a semicolon or a newline, e.g. + +```console + --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd' +``` + +Remote build machines can also be configured in [`nix.conf`](@docroot@/command-ref/conf-file.md), e.g. + + builders = ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd + +After making changes to `nix.conf`, restart the Nix daemon for changes to take effect. + +Finally, remote build machines can be configured in a separate configuration +file included in `builders` via the syntax `@/path/to/file`. For example, + + builders = @/etc/nix/machines + +causes the list of machines in `/etc/nix/machines` to be included. +(This is the default.) diff --git a/doc/manual/src/advanced-topics/index.md b/doc/manual/source/advanced-topics/index.md similarity index 100% rename from doc/manual/src/advanced-topics/index.md rename to doc/manual/source/advanced-topics/index.md diff --git a/doc/manual/src/advanced-topics/post-build-hook.md b/doc/manual/source/advanced-topics/post-build-hook.md similarity index 100% rename from doc/manual/src/advanced-topics/post-build-hook.md rename to doc/manual/source/advanced-topics/post-build-hook.md diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/source/architecture/architecture.md similarity index 100% rename from doc/manual/src/architecture/architecture.md rename to doc/manual/source/architecture/architecture.md diff --git a/doc/manual/src/c-api.md b/doc/manual/source/c-api.md similarity index 100% rename from doc/manual/src/c-api.md rename to doc/manual/source/c-api.md diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/source/command-ref/conf-file-prefix.md similarity index 100% rename from doc/manual/src/command-ref/conf-file-prefix.md rename to doc/manual/source/command-ref/conf-file-prefix.md diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/source/command-ref/env-common.md similarity index 100% rename from doc/manual/src/command-ref/env-common.md rename to doc/manual/source/command-ref/env-common.md diff --git a/doc/manual/src/command-ref/experimental-commands.md b/doc/manual/source/command-ref/experimental-commands.md similarity index 100% rename from doc/manual/src/command-ref/experimental-commands.md rename to doc/manual/source/command-ref/experimental-commands.md diff --git a/doc/manual/src/command-ref/files.md b/doc/manual/source/command-ref/files.md similarity index 100% rename from doc/manual/src/command-ref/files.md rename to doc/manual/source/command-ref/files.md diff --git a/doc/manual/src/command-ref/files/channels.md b/doc/manual/source/command-ref/files/channels.md similarity index 100% rename from doc/manual/src/command-ref/files/channels.md rename to doc/manual/source/command-ref/files/channels.md diff --git a/doc/manual/src/command-ref/files/default-nix-expression.md b/doc/manual/source/command-ref/files/default-nix-expression.md similarity index 100% rename from doc/manual/src/command-ref/files/default-nix-expression.md rename to doc/manual/source/command-ref/files/default-nix-expression.md diff --git a/doc/manual/src/command-ref/files/manifest.json.md b/doc/manual/source/command-ref/files/manifest.json.md similarity index 100% rename from doc/manual/src/command-ref/files/manifest.json.md rename to doc/manual/source/command-ref/files/manifest.json.md diff --git a/doc/manual/src/command-ref/files/manifest.nix.md b/doc/manual/source/command-ref/files/manifest.nix.md similarity index 100% rename from doc/manual/src/command-ref/files/manifest.nix.md rename to doc/manual/source/command-ref/files/manifest.nix.md diff --git a/doc/manual/src/command-ref/files/profiles.md b/doc/manual/source/command-ref/files/profiles.md similarity index 100% rename from doc/manual/src/command-ref/files/profiles.md rename to doc/manual/source/command-ref/files/profiles.md diff --git a/doc/manual/src/command-ref/index.md b/doc/manual/source/command-ref/index.md similarity index 100% rename from doc/manual/src/command-ref/index.md rename to doc/manual/source/command-ref/index.md diff --git a/doc/manual/src/command-ref/main-commands.md b/doc/manual/source/command-ref/main-commands.md similarity index 100% rename from doc/manual/src/command-ref/main-commands.md rename to doc/manual/source/command-ref/main-commands.md diff --git a/doc/manual/source/command-ref/meson.build b/doc/manual/source/command-ref/meson.build new file mode 100644 index 000000000..2976f69ff --- /dev/null +++ b/doc/manual/source/command-ref/meson.build @@ -0,0 +1,63 @@ +xp_features_json = custom_target( + command : [nix, '__dump-xp-features'], + capture : true, + output : 'xp-features.json', +) + +experimental_features_shortlist_md = custom_target( + command : nix_eval_for_docs + [ + '--expr', + 'import @INPUT0@ (builtins.fromJSON (builtins.readFile ./@INPUT1@))', + ], + input : [ + '../../generate-xp-features-shortlist.nix', + xp_features_json, + ], + output : 'experimental-features-shortlist.md', + capture : true, + env : nix_env_for_docs, +) + +nix3_cli_files = custom_target( + command : [ + python.full_path(), + '@INPUT0@', + '@OUTPUT@', + '--' + ] + nix_eval_for_docs + [ + '--expr', + 'import @INPUT1@ true (builtins.readFile ./@INPUT2@)', + ], + input : [ + '../../remove_before_wrapper.py', + '../../generate-manpage.nix', + nix3_cli_json, + ], + output : 'new-cli', + env : nix_env_for_docs, +) + +conf_file_md_body = custom_target( + command : [ + nix_eval_for_docs, + '--expr', + 'import @INPUT0@ { prefix = "conf"; } (builtins.fromJSON (builtins.readFile ./@INPUT1@))', + ], + capture : true, + input : [ + '../../generate-settings.nix', + conf_file_json, + ], + output : 'conf-file.body.md', + env : nix_env_for_docs, +) + +conf_file_md = custom_target( + command : [ 'cat', '@INPUT0@', '@INPUT1@' ], + capture : true, + input : [ + 'conf-file-prefix.md', + conf_file_md_body, + ], + output : 'conf-file.md', +) diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/source/command-ref/nix-build.md similarity index 100% rename from doc/manual/src/command-ref/nix-build.md rename to doc/manual/source/command-ref/nix-build.md diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/source/command-ref/nix-channel.md similarity index 100% rename from doc/manual/src/command-ref/nix-channel.md rename to doc/manual/source/command-ref/nix-channel.md diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/source/command-ref/nix-collect-garbage.md similarity index 100% rename from doc/manual/src/command-ref/nix-collect-garbage.md rename to doc/manual/source/command-ref/nix-collect-garbage.md diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/source/command-ref/nix-copy-closure.md similarity index 100% rename from doc/manual/src/command-ref/nix-copy-closure.md rename to doc/manual/source/command-ref/nix-copy-closure.md diff --git a/doc/manual/src/command-ref/nix-daemon.md b/doc/manual/source/command-ref/nix-daemon.md similarity index 100% rename from doc/manual/src/command-ref/nix-daemon.md rename to doc/manual/source/command-ref/nix-daemon.md diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/source/command-ref/nix-env.md similarity index 100% rename from doc/manual/src/command-ref/nix-env.md rename to doc/manual/source/command-ref/nix-env.md diff --git a/doc/manual/src/command-ref/nix-env/delete-generations.md b/doc/manual/source/command-ref/nix-env/delete-generations.md similarity index 100% rename from doc/manual/src/command-ref/nix-env/delete-generations.md rename to doc/manual/source/command-ref/nix-env/delete-generations.md diff --git a/doc/manual/src/command-ref/nix-env/env-common.md b/doc/manual/source/command-ref/nix-env/env-common.md similarity index 100% rename from doc/manual/src/command-ref/nix-env/env-common.md rename to doc/manual/source/command-ref/nix-env/env-common.md diff --git a/doc/manual/src/command-ref/nix-env/install.md b/doc/manual/source/command-ref/nix-env/install.md similarity index 100% rename from doc/manual/src/command-ref/nix-env/install.md rename to doc/manual/source/command-ref/nix-env/install.md diff --git a/doc/manual/src/command-ref/nix-env/list-generations.md b/doc/manual/source/command-ref/nix-env/list-generations.md similarity index 100% rename from doc/manual/src/command-ref/nix-env/list-generations.md rename to doc/manual/source/command-ref/nix-env/list-generations.md diff --git a/doc/manual/src/command-ref/nix-env/opt-common.md b/doc/manual/source/command-ref/nix-env/opt-common.md similarity index 100% rename from doc/manual/src/command-ref/nix-env/opt-common.md rename to doc/manual/source/command-ref/nix-env/opt-common.md diff --git a/doc/manual/src/command-ref/nix-env/query.md b/doc/manual/source/command-ref/nix-env/query.md similarity index 100% rename from doc/manual/src/command-ref/nix-env/query.md rename to doc/manual/source/command-ref/nix-env/query.md diff --git a/doc/manual/src/command-ref/nix-env/rollback.md b/doc/manual/source/command-ref/nix-env/rollback.md similarity index 100% rename from doc/manual/src/command-ref/nix-env/rollback.md rename to doc/manual/source/command-ref/nix-env/rollback.md diff --git a/doc/manual/src/command-ref/nix-env/set-flag.md b/doc/manual/source/command-ref/nix-env/set-flag.md similarity index 100% rename from doc/manual/src/command-ref/nix-env/set-flag.md rename to doc/manual/source/command-ref/nix-env/set-flag.md diff --git a/doc/manual/src/command-ref/nix-env/set.md b/doc/manual/source/command-ref/nix-env/set.md similarity index 100% rename from doc/manual/src/command-ref/nix-env/set.md rename to doc/manual/source/command-ref/nix-env/set.md diff --git a/doc/manual/src/command-ref/nix-env/switch-generation.md b/doc/manual/source/command-ref/nix-env/switch-generation.md similarity index 100% rename from doc/manual/src/command-ref/nix-env/switch-generation.md rename to doc/manual/source/command-ref/nix-env/switch-generation.md diff --git a/doc/manual/src/command-ref/nix-env/switch-profile.md b/doc/manual/source/command-ref/nix-env/switch-profile.md similarity index 100% rename from doc/manual/src/command-ref/nix-env/switch-profile.md rename to doc/manual/source/command-ref/nix-env/switch-profile.md diff --git a/doc/manual/src/command-ref/nix-env/uninstall.md b/doc/manual/source/command-ref/nix-env/uninstall.md similarity index 100% rename from doc/manual/src/command-ref/nix-env/uninstall.md rename to doc/manual/source/command-ref/nix-env/uninstall.md diff --git a/doc/manual/src/command-ref/nix-env/upgrade.md b/doc/manual/source/command-ref/nix-env/upgrade.md similarity index 100% rename from doc/manual/src/command-ref/nix-env/upgrade.md rename to doc/manual/source/command-ref/nix-env/upgrade.md diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/source/command-ref/nix-hash.md similarity index 100% rename from doc/manual/src/command-ref/nix-hash.md rename to doc/manual/source/command-ref/nix-hash.md diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/source/command-ref/nix-instantiate.md similarity index 100% rename from doc/manual/src/command-ref/nix-instantiate.md rename to doc/manual/source/command-ref/nix-instantiate.md diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/source/command-ref/nix-prefetch-url.md similarity index 100% rename from doc/manual/src/command-ref/nix-prefetch-url.md rename to doc/manual/source/command-ref/nix-prefetch-url.md diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/source/command-ref/nix-shell.md similarity index 100% rename from doc/manual/src/command-ref/nix-shell.md rename to doc/manual/source/command-ref/nix-shell.md diff --git a/doc/manual/src/command-ref/nix-store.md b/doc/manual/source/command-ref/nix-store.md similarity index 100% rename from doc/manual/src/command-ref/nix-store.md rename to doc/manual/source/command-ref/nix-store.md diff --git a/doc/manual/src/command-ref/nix-store/add-fixed.md b/doc/manual/source/command-ref/nix-store/add-fixed.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/add-fixed.md rename to doc/manual/source/command-ref/nix-store/add-fixed.md diff --git a/doc/manual/src/command-ref/nix-store/add.md b/doc/manual/source/command-ref/nix-store/add.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/add.md rename to doc/manual/source/command-ref/nix-store/add.md diff --git a/doc/manual/src/command-ref/nix-store/delete.md b/doc/manual/source/command-ref/nix-store/delete.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/delete.md rename to doc/manual/source/command-ref/nix-store/delete.md diff --git a/doc/manual/src/command-ref/nix-store/dump-db.md b/doc/manual/source/command-ref/nix-store/dump-db.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/dump-db.md rename to doc/manual/source/command-ref/nix-store/dump-db.md diff --git a/doc/manual/src/command-ref/nix-store/dump.md b/doc/manual/source/command-ref/nix-store/dump.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/dump.md rename to doc/manual/source/command-ref/nix-store/dump.md diff --git a/doc/manual/src/command-ref/nix-store/export.md b/doc/manual/source/command-ref/nix-store/export.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/export.md rename to doc/manual/source/command-ref/nix-store/export.md diff --git a/doc/manual/src/command-ref/nix-store/gc.md b/doc/manual/source/command-ref/nix-store/gc.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/gc.md rename to doc/manual/source/command-ref/nix-store/gc.md diff --git a/doc/manual/src/command-ref/nix-store/generate-binary-cache-key.md b/doc/manual/source/command-ref/nix-store/generate-binary-cache-key.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/generate-binary-cache-key.md rename to doc/manual/source/command-ref/nix-store/generate-binary-cache-key.md diff --git a/doc/manual/src/command-ref/nix-store/import.md b/doc/manual/source/command-ref/nix-store/import.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/import.md rename to doc/manual/source/command-ref/nix-store/import.md diff --git a/doc/manual/src/command-ref/nix-store/load-db.md b/doc/manual/source/command-ref/nix-store/load-db.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/load-db.md rename to doc/manual/source/command-ref/nix-store/load-db.md diff --git a/doc/manual/src/command-ref/nix-store/opt-common.md b/doc/manual/source/command-ref/nix-store/opt-common.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/opt-common.md rename to doc/manual/source/command-ref/nix-store/opt-common.md diff --git a/doc/manual/src/command-ref/nix-store/optimise.md b/doc/manual/source/command-ref/nix-store/optimise.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/optimise.md rename to doc/manual/source/command-ref/nix-store/optimise.md diff --git a/doc/manual/src/command-ref/nix-store/print-env.md b/doc/manual/source/command-ref/nix-store/print-env.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/print-env.md rename to doc/manual/source/command-ref/nix-store/print-env.md diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/source/command-ref/nix-store/query.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/query.md rename to doc/manual/source/command-ref/nix-store/query.md diff --git a/doc/manual/src/command-ref/nix-store/read-log.md b/doc/manual/source/command-ref/nix-store/read-log.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/read-log.md rename to doc/manual/source/command-ref/nix-store/read-log.md diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/source/command-ref/nix-store/realise.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/realise.md rename to doc/manual/source/command-ref/nix-store/realise.md diff --git a/doc/manual/src/command-ref/nix-store/repair-path.md b/doc/manual/source/command-ref/nix-store/repair-path.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/repair-path.md rename to doc/manual/source/command-ref/nix-store/repair-path.md diff --git a/doc/manual/src/command-ref/nix-store/restore.md b/doc/manual/source/command-ref/nix-store/restore.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/restore.md rename to doc/manual/source/command-ref/nix-store/restore.md diff --git a/doc/manual/src/command-ref/nix-store/serve.md b/doc/manual/source/command-ref/nix-store/serve.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/serve.md rename to doc/manual/source/command-ref/nix-store/serve.md diff --git a/doc/manual/src/command-ref/nix-store/verify-path.md b/doc/manual/source/command-ref/nix-store/verify-path.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/verify-path.md rename to doc/manual/source/command-ref/nix-store/verify-path.md diff --git a/doc/manual/src/command-ref/nix-store/verify.md b/doc/manual/source/command-ref/nix-store/verify.md similarity index 100% rename from doc/manual/src/command-ref/nix-store/verify.md rename to doc/manual/source/command-ref/nix-store/verify.md diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/source/command-ref/opt-common.md similarity index 92% rename from doc/manual/src/command-ref/opt-common.md rename to doc/manual/source/command-ref/opt-common.md index 69a700207..70ae03959 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/source/command-ref/opt-common.md @@ -1,3 +1,7 @@ + + # Common Options Most Nix commands accept the following command-line options: @@ -161,6 +165,14 @@ Most Nix commands accept the following command-line options: You can override this using `--arg`, e.g., `nix-env --install --attr pkgname --arg system \"i686-freebsd\"`. (Note that since the argument is a Nix string literal, you have to escape the quotes.) +- [`--arg-from-file`](#opt-arg-from-file) *name* *path* + + Pass the contents of file *path* as the argument *name* to Nix functions. + +- [`--arg-from-stdin`](#opt-arg-from-stdin) *name* + + Pass the contents of stdin as the argument *name* to Nix functions. + - [`--argstr`](#opt-argstr) *name* *value* This option is like `--arg`, only the value is not a Nix expression but a string. @@ -179,6 +191,10 @@ Most Nix commands accept the following command-line options: attribute of the fourth element of the array in the `foo` attribute of the top-level expression. +- [`--eval-store`](#opt-eval-store) *store-url* + + The [URL to the Nix store](@docroot@/store/types/index.md#store-url-format) to use for evaluation, i.e. where to store derivations (`.drv` files) and inputs referenced by them. + - [`--expr`](#opt-expr) / `-E` Interpret the command line arguments as a list of Nix expressions to be parsed and evaluated, rather than as a list of file names of Nix expressions. @@ -194,6 +210,10 @@ Most Nix commands accept the following command-line options: Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). +- [`--impure`](#opt-impure) + + Allow access to mutable paths and repositories. + - [`--option`](#opt-option) *name* *value* Set the Nix configuration option *name* to *value*. diff --git a/doc/manual/src/command-ref/status-build-failure.md b/doc/manual/source/command-ref/status-build-failure.md similarity index 100% rename from doc/manual/src/command-ref/status-build-failure.md rename to doc/manual/source/command-ref/status-build-failure.md diff --git a/doc/manual/src/command-ref/utilities.md b/doc/manual/source/command-ref/utilities.md similarity index 100% rename from doc/manual/src/command-ref/utilities.md rename to doc/manual/source/command-ref/utilities.md diff --git a/doc/manual/src/development/building.md b/doc/manual/source/development/building.md similarity index 100% rename from doc/manual/src/development/building.md rename to doc/manual/source/development/building.md diff --git a/doc/manual/src/development/cli-guideline.md b/doc/manual/source/development/cli-guideline.md similarity index 100% rename from doc/manual/src/development/cli-guideline.md rename to doc/manual/source/development/cli-guideline.md diff --git a/doc/manual/src/development/contributing.md b/doc/manual/source/development/contributing.md similarity index 100% rename from doc/manual/src/development/contributing.md rename to doc/manual/source/development/contributing.md diff --git a/doc/manual/src/development/cxx.md b/doc/manual/source/development/cxx.md similarity index 100% rename from doc/manual/src/development/cxx.md rename to doc/manual/source/development/cxx.md diff --git a/doc/manual/source/development/debugging.md b/doc/manual/source/development/debugging.md new file mode 100644 index 000000000..ce623110b --- /dev/null +++ b/doc/manual/source/development/debugging.md @@ -0,0 +1,62 @@ +# Debugging Nix + +This section shows how to build and debug Nix with debug symbols enabled. + +## Building Nix with Debug Symbols + +In the development shell, set the `mesonBuildType` environment variable to `debug` before configuring the build: + +```console +[nix-shell]$ export mesonBuildType=debugoptimized +``` + +Then, proceed to build Nix as described in [Building Nix](./building.md). +This will build Nix with debug symbols, which are essential for effective debugging. + +## Debugging the Nix Binary + +Obtain your preferred debugger within the development shell: + +```console +[nix-shell]$ nix-shell -p gdb +``` + +On macOS, use `lldb`: + +```console +[nix-shell]$ nix-shell -p lldb +``` + +### Launching the Debugger + +To debug the Nix binary, run: + +```console +[nix-shell]$ gdb --args ../outputs/out/bin/nix +``` + +On macOS, use `lldb`: + +```console +[nix-shell]$ lldb -- ../outputs/out/bin/nix +``` + +### Using the Debugger + +Inside the debugger, you can set breakpoints, run the program, and inspect variables. + +```gdb +(gdb) break main +(gdb) run +``` + +Refer to the [GDB Documentation](https://www.gnu.org/software/gdb/documentation/) for comprehensive usage instructions. + +On macOS, use `lldb`: + +```lldb +(lldb) breakpoint set --name main +(lldb) process launch -- +``` + +Refer to the [LLDB Tutorial](https://lldb.llvm.org/use/tutorial.html) for comprehensive usage instructions. diff --git a/doc/manual/src/development/documentation.md b/doc/manual/source/development/documentation.md similarity index 98% rename from doc/manual/src/development/documentation.md rename to doc/manual/source/development/documentation.md index d5a95e0c1..d51373e7b 100644 --- a/doc/manual/src/development/documentation.md +++ b/doc/manual/source/development/documentation.md @@ -35,7 +35,7 @@ In order to reflect changes to the [Makefile for the manual], clear all generate [Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk ```console -rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/src/command-ref/new-cli && make manual-html -j $NIX_BUILD_CORES +rm $(git ls-files doc/manual/ -o | grep -F '.md') && rmdir doc/manual/source/command-ref/new-cli && make manual-html -j $NIX_BUILD_CORES ``` ## Style guide @@ -182,7 +182,7 @@ Please observe these guidelines to ease reviews: `@docroot@` provides a base path for links that occur in reusable snippets or other documentation that doesn't have a base path of its own. -If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/src` directory. +If a broken link occurs in a snippet that was inserted into multiple generated files in different directories, use `@docroot@` to reference the `doc/manual/source` directory. If the `@docroot@` literal appears in an error message from the [`mdbook-linkcheck`] tool, the `@docroot@` replacement needs to be applied to the generated source file that mentions it. See existing `@docroot@` logic in the [Makefile for the manual]. diff --git a/doc/manual/src/development/experimental-features.md b/doc/manual/source/development/experimental-features.md similarity index 100% rename from doc/manual/src/development/experimental-features.md rename to doc/manual/source/development/experimental-features.md diff --git a/doc/manual/src/development/index.md b/doc/manual/source/development/index.md similarity index 100% rename from doc/manual/src/development/index.md rename to doc/manual/source/development/index.md diff --git a/doc/manual/src/development/json-guideline.md b/doc/manual/source/development/json-guideline.md similarity index 100% rename from doc/manual/src/development/json-guideline.md rename to doc/manual/source/development/json-guideline.md diff --git a/doc/manual/source/development/meson.build b/doc/manual/source/development/meson.build new file mode 100644 index 000000000..5ffbfe394 --- /dev/null +++ b/doc/manual/source/development/meson.build @@ -0,0 +1,12 @@ +experimental_feature_descriptions_md = custom_target( + command : nix_eval_for_docs + [ + '--expr', + 'import @INPUT0@ (builtins.fromJSON (builtins.readFile @INPUT1@))', + ], + input : [ + '../../generate-xp-features.nix', + xp_features_json, + ], + capture : true, + output : 'experimental-feature-descriptions.md', +) diff --git a/doc/manual/src/development/testing.md b/doc/manual/source/development/testing.md similarity index 98% rename from doc/manual/src/development/testing.md rename to doc/manual/source/development/testing.md index 0df72cc38..9d228fd16 100644 --- a/doc/manual/src/development/testing.md +++ b/doc/manual/source/development/testing.md @@ -60,10 +60,10 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > ``` The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_name_without-nix}-test`. -Given an interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `src/nix-expr-tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `src/nix-expr-test-support/tests/value/context.{hh,cc}`. +Given an interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `src/libexpr-tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `src/libexpr-test-support/tests/value/context.{hh,cc}`. Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. -For example, `libnixstore` code is in `src/libstore`, and its test data is in `src/nix-store-tests/data`. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `src/libstore-tests/data`. The path to the `src/${library_name_without-nix}-test/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. Note that each executable only gets the data for its tests. diff --git a/doc/manual/src/favicon.png b/doc/manual/source/favicon.png similarity index 100% rename from doc/manual/src/favicon.png rename to doc/manual/source/favicon.png diff --git a/doc/manual/src/favicon.svg b/doc/manual/source/favicon.svg similarity index 100% rename from doc/manual/src/favicon.svg rename to doc/manual/source/favicon.svg diff --git a/doc/manual/src/figures/user-environments.png b/doc/manual/source/figures/user-environments.png similarity index 100% rename from doc/manual/src/figures/user-environments.png rename to doc/manual/source/figures/user-environments.png diff --git a/doc/manual/src/figures/user-environments.sxd b/doc/manual/source/figures/user-environments.sxd similarity index 100% rename from doc/manual/src/figures/user-environments.sxd rename to doc/manual/source/figures/user-environments.sxd diff --git a/doc/manual/src/glossary.md b/doc/manual/source/glossary.md similarity index 100% rename from doc/manual/src/glossary.md rename to doc/manual/source/glossary.md diff --git a/doc/manual/src/installation/building-source.md b/doc/manual/source/installation/building-source.md similarity index 100% rename from doc/manual/src/installation/building-source.md rename to doc/manual/source/installation/building-source.md diff --git a/doc/manual/src/installation/env-variables.md b/doc/manual/source/installation/env-variables.md similarity index 100% rename from doc/manual/src/installation/env-variables.md rename to doc/manual/source/installation/env-variables.md diff --git a/doc/manual/src/installation/index.md b/doc/manual/source/installation/index.md similarity index 100% rename from doc/manual/src/installation/index.md rename to doc/manual/source/installation/index.md diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/source/installation/installing-binary.md similarity index 100% rename from doc/manual/src/installation/installing-binary.md rename to doc/manual/source/installation/installing-binary.md diff --git a/doc/manual/src/installation/installing-docker.md b/doc/manual/source/installation/installing-docker.md similarity index 100% rename from doc/manual/src/installation/installing-docker.md rename to doc/manual/source/installation/installing-docker.md diff --git a/doc/manual/src/installation/installing-source.md b/doc/manual/source/installation/installing-source.md similarity index 100% rename from doc/manual/src/installation/installing-source.md rename to doc/manual/source/installation/installing-source.md diff --git a/doc/manual/src/installation/multi-user.md b/doc/manual/source/installation/multi-user.md similarity index 100% rename from doc/manual/src/installation/multi-user.md rename to doc/manual/source/installation/multi-user.md diff --git a/doc/manual/src/installation/nix-security.md b/doc/manual/source/installation/nix-security.md similarity index 100% rename from doc/manual/src/installation/nix-security.md rename to doc/manual/source/installation/nix-security.md diff --git a/doc/manual/src/installation/obtaining-source.md b/doc/manual/source/installation/obtaining-source.md similarity index 100% rename from doc/manual/src/installation/obtaining-source.md rename to doc/manual/source/installation/obtaining-source.md diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/source/installation/prerequisites-source.md similarity index 100% rename from doc/manual/src/installation/prerequisites-source.md rename to doc/manual/source/installation/prerequisites-source.md diff --git a/doc/manual/src/installation/single-user.md b/doc/manual/source/installation/single-user.md similarity index 100% rename from doc/manual/src/installation/single-user.md rename to doc/manual/source/installation/single-user.md diff --git a/doc/manual/src/installation/supported-platforms.md b/doc/manual/source/installation/supported-platforms.md similarity index 100% rename from doc/manual/src/installation/supported-platforms.md rename to doc/manual/source/installation/supported-platforms.md diff --git a/doc/manual/src/installation/uninstall.md b/doc/manual/source/installation/uninstall.md similarity index 100% rename from doc/manual/src/installation/uninstall.md rename to doc/manual/source/installation/uninstall.md diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/source/installation/upgrading.md similarity index 100% rename from doc/manual/src/installation/upgrading.md rename to doc/manual/source/installation/upgrading.md diff --git a/doc/manual/src/introduction.md b/doc/manual/source/introduction.md similarity index 100% rename from doc/manual/src/introduction.md rename to doc/manual/source/introduction.md diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/source/language/advanced-attributes.md similarity index 100% rename from doc/manual/src/language/advanced-attributes.md rename to doc/manual/source/language/advanced-attributes.md diff --git a/doc/manual/src/language/builtins-prefix.md b/doc/manual/source/language/builtins-prefix.md similarity index 100% rename from doc/manual/src/language/builtins-prefix.md rename to doc/manual/source/language/builtins-prefix.md diff --git a/doc/manual/src/language/builtins-suffix.md b/doc/manual/source/language/builtins-suffix.md similarity index 100% rename from doc/manual/src/language/builtins-suffix.md rename to doc/manual/source/language/builtins-suffix.md diff --git a/doc/manual/source/language/constructs.md b/doc/manual/source/language/constructs.md new file mode 100644 index 000000000..41a180246 --- /dev/null +++ b/doc/manual/source/language/constructs.md @@ -0,0 +1 @@ +# Language Constructs diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/source/language/constructs/lookup-path.md similarity index 100% rename from doc/manual/src/language/constructs/lookup-path.md rename to doc/manual/source/language/constructs/lookup-path.md diff --git a/doc/manual/src/language/derivations.md b/doc/manual/source/language/derivations.md similarity index 100% rename from doc/manual/src/language/derivations.md rename to doc/manual/source/language/derivations.md diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/source/language/identifiers.md similarity index 100% rename from doc/manual/src/language/identifiers.md rename to doc/manual/source/language/identifiers.md diff --git a/doc/manual/src/language/import-from-derivation.md b/doc/manual/source/language/import-from-derivation.md similarity index 100% rename from doc/manual/src/language/import-from-derivation.md rename to doc/manual/source/language/import-from-derivation.md diff --git a/doc/manual/src/language/index.md b/doc/manual/source/language/index.md similarity index 100% rename from doc/manual/src/language/index.md rename to doc/manual/source/language/index.md diff --git a/doc/manual/source/language/meson.build b/doc/manual/source/language/meson.build new file mode 100644 index 000000000..97469e2f3 --- /dev/null +++ b/doc/manual/source/language/meson.build @@ -0,0 +1,20 @@ +builtins_md = custom_target( + command : [ + python.full_path(), + '@INPUT0@', + '@OUTPUT@', + '--' + ] + nix_eval_for_docs + [ + '--expr', + '(builtins.readFile @INPUT3@) + import @INPUT1@ (builtins.fromJSON (builtins.readFile ./@INPUT2@)) + (builtins.readFile @INPUT4@)', + ], + input : [ + '../../remove_before_wrapper.py', + '../../generate-builtins.nix', + language_json, + 'builtins-prefix.md', + 'builtins-suffix.md' + ], + output : 'builtins.md', + env : nix_env_for_docs, +) diff --git a/doc/manual/src/language/operators.md b/doc/manual/source/language/operators.md similarity index 97% rename from doc/manual/src/language/operators.md rename to doc/manual/source/language/operators.md index e2ed3fbed..dbf2441cb 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/source/language/operators.md @@ -102,7 +102,7 @@ The `+` operator is overloaded to also work on strings and paths. > > *string* `+` *string* -Concatenate two [strings][string] and merge their string contexts. +Concatenate two [strings][string] and merge their [string contexts](./string-context.md). [String concatenation]: #string-concatenation @@ -128,7 +128,7 @@ The result is a path. > **Note** > -> The string must not have a string context that refers to a [store path]. +> The string must not have a [string context](./string-context.md) that refers to a [store path]. [Path and string concatenation]: #path-and-string-concatenation diff --git a/doc/manual/src/language/scope.md b/doc/manual/source/language/scope.md similarity index 100% rename from doc/manual/src/language/scope.md rename to doc/manual/source/language/scope.md diff --git a/doc/manual/src/language/string-context.md b/doc/manual/source/language/string-context.md similarity index 100% rename from doc/manual/src/language/string-context.md rename to doc/manual/source/language/string-context.md diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/source/language/string-interpolation.md similarity index 100% rename from doc/manual/src/language/string-interpolation.md rename to doc/manual/source/language/string-interpolation.md diff --git a/doc/manual/src/language/string-literals.md b/doc/manual/source/language/string-literals.md similarity index 93% rename from doc/manual/src/language/string-literals.md rename to doc/manual/source/language/string-literals.md index 8f4b75f3e..ca064989a 100644 --- a/doc/manual/src/language/string-literals.md +++ b/doc/manual/source/language/string-literals.md @@ -150,6 +150,21 @@ These special characters are escaped as follows: `''\` escapes any other character. +A "dollar-curly" (`${`) can be written as follows: +> **Example** +> +> ```nix +> '' +> echo ''${PATH} +> '' +> ``` +> +> "echo ${PATH}\n" + +> **Note** +> +> This differs from the syntax for escaping a dollar-curly within double quotes (`"\${"`). Be aware of which one is needed at a given moment. + A "double-dollar-curly" (`$${`) can be written literally. > **Example** diff --git a/doc/manual/src/language/syntax.md b/doc/manual/source/language/syntax.md similarity index 100% rename from doc/manual/src/language/syntax.md rename to doc/manual/source/language/syntax.md diff --git a/doc/manual/src/language/types.md b/doc/manual/source/language/types.md similarity index 100% rename from doc/manual/src/language/types.md rename to doc/manual/source/language/types.md diff --git a/doc/manual/source/language/values.md b/doc/manual/source/language/values.md new file mode 100644 index 000000000..e05f56025 --- /dev/null +++ b/doc/manual/source/language/values.md @@ -0,0 +1 @@ +# Data Types diff --git a/doc/manual/src/language/variables.md b/doc/manual/source/language/variables.md similarity index 100% rename from doc/manual/src/language/variables.md rename to doc/manual/source/language/variables.md diff --git a/doc/manual/source/meson.build b/doc/manual/source/meson.build new file mode 100644 index 000000000..098a29897 --- /dev/null +++ b/doc/manual/source/meson.build @@ -0,0 +1,17 @@ +summary_rl_next = custom_target( + command : [ + bash, + '-euo', 'pipefail', + '-c', + ''' + if [ -e "@INPUT@" ]; then + echo ' - [Upcoming release](release-notes/rl-next.md)' + fi + ''', + ], + input : [ + rl_next_generated, + ], + capture: true, + output : 'SUMMARY-rl-next.md', +) diff --git a/doc/manual/src/package-management/binary-cache-substituter.md b/doc/manual/source/package-management/binary-cache-substituter.md similarity index 100% rename from doc/manual/src/package-management/binary-cache-substituter.md rename to doc/manual/source/package-management/binary-cache-substituter.md diff --git a/doc/manual/src/package-management/garbage-collection.md b/doc/manual/source/package-management/garbage-collection.md similarity index 100% rename from doc/manual/src/package-management/garbage-collection.md rename to doc/manual/source/package-management/garbage-collection.md diff --git a/doc/manual/src/package-management/garbage-collector-roots.md b/doc/manual/source/package-management/garbage-collector-roots.md similarity index 100% rename from doc/manual/src/package-management/garbage-collector-roots.md rename to doc/manual/source/package-management/garbage-collector-roots.md diff --git a/doc/manual/src/package-management/index.md b/doc/manual/source/package-management/index.md similarity index 100% rename from doc/manual/src/package-management/index.md rename to doc/manual/source/package-management/index.md diff --git a/doc/manual/src/package-management/profiles.md b/doc/manual/source/package-management/profiles.md similarity index 100% rename from doc/manual/src/package-management/profiles.md rename to doc/manual/source/package-management/profiles.md diff --git a/doc/manual/src/package-management/sharing-packages.md b/doc/manual/source/package-management/sharing-packages.md similarity index 100% rename from doc/manual/src/package-management/sharing-packages.md rename to doc/manual/source/package-management/sharing-packages.md diff --git a/doc/manual/src/package-management/ssh-substituter.md b/doc/manual/source/package-management/ssh-substituter.md similarity index 100% rename from doc/manual/src/package-management/ssh-substituter.md rename to doc/manual/source/package-management/ssh-substituter.md diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/source/protocols/derivation-aterm.md similarity index 100% rename from doc/manual/src/protocols/derivation-aterm.md rename to doc/manual/source/protocols/derivation-aterm.md diff --git a/doc/manual/src/protocols/index.md b/doc/manual/source/protocols/index.md similarity index 100% rename from doc/manual/src/protocols/index.md rename to doc/manual/source/protocols/index.md diff --git a/doc/manual/src/protocols/json/derivation.md b/doc/manual/source/protocols/json/derivation.md similarity index 100% rename from doc/manual/src/protocols/json/derivation.md rename to doc/manual/source/protocols/json/derivation.md diff --git a/doc/manual/source/protocols/json/index.md b/doc/manual/source/protocols/json/index.md new file mode 100644 index 000000000..1fcd1e62d --- /dev/null +++ b/doc/manual/source/protocols/json/index.md @@ -0,0 +1 @@ +# JSON Formats diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/source/protocols/json/store-object-info.md similarity index 100% rename from doc/manual/src/protocols/json/store-object-info.md rename to doc/manual/source/protocols/json/store-object-info.md diff --git a/doc/manual/src/protocols/nix-archive.md b/doc/manual/source/protocols/nix-archive.md similarity index 100% rename from doc/manual/src/protocols/nix-archive.md rename to doc/manual/source/protocols/nix-archive.md diff --git a/doc/manual/src/protocols/store-path.md b/doc/manual/source/protocols/store-path.md similarity index 100% rename from doc/manual/src/protocols/store-path.md rename to doc/manual/source/protocols/store-path.md diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/source/protocols/tarball-fetcher.md similarity index 100% rename from doc/manual/src/protocols/tarball-fetcher.md rename to doc/manual/source/protocols/tarball-fetcher.md diff --git a/doc/manual/src/quick-start.md b/doc/manual/source/quick-start.md similarity index 100% rename from doc/manual/src/quick-start.md rename to doc/manual/source/quick-start.md diff --git a/doc/manual/src/release-notes/index.md b/doc/manual/source/release-notes/index.md similarity index 100% rename from doc/manual/src/release-notes/index.md rename to doc/manual/source/release-notes/index.md diff --git a/doc/manual/source/release-notes/meson.build b/doc/manual/source/release-notes/meson.build new file mode 100644 index 000000000..d8bf154e1 --- /dev/null +++ b/doc/manual/source/release-notes/meson.build @@ -0,0 +1,24 @@ +rl_next_generated = custom_target( + command : [ + 'bash', + '-euo', + 'pipefail', + '-c', + ''' + if type -p build-release-notes > /dev/null; then + build-release-notes --change-authors @CURRENT_SOURCE_DIR@/../../change-authors.yml @CURRENT_SOURCE_DIR@/../../rl-next + elif type -p changelog-d > /dev/null; then + changelog-d @CURRENT_SOURCE_DIR@/../../rl-next + fi + @0@ @INPUT0@ @CURRENT_SOURCE_DIR@/../../rl-next > @DEPFILE@ + '''.format( + python.full_path(), + ), + ], + input : [ + generate_manual_deps, + ], + output : 'rl-next.md', + capture : true, + depfile : 'rl-next.d', +) diff --git a/doc/manual/src/release-notes/rl-0.10.1.md b/doc/manual/source/release-notes/rl-0.10.1.md similarity index 100% rename from doc/manual/src/release-notes/rl-0.10.1.md rename to doc/manual/source/release-notes/rl-0.10.1.md diff --git a/doc/manual/src/release-notes/rl-0.10.md b/doc/manual/source/release-notes/rl-0.10.md similarity index 100% rename from doc/manual/src/release-notes/rl-0.10.md rename to doc/manual/source/release-notes/rl-0.10.md diff --git a/doc/manual/src/release-notes/rl-0.11.md b/doc/manual/source/release-notes/rl-0.11.md similarity index 100% rename from doc/manual/src/release-notes/rl-0.11.md rename to doc/manual/source/release-notes/rl-0.11.md diff --git a/doc/manual/src/release-notes/rl-0.12.md b/doc/manual/source/release-notes/rl-0.12.md similarity index 100% rename from doc/manual/src/release-notes/rl-0.12.md rename to doc/manual/source/release-notes/rl-0.12.md diff --git a/doc/manual/src/release-notes/rl-0.13.md b/doc/manual/source/release-notes/rl-0.13.md similarity index 100% rename from doc/manual/src/release-notes/rl-0.13.md rename to doc/manual/source/release-notes/rl-0.13.md diff --git a/doc/manual/src/release-notes/rl-0.14.md b/doc/manual/source/release-notes/rl-0.14.md similarity index 100% rename from doc/manual/src/release-notes/rl-0.14.md rename to doc/manual/source/release-notes/rl-0.14.md diff --git a/doc/manual/src/release-notes/rl-0.15.md b/doc/manual/source/release-notes/rl-0.15.md similarity index 100% rename from doc/manual/src/release-notes/rl-0.15.md rename to doc/manual/source/release-notes/rl-0.15.md diff --git a/doc/manual/src/release-notes/rl-0.16.md b/doc/manual/source/release-notes/rl-0.16.md similarity index 100% rename from doc/manual/src/release-notes/rl-0.16.md rename to doc/manual/source/release-notes/rl-0.16.md diff --git a/doc/manual/src/release-notes/rl-0.5.md b/doc/manual/source/release-notes/rl-0.5.md similarity index 100% rename from doc/manual/src/release-notes/rl-0.5.md rename to doc/manual/source/release-notes/rl-0.5.md diff --git a/doc/manual/src/release-notes/rl-0.6.md b/doc/manual/source/release-notes/rl-0.6.md similarity index 100% rename from doc/manual/src/release-notes/rl-0.6.md rename to doc/manual/source/release-notes/rl-0.6.md diff --git a/doc/manual/src/release-notes/rl-0.7.md b/doc/manual/source/release-notes/rl-0.7.md similarity index 100% rename from doc/manual/src/release-notes/rl-0.7.md rename to doc/manual/source/release-notes/rl-0.7.md diff --git a/doc/manual/src/release-notes/rl-0.8.1.md b/doc/manual/source/release-notes/rl-0.8.1.md similarity index 100% rename from doc/manual/src/release-notes/rl-0.8.1.md rename to doc/manual/source/release-notes/rl-0.8.1.md diff --git a/doc/manual/src/release-notes/rl-0.8.md b/doc/manual/source/release-notes/rl-0.8.md similarity index 100% rename from doc/manual/src/release-notes/rl-0.8.md rename to doc/manual/source/release-notes/rl-0.8.md diff --git a/doc/manual/src/release-notes/rl-0.9.1.md b/doc/manual/source/release-notes/rl-0.9.1.md similarity index 100% rename from doc/manual/src/release-notes/rl-0.9.1.md rename to doc/manual/source/release-notes/rl-0.9.1.md diff --git a/doc/manual/src/release-notes/rl-0.9.2.md b/doc/manual/source/release-notes/rl-0.9.2.md similarity index 100% rename from doc/manual/src/release-notes/rl-0.9.2.md rename to doc/manual/source/release-notes/rl-0.9.2.md diff --git a/doc/manual/src/release-notes/rl-0.9.md b/doc/manual/source/release-notes/rl-0.9.md similarity index 100% rename from doc/manual/src/release-notes/rl-0.9.md rename to doc/manual/source/release-notes/rl-0.9.md diff --git a/doc/manual/src/release-notes/rl-1.0.md b/doc/manual/source/release-notes/rl-1.0.md similarity index 100% rename from doc/manual/src/release-notes/rl-1.0.md rename to doc/manual/source/release-notes/rl-1.0.md diff --git a/doc/manual/src/release-notes/rl-1.1.md b/doc/manual/source/release-notes/rl-1.1.md similarity index 100% rename from doc/manual/src/release-notes/rl-1.1.md rename to doc/manual/source/release-notes/rl-1.1.md diff --git a/doc/manual/src/release-notes/rl-1.10.md b/doc/manual/source/release-notes/rl-1.10.md similarity index 100% rename from doc/manual/src/release-notes/rl-1.10.md rename to doc/manual/source/release-notes/rl-1.10.md diff --git a/doc/manual/src/release-notes/rl-1.11.10.md b/doc/manual/source/release-notes/rl-1.11.10.md similarity index 100% rename from doc/manual/src/release-notes/rl-1.11.10.md rename to doc/manual/source/release-notes/rl-1.11.10.md diff --git a/doc/manual/src/release-notes/rl-1.11.md b/doc/manual/source/release-notes/rl-1.11.md similarity index 100% rename from doc/manual/src/release-notes/rl-1.11.md rename to doc/manual/source/release-notes/rl-1.11.md diff --git a/doc/manual/src/release-notes/rl-1.2.md b/doc/manual/source/release-notes/rl-1.2.md similarity index 100% rename from doc/manual/src/release-notes/rl-1.2.md rename to doc/manual/source/release-notes/rl-1.2.md diff --git a/doc/manual/src/release-notes/rl-1.3.md b/doc/manual/source/release-notes/rl-1.3.md similarity index 100% rename from doc/manual/src/release-notes/rl-1.3.md rename to doc/manual/source/release-notes/rl-1.3.md diff --git a/doc/manual/src/release-notes/rl-1.4.md b/doc/manual/source/release-notes/rl-1.4.md similarity index 100% rename from doc/manual/src/release-notes/rl-1.4.md rename to doc/manual/source/release-notes/rl-1.4.md diff --git a/doc/manual/src/release-notes/rl-1.5.1.md b/doc/manual/source/release-notes/rl-1.5.1.md similarity index 100% rename from doc/manual/src/release-notes/rl-1.5.1.md rename to doc/manual/source/release-notes/rl-1.5.1.md diff --git a/doc/manual/src/release-notes/rl-1.5.2.md b/doc/manual/source/release-notes/rl-1.5.2.md similarity index 100% rename from doc/manual/src/release-notes/rl-1.5.2.md rename to doc/manual/source/release-notes/rl-1.5.2.md diff --git a/doc/manual/src/release-notes/rl-1.5.md b/doc/manual/source/release-notes/rl-1.5.md similarity index 100% rename from doc/manual/src/release-notes/rl-1.5.md rename to doc/manual/source/release-notes/rl-1.5.md diff --git a/doc/manual/src/release-notes/rl-1.6.1.md b/doc/manual/source/release-notes/rl-1.6.1.md similarity index 100% rename from doc/manual/src/release-notes/rl-1.6.1.md rename to doc/manual/source/release-notes/rl-1.6.1.md diff --git a/doc/manual/src/release-notes/rl-1.6.md b/doc/manual/source/release-notes/rl-1.6.md similarity index 100% rename from doc/manual/src/release-notes/rl-1.6.md rename to doc/manual/source/release-notes/rl-1.6.md diff --git a/doc/manual/src/release-notes/rl-1.7.md b/doc/manual/source/release-notes/rl-1.7.md similarity index 100% rename from doc/manual/src/release-notes/rl-1.7.md rename to doc/manual/source/release-notes/rl-1.7.md diff --git a/doc/manual/src/release-notes/rl-1.8.md b/doc/manual/source/release-notes/rl-1.8.md similarity index 100% rename from doc/manual/src/release-notes/rl-1.8.md rename to doc/manual/source/release-notes/rl-1.8.md diff --git a/doc/manual/src/release-notes/rl-1.9.md b/doc/manual/source/release-notes/rl-1.9.md similarity index 100% rename from doc/manual/src/release-notes/rl-1.9.md rename to doc/manual/source/release-notes/rl-1.9.md diff --git a/doc/manual/src/release-notes/rl-2.0.md b/doc/manual/source/release-notes/rl-2.0.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.0.md rename to doc/manual/source/release-notes/rl-2.0.md diff --git a/doc/manual/src/release-notes/rl-2.1.md b/doc/manual/source/release-notes/rl-2.1.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.1.md rename to doc/manual/source/release-notes/rl-2.1.md diff --git a/doc/manual/src/release-notes/rl-2.10.md b/doc/manual/source/release-notes/rl-2.10.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.10.md rename to doc/manual/source/release-notes/rl-2.10.md diff --git a/doc/manual/src/release-notes/rl-2.11.md b/doc/manual/source/release-notes/rl-2.11.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.11.md rename to doc/manual/source/release-notes/rl-2.11.md diff --git a/doc/manual/src/release-notes/rl-2.12.md b/doc/manual/source/release-notes/rl-2.12.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.12.md rename to doc/manual/source/release-notes/rl-2.12.md diff --git a/doc/manual/src/release-notes/rl-2.13.md b/doc/manual/source/release-notes/rl-2.13.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.13.md rename to doc/manual/source/release-notes/rl-2.13.md diff --git a/doc/manual/src/release-notes/rl-2.14.md b/doc/manual/source/release-notes/rl-2.14.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.14.md rename to doc/manual/source/release-notes/rl-2.14.md diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/source/release-notes/rl-2.15.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.15.md rename to doc/manual/source/release-notes/rl-2.15.md diff --git a/doc/manual/src/release-notes/rl-2.16.md b/doc/manual/source/release-notes/rl-2.16.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.16.md rename to doc/manual/source/release-notes/rl-2.16.md diff --git a/doc/manual/src/release-notes/rl-2.17.md b/doc/manual/source/release-notes/rl-2.17.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.17.md rename to doc/manual/source/release-notes/rl-2.17.md diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/source/release-notes/rl-2.18.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.18.md rename to doc/manual/source/release-notes/rl-2.18.md diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/source/release-notes/rl-2.19.md similarity index 94% rename from doc/manual/src/release-notes/rl-2.19.md rename to doc/manual/source/release-notes/rl-2.19.md index e2e2f85cc..e6a93c7ea 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/source/release-notes/rl-2.19.md @@ -75,3 +75,7 @@ (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). - A new command `nix store add` has been added. It replaces `nix store add-file` and `nix store add-path` which are now deprecated. + +- A new option [`always-allow-substitutes`](@docroot@/command-ref/conf-file.md#conf-always-allow-substitutes) has been added. + + When set to `true`, Nix will always try to substitute a derivation, even if it has the [`allowSubstitutes`]{#adv-attr-allowSubstitutes} attribute set to `false`. diff --git a/doc/manual/src/release-notes/rl-2.2.md b/doc/manual/source/release-notes/rl-2.2.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.2.md rename to doc/manual/source/release-notes/rl-2.2.md diff --git a/doc/manual/src/release-notes/rl-2.20.md b/doc/manual/source/release-notes/rl-2.20.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.20.md rename to doc/manual/source/release-notes/rl-2.20.md diff --git a/doc/manual/src/release-notes/rl-2.21.md b/doc/manual/source/release-notes/rl-2.21.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.21.md rename to doc/manual/source/release-notes/rl-2.21.md diff --git a/doc/manual/src/release-notes/rl-2.22.md b/doc/manual/source/release-notes/rl-2.22.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.22.md rename to doc/manual/source/release-notes/rl-2.22.md diff --git a/doc/manual/src/release-notes/rl-2.23.md b/doc/manual/source/release-notes/rl-2.23.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.23.md rename to doc/manual/source/release-notes/rl-2.23.md diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/source/release-notes/rl-2.24.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.24.md rename to doc/manual/source/release-notes/rl-2.24.md diff --git a/doc/manual/src/release-notes/rl-2.3.md b/doc/manual/source/release-notes/rl-2.3.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.3.md rename to doc/manual/source/release-notes/rl-2.3.md diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/source/release-notes/rl-2.4.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.4.md rename to doc/manual/source/release-notes/rl-2.4.md diff --git a/doc/manual/src/release-notes/rl-2.5.md b/doc/manual/source/release-notes/rl-2.5.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.5.md rename to doc/manual/source/release-notes/rl-2.5.md diff --git a/doc/manual/src/release-notes/rl-2.6.md b/doc/manual/source/release-notes/rl-2.6.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.6.md rename to doc/manual/source/release-notes/rl-2.6.md diff --git a/doc/manual/src/release-notes/rl-2.7.md b/doc/manual/source/release-notes/rl-2.7.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.7.md rename to doc/manual/source/release-notes/rl-2.7.md diff --git a/doc/manual/src/release-notes/rl-2.8.md b/doc/manual/source/release-notes/rl-2.8.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.8.md rename to doc/manual/source/release-notes/rl-2.8.md diff --git a/doc/manual/src/release-notes/rl-2.9.md b/doc/manual/source/release-notes/rl-2.9.md similarity index 100% rename from doc/manual/src/release-notes/rl-2.9.md rename to doc/manual/source/release-notes/rl-2.9.md diff --git a/doc/manual/src/store/file-system-object.md b/doc/manual/source/store/file-system-object.md similarity index 100% rename from doc/manual/src/store/file-system-object.md rename to doc/manual/source/store/file-system-object.md diff --git a/doc/manual/src/store/file-system-object/content-address.md b/doc/manual/source/store/file-system-object/content-address.md similarity index 97% rename from doc/manual/src/store/file-system-object/content-address.md rename to doc/manual/source/store/file-system-object/content-address.md index 410d7fb7c..72b087fe9 100644 --- a/doc/manual/src/store/file-system-object/content-address.md +++ b/doc/manual/source/store/file-system-object/content-address.md @@ -1,13 +1,13 @@ # Content-Addressing File System Objects -For many operations, Nix needs to calculate [a content addresses](@docroot@/glossary.md#gloss-content-address) of [a file system object][file system object]. +For many operations, Nix needs to calculate [a content addresses](@docroot@/glossary.md#gloss-content-address) of [a file system object][file system object] (FSO). Usually this is needed as part of [content addressing store objects](../store-object/content-address.md), since store objects always have a root file system object. But some command-line utilities also just work on "raw" file system objects, not part of any store object. Every content addressing scheme Nix uses ultimately involves feeding data into a [hash function](https://en.wikipedia.org/wiki/Hash_function), and getting back an opaque fixed-size digest which is deemed a content address. -The various *methods* of content addressing thus differ in how abstract data (in this case, a file system object and its descendents) are fed into the hash function. +The various *methods* of content addressing thus differ in how abstract data (in this case, a file system object and its descendants) are fed into the hash function. ## Serialising File System Objects { #serial } @@ -25,7 +25,7 @@ For example, Unix commands like `sha256sum` or `sha1sum` will produce hashes for ### Nix Archive (NAR) { #serial-nix-archive } -For the other cases of [file system objects][file system object], especially directories with arbitrary descendents, we need a more complex serialisation format. +For the other cases of [file system objects][file system object], especially directories with arbitrary descendants, we need a more complex serialisation format. Examples of such serialisations are the ZIP and TAR file formats. However, for our purposes these formats have two problems: diff --git a/doc/manual/src/store/index.md b/doc/manual/source/store/index.md similarity index 100% rename from doc/manual/src/store/index.md rename to doc/manual/source/store/index.md diff --git a/doc/manual/source/store/meson.build b/doc/manual/source/store/meson.build new file mode 100644 index 000000000..e3006020d --- /dev/null +++ b/doc/manual/source/store/meson.build @@ -0,0 +1,18 @@ +types_dir = custom_target( + command : [ + python.full_path(), + '@INPUT0@', + '@OUTPUT@', + '--' + ] + nix_eval_for_docs + [ + '--expr', + 'import @INPUT1@ (builtins.fromJSON (builtins.readFile ./@INPUT2@)).stores', + ], + input : [ + '../../remove_before_wrapper.py', + '../../generate-store-types.nix', + nix3_cli_json, + ], + output : 'types', + env : nix_env_for_docs, +) diff --git a/doc/manual/src/store/store-object.md b/doc/manual/source/store/store-object.md similarity index 100% rename from doc/manual/src/store/store-object.md rename to doc/manual/source/store/store-object.md diff --git a/doc/manual/src/store/store-object/content-address.md b/doc/manual/source/store/store-object/content-address.md similarity index 100% rename from doc/manual/src/store/store-object/content-address.md rename to doc/manual/source/store/store-object/content-address.md diff --git a/doc/manual/src/store/store-path.md b/doc/manual/source/store/store-path.md similarity index 100% rename from doc/manual/src/store/store-path.md rename to doc/manual/source/store/store-path.md diff --git a/doc/manual/src/store/types/index.md.in b/doc/manual/source/store/types/index.md.in similarity index 100% rename from doc/manual/src/store/types/index.md.in rename to doc/manual/source/store/types/index.md.in diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md deleted file mode 100644 index ddabaeb4d..000000000 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ /dev/null @@ -1,71 +0,0 @@ -# Remote Builds - -Nix supports remote builds, where a local Nix installation can forward -Nix builds to other machines. This allows multiple builds to be -performed in parallel and allows Nix to perform multi-platform builds in -a semi-transparent way. For instance, if you perform a build for a -`x86_64-darwin` on an `i686-linux` machine, Nix can automatically -forward the build to a `x86_64-darwin` machine, if available. - -To forward a build to a remote machine, it’s required that the remote -machine is accessible via SSH and that it has Nix installed. You can -test whether connecting to the remote Nix instance works, e.g. - -```console -$ nix store ping --store ssh://mac -``` - -will try to connect to the machine named `mac`. It is possible to -specify an SSH identity file as part of the remote store URI, e.g. - -```console -$ nix store ping --store ssh://mac?ssh-key=/home/alice/my-key -``` - -Since builds should be non-interactive, the key should not have a -passphrase. Alternatively, you can load identities ahead of time into -`ssh-agent` or `gpg-agent`. - -If you get the error - -```console -bash: nix-store: command not found -error: cannot connect to 'mac' -``` - -then you need to ensure that the `PATH` of non-interactive login shells -contains Nix. - -The [list of remote build machines](@docroot@/command-ref/conf-file.md#conf-builders) can be specified on the command line or in the Nix configuration file. -For example, the following command allows you to build a derivation for `x86_64-darwin` on a Linux machine: - -```console -$ uname -Linux - -$ nix build --impure \ - --expr '(with import { system = "x86_64-darwin"; }; runCommand "foo" {} "uname > $out")' \ - --builders 'ssh://mac x86_64-darwin' -[1/0/1 built, 0.0 MiB DL] building foo on ssh://mac - -$ cat ./result -Darwin -``` - -It is possible to specify multiple build machines separated by a semicolon or a newline, e.g. - -```console - --builders 'ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd' -``` - -Remote build machines can also be configured in [`nix.conf`](@docroot@/command-ref/conf-file.md), e.g. - - builders = ssh://mac x86_64-darwin ; ssh://beastie x86_64-freebsd - -Finally, remote build machines can be configured in a separate configuration -file included in `builders` via the syntax `@/path/to/file`. For example, - - builders = @/etc/nix/machines - -causes the list of machines in `/etc/nix/machines` to be included. -(This is the default.) diff --git a/doc/manual/substitute.py b/doc/manual/substitute.py new file mode 100644 index 000000000..a8b11d932 --- /dev/null +++ b/doc/manual/substitute.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import json +import os, os.path +import sys +import typing as t + +name = 'substitute.py' + +def log(*args: t.Any, **kwargs: t.Any) -> None: + kwargs['file'] = sys.stderr + print(f'{name}:', *args, **kwargs) + +def do_include(content: str, relative_md_path: Path, source_root: Path, search_path: Path) -> str: + assert not relative_md_path.is_absolute(), f'{relative_md_path=} from mdbook should be relative' + + md_path_abs = source_root / relative_md_path + var_abs = md_path_abs.parent + assert var_abs.is_dir(), f'supposed directory {var_abs} is not a directory (cwd={os.getcwd()})' + + lines = [] + for l in content.splitlines(keepends=True): + if l.strip().startswith("{{#include "): + requested = l.strip()[11:][:-2] + if requested.startswith("@generated@/"): + included = search_path / Path(requested[12:]) + requested = included.relative_to(search_path) + else: + included = source_root / relative_md_path.parent / requested + requested = included.resolve().relative_to(source_root) + assert included.exists(), f"{requested} not found at {included}" + lines.append(do_include(included.read_text(), requested, source_root, search_path) + "\n") + else: + lines.append(l) + return "".join(lines) + +def recursive_replace(data: dict[str, t.Any], book_root: Path, search_path: Path) -> dict[str, t.Any]: + match data: + case {'sections': sections}: + return data | dict( + sections = [recursive_replace(section, book_root, search_path) for section in sections], + ) + case {'Chapter': chapter}: + path_to_chapter = Path(chapter['path']) + chapter_content = chapter['content'] + + return data | dict( + Chapter = chapter | dict( + # first process includes. this must happen before docroot processing since + # mdbook does not see these included files, only the final agglomeration. + content = do_include( + chapter_content, + path_to_chapter, + book_root, + search_path + ).replace( + '@docroot@', + ("../" * len(path_to_chapter.parent.parts) or "./")[:-1] + ), + sub_items = [ + recursive_replace(sub_item, book_root, search_path) + for sub_item in chapter['sub_items'] + ], + ), + ) + + case rest: + assert False, f'should have been called on a dict, not {type(rest)=}\n\t{rest=}' + +def main() -> None: + + + if len(sys.argv) > 1 and sys.argv[1] == 'supports': + return 0 + + # includes pointing into @generated@ will look here + search_path = Path(os.environ['MDBOOK_SUBSTITUTE_SEARCH']) + + if len(sys.argv) > 1 and sys.argv[1] == 'summary': + print(do_include( + sys.stdin.read(), + Path('source/SUMMARY.md'), + Path(sys.argv[2]).resolve(), + search_path)) + return + + # mdbook communicates with us over stdin and stdout. + # It splorks us a JSON array, the first element describing the context, + # the second element describing the book itself, + # and then expects us to send it the modified book JSON over stdout. + + context, book = json.load(sys.stdin) + + # book_root is the directory where book contents leave (ie, source/) + book_root = Path(context['root']) / context['config']['book']['src'] + + # Find @var@ in all parts of our recursive book structure. + replaced_content = recursive_replace(book, book_root, search_path) + + replaced_content_str = json.dumps(replaced_content) + + # Give mdbook our changes. + print(replaced_content_str) + +try: + sys.exit(main()) +except AssertionError as e: + print(f'{name}: INTERNAL ERROR in mdbook preprocessor: {e}', file=sys.stderr) + print(f'this is a bug in {name}', file=sys.stderr) + raise diff --git a/flake.nix b/flake.nix index cbcf10021..303779c2b 100644 --- a/flake.nix +++ b/flake.nix @@ -221,6 +221,7 @@ inherit (nixpkgsFor.${system}.native) changelog-d; default = self.packages.${system}.nix-ng; + nix-manual = nixpkgsFor.${system}.native.nixComponents.nix-manual; nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; } @@ -349,9 +350,10 @@ ++ pkgs.nixComponents.nix-store.nativeBuildInputs ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs + ++ lib.optionals buildCanExecuteHost pkgs.nixComponents.nix-manual.externalNativeBuildInputs ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs - ++ pkgs.nixComponents.nix-functional-tests.baseNativeBuildInputs + ++ pkgs.nixComponents.nix-functional-tests.externalNativeBuildInputs ++ lib.optional (!buildCanExecuteHost # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 diff --git a/maintainers/README.md b/maintainers/README.md index b92833497..fd9bbed8e 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -29,11 +29,7 @@ We aim to achieve this by improving the contributor experience and attracting mo ## Members -- Eelco Dolstra (@edolstra) – Team lead -- Valentin Gagarin (@fricklerhandwerk) -- Thomas Bereknyei (@tomberek) -- Robert Hensing (@roberth) -- John Ericson (@Ericson2314) +See https://nixos.org/community/teams/nix/ for the current team membership. The team is on Github as [@NixOS/nix-team](https://github.com/orgs/NixOS/teams/nix-team). @@ -63,6 +59,9 @@ Team meetings are generally open to anyone interested. We can make exceptions to discuss sensitive issues, such as security incidents or people matters. Contact any team member to get a calendar invite for reminders and updates. +> [!IMPORTANT] +> [Handling security reports](./security-reports.md) always takes priority. + ## Project board protocol The team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19/views/1) for tracking its work. diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index fb286208d..225b0b300 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -7,7 +7,7 @@ perSystem = { config, pkgs, ... }: { - # https://flake.parts/options/pre-commit-hooks-nix.html#options + # https://flake.parts/options/git-hooks-nix#options pre-commit.settings = { hooks = { clang-format = { @@ -15,7 +15,7 @@ excludes = [ # We don't want to format test data # ''tests/(?!nixos/).*\.nix'' - ''^tests/unit/[^/]*/data/.*$'' + ''^src/[^/]*-tests/data/.*$'' # Don't format vendored code ''^doc/manual/redirects\.js$'' @@ -427,64 +427,64 @@ ''^tests/nixos/ca-fd-leak/sender\.c'' ''^tests/nixos/ca-fd-leak/smuggler\.c'' ''^tests/nixos/user-sandboxing/attacker\.c'' - ''^tests/unit/libexpr-support/tests/libexpr\.hh'' - ''^tests/unit/libexpr-support/tests/value/context\.cc'' - ''^tests/unit/libexpr-support/tests/value/context\.hh'' - ''^tests/unit/libexpr/derived-path\.cc'' - ''^tests/unit/libexpr/error_traces\.cc'' - ''^tests/unit/libexpr/eval\.cc'' - ''^tests/unit/libexpr/json\.cc'' - ''^tests/unit/libexpr/main\.cc'' - ''^tests/unit/libexpr/primops\.cc'' - ''^tests/unit/libexpr/search-path\.cc'' - ''^tests/unit/libexpr/trivial\.cc'' - ''^tests/unit/libexpr/value/context\.cc'' - ''^tests/unit/libexpr/value/print\.cc'' - ''^tests/unit/libfetchers/public-key\.cc'' - ''^tests/unit/libflake/flakeref\.cc'' - ''^tests/unit/libflake/url-name\.cc'' - ''^tests/unit/libstore-support/tests/derived-path\.cc'' - ''^tests/unit/libstore-support/tests/derived-path\.hh'' - ''^tests/unit/libstore-support/tests/nix_api_store\.hh'' - ''^tests/unit/libstore-support/tests/outputs-spec\.cc'' - ''^tests/unit/libstore-support/tests/outputs-spec\.hh'' - ''^tests/unit/libstore-support/tests/path\.cc'' - ''^tests/unit/libstore-support/tests/path\.hh'' - ''^tests/unit/libstore-support/tests/protocol\.hh'' - ''^tests/unit/libstore/common-protocol\.cc'' - ''^tests/unit/libstore/content-address\.cc'' - ''^tests/unit/libstore/derivation\.cc'' - ''^tests/unit/libstore/derived-path\.cc'' - ''^tests/unit/libstore/downstream-placeholder\.cc'' - ''^tests/unit/libstore/machines\.cc'' - ''^tests/unit/libstore/nar-info-disk-cache\.cc'' - ''^tests/unit/libstore/nar-info\.cc'' - ''^tests/unit/libstore/outputs-spec\.cc'' - ''^tests/unit/libstore/path-info\.cc'' - ''^tests/unit/libstore/path\.cc'' - ''^tests/unit/libstore/serve-protocol\.cc'' - ''^tests/unit/libstore/worker-protocol\.cc'' - ''^tests/unit/libutil-support/tests/characterization\.hh'' - ''^tests/unit/libutil-support/tests/hash\.cc'' - ''^tests/unit/libutil-support/tests/hash\.hh'' - ''^tests/unit/libutil/args\.cc'' - ''^tests/unit/libutil/canon-path\.cc'' - ''^tests/unit/libutil/chunked-vector\.cc'' - ''^tests/unit/libutil/closure\.cc'' - ''^tests/unit/libutil/compression\.cc'' - ''^tests/unit/libutil/config\.cc'' - ''^tests/unit/libutil/file-content-address\.cc'' - ''^tests/unit/libutil/git\.cc'' - ''^tests/unit/libutil/hash\.cc'' - ''^tests/unit/libutil/hilite\.cc'' - ''^tests/unit/libutil/json-utils\.cc'' - ''^tests/unit/libutil/logging\.cc'' - ''^tests/unit/libutil/lru-cache\.cc'' - ''^tests/unit/libutil/pool\.cc'' - ''^tests/unit/libutil/references\.cc'' - ''^tests/unit/libutil/suggestions\.cc'' - ''^tests/unit/libutil/url\.cc'' - ''^tests/unit/libutil/xml-writer\.cc'' + ''^src/libexpr-test-support/tests/libexpr\.hh'' + ''^src/libexpr-test-support/tests/value/context\.cc'' + ''^src/libexpr-test-support/tests/value/context\.hh'' + ''^src/libexpr-tests/derived-path\.cc'' + ''^src/libexpr-tests/error_traces\.cc'' + ''^src/libexpr-tests/eval\.cc'' + ''^src/libexpr-tests/json\.cc'' + ''^src/libexpr-tests/main\.cc'' + ''^src/libexpr-tests/primops\.cc'' + ''^src/libexpr-tests/search-path\.cc'' + ''^src/libexpr-tests/trivial\.cc'' + ''^src/libexpr-tests/value/context\.cc'' + ''^src/libexpr-tests/value/print\.cc'' + ''^src/libfetchers-tests/public-key\.cc'' + ''^src/libflake-tests/flakeref\.cc'' + ''^src/libflake-tests/url-name\.cc'' + ''^src/libstore-test-support/tests/derived-path\.cc'' + ''^src/libstore-test-support/tests/derived-path\.hh'' + ''^src/libstore-test-support/tests/nix_api_store\.hh'' + ''^src/libstore-test-support/tests/outputs-spec\.cc'' + ''^src/libstore-test-support/tests/outputs-spec\.hh'' + ''^src/libstore-test-support/tests/path\.cc'' + ''^src/libstore-test-support/tests/path\.hh'' + ''^src/libstore-test-support/tests/protocol\.hh'' + ''^src/libstore-tests/common-protocol\.cc'' + ''^src/libstore-tests/content-address\.cc'' + ''^src/libstore-tests/derivation\.cc'' + ''^src/libstore-tests/derived-path\.cc'' + ''^src/libstore-tests/downstream-placeholder\.cc'' + ''^src/libstore-tests/machines\.cc'' + ''^src/libstore-tests/nar-info-disk-cache\.cc'' + ''^src/libstore-tests/nar-info\.cc'' + ''^src/libstore-tests/outputs-spec\.cc'' + ''^src/libstore-tests/path-info\.cc'' + ''^src/libstore-tests/path\.cc'' + ''^src/libstore-tests/serve-protocol\.cc'' + ''^src/libstore-tests/worker-protocol\.cc'' + ''^src/libutil-test-support/tests/characterization\.hh'' + ''^src/libutil-test-support/tests/hash\.cc'' + ''^src/libutil-test-support/tests/hash\.hh'' + ''^src/libutil-tests/args\.cc'' + ''^src/libutil-tests/canon-path\.cc'' + ''^src/libutil-tests/chunked-vector\.cc'' + ''^src/libutil-tests/closure\.cc'' + ''^src/libutil-tests/compression\.cc'' + ''^src/libutil-tests/config\.cc'' + ''^src/libutil-tests/file-content-address\.cc'' + ''^src/libutil-tests/git\.cc'' + ''^src/libutil-tests/hash\.cc'' + ''^src/libutil-tests/hilite\.cc'' + ''^src/libutil-tests/json-utils\.cc'' + ''^src/libutil-tests/logging\.cc'' + ''^src/libutil-tests/lru-cache\.cc'' + ''^src/libutil-tests/pool\.cc'' + ''^src/libutil-tests/references\.cc'' + ''^src/libutil-tests/suggestions\.cc'' + ''^src/libutil-tests/url\.cc'' + ''^src/libutil-tests/xml-writer\.cc'' ]; }; shellcheck = { @@ -501,7 +501,6 @@ ''^scripts/install-nix-from-closure\.sh$'' ''^scripts/install-systemd-multi-user\.sh$'' ''^src/nix/get-env\.sh$'' - ''^tests/functional/build\.sh$'' ''^tests/functional/ca/build-dry\.sh$'' ''^tests/functional/ca/build-with-garbage-path\.sh$'' ''^tests/functional/ca/common\.sh$'' @@ -517,7 +516,6 @@ ''^tests/functional/ca/selfref-gc\.sh$'' ''^tests/functional/ca/why-depends\.sh$'' ''^tests/functional/characterisation-test-infra\.sh$'' - ''^tests/functional/check\.sh$'' ''^tests/functional/common/vars-and-functions\.sh$'' ''^tests/functional/completions\.sh$'' ''^tests/functional/compute-levels\.sh$'' @@ -534,7 +532,6 @@ ''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$'' ''^tests/functional/dyn-drv/recursive-mod-json\.sh$'' ''^tests/functional/eval-store\.sh$'' - ''^tests/functional/eval\.sh$'' ''^tests/functional/export-graph\.sh$'' ''^tests/functional/export\.sh$'' ''^tests/functional/extra-sandbox-profile\.sh$'' @@ -544,13 +541,11 @@ ''^tests/functional/fetchGitSubmodules\.sh$'' ''^tests/functional/fetchGitVerification\.sh$'' ''^tests/functional/fetchMercurial\.sh$'' - ''^tests/functional/fetchurl\.sh$'' ''^tests/functional/fixed\.builder1\.sh$'' ''^tests/functional/fixed\.builder2\.sh$'' ''^tests/functional/fixed\.sh$'' ''^tests/functional/flakes/absolute-paths\.sh$'' ''^tests/functional/flakes/check\.sh$'' - ''^tests/functional/flakes/common\.sh$'' ''^tests/functional/flakes/config\.sh$'' ''^tests/functional/flakes/develop\.sh$'' ''^tests/functional/flakes/flakes\.sh$'' @@ -565,16 +560,12 @@ ''^tests/functional/gc-concurrent\.sh$'' ''^tests/functional/gc-concurrent2\.builder\.sh$'' ''^tests/functional/gc-non-blocking\.sh$'' - ''^tests/functional/gc\.sh$'' ''^tests/functional/git-hashing/common\.sh$'' ''^tests/functional/git-hashing/simple\.sh$'' ''^tests/functional/hash-convert\.sh$'' - ''^tests/functional/help\.sh$'' ''^tests/functional/impure-derivations\.sh$'' - ''^tests/functional/impure-env\.sh$'' ''^tests/functional/impure-eval\.sh$'' ''^tests/functional/install-darwin\.sh$'' - ''^tests/functional/lang\.sh$'' ''^tests/functional/legacy-ssh-store\.sh$'' ''^tests/functional/linux-sandbox\.sh$'' ''^tests/functional/local-overlay-store/add-lower-inner\.sh$'' @@ -603,7 +594,6 @@ ''^tests/functional/logging\.sh$'' ''^tests/functional/misc\.sh$'' ''^tests/functional/multiple-outputs\.sh$'' - ''^tests/functional/nar-access\.sh$'' ''^tests/functional/nested-sandboxing\.sh$'' ''^tests/functional/nested-sandboxing/command\.sh$'' ''^tests/functional/nix-build\.sh$'' @@ -624,7 +614,6 @@ ''^tests/functional/path-from-hash-part\.sh$'' ''^tests/functional/path-info\.sh$'' ''^tests/functional/placeholders\.sh$'' - ''^tests/functional/plugins\.sh$'' ''^tests/functional/post-hook\.sh$'' ''^tests/functional/pure-eval\.sh$'' ''^tests/functional/push-to-store-old\.sh$'' @@ -639,7 +628,6 @@ ''^tests/functional/search\.sh$'' ''^tests/functional/secure-drv-outputs\.sh$'' ''^tests/functional/selfref-gc\.sh$'' - ''^tests/functional/shell\.sh$'' ''^tests/functional/shell\.shebang\.sh$'' ''^tests/functional/simple\.builder\.sh$'' ''^tests/functional/supplementary-groups\.sh$'' @@ -649,8 +637,7 @@ ''^tests/functional/user-envs\.builder\.sh$'' ''^tests/functional/user-envs\.sh$'' ''^tests/functional/why-depends\.sh$'' - ''^tests/functional/zstd\.sh$'' - ''^tests/unit/libutil/data/git/check-data\.sh$'' + ''^src/libutil-tests/data/git/check-data\.sh$'' ]; }; # TODO: nixfmt, https://github.com/NixOS/nixfmt/issues/153 diff --git a/maintainers/onboarding.md b/maintainers/onboarding.md new file mode 100644 index 000000000..e750bd8a7 --- /dev/null +++ b/maintainers/onboarding.md @@ -0,0 +1,6 @@ + +# Onboarding a new team member + +- https://github.com/NixOS/nixos-homepage/ +- https://github.com/orgs/NixOS/teams/nix-team +- Matrix room diff --git a/maintainers/release-notes b/maintainers/release-notes index c0c4ee734..0cdcd517b 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -78,7 +78,7 @@ if ! git diff --quiet --cached; then die "repo has staged changes, please commit or stash them" fi -if ! grep "$SUMMARY_MARKER_LINE" doc/manual/src/SUMMARY.md.in >/dev/null; then +if ! grep "$SUMMARY_MARKER_LINE" doc/manual/source/SUMMARY.md.in >/dev/null; then # would have been nice to catch this early, but won't be worth the extra infra die "SUMMARY.md.in is missing the marker line '$SUMMARY_MARKER_LINE', which would be used for inserting a new release notes page. Please fix the script." fi @@ -117,7 +117,7 @@ log "version_full=$version_full" log "IS_PATCH=$IS_PATCH" basename=rl-${version_major_minor}.md -file=doc/manual/src/release-notes/$basename +file=doc/manual/source/release-notes/$basename if ! $IS_PATCH; then if [[ -e $file ]]; then @@ -169,7 +169,7 @@ if ! $IS_PATCH; then # find the marker line, insert new link after it escaped_marker="$(echo "$SUMMARY_MARKER_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" escaped_line="$(echo "$NEW_SUMMARY_LINE" | sed -e 's/\//\\\//g' -e 's/ /\\ /g')" - logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/src/SUMMARY.md.in + logcmd sed -i -e "/$escaped_marker/a $escaped_line" doc/manual/source/SUMMARY.md.in fi for f in doc/manual/rl-next/*.md; do @@ -178,7 +178,7 @@ for f in doc/manual/rl-next/*.md; do fi done -logcmd git add $file doc/manual/src/SUMMARY.md.in +logcmd git add $file doc/manual/source/SUMMARY.md.in logcmd git status logcmd git commit -m "release notes: $version_full" diff --git a/maintainers/release-process.md b/maintainers/release-process.md index 7a2b3c0a7..bf3c308cf 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -15,7 +15,7 @@ release: * (Optionally) Updated `fallback-paths.nix` in Nixpkgs -* An updated manual on https://nixos.org/manual/nix/stable/ +* An updated manual on https://nix.dev/manual/nix/latest/ ## Creating a new release from the `master` branch @@ -194,8 +194,58 @@ release: * Bump the version number of the release branch as above (e.g. to `2.12.2`). - + ## Recovering from mistakes `upload-release.pl` should be idempotent. For instance a wrong `IS_LATEST` value can be fixed that way, by running the script on the actual latest release. +## Security releases + +> See also the instructions for [handling security reports](./security-reports.md). + +Once a security fix is ready for merging: + +1. Summarize *all* past communication in the report. + +1. Request a CVE in the [GitHub security advisory](https://github.com/NixOS/nix/security/advisories) for the security fix. + +1. Notify all collaborators on the advisory with a timeline for the release. + +1. Merge the fix. Publish the advisory. + +1. [Make point releases](#creating-point-releases) for all affected versions. + +1. Update the affected Nix releases in Nixpkgs to the patched version. + + For each Nix release, change the `version = ` strings and run + + ```shell-session + nix-build -A nixVersions.nix__ + ``` + + to get the correct hash for the `hash =` field. + +1. Once the release is built by Hydra, update fallback paths. + + For the Nix release `${version}` shipped with Nixpkgs, run: + + ```shell-session + curl https://releases.nixos.org/nix/nix-${version}/fallback-paths.nix > nixos/modules/installer/tools/nix-fallback-paths.nix + ``` + + Starting with Nixpkgs 24.11, there is an automatic check that fallback paths with Nix binaries match the Nix release shipped with Nixpkgs. + +1. Backport the updates to the two most recent stable releases of Nixpkgs. + + Add `backport release-` labels, which will trigger GitHub Actions to attempt automatic backports. + +1. Once the pull request against `master` lands on `nixpkgs-unstable`, post a Discourse announcement with + + - Links to the CVE and GitHub security advisory + - A description of the vulnerability and its fix + - Credits to the reporters of the vulnerability and contributors of the fix + - A list of affected and patched Nix releases + - Instructions for updating + - A link to the [pull request tracker](https://nixpk.gs/pr-tracker.html) to follow when the patched Nix versions will appear on the various release channels + + Check [past announcements](https://discourse.nixos.org/search?expanded=true&q=Security%20fix%20in%3Atitle%20order%3Alatest_topic) for reference. diff --git a/maintainers/security-reports.md b/maintainers/security-reports.md new file mode 100644 index 000000000..700590cf7 --- /dev/null +++ b/maintainers/security-reports.md @@ -0,0 +1,31 @@ +# Handling security reports + +Reports can be expected to be submitted following the [security policy](https://github.com/NixOS/nix/security/policy), but may reach maintainers on various other channels. + +In case a vulnerability is reported: + +1. [Create a GitHub security advisory](https://github.com/NixOS/nix/security/advisories/new) + + > [!IMPORTANT] + > Add the reporter as a collaborator so they get notified of all activities. + + In addition to the details in the advisory template, the initial report should: + + - Include sufficient details of the vulnerability to allow it to be understood and reproduced. + - Redact any personal data. + - Set a deadline (if applicable). + - Provide proof of concept code (if available). + - Reference any further reading material that may be appropriate. + +1. Establish a private communication channel (e.g. a Matrix room) with the reporter and all Nix maintainers. + +1. Communicate with the reporter which team members are assigned and when they are available. + +1. Consider which immediate preliminary measures should be taken before working on a fix. + +1. Prioritize fixing the security issue over ongoing work. + +1. Keep everyone involved up to date on progress and the estimated timeline for releasing the fix. + +> See also the instructions for [security releases](./release-process.md#security-releases). + diff --git a/meson.build b/meson.build index 8dd44cc10..d68db0a0d 100644 --- a/meson.build +++ b/meson.build @@ -3,6 +3,7 @@ project('nix-dev-shell', 'cpp', version : files('.version'), + subproject_dir : 'src', default_options : [ 'localstatedir=/nix/var', ] @@ -23,6 +24,9 @@ subproject('nix') # Docs subproject('internal-api-docs') subproject('external-api-docs') +if not meson.is_cross_build() + subproject('nix-manual') +endif # External C wrapper libraries subproject('libutil-c') @@ -36,12 +40,12 @@ if not meson.is_cross_build() endif # Testing -subproject('nix-util-test-support') -subproject('nix-util-tests') -subproject('nix-store-test-support') -subproject('nix-store-tests') -subproject('nix-fetchers-tests') -subproject('nix-expr-test-support') -subproject('nix-expr-tests') -subproject('nix-flake-tests') +subproject('libutil-test-support') +subproject('libutil-tests') +subproject('libstore-test-support') +subproject('libstore-tests') +subproject('libfetchers-tests') +subproject('libexpr-test-support') +subproject('libexpr-tests') +subproject('libflake-tests') subproject('nix-functional-tests') diff --git a/mk/common-test.sh b/mk/common-test.sh index 817422c40..dd899e869 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -9,13 +9,21 @@ test_name=$(echo -n "${test?must be defined by caller (test runner)}" | sed \ -e "s|\.sh$||" \ ) +# Layer violation, but I am not inclined to care too much, as this code +# is about to be deleted. +src_dir=$(realpath tests/functional) + # shellcheck disable=SC2016 TESTS_ENVIRONMENT=( "TEST_NAME=$test_name" 'NIX_REMOTE=' 'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) ' + "_NIX_TEST_SOURCE_DIR=${src_dir}" + "_NIX_TEST_BUILD_DIR=${src_dir}" ) +unset src_dir + read -r -a bash <<< "${BASH:-/usr/bin/env bash}" run () { diff --git a/mk/libraries.mk b/mk/libraries.mk index b99ba2782..a7848ba35 100644 --- a/mk/libraries.mk +++ b/mk/libraries.mk @@ -86,7 +86,9 @@ define build-library else ifndef HOST_DARWIN ifndef HOST_WINDOWS - $(1)_LDFLAGS += -Wl,-z,defs + ifndef HOST_OPENBSD + $(1)_LDFLAGS += -Wl,-z,defs + endif endif endif endif diff --git a/mk/platform.mk b/mk/platform.mk index 22c114a20..3c4fff780 100644 --- a/mk/platform.mk +++ b/mk/platform.mk @@ -21,6 +21,10 @@ ifdef HOST_OS HOST_NETBSD = 1 HOST_UNIX = 1 endif + ifeq ($(patsubst openbsd%,,$(HOST_KERNEL)),) + HOST_OPENBSD = 1 + HOST_UNIX = 1 + endif ifeq ($(HOST_KERNEL), linux) HOST_LINUX = 1 HOST_UNIX = 1 diff --git a/package.nix b/package.nix index 8ab184667..8cbe325e9 100644 --- a/package.nix +++ b/package.nix @@ -23,6 +23,7 @@ , libseccomp , libsodium , man +, darwin , lowdown , mdbook , mdbook-linkcheck @@ -76,7 +77,9 @@ # # Temporarily disabled on Windows because the `GC_throw_bad_alloc` # symbol is missing during linking. -, enableGC ? !stdenv.hostPlatform.isWindows +# +# Disabled on OpenBSD because of missing `_data_start` symbol while linking +, enableGC ? !stdenv.hostPlatform.isWindows && !stdenv.hostPlatform.isOpenBSD # Whether to enable Markdown rendering in the Nix binary. , enableMarkdown ? !stdenv.hostPlatform.isWindows @@ -176,8 +179,6 @@ in { ./scripts/local.mk ] ++ lib.optionals enableManual [ ./doc/manual - ] ++ lib.optionals buildUnitTests [ - ./tests/unit ] ++ lib.optionals doInstallCheck [ ./tests/functional ])); @@ -209,9 +210,10 @@ in { git mercurial openssh - man # for testing `nix-* --help` ] ++ lib.optionals (doInstallCheck || enableManual) [ jq # Also for custom mdBook preprocessor. + ] ++ lib.optionals enableManual [ + man ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump ; @@ -234,6 +236,7 @@ in { gtest rapidcheck ] ++ lib.optional stdenv.isLinux libseccomp + ++ lib.optional stdenv.hostPlatform.isDarwin darwin.apple_sdk.libs.sandbox ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) diff --git a/packaging/components.nix b/packaging/components.nix index 5fc3236cf..5cc0be784 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -32,24 +32,24 @@ in nix-util = callPackage ../src/libutil/package.nix { }; nix-util-c = callPackage ../src/libutil-c/package.nix { }; - nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { }; - nix-util-tests = callPackage ../tests/unit/libutil/package.nix { }; + nix-util-test-support = callPackage ../src/libutil-test-support/package.nix { }; + nix-util-tests = callPackage ../src/libutil-tests/package.nix { }; nix-store = callPackage ../src/libstore/package.nix { }; nix-store-c = callPackage ../src/libstore-c/package.nix { }; - nix-store-test-support = callPackage ../tests/unit/libstore-support/package.nix { }; - nix-store-tests = callPackage ../tests/unit/libstore/package.nix { }; + nix-store-test-support = callPackage ../src/libstore-test-support/package.nix { }; + nix-store-tests = callPackage ../src/libstore-tests/package.nix { }; nix-fetchers = callPackage ../src/libfetchers/package.nix { }; - nix-fetchers-tests = callPackage ../tests/unit/libfetchers/package.nix { }; + nix-fetchers-tests = callPackage ../src/libfetchers-tests/package.nix { }; nix-expr = callPackage ../src/libexpr/package.nix { }; nix-expr-c = callPackage ../src/libexpr-c/package.nix { }; - nix-expr-test-support = callPackage ../tests/unit/libexpr-support/package.nix { }; - nix-expr-tests = callPackage ../tests/unit/libexpr/package.nix { }; + nix-expr-test-support = callPackage ../src/libexpr-test-support/package.nix { }; + nix-expr-tests = callPackage ../src/libexpr-tests/package.nix { }; nix-flake = callPackage ../src/libflake/package.nix { }; - nix-flake-tests = callPackage ../tests/unit/libflake/package.nix { }; + nix-flake-tests = callPackage ../src/libflake-tests/package.nix { }; nix-main = callPackage ../src/libmain/package.nix { }; nix-main-c = callPackage ../src/libmain-c/package.nix { }; @@ -60,6 +60,7 @@ in nix-functional-tests = callPackage ../src/nix-functional-tests/package.nix { version = fineVersion; }; + nix-manual = callPackage ../doc/manual/package.nix { version = fineVersion; }; nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { version = fineVersion; }; nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; }; diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index e5f4c0f91..13766f2c0 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -38,6 +38,10 @@ let # Indirection for Nixpkgs to override when package.nix files are vendored filesetToSource = lib.fileset.toSource; + /** Given a set of layers, create a mkDerivation-like function */ + mkPackageBuilder = exts: userFn: + stdenv.mkDerivation (lib.extends (lib.composeManyExtensions exts) userFn); + localSourceLayer = finalAttrs: prevAttrs: let workDirPath = @@ -60,6 +64,28 @@ let workDir = null; }; + mesonLayer = finalAttrs: prevAttrs: + { + nativeBuildInputs = [ + pkgs.buildPackages.meson + pkgs.buildPackages.ninja + ] ++ prevAttrs.nativeBuildInputs or []; + }; + + mesonBuildLayer = finalAttrs: prevAttrs: + { + nativeBuildInputs = prevAttrs.nativeBuildInputs or [] ++ [ + pkgs.buildPackages.pkg-config + ]; + separateDebugInfo = !stdenv.hostPlatform.isStatic; + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + }; + + mesonLibraryLayer = finalAttrs: prevAttrs: + { + outputs = prevAttrs.outputs or [ "out" ] ++ [ "dev" ]; + }; + # Work around weird `--as-needed` linker behavior with BSD, see # https://github.com/mesonbuild/meson/issues/3593 bsdNoLinkAsNeeded = finalAttrs: prevAttrs: @@ -172,14 +198,27 @@ scope: { inherit resolvePath filesetToSource; - mkMesonDerivation = f: let - exts = [ + mkMesonDerivation = + mkPackageBuilder [ + miscGoodPractice + localSourceLayer + mesonLayer + ]; + mkMesonExecutable = + mkPackageBuilder [ miscGoodPractice bsdNoLinkAsNeeded localSourceLayer + mesonLayer + mesonBuildLayer + ]; + mkMesonLibrary = + mkPackageBuilder [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + mesonLayer + mesonBuildLayer + mesonLibraryLayer ]; - in stdenv.mkDerivation - (lib.extends - (lib.foldr lib.composeExtensions (_: _: {}) exts) - f); } diff --git a/packaging/everything.nix b/packaging/everything.nix index d26c81572..ae2f93da0 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -33,6 +33,7 @@ nix-functional-tests, + nix-manual, nix-internal-api-docs, nix-external-api-docs, @@ -70,6 +71,7 @@ nix-cli + nix-manual nix-internal-api-docs nix-external-api-docs diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 65978835c..cba1b2583 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -146,6 +146,9 @@ in withCoverageChecks = true; }; + # Nix's manual + manual = nixpkgsFor.x86_64-linux.native.nixComponents.nix-manual; + # API docs for Nix's unstable internal C++ interfaces. internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs; diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index a79a69990..dc373f4db 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -96,6 +96,9 @@ poly_configure_nix_daemon_service() { if [ -e /run/systemd/system ]; then task "Setting up the nix-daemon systemd service" + _sudo "to create parent of the nix-daemon tmpfiles config" \ + mkdir -p "$(dirname "$TMPFILES_DEST")" + _sudo "to create the nix-daemon tmpfiles config" \ ln -sfn "/nix/var/nix/profiles/default$TMPFILES_SRC" "$TMPFILES_DEST" diff --git a/scripts/meson.build b/scripts/meson.build index 2671e6a13..777da42b1 100644 --- a/scripts/meson.build +++ b/scripts/meson.build @@ -1,5 +1,3 @@ -# configures `scripts/nix-profile.sh.in` (and copies the original to the build directory). -# this is only needed for tests, but running it unconditionally does not hurt enough to care. configure_file( input : 'nix-profile.sh.in', output : 'nix-profile.sh', @@ -8,13 +6,6 @@ configure_file( } ) -# https://github.com/mesonbuild/meson/issues/860 -configure_file( - input : 'nix-profile.sh.in', - output : 'nix-profile.sh.in', - copy : true, -) - foreach rc : [ '.sh', '.fish', '-daemon.sh', '-daemon.fish' ] configure_file( input : 'nix-profile' + rc + '.in', diff --git a/src/external-api-docs/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in index 1be71d895..7ae4c83df 100644 --- a/src/external-api-docs/doxygen.cfg.in +++ b/src/external-api-docs/doxygen.cfg.in @@ -56,3 +56,5 @@ GENERATE_TREEVIEW = YES OPTIMIZE_OUTPUT_FOR_C = YES USE_MDFILE_AS_MAINPAGE = doc/external-api/README.md + +QUIET = YES diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix index 743b3e9b7..0c592955a 100644 --- a/src/external-api-docs/package.nix +++ b/src/external-api-docs/package.nix @@ -1,8 +1,6 @@ { lib , mkMesonDerivation -, meson -, ninja , doxygen # Configuration Options @@ -37,8 +35,6 @@ mkMesonDerivation (finalAttrs: { ]; nativeBuildInputs = [ - meson - ninja doxygen ]; diff --git a/src/internal-api-docs/doxygen.cfg.in b/src/internal-api-docs/doxygen.cfg.in index f1ef75b38..bf4c42d11 100644 --- a/src/internal-api-docs/doxygen.cfg.in +++ b/src/internal-api-docs/doxygen.cfg.in @@ -41,21 +41,21 @@ INPUT = \ @src@/libcmd \ @src@/libexpr \ @src@/libexpr/flake \ - @src@/nix-expr-tests \ - @src@/nix-expr-tests/value \ - @src@/nix-expr-test-support/test \ - @src@/nix-expr-test-support/test/value \ + @src@/libexpr-tests \ + @src@/libexpr-tests/value \ + @src@/libexpr-test-support/test \ + @src@/libexpr-test-support/test/value \ @src@/libexpr/value \ @src@/libfetchers \ @src@/libmain \ @src@/libstore \ @src@/libstore/build \ @src@/libstore/builtins \ - @src@/nix-store-tests \ - @src@/nix-store-test-support/test \ + @src@/libstore-tests \ + @src@/libstore-test-support/test \ @src@/libutil \ - @src@/nix-util-tests \ - @src@/nix-util-test-support/test \ + @src@/libutil-tests \ + @src@/libutil-test-support/test \ @src@/nix \ @src@/nix-env \ @src@/nix-store @@ -97,3 +97,6 @@ EXPAND_AS_DEFINED = \ DECLARE_WORKER_SERIALISER \ DECLARE_SERVE_SERIALISER \ LENGTH_PREFIXED_PROTO_HELPER + +WARN_IF_UNDOCUMENTED = NO +QUIET = YES diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix index 07ca6d4d9..993a257a6 100644 --- a/src/internal-api-docs/package.nix +++ b/src/internal-api-docs/package.nix @@ -1,8 +1,6 @@ { lib , mkMesonDerivation -, meson -, ninja , doxygen # Configuration Options @@ -32,8 +30,6 @@ mkMesonDerivation (finalAttrs: { ]; nativeBuildInputs = [ - meson - ninja doxygen ]; diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index cde494901..244179ee4 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -1,11 +1,6 @@ { lib , stdenv -, mkMesonDerivation -, releaseTools - -, meson -, ninja -, pkg-config +, mkMesonLibrary , nix-util , nix-store @@ -38,7 +33,7 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonLibrary (finalAttrs: { pname = "nix-cmd"; inherit version; @@ -54,14 +49,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - outputs = [ "out" "dev" ]; - - nativeBuildInputs = [ - meson - ninja - pkg-config - ]; - buildInputs = [ ({ inherit editline readline; }.${readlineFlavor}) ] ++ lib.optional enableMarkdown lowdown; @@ -93,10 +80,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 940b16dfd..f292f06bb 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -645,7 +645,7 @@ ProcessLineResult NixRepl::processLine(std::string line) logger->cout(trim(renderMarkdownToTerminal(markdown))); } else if (fallbackPos) { - std::stringstream ss; + std::ostringstream ss; ss << "Attribute `" << fallbackName << "`\n\n"; ss << " … defined at " << state->positions[fallbackPos] << "\n\n"; if (fallbackDoc) { @@ -654,7 +654,7 @@ ProcessLineResult NixRepl::processLine(std::string line) ss << "No documentation found.\n\n"; } - auto markdown = ss.str(); + auto markdown = toView(ss); logger->cout(trim(renderMarkdownToTerminal(markdown))); } else @@ -826,7 +826,7 @@ void NixRepl::runNix(Path program, const Strings & args, const std::optional ../../../.version + echo ${version} > ../../.version ''; mesonFlags = [ @@ -66,10 +53,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/tests/unit/libexpr-support/tests/libexpr.hh b/src/libexpr-test-support/tests/libexpr.hh similarity index 100% rename from tests/unit/libexpr-support/tests/libexpr.hh rename to src/libexpr-test-support/tests/libexpr.hh diff --git a/tests/unit/libexpr-support/tests/nix_api_expr.hh b/src/libexpr-test-support/tests/nix_api_expr.hh similarity index 100% rename from tests/unit/libexpr-support/tests/nix_api_expr.hh rename to src/libexpr-test-support/tests/nix_api_expr.hh diff --git a/tests/unit/libexpr-support/tests/value/context.cc b/src/libexpr-test-support/tests/value/context.cc similarity index 100% rename from tests/unit/libexpr-support/tests/value/context.cc rename to src/libexpr-test-support/tests/value/context.cc diff --git a/tests/unit/libexpr-support/tests/value/context.hh b/src/libexpr-test-support/tests/value/context.hh similarity index 100% rename from tests/unit/libexpr-support/tests/value/context.hh rename to src/libexpr-test-support/tests/value/context.hh diff --git a/src/libexpr-tests/.version b/src/libexpr-tests/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libexpr-tests/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libexpr-tests/build-utils-meson b/src/libexpr-tests/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libexpr-tests/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/tests/unit/libexpr/data/.gitkeep b/src/libexpr-tests/data/.gitkeep similarity index 100% rename from tests/unit/libexpr/data/.gitkeep rename to src/libexpr-tests/data/.gitkeep diff --git a/tests/unit/libexpr/derived-path.cc b/src/libexpr-tests/derived-path.cc similarity index 100% rename from tests/unit/libexpr/derived-path.cc rename to src/libexpr-tests/derived-path.cc diff --git a/tests/unit/libexpr/error_traces.cc b/src/libexpr-tests/error_traces.cc similarity index 100% rename from tests/unit/libexpr/error_traces.cc rename to src/libexpr-tests/error_traces.cc diff --git a/tests/unit/libexpr/eval.cc b/src/libexpr-tests/eval.cc similarity index 100% rename from tests/unit/libexpr/eval.cc rename to src/libexpr-tests/eval.cc diff --git a/tests/unit/libexpr/json.cc b/src/libexpr-tests/json.cc similarity index 100% rename from tests/unit/libexpr/json.cc rename to src/libexpr-tests/json.cc diff --git a/tests/unit/libexpr/local.mk b/src/libexpr-tests/local.mk similarity index 91% rename from tests/unit/libexpr/local.mk rename to src/libexpr-tests/local.mk index 1617e2823..79583a9ee 100644 --- a/tests/unit/libexpr/local.mk +++ b/src/libexpr-tests/local.mk @@ -20,9 +20,9 @@ libexpr-tests_SOURCES := \ $(wildcard $(d)/flake/*.cc) libexpr-tests_EXTRA_INCLUDES = \ - -I tests/unit/libexpr-support \ - -I tests/unit/libstore-support \ - -I tests/unit/libutil-support \ + -I src/libexpr-test-support \ + -I src/libstore-test-support \ + -I src/libutil-test-support \ $(INCLUDE_libexpr) \ $(INCLUDE_libexprc) \ $(INCLUDE_libfetchers) \ diff --git a/tests/unit/libexpr/main.cc b/src/libexpr-tests/main.cc similarity index 100% rename from tests/unit/libexpr/main.cc rename to src/libexpr-tests/main.cc diff --git a/tests/unit/libexpr/meson.build b/src/libexpr-tests/meson.build similarity index 100% rename from tests/unit/libexpr/meson.build rename to src/libexpr-tests/meson.build diff --git a/tests/unit/libexpr/nix_api_expr.cc b/src/libexpr-tests/nix_api_expr.cc similarity index 100% rename from tests/unit/libexpr/nix_api_expr.cc rename to src/libexpr-tests/nix_api_expr.cc diff --git a/tests/unit/libexpr/nix_api_external.cc b/src/libexpr-tests/nix_api_external.cc similarity index 100% rename from tests/unit/libexpr/nix_api_external.cc rename to src/libexpr-tests/nix_api_external.cc diff --git a/tests/unit/libexpr/nix_api_value.cc b/src/libexpr-tests/nix_api_value.cc similarity index 100% rename from tests/unit/libexpr/nix_api_value.cc rename to src/libexpr-tests/nix_api_value.cc diff --git a/tests/unit/libexpr/package.nix b/src/libexpr-tests/package.nix similarity index 80% rename from tests/unit/libexpr/package.nix rename to src/libexpr-tests/package.nix index e70ed7836..959d6b84e 100644 --- a/tests/unit/libexpr/package.nix +++ b/src/libexpr-tests/package.nix @@ -1,12 +1,7 @@ { lib , buildPackages , stdenv -, mkMesonDerivation -, releaseTools - -, meson -, ninja -, pkg-config +, mkMesonExecutable , nix-expr , nix-expr-c @@ -26,15 +21,15 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonExecutable (finalAttrs: { pname = "nix-expr-tests"; inherit version; workDir = ./.; fileset = fileset.unions [ - ../../../build-utils-meson + ../../build-utils-meson ./build-utils-meson - ../../../.version + ../../.version ./.version ./meson.build # ./meson.options @@ -42,12 +37,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - nativeBuildInputs = [ - meson - ninja - pkg-config - ]; - buildInputs = [ nix-expr nix-expr-c @@ -61,7 +50,7 @@ mkMesonDerivation (finalAttrs: { # Do the meson utils, without modification. '' chmod u+w ./.version - echo ${version} > ../../../.version + echo ${version} > ../../.version ''; mesonFlags = [ @@ -71,10 +60,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { diff --git a/tests/unit/libexpr/primops.cc b/src/libexpr-tests/primops.cc similarity index 100% rename from tests/unit/libexpr/primops.cc rename to src/libexpr-tests/primops.cc diff --git a/tests/unit/libexpr/search-path.cc b/src/libexpr-tests/search-path.cc similarity index 100% rename from tests/unit/libexpr/search-path.cc rename to src/libexpr-tests/search-path.cc diff --git a/tests/unit/libexpr/trivial.cc b/src/libexpr-tests/trivial.cc similarity index 100% rename from tests/unit/libexpr/trivial.cc rename to src/libexpr-tests/trivial.cc diff --git a/tests/unit/libexpr/value/context.cc b/src/libexpr-tests/value/context.cc similarity index 100% rename from tests/unit/libexpr/value/context.cc rename to src/libexpr-tests/value/context.cc diff --git a/tests/unit/libexpr/value/print.cc b/src/libexpr-tests/value/print.cc similarity index 100% rename from tests/unit/libexpr/value/print.cc rename to src/libexpr-tests/value/print.cc diff --git a/tests/unit/libexpr/value/value.cc b/src/libexpr-tests/value/value.cc similarity index 100% rename from tests/unit/libexpr/value/value.cc rename to src/libexpr-tests/value/value.cc diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 9eae6078b..e4937735b 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -539,7 +539,7 @@ std::optional EvalState::getDoc(Value & v) if (v.isLambda()) { auto exprLambda = v.payload.lambda.fun; - std::stringstream s(std::ios_base::out); + std::ostringstream s; std::string name; auto pos = positions[exprLambda->getPos()]; std::string docStr; @@ -571,17 +571,12 @@ std::optional EvalState::getDoc(Value & v) s << docStr; - s << '\0'; // for making a c string below - std::string ss = s.str(); - return Doc { .pos = pos, .name = name, .arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though... .args = {}, - .doc = - // FIXME: this leaks; make the field std::string? - strdup(ss.data()), + .doc = makeImmutableString(toView(s)), // NOTE: memory leak when compiled without GC }; } if (isFunctor(v)) { @@ -1805,11 +1800,9 @@ void ExprIf::eval(EvalState & state, Env & env, Value & v) void ExprAssert::eval(EvalState & state, Env & env, Value & v) { if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) { - auto exprStr = ({ - std::ostringstream out; - cond->show(state.symbols, out); - out.str(); - }); + std::ostringstream out; + cond->show(state.symbols, out); + auto exprStr = toView(out); if (auto eq = dynamic_cast(cond)) { try { @@ -2841,7 +2834,9 @@ void EvalState::printStatistics() #endif #if HAVE_BOEHMGC {GC_is_incremental_mode() ? "gcNonIncremental" : "gc", gcFullOnlyTime}, +#ifndef _WIN32 // TODO implement {GC_is_incremental_mode() ? "gcNonIncrementalFraction" : "gcFraction", gcFullOnlyTime / cpuTime}, +#endif #endif }; topObj["envs"] = { diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 20963ec91..1ac13fcd2 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -374,11 +374,12 @@ static void getDerivations(EvalState & state, Value & vIn, bound to the attribute with the "lower" name should take precedence). */ for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) { + std::string_view symbol{state.symbols[i->name]}; try { - debug("evaluating attribute '%1%'", state.symbols[i->name]); - if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex)) + debug("evaluating attribute '%1%'", symbol); + if (!std::regex_match(symbol.begin(), symbol.end(), attrRegex)) continue; - std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]); + std::string pathPrefix2 = addToPath(pathPrefix, symbol); if (combineChannels) getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) { @@ -392,7 +393,7 @@ static void getDerivations(EvalState & state, Value & vIn, } } } catch (Error & e) { - e.addTrace(state.positions[i->pos], "while evaluating the attribute '%s'", state.symbols[i->name]); + e.addTrace(state.positions[i->pos], "while evaluating the attribute '%s'", symbol); throw; } } diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index 4d10079ff..ca1f8bf21 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -1,11 +1,7 @@ { lib , stdenv -, mkMesonDerivation -, releaseTools +, mkMesonLibrary -, meson -, ninja -, pkg-config , bison , flex , cmake # for resolving toml11 dep @@ -38,7 +34,7 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonLibrary (finalAttrs: { pname = "nix-expr"; inherit version; @@ -55,15 +51,13 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ./lexer.l ./parser.y - (fileset.fileFilter (file: file.hasExt "nix") ./.) + (fileset.difference + (fileset.fileFilter (file: file.hasExt "nix") ./.) + ./package.nix + ) ]; - outputs = [ "out" "dev" ]; - nativeBuildInputs = [ - meson - ninja - pkg-config bison flex cmake @@ -102,10 +96,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7b6f222a8..84aa6bac9 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -40,6 +40,13 @@ namespace nix { * Miscellaneous *************************************************************/ +static inline Value * mkString(EvalState & state, const std::csub_match & match) +{ + Value * v = state.allocValue(); + v->mkString({match.first, match.second}); + return v; +} + StringMap EvalState::realiseContext(const NixStringContext & context, StorePathSet * maybePathsOut, bool isIFD) { std::vector drvs; @@ -171,6 +178,78 @@ static void mkOutputString( o.second.path(*state.store, Derivation::nameFromPath(drvPath), o.first)); } +/** + * `import` will parse a derivation when it imports a `.drv` file from the store. + * + * @param state The evaluation state. + * @param pos The position of the `import` call. + * @param path The path to the `.drv` to import. + * @param storePath The path to the `.drv` to import. + * @param v Return value + */ +void derivationToValue(EvalState & state, const PosIdx pos, const SourcePath & path, const StorePath & storePath, Value & v) { + auto path2 = path.path.abs(); + Derivation drv = state.store->readDerivation(storePath); + auto attrs = state.buildBindings(3 + drv.outputs.size()); + attrs.alloc(state.sDrvPath).mkString(path2, { + NixStringContextElem::DrvDeep { .drvPath = storePath }, + }); + attrs.alloc(state.sName).mkString(drv.env["name"]); + + auto list = state.buildList(drv.outputs.size()); + for (const auto & [i, o] : enumerate(drv.outputs)) { + mkOutputString(state, attrs, storePath, o); + (list[i] = state.allocValue())->mkString(o.first); + } + attrs.alloc(state.sOutputs).mkList(list); + + auto w = state.allocValue(); + w->mkAttrs(attrs); + + if (!state.vImportedDrvToDerivation) { + state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); + state.eval(state.parseExprFromString( + #include "imported-drv-to-derivation.nix.gen.hh" + , state.rootPath(CanonPath::root)), **state.vImportedDrvToDerivation); + } + + state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); + v.mkApp(*state.vImportedDrvToDerivation, w); + state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); +} + +/** + * Import a Nix file with an alternate base scope, as `builtins.scopedImport` does. + * + * @param state The evaluation state. + * @param pos The position of the import call. + * @param path The path to the file to import. + * @param vScope The base scope to use for the import. + * @param v Return value + */ +static void scopedImport(EvalState & state, const PosIdx pos, SourcePath & path, Value * vScope, Value & v) { + state.forceAttrs(*vScope, pos, "while evaluating the first argument passed to builtins.scopedImport"); + + Env * env = &state.allocEnv(vScope->attrs()->size()); + env->up = &state.baseEnv; + + auto staticEnv = std::make_shared(nullptr, state.staticBaseEnv.get(), vScope->attrs()->size()); + + unsigned int displ = 0; + for (auto & attr : *vScope->attrs()) { + staticEnv->vars.emplace_back(attr.name, displ); + env->values[displ++] = attr.value; + } + + // No need to call staticEnv.sort(), because + // args[0]->attrs is already sorted. + + printTalkative("evaluating file '%1%'", path); + Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv); + + e->eval(state, *env, v); +} + /* Load and evaluate an expression from path specified by the argument. */ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v) @@ -189,60 +268,13 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v }; if (auto storePath = isValidDerivationInStore()) { - Derivation drv = state.store->readDerivation(*storePath); - auto attrs = state.buildBindings(3 + drv.outputs.size()); - attrs.alloc(state.sDrvPath).mkString(path2, { - NixStringContextElem::DrvDeep { .drvPath = *storePath }, - }); - attrs.alloc(state.sName).mkString(drv.env["name"]); - - auto list = state.buildList(drv.outputs.size()); - for (const auto & [i, o] : enumerate(drv.outputs)) { - mkOutputString(state, attrs, *storePath, o); - (list[i] = state.allocValue())->mkString(o.first); - } - attrs.alloc(state.sOutputs).mkList(list); - - auto w = state.allocValue(); - w->mkAttrs(attrs); - - if (!state.vImportedDrvToDerivation) { - state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); - state.eval(state.parseExprFromString( - #include "imported-drv-to-derivation.nix.gen.hh" - , state.rootPath(CanonPath::root)), **state.vImportedDrvToDerivation); - } - - state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); - v.mkApp(*state.vImportedDrvToDerivation, w); - state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); + derivationToValue(state, pos, path, *storePath, v); + } + else if (vScope) { + scopedImport(state, pos, path, vScope, v); } - else { - if (!vScope) - state.evalFile(path, v); - else { - state.forceAttrs(*vScope, pos, "while evaluating the first argument passed to builtins.scopedImport"); - - Env * env = &state.allocEnv(vScope->attrs()->size()); - env->up = &state.baseEnv; - - auto staticEnv = std::make_shared(nullptr, state.staticBaseEnv.get(), vScope->attrs()->size()); - - unsigned int displ = 0; - for (auto & attr : *vScope->attrs()) { - staticEnv->vars.emplace_back(attr.name, displ); - env->values[displ++] = attr.value; - } - - // No need to call staticEnv.sort(), because - // args[0]->attrs is already sorted. - - printTalkative("evaluating file '%1%'", path); - Expr * e = state.parseExprFromFile(resolveExprPath(path), staticEnv); - - e->eval(state, *env, v); - } + state.evalFile(path, v); } } @@ -937,6 +969,9 @@ static RegisterPrimOp primop_tryEval({ `let e = { x = throw ""; }; in (builtins.tryEval (builtins.deepSeq e e)).success` will be `false`. + + `tryEval` intentionally does not return the error message, because that risks bringing non-determinism into the evaluation result, and it would become very difficult to improve error reporting without breaking existing expressions. + Instead, use [`builtins.addErrorContext`](@docroot@/language/builtins.md#builtins-addErrorContext) to add context to the error message, and use a Nix unit testing tool for testing. )", .fun = prim_tryEval, }); @@ -2129,7 +2164,7 @@ static void prim_toXML(EvalState & state, const PosIdx pos, Value * * args, Valu std::ostringstream out; NixStringContext context; printValueAsXML(state, true, false, *args[0], out, context, pos); - v.mkString(out.str(), context); + v.mkString(toView(out), context); } static RegisterPrimOp primop_toXML({ @@ -2237,7 +2272,7 @@ static void prim_toJSON(EvalState & state, const PosIdx pos, Value * * args, Val std::ostringstream out; NixStringContext context; printValueAsJSON(state, true, *args[0], pos, out, context); - v.mkString(out.str(), context); + v.mkString(toView(out), context); } static RegisterPrimOp primop_toJSON({ @@ -4268,7 +4303,7 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) if (!match[i + 1].matched) v2 = &state.vNull; else - (v2 = state.allocValue())->mkString(match[i + 1].str()); + v2 = mkString(state, match[i + 1]); v.mkList(list); } catch (std::regex_error & e) { @@ -4352,7 +4387,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto match = *i; // Add a string for non-matched characters. - (list[idx++] = state.allocValue())->mkString(match.prefix().str()); + list[idx++] = mkString(state, match.prefix()); // Add a list for matched substrings. const size_t slen = match.size() - 1; @@ -4363,14 +4398,14 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) if (!match[si + 1].matched) v2 = &state.vNull; else - (v2 = state.allocValue())->mkString(match[si + 1].str()); + v2 = mkString(state, match[si + 1]); } (list[idx++] = state.allocValue())->mkList(list2); // Add a string for non-matched suffix characters. if (idx == 2 * len) - (list[idx++] = state.allocValue())->mkString(match.suffix().str()); + list[idx++] = mkString(state, match.suffix()); } assert(idx == 2 * len + 1); diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index b4f1df7a8..264046711 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -66,7 +66,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V attrs.alloc("_type").mkString("timestamp"); std::ostringstream s; s << t; - attrs.alloc("value").mkString(s.str()); + attrs.alloc("value").mkString(toView(s)); v.mkAttrs(attrs); } else { throw std::runtime_error("Dates and times are not supported"); diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index 4d1a6868c..d62aaf25f 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -460,7 +460,7 @@ private: std::ostringstream s; s << state.positions[v.payload.lambda.fun->pos]; - output << " @ " << filterANSIEscapes(s.str()); + output << " @ " << filterANSIEscapes(toView(s)); } } else if (v.isPrimOp()) { if (v.primOp()) diff --git a/src/libfetchers-tests/.version b/src/libfetchers-tests/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libfetchers-tests/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libfetchers-tests/build-utils-meson b/src/libfetchers-tests/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libfetchers-tests/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/tests/unit/libfetchers/data/public-key/defaultType.json b/src/libfetchers-tests/data/public-key/defaultType.json similarity index 100% rename from tests/unit/libfetchers/data/public-key/defaultType.json rename to src/libfetchers-tests/data/public-key/defaultType.json diff --git a/tests/unit/libfetchers/data/public-key/noRoundTrip.json b/src/libfetchers-tests/data/public-key/noRoundTrip.json similarity index 100% rename from tests/unit/libfetchers/data/public-key/noRoundTrip.json rename to src/libfetchers-tests/data/public-key/noRoundTrip.json diff --git a/tests/unit/libfetchers/data/public-key/simple.json b/src/libfetchers-tests/data/public-key/simple.json similarity index 100% rename from tests/unit/libfetchers/data/public-key/simple.json rename to src/libfetchers-tests/data/public-key/simple.json diff --git a/tests/unit/libfetchers/git-utils.cc b/src/libfetchers-tests/git-utils.cc similarity index 100% rename from tests/unit/libfetchers/git-utils.cc rename to src/libfetchers-tests/git-utils.cc diff --git a/tests/unit/libfetchers/local.mk b/src/libfetchers-tests/local.mk similarity index 93% rename from tests/unit/libfetchers/local.mk rename to src/libfetchers-tests/local.mk index 30aa142a5..5c90f1fc7 100644 --- a/tests/unit/libfetchers/local.mk +++ b/src/libfetchers-tests/local.mk @@ -17,8 +17,8 @@ endif libfetchers-tests_SOURCES := $(wildcard $(d)/*.cc) libfetchers-tests_EXTRA_INCLUDES = \ - -I tests/unit/libstore-support \ - -I tests/unit/libutil-support \ + -I src/libstore-test-support \ + -I src/libutil-test-support \ $(INCLUDE_libfetchers) \ $(INCLUDE_libstore) \ $(INCLUDE_libutil) diff --git a/tests/unit/libfetchers/meson.build b/src/libfetchers-tests/meson.build similarity index 100% rename from tests/unit/libfetchers/meson.build rename to src/libfetchers-tests/meson.build diff --git a/tests/unit/libfetchers/package.nix b/src/libfetchers-tests/package.nix similarity index 80% rename from tests/unit/libfetchers/package.nix rename to src/libfetchers-tests/package.nix index ad512f562..7b2ba8f2c 100644 --- a/tests/unit/libfetchers/package.nix +++ b/src/libfetchers-tests/package.nix @@ -1,12 +1,7 @@ { lib , buildPackages , stdenv -, mkMesonDerivation -, releaseTools - -, meson -, ninja -, pkg-config +, mkMesonExecutable , nix-fetchers , nix-store-test-support @@ -25,15 +20,15 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonExecutable (finalAttrs: { pname = "nix-fetchers-tests"; inherit version; workDir = ./.; fileset = fileset.unions [ - ../../../build-utils-meson + ../../build-utils-meson ./build-utils-meson - ../../../.version + ../../.version ./.version ./meson.build # ./meson.options @@ -41,12 +36,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - nativeBuildInputs = [ - meson - ninja - pkg-config - ]; - buildInputs = [ nix-fetchers nix-store-test-support @@ -59,7 +48,7 @@ mkMesonDerivation (finalAttrs: { # Do the meson utils, without modification. '' chmod u+w ./.version - echo ${version} > ../../../.version + echo ${version} > ../../.version ''; mesonFlags = [ @@ -69,10 +58,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { diff --git a/tests/unit/libfetchers/public-key.cc b/src/libfetchers-tests/public-key.cc similarity index 100% rename from tests/unit/libfetchers/public-key.cc rename to src/libfetchers-tests/public-key.cc diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 582686412..95ee33089 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -208,7 +208,7 @@ static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuil static void initRepoAtomically(std::filesystem::path &path, bool bare) { if (pathExists(path.string())) return; - Path tmpDir = createTempDir(std::filesystem::path(path).parent_path()); + Path tmpDir = createTempDir(os_string_to_string(PathViewNG { std::filesystem::path(path).parent_path() })); AutoDelete delTmpDir(tmpDir, true); Repository tmpRepo; @@ -977,8 +977,24 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink void pushBuilder(std::string name) { + const git_tree_entry * entry; + Tree prevTree = nullptr; + + if (!pendingDirs.empty() && + (entry = git_treebuilder_get(pendingDirs.back().builder.get(), name.c_str()))) + { + /* Clone a tree that we've already finished. This happens + if a tarball has directory entries that are not + contiguous. */ + if (git_tree_entry_type(entry) != GIT_OBJECT_TREE) + throw Error("parent of '%s' is not a directory", name); + + if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(prevTree), *repo, entry)) + throw Error("looking up parent of '%s': %s", name, git_error_last()->message); + } + git_treebuilder * b; - if (git_treebuilder_new(&b, *repo, nullptr)) + if (git_treebuilder_new(&b, *repo, prevTree.get())) throw Error("creating a tree builder: %s", git_error_last()->message); pendingDirs.push_back({ .name = std::move(name), .builder = TreeBuilder(b) }); }; diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index 9b5d8bff7..70973bdb2 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -1,17 +1,11 @@ { lib , stdenv -, mkMesonDerivation -, releaseTools - -, meson -, ninja -, pkg-config +, mkMesonLibrary , nix-util , nix-store , nlohmann_json , libgit2 -, man # Configuration Options @@ -22,7 +16,7 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonLibrary (finalAttrs: { pname = "nix-fetchers"; inherit version; @@ -37,14 +31,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - outputs = [ "out" "dev" ]; - - nativeBuildInputs = [ - meson - ninja - pkg-config - ]; - buildInputs = [ libgit2 ]; @@ -67,10 +53,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index fca0df84b..fe1534aba 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -157,7 +157,11 @@ struct PathInputScheme : InputScheme }); storePath = store->addToStoreFromDump(*src, "source"); } - input.attrs.insert_or_assign("lastModified", uint64_t(mtime)); + + /* Trust the lastModified value supplied by the user, if + any. It's not a "secure" attribute so we don't care. */ + if (!input.getLastModified()) + input.attrs.insert_or_assign("lastModified", uint64_t(mtime)); return {makeStorePathAccessor(store, *storePath), std::move(input)}; } diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index aa5d61bc5..28574e7b1 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -90,6 +90,7 @@ DownloadFileResult downloadFile( /* Cache metadata for all URLs in the redirect chain. */ for (auto & url : res.urls) { key.second.insert_or_assign("url", url); + assert(!res.urls.empty()); infoAttrs.insert_or_assign("url", *res.urls.rbegin()); getCache()->upsert(key, *store, infoAttrs, *storePath); } diff --git a/src/libflake-tests/.version b/src/libflake-tests/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libflake-tests/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libflake-tests/build-utils-meson b/src/libflake-tests/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libflake-tests/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/tests/unit/libflake/data/.gitkeep b/src/libflake-tests/data/.gitkeep similarity index 100% rename from tests/unit/libflake/data/.gitkeep rename to src/libflake-tests/data/.gitkeep diff --git a/tests/unit/libflake/flakeref.cc b/src/libflake-tests/flakeref.cc similarity index 100% rename from tests/unit/libflake/flakeref.cc rename to src/libflake-tests/flakeref.cc diff --git a/tests/unit/libflake/local.mk b/src/libflake-tests/local.mk similarity index 90% rename from tests/unit/libflake/local.mk rename to src/libflake-tests/local.mk index 590bcf7c0..8599b43f6 100644 --- a/tests/unit/libflake/local.mk +++ b/src/libflake-tests/local.mk @@ -20,9 +20,9 @@ libflake-tests_SOURCES := \ $(wildcard $(d)/flake/*.cc) libflake-tests_EXTRA_INCLUDES = \ - -I tests/unit/libflake-support \ - -I tests/unit/libstore-support \ - -I tests/unit/libutil-support \ + -I src/libflake-test-support \ + -I src/libstore-test-support \ + -I src/libutil-test-support \ $(INCLUDE_libflake) \ $(INCLUDE_libexpr) \ $(INCLUDE_libfetchers) \ diff --git a/tests/unit/libflake/meson.build b/src/libflake-tests/meson.build similarity index 100% rename from tests/unit/libflake/meson.build rename to src/libflake-tests/meson.build diff --git a/tests/unit/libflake/package.nix b/src/libflake-tests/package.nix similarity index 80% rename from tests/unit/libflake/package.nix rename to src/libflake-tests/package.nix index 0d63d2ff7..67e716979 100644 --- a/tests/unit/libflake/package.nix +++ b/src/libflake-tests/package.nix @@ -1,12 +1,7 @@ { lib , buildPackages , stdenv -, mkMesonDerivation -, releaseTools - -, meson -, ninja -, pkg-config +, mkMesonExecutable , nix-flake , nix-expr-test-support @@ -25,15 +20,15 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonExecutable (finalAttrs: { pname = "nix-flake-tests"; inherit version; workDir = ./.; fileset = fileset.unions [ - ../../../build-utils-meson + ../../build-utils-meson ./build-utils-meson - ../../../.version + ../../.version ./.version ./meson.build # ./meson.options @@ -41,12 +36,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - nativeBuildInputs = [ - meson - ninja - pkg-config - ]; - buildInputs = [ nix-flake nix-expr-test-support @@ -59,7 +48,7 @@ mkMesonDerivation (finalAttrs: { # Do the meson utils, without modification. '' chmod u+w ./.version - echo ${version} > ../../../.version + echo ${version} > ../../.version ''; mesonFlags = [ @@ -69,10 +58,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { diff --git a/tests/unit/libflake/url-name.cc b/src/libflake-tests/url-name.cc similarity index 100% rename from tests/unit/libflake/url-name.cc rename to src/libflake-tests/url-name.cc diff --git a/src/libflake/package.nix b/src/libflake/package.nix index 851adf07e..fff481720 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -1,19 +1,12 @@ { lib , stdenv -, mkMesonDerivation -, releaseTools - -, meson -, ninja -, pkg-config +, mkMesonLibrary , nix-util , nix-store , nix-fetchers , nix-expr , nlohmann_json -, libgit2 -, man # Configuration Options @@ -24,7 +17,7 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonLibrary (finalAttrs: { pname = "nix-flake"; inherit version; @@ -39,14 +32,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - outputs = [ "out" "dev" ]; - - nativeBuildInputs = [ - meson - ninja - pkg-config - ]; - propagatedBuildInputs = [ nix-store nix-util @@ -67,10 +52,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index ce6f67300..5522037f3 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -1,11 +1,6 @@ { lib , stdenv -, mkMesonDerivation -, releaseTools - -, meson -, ninja -, pkg-config +, mkMesonLibrary , nix-util-c , nix-store @@ -21,7 +16,7 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonLibrary (finalAttrs: { pname = "nix-main-c"; inherit version; @@ -38,14 +33,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "h") ./.) ]; - outputs = [ "out" "dev" ]; - - nativeBuildInputs = [ - meson - ninja - pkg-config - ]; - propagatedBuildInputs = [ nix-util-c nix-store @@ -68,10 +55,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libmain/package.nix b/src/libmain/package.nix index 47513dbdc..7e7b80472 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -1,11 +1,6 @@ { lib , stdenv -, mkMesonDerivation -, releaseTools - -, meson -, ninja -, pkg-config +, mkMesonLibrary , openssl @@ -21,7 +16,7 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonLibrary (finalAttrs: { pname = "nix-main"; inherit version; @@ -36,14 +31,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - outputs = [ "out" "dev" ]; - - nativeBuildInputs = [ - meson - ninja - pkg-config - ]; - propagatedBuildInputs = [ nix-util nix-store @@ -62,10 +49,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index e63d4f13f..22f890f7d 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -158,10 +158,10 @@ public: { auto state(state_.lock()); - std::stringstream oss; + std::ostringstream oss; showErrorInfo(oss, ei, loggerSettings.showTrace.get()); - log(*state, ei.level, oss.str()); + log(*state, ei.level, toView(oss)); } void log(State & state, Verbosity lvl, std::string_view s) diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index e4f372236..896a1a39f 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -1,11 +1,6 @@ { lib , stdenv -, mkMesonDerivation -, releaseTools - -, meson -, ninja -, pkg-config +, mkMesonLibrary , nix-util-c , nix-store @@ -19,7 +14,7 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonLibrary (finalAttrs: { pname = "nix-store-c"; inherit version; @@ -36,14 +31,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "h") ./.) ]; - outputs = [ "out" "dev" ]; - - nativeBuildInputs = [ - meson - ninja - pkg-config - ]; - propagatedBuildInputs = [ nix-util-c nix-store @@ -64,10 +51,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libstore-test-support/.version b/src/libstore-test-support/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libstore-test-support/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libstore-test-support/build-utils-meson b/src/libstore-test-support/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libstore-test-support/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/tests/unit/libstore-support/local.mk b/src/libstore-test-support/local.mk similarity index 100% rename from tests/unit/libstore-support/local.mk rename to src/libstore-test-support/local.mk diff --git a/tests/unit/libstore-support/meson.build b/src/libstore-test-support/meson.build similarity index 100% rename from tests/unit/libstore-support/meson.build rename to src/libstore-test-support/meson.build diff --git a/tests/unit/libstore-support/package.nix b/src/libstore-test-support/package.nix similarity index 69% rename from tests/unit/libstore-support/package.nix rename to src/libstore-test-support/package.nix index f512db3ee..48f8b5e6b 100644 --- a/tests/unit/libstore-support/package.nix +++ b/src/libstore-test-support/package.nix @@ -1,11 +1,6 @@ { lib , stdenv -, mkMesonDerivation -, releaseTools - -, meson -, ninja -, pkg-config +, mkMesonLibrary , nix-util-test-support , nix-store @@ -21,15 +16,15 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonLibrary (finalAttrs: { pname = "nix-store-test-support"; inherit version; workDir = ./.; fileset = fileset.unions [ - ../../../build-utils-meson + ../../build-utils-meson ./build-utils-meson - ../../../.version + ../../.version ./.version ./meson.build # ./meson.options @@ -37,14 +32,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - outputs = [ "out" "dev" ]; - - nativeBuildInputs = [ - meson - ninja - pkg-config - ]; - propagatedBuildInputs = [ nix-util-test-support nix-store @@ -56,7 +43,7 @@ mkMesonDerivation (finalAttrs: { # Do the meson utils, without modification. '' chmod u+w ./.version - echo ${version} > ../../../.version + echo ${version} > ../../.version ''; mesonFlags = [ @@ -66,10 +53,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/tests/unit/libstore-support/tests/derived-path.cc b/src/libstore-test-support/tests/derived-path.cc similarity index 100% rename from tests/unit/libstore-support/tests/derived-path.cc rename to src/libstore-test-support/tests/derived-path.cc diff --git a/tests/unit/libstore-support/tests/derived-path.hh b/src/libstore-test-support/tests/derived-path.hh similarity index 100% rename from tests/unit/libstore-support/tests/derived-path.hh rename to src/libstore-test-support/tests/derived-path.hh diff --git a/tests/unit/libstore-support/tests/libstore.hh b/src/libstore-test-support/tests/libstore.hh similarity index 100% rename from tests/unit/libstore-support/tests/libstore.hh rename to src/libstore-test-support/tests/libstore.hh diff --git a/tests/unit/libstore-support/tests/nix_api_store.hh b/src/libstore-test-support/tests/nix_api_store.hh similarity index 100% rename from tests/unit/libstore-support/tests/nix_api_store.hh rename to src/libstore-test-support/tests/nix_api_store.hh diff --git a/tests/unit/libstore-support/tests/outputs-spec.cc b/src/libstore-test-support/tests/outputs-spec.cc similarity index 100% rename from tests/unit/libstore-support/tests/outputs-spec.cc rename to src/libstore-test-support/tests/outputs-spec.cc diff --git a/tests/unit/libstore-support/tests/outputs-spec.hh b/src/libstore-test-support/tests/outputs-spec.hh similarity index 100% rename from tests/unit/libstore-support/tests/outputs-spec.hh rename to src/libstore-test-support/tests/outputs-spec.hh diff --git a/tests/unit/libstore-support/tests/path.cc b/src/libstore-test-support/tests/path.cc similarity index 100% rename from tests/unit/libstore-support/tests/path.cc rename to src/libstore-test-support/tests/path.cc diff --git a/tests/unit/libstore-support/tests/path.hh b/src/libstore-test-support/tests/path.hh similarity index 100% rename from tests/unit/libstore-support/tests/path.hh rename to src/libstore-test-support/tests/path.hh diff --git a/tests/unit/libstore-support/tests/protocol.hh b/src/libstore-test-support/tests/protocol.hh similarity index 100% rename from tests/unit/libstore-support/tests/protocol.hh rename to src/libstore-test-support/tests/protocol.hh diff --git a/src/libstore-tests/.version b/src/libstore-tests/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libstore-tests/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libstore-tests/build-utils-meson b/src/libstore-tests/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libstore-tests/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/tests/unit/libstore/common-protocol.cc b/src/libstore-tests/common-protocol.cc similarity index 100% rename from tests/unit/libstore/common-protocol.cc rename to src/libstore-tests/common-protocol.cc diff --git a/tests/unit/libstore/content-address.cc b/src/libstore-tests/content-address.cc similarity index 100% rename from tests/unit/libstore/content-address.cc rename to src/libstore-tests/content-address.cc diff --git a/tests/unit/libstore/data/common-protocol/content-address.bin b/src/libstore-tests/data/common-protocol/content-address.bin similarity index 100% rename from tests/unit/libstore/data/common-protocol/content-address.bin rename to src/libstore-tests/data/common-protocol/content-address.bin diff --git a/tests/unit/libstore/data/common-protocol/drv-output.bin b/src/libstore-tests/data/common-protocol/drv-output.bin similarity index 100% rename from tests/unit/libstore/data/common-protocol/drv-output.bin rename to src/libstore-tests/data/common-protocol/drv-output.bin diff --git a/tests/unit/libstore/data/common-protocol/optional-content-address.bin b/src/libstore-tests/data/common-protocol/optional-content-address.bin similarity index 100% rename from tests/unit/libstore/data/common-protocol/optional-content-address.bin rename to src/libstore-tests/data/common-protocol/optional-content-address.bin diff --git a/tests/unit/libstore/data/common-protocol/optional-store-path.bin b/src/libstore-tests/data/common-protocol/optional-store-path.bin similarity index 100% rename from tests/unit/libstore/data/common-protocol/optional-store-path.bin rename to src/libstore-tests/data/common-protocol/optional-store-path.bin diff --git a/tests/unit/libstore/data/common-protocol/realisation.bin b/src/libstore-tests/data/common-protocol/realisation.bin similarity index 100% rename from tests/unit/libstore/data/common-protocol/realisation.bin rename to src/libstore-tests/data/common-protocol/realisation.bin diff --git a/tests/unit/libstore/data/common-protocol/set.bin b/src/libstore-tests/data/common-protocol/set.bin similarity index 100% rename from tests/unit/libstore/data/common-protocol/set.bin rename to src/libstore-tests/data/common-protocol/set.bin diff --git a/tests/unit/libstore/data/common-protocol/store-path.bin b/src/libstore-tests/data/common-protocol/store-path.bin similarity index 100% rename from tests/unit/libstore/data/common-protocol/store-path.bin rename to src/libstore-tests/data/common-protocol/store-path.bin diff --git a/tests/unit/libstore/data/common-protocol/string.bin b/src/libstore-tests/data/common-protocol/string.bin similarity index 100% rename from tests/unit/libstore/data/common-protocol/string.bin rename to src/libstore-tests/data/common-protocol/string.bin diff --git a/tests/unit/libstore/data/common-protocol/vector.bin b/src/libstore-tests/data/common-protocol/vector.bin similarity index 100% rename from tests/unit/libstore/data/common-protocol/vector.bin rename to src/libstore-tests/data/common-protocol/vector.bin diff --git a/src/libstore-tests/data/derivation/advanced-attributes-defaults.drv b/src/libstore-tests/data/derivation/advanced-attributes-defaults.drv new file mode 120000 index 000000000..f8f30ac32 --- /dev/null +++ b/src/libstore-tests/data/derivation/advanced-attributes-defaults.drv @@ -0,0 +1 @@ +../../../../tests/functional/derivation/advanced-attributes-defaults.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/advanced-attributes-defaults.json similarity index 100% rename from tests/unit/libstore/data/derivation/advanced-attributes-defaults.json rename to src/libstore-tests/data/derivation/advanced-attributes-defaults.json diff --git a/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs-defaults.drv b/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs-defaults.drv new file mode 120000 index 000000000..837e9a0e4 --- /dev/null +++ b/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs-defaults.drv @@ -0,0 +1 @@ +../../../../tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs-defaults.json similarity index 100% rename from tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json rename to src/libstore-tests/data/derivation/advanced-attributes-structured-attrs-defaults.json diff --git a/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.drv b/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.drv new file mode 120000 index 000000000..e08bb5737 --- /dev/null +++ b/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +../../../../tests/functional/derivation/advanced-attributes-structured-attrs.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.json similarity index 100% rename from tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json rename to src/libstore-tests/data/derivation/advanced-attributes-structured-attrs.json diff --git a/src/libstore-tests/data/derivation/advanced-attributes.drv b/src/libstore-tests/data/derivation/advanced-attributes.drv new file mode 120000 index 000000000..1dc394a0a --- /dev/null +++ b/src/libstore-tests/data/derivation/advanced-attributes.drv @@ -0,0 +1 @@ +../../../../tests/functional/derivation/advanced-attributes.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv b/src/libstore-tests/data/derivation/bad-old-version-dyn-deps.drv similarity index 100% rename from tests/unit/libstore/data/derivation/bad-old-version-dyn-deps.drv rename to src/libstore-tests/data/derivation/bad-old-version-dyn-deps.drv diff --git a/tests/unit/libstore/data/derivation/bad-version.drv b/src/libstore-tests/data/derivation/bad-version.drv similarity index 100% rename from tests/unit/libstore/data/derivation/bad-version.drv rename to src/libstore-tests/data/derivation/bad-version.drv diff --git a/tests/unit/libstore/data/derivation/dynDerivationDeps.drv b/src/libstore-tests/data/derivation/dynDerivationDeps.drv similarity index 100% rename from tests/unit/libstore/data/derivation/dynDerivationDeps.drv rename to src/libstore-tests/data/derivation/dynDerivationDeps.drv diff --git a/tests/unit/libstore/data/derivation/dynDerivationDeps.json b/src/libstore-tests/data/derivation/dynDerivationDeps.json similarity index 100% rename from tests/unit/libstore/data/derivation/dynDerivationDeps.json rename to src/libstore-tests/data/derivation/dynDerivationDeps.json diff --git a/tests/unit/libstore/data/derivation/output-caFixedFlat.json b/src/libstore-tests/data/derivation/output-caFixedFlat.json similarity index 100% rename from tests/unit/libstore/data/derivation/output-caFixedFlat.json rename to src/libstore-tests/data/derivation/output-caFixedFlat.json diff --git a/tests/unit/libstore/data/derivation/output-caFixedNAR.json b/src/libstore-tests/data/derivation/output-caFixedNAR.json similarity index 100% rename from tests/unit/libstore/data/derivation/output-caFixedNAR.json rename to src/libstore-tests/data/derivation/output-caFixedNAR.json diff --git a/tests/unit/libstore/data/derivation/output-caFixedText.json b/src/libstore-tests/data/derivation/output-caFixedText.json similarity index 100% rename from tests/unit/libstore/data/derivation/output-caFixedText.json rename to src/libstore-tests/data/derivation/output-caFixedText.json diff --git a/tests/unit/libstore/data/derivation/output-caFloating.json b/src/libstore-tests/data/derivation/output-caFloating.json similarity index 100% rename from tests/unit/libstore/data/derivation/output-caFloating.json rename to src/libstore-tests/data/derivation/output-caFloating.json diff --git a/tests/unit/libstore/data/derivation/output-deferred.json b/src/libstore-tests/data/derivation/output-deferred.json similarity index 100% rename from tests/unit/libstore/data/derivation/output-deferred.json rename to src/libstore-tests/data/derivation/output-deferred.json diff --git a/tests/unit/libstore/data/derivation/output-impure.json b/src/libstore-tests/data/derivation/output-impure.json similarity index 100% rename from tests/unit/libstore/data/derivation/output-impure.json rename to src/libstore-tests/data/derivation/output-impure.json diff --git a/tests/unit/libstore/data/derivation/output-inputAddressed.json b/src/libstore-tests/data/derivation/output-inputAddressed.json similarity index 100% rename from tests/unit/libstore/data/derivation/output-inputAddressed.json rename to src/libstore-tests/data/derivation/output-inputAddressed.json diff --git a/tests/unit/libstore/data/derivation/simple.drv b/src/libstore-tests/data/derivation/simple.drv similarity index 100% rename from tests/unit/libstore/data/derivation/simple.drv rename to src/libstore-tests/data/derivation/simple.drv diff --git a/tests/unit/libstore/data/derivation/simple.json b/src/libstore-tests/data/derivation/simple.json similarity index 100% rename from tests/unit/libstore/data/derivation/simple.json rename to src/libstore-tests/data/derivation/simple.json diff --git a/tests/unit/libstore/data/machines/bad_format b/src/libstore-tests/data/machines/bad_format similarity index 100% rename from tests/unit/libstore/data/machines/bad_format rename to src/libstore-tests/data/machines/bad_format diff --git a/tests/unit/libstore/data/machines/valid b/src/libstore-tests/data/machines/valid similarity index 100% rename from tests/unit/libstore/data/machines/valid rename to src/libstore-tests/data/machines/valid diff --git a/tests/unit/libstore/data/nar-info/impure.json b/src/libstore-tests/data/nar-info/impure.json similarity index 100% rename from tests/unit/libstore/data/nar-info/impure.json rename to src/libstore-tests/data/nar-info/impure.json diff --git a/tests/unit/libstore/data/nar-info/pure.json b/src/libstore-tests/data/nar-info/pure.json similarity index 100% rename from tests/unit/libstore/data/nar-info/pure.json rename to src/libstore-tests/data/nar-info/pure.json diff --git a/tests/unit/libstore/data/path-info/empty_impure.json b/src/libstore-tests/data/path-info/empty_impure.json similarity index 100% rename from tests/unit/libstore/data/path-info/empty_impure.json rename to src/libstore-tests/data/path-info/empty_impure.json diff --git a/tests/unit/libstore/data/path-info/empty_pure.json b/src/libstore-tests/data/path-info/empty_pure.json similarity index 100% rename from tests/unit/libstore/data/path-info/empty_pure.json rename to src/libstore-tests/data/path-info/empty_pure.json diff --git a/tests/unit/libstore/data/path-info/impure.json b/src/libstore-tests/data/path-info/impure.json similarity index 100% rename from tests/unit/libstore/data/path-info/impure.json rename to src/libstore-tests/data/path-info/impure.json diff --git a/tests/unit/libstore/data/path-info/pure.json b/src/libstore-tests/data/path-info/pure.json similarity index 100% rename from tests/unit/libstore/data/path-info/pure.json rename to src/libstore-tests/data/path-info/pure.json diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.1.bin b/src/libstore-tests/data/serve-protocol/build-options-2.1.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/build-options-2.1.bin rename to src/libstore-tests/data/serve-protocol/build-options-2.1.bin diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.2.bin b/src/libstore-tests/data/serve-protocol/build-options-2.2.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/build-options-2.2.bin rename to src/libstore-tests/data/serve-protocol/build-options-2.2.bin diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.3.bin b/src/libstore-tests/data/serve-protocol/build-options-2.3.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/build-options-2.3.bin rename to src/libstore-tests/data/serve-protocol/build-options-2.3.bin diff --git a/tests/unit/libstore/data/serve-protocol/build-options-2.7.bin b/src/libstore-tests/data/serve-protocol/build-options-2.7.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/build-options-2.7.bin rename to src/libstore-tests/data/serve-protocol/build-options-2.7.bin diff --git a/tests/unit/libstore/data/serve-protocol/build-result-2.2.bin b/src/libstore-tests/data/serve-protocol/build-result-2.2.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/build-result-2.2.bin rename to src/libstore-tests/data/serve-protocol/build-result-2.2.bin diff --git a/tests/unit/libstore/data/serve-protocol/build-result-2.3.bin b/src/libstore-tests/data/serve-protocol/build-result-2.3.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/build-result-2.3.bin rename to src/libstore-tests/data/serve-protocol/build-result-2.3.bin diff --git a/tests/unit/libstore/data/serve-protocol/build-result-2.6.bin b/src/libstore-tests/data/serve-protocol/build-result-2.6.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/build-result-2.6.bin rename to src/libstore-tests/data/serve-protocol/build-result-2.6.bin diff --git a/tests/unit/libstore/data/serve-protocol/content-address.bin b/src/libstore-tests/data/serve-protocol/content-address.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/content-address.bin rename to src/libstore-tests/data/serve-protocol/content-address.bin diff --git a/tests/unit/libstore/data/serve-protocol/drv-output.bin b/src/libstore-tests/data/serve-protocol/drv-output.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/drv-output.bin rename to src/libstore-tests/data/serve-protocol/drv-output.bin diff --git a/tests/unit/libstore/data/serve-protocol/handshake-to-client.bin b/src/libstore-tests/data/serve-protocol/handshake-to-client.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/handshake-to-client.bin rename to src/libstore-tests/data/serve-protocol/handshake-to-client.bin diff --git a/tests/unit/libstore/data/serve-protocol/optional-content-address.bin b/src/libstore-tests/data/serve-protocol/optional-content-address.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/optional-content-address.bin rename to src/libstore-tests/data/serve-protocol/optional-content-address.bin diff --git a/tests/unit/libstore/data/serve-protocol/optional-store-path.bin b/src/libstore-tests/data/serve-protocol/optional-store-path.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/optional-store-path.bin rename to src/libstore-tests/data/serve-protocol/optional-store-path.bin diff --git a/tests/unit/libstore/data/serve-protocol/realisation.bin b/src/libstore-tests/data/serve-protocol/realisation.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/realisation.bin rename to src/libstore-tests/data/serve-protocol/realisation.bin diff --git a/tests/unit/libstore/data/serve-protocol/set.bin b/src/libstore-tests/data/serve-protocol/set.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/set.bin rename to src/libstore-tests/data/serve-protocol/set.bin diff --git a/tests/unit/libstore/data/serve-protocol/store-path.bin b/src/libstore-tests/data/serve-protocol/store-path.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/store-path.bin rename to src/libstore-tests/data/serve-protocol/store-path.bin diff --git a/tests/unit/libstore/data/serve-protocol/string.bin b/src/libstore-tests/data/serve-protocol/string.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/string.bin rename to src/libstore-tests/data/serve-protocol/string.bin diff --git a/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.3.bin rename to src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.3.bin diff --git a/tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.4.bin b/src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/unkeyed-valid-path-info-2.4.bin rename to src/libstore-tests/data/serve-protocol/unkeyed-valid-path-info-2.4.bin diff --git a/tests/unit/libstore/data/serve-protocol/vector.bin b/src/libstore-tests/data/serve-protocol/vector.bin similarity index 100% rename from tests/unit/libstore/data/serve-protocol/vector.bin rename to src/libstore-tests/data/serve-protocol/vector.bin diff --git a/tests/unit/libstore/data/store-reference/auto.txt b/src/libstore-tests/data/store-reference/auto.txt similarity index 100% rename from tests/unit/libstore/data/store-reference/auto.txt rename to src/libstore-tests/data/store-reference/auto.txt diff --git a/tests/unit/libstore/data/store-reference/auto_param.txt b/src/libstore-tests/data/store-reference/auto_param.txt similarity index 100% rename from tests/unit/libstore/data/store-reference/auto_param.txt rename to src/libstore-tests/data/store-reference/auto_param.txt diff --git a/tests/unit/libstore/data/store-reference/local_1.txt b/src/libstore-tests/data/store-reference/local_1.txt similarity index 100% rename from tests/unit/libstore/data/store-reference/local_1.txt rename to src/libstore-tests/data/store-reference/local_1.txt diff --git a/tests/unit/libstore/data/store-reference/local_2.txt b/src/libstore-tests/data/store-reference/local_2.txt similarity index 100% rename from tests/unit/libstore/data/store-reference/local_2.txt rename to src/libstore-tests/data/store-reference/local_2.txt diff --git a/tests/unit/libstore/data/store-reference/local_shorthand_1.txt b/src/libstore-tests/data/store-reference/local_shorthand_1.txt similarity index 100% rename from tests/unit/libstore/data/store-reference/local_shorthand_1.txt rename to src/libstore-tests/data/store-reference/local_shorthand_1.txt diff --git a/tests/unit/libstore/data/store-reference/local_shorthand_2.txt b/src/libstore-tests/data/store-reference/local_shorthand_2.txt similarity index 100% rename from tests/unit/libstore/data/store-reference/local_shorthand_2.txt rename to src/libstore-tests/data/store-reference/local_shorthand_2.txt diff --git a/tests/unit/libstore/data/store-reference/ssh.txt b/src/libstore-tests/data/store-reference/ssh.txt similarity index 100% rename from tests/unit/libstore/data/store-reference/ssh.txt rename to src/libstore-tests/data/store-reference/ssh.txt diff --git a/tests/unit/libstore/data/store-reference/unix.txt b/src/libstore-tests/data/store-reference/unix.txt similarity index 100% rename from tests/unit/libstore/data/store-reference/unix.txt rename to src/libstore-tests/data/store-reference/unix.txt diff --git a/tests/unit/libstore/data/store-reference/unix_shorthand.txt b/src/libstore-tests/data/store-reference/unix_shorthand.txt similarity index 100% rename from tests/unit/libstore/data/store-reference/unix_shorthand.txt rename to src/libstore-tests/data/store-reference/unix_shorthand.txt diff --git a/tests/unit/libstore/data/worker-protocol/build-mode.bin b/src/libstore-tests/data/worker-protocol/build-mode.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/build-mode.bin rename to src/libstore-tests/data/worker-protocol/build-mode.bin diff --git a/tests/unit/libstore/data/worker-protocol/build-result-1.27.bin b/src/libstore-tests/data/worker-protocol/build-result-1.27.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/build-result-1.27.bin rename to src/libstore-tests/data/worker-protocol/build-result-1.27.bin diff --git a/tests/unit/libstore/data/worker-protocol/build-result-1.28.bin b/src/libstore-tests/data/worker-protocol/build-result-1.28.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/build-result-1.28.bin rename to src/libstore-tests/data/worker-protocol/build-result-1.28.bin diff --git a/tests/unit/libstore/data/worker-protocol/build-result-1.29.bin b/src/libstore-tests/data/worker-protocol/build-result-1.29.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/build-result-1.29.bin rename to src/libstore-tests/data/worker-protocol/build-result-1.29.bin diff --git a/tests/unit/libstore/data/worker-protocol/build-result-1.37.bin b/src/libstore-tests/data/worker-protocol/build-result-1.37.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/build-result-1.37.bin rename to src/libstore-tests/data/worker-protocol/build-result-1.37.bin diff --git a/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_30.bin b/src/libstore-tests/data/worker-protocol/client-handshake-info_1_30.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/client-handshake-info_1_30.bin rename to src/libstore-tests/data/worker-protocol/client-handshake-info_1_30.bin diff --git a/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_33.bin b/src/libstore-tests/data/worker-protocol/client-handshake-info_1_33.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/client-handshake-info_1_33.bin rename to src/libstore-tests/data/worker-protocol/client-handshake-info_1_33.bin diff --git a/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_35.bin b/src/libstore-tests/data/worker-protocol/client-handshake-info_1_35.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/client-handshake-info_1_35.bin rename to src/libstore-tests/data/worker-protocol/client-handshake-info_1_35.bin diff --git a/tests/unit/libstore/data/worker-protocol/content-address.bin b/src/libstore-tests/data/worker-protocol/content-address.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/content-address.bin rename to src/libstore-tests/data/worker-protocol/content-address.bin diff --git a/tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin b/src/libstore-tests/data/worker-protocol/derived-path-1.29.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/derived-path-1.29.bin rename to src/libstore-tests/data/worker-protocol/derived-path-1.29.bin diff --git a/tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin b/src/libstore-tests/data/worker-protocol/derived-path-1.30.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/derived-path-1.30.bin rename to src/libstore-tests/data/worker-protocol/derived-path-1.30.bin diff --git a/tests/unit/libstore/data/worker-protocol/drv-output.bin b/src/libstore-tests/data/worker-protocol/drv-output.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/drv-output.bin rename to src/libstore-tests/data/worker-protocol/drv-output.bin diff --git a/tests/unit/libstore/data/worker-protocol/handshake-to-client.bin b/src/libstore-tests/data/worker-protocol/handshake-to-client.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/handshake-to-client.bin rename to src/libstore-tests/data/worker-protocol/handshake-to-client.bin diff --git a/tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin b/src/libstore-tests/data/worker-protocol/keyed-build-result-1.29.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/keyed-build-result-1.29.bin rename to src/libstore-tests/data/worker-protocol/keyed-build-result-1.29.bin diff --git a/tests/unit/libstore/data/worker-protocol/optional-content-address.bin b/src/libstore-tests/data/worker-protocol/optional-content-address.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/optional-content-address.bin rename to src/libstore-tests/data/worker-protocol/optional-content-address.bin diff --git a/tests/unit/libstore/data/worker-protocol/optional-store-path.bin b/src/libstore-tests/data/worker-protocol/optional-store-path.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/optional-store-path.bin rename to src/libstore-tests/data/worker-protocol/optional-store-path.bin diff --git a/tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin b/src/libstore-tests/data/worker-protocol/optional-trusted-flag.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/optional-trusted-flag.bin rename to src/libstore-tests/data/worker-protocol/optional-trusted-flag.bin diff --git a/tests/unit/libstore/data/worker-protocol/realisation.bin b/src/libstore-tests/data/worker-protocol/realisation.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/realisation.bin rename to src/libstore-tests/data/worker-protocol/realisation.bin diff --git a/tests/unit/libstore/data/worker-protocol/set.bin b/src/libstore-tests/data/worker-protocol/set.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/set.bin rename to src/libstore-tests/data/worker-protocol/set.bin diff --git a/tests/unit/libstore/data/worker-protocol/store-path.bin b/src/libstore-tests/data/worker-protocol/store-path.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/store-path.bin rename to src/libstore-tests/data/worker-protocol/store-path.bin diff --git a/tests/unit/libstore/data/worker-protocol/string.bin b/src/libstore-tests/data/worker-protocol/string.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/string.bin rename to src/libstore-tests/data/worker-protocol/string.bin diff --git a/tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin b/src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/unkeyed-valid-path-info-1.15.bin rename to src/libstore-tests/data/worker-protocol/unkeyed-valid-path-info-1.15.bin diff --git a/tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin b/src/libstore-tests/data/worker-protocol/valid-path-info-1.15.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/valid-path-info-1.15.bin rename to src/libstore-tests/data/worker-protocol/valid-path-info-1.15.bin diff --git a/tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin b/src/libstore-tests/data/worker-protocol/valid-path-info-1.16.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/valid-path-info-1.16.bin rename to src/libstore-tests/data/worker-protocol/valid-path-info-1.16.bin diff --git a/tests/unit/libstore/data/worker-protocol/vector.bin b/src/libstore-tests/data/worker-protocol/vector.bin similarity index 100% rename from tests/unit/libstore/data/worker-protocol/vector.bin rename to src/libstore-tests/data/worker-protocol/vector.bin diff --git a/tests/unit/libstore/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc similarity index 100% rename from tests/unit/libstore/derivation-advanced-attrs.cc rename to src/libstore-tests/derivation-advanced-attrs.cc diff --git a/tests/unit/libstore/derivation.cc b/src/libstore-tests/derivation.cc similarity index 100% rename from tests/unit/libstore/derivation.cc rename to src/libstore-tests/derivation.cc diff --git a/tests/unit/libstore/derived-path.cc b/src/libstore-tests/derived-path.cc similarity index 100% rename from tests/unit/libstore/derived-path.cc rename to src/libstore-tests/derived-path.cc diff --git a/tests/unit/libstore/downstream-placeholder.cc b/src/libstore-tests/downstream-placeholder.cc similarity index 100% rename from tests/unit/libstore/downstream-placeholder.cc rename to src/libstore-tests/downstream-placeholder.cc diff --git a/tests/unit/libstore/http-binary-cache-store.cc b/src/libstore-tests/http-binary-cache-store.cc similarity index 100% rename from tests/unit/libstore/http-binary-cache-store.cc rename to src/libstore-tests/http-binary-cache-store.cc diff --git a/tests/unit/libstore/legacy-ssh-store.cc b/src/libstore-tests/legacy-ssh-store.cc similarity index 100% rename from tests/unit/libstore/legacy-ssh-store.cc rename to src/libstore-tests/legacy-ssh-store.cc diff --git a/tests/unit/libstore/local-binary-cache-store.cc b/src/libstore-tests/local-binary-cache-store.cc similarity index 100% rename from tests/unit/libstore/local-binary-cache-store.cc rename to src/libstore-tests/local-binary-cache-store.cc diff --git a/tests/unit/libstore/local-overlay-store.cc b/src/libstore-tests/local-overlay-store.cc similarity index 100% rename from tests/unit/libstore/local-overlay-store.cc rename to src/libstore-tests/local-overlay-store.cc diff --git a/tests/unit/libstore/local-store.cc b/src/libstore-tests/local-store.cc similarity index 100% rename from tests/unit/libstore/local-store.cc rename to src/libstore-tests/local-store.cc diff --git a/tests/unit/libstore/local.mk b/src/libstore-tests/local.mk similarity index 92% rename from tests/unit/libstore/local.mk rename to src/libstore-tests/local.mk index 8d3d6b0af..b565ff0be 100644 --- a/tests/unit/libstore/local.mk +++ b/src/libstore-tests/local.mk @@ -17,8 +17,8 @@ endif libstore-tests_SOURCES := $(wildcard $(d)/*.cc) libstore-tests_EXTRA_INCLUDES = \ - -I tests/unit/libstore-support \ - -I tests/unit/libutil-support \ + -I src/libstore-test-support \ + -I src/libutil-test-support \ $(INCLUDE_libstore) \ $(INCLUDE_libstorec) \ $(INCLUDE_libutil) \ diff --git a/tests/unit/libstore/machines.cc b/src/libstore-tests/machines.cc similarity index 100% rename from tests/unit/libstore/machines.cc rename to src/libstore-tests/machines.cc diff --git a/tests/unit/libstore/meson.build b/src/libstore-tests/meson.build similarity index 100% rename from tests/unit/libstore/meson.build rename to src/libstore-tests/meson.build diff --git a/tests/unit/libstore/nar-info-disk-cache.cc b/src/libstore-tests/nar-info-disk-cache.cc similarity index 100% rename from tests/unit/libstore/nar-info-disk-cache.cc rename to src/libstore-tests/nar-info-disk-cache.cc diff --git a/tests/unit/libstore/nar-info.cc b/src/libstore-tests/nar-info.cc similarity index 100% rename from tests/unit/libstore/nar-info.cc rename to src/libstore-tests/nar-info.cc diff --git a/tests/unit/libstore/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc similarity index 100% rename from tests/unit/libstore/nix_api_store.cc rename to src/libstore-tests/nix_api_store.cc diff --git a/tests/unit/libstore/outputs-spec.cc b/src/libstore-tests/outputs-spec.cc similarity index 100% rename from tests/unit/libstore/outputs-spec.cc rename to src/libstore-tests/outputs-spec.cc diff --git a/tests/unit/libstore/package.nix b/src/libstore-tests/package.nix similarity index 78% rename from tests/unit/libstore/package.nix rename to src/libstore-tests/package.nix index 7560a5b79..5b2fd108b 100644 --- a/tests/unit/libstore/package.nix +++ b/src/libstore-tests/package.nix @@ -1,12 +1,7 @@ { lib , buildPackages , stdenv -, mkMesonDerivation -, releaseTools - -, meson -, ninja -, pkg-config +, mkMesonExecutable , nix-store , nix-store-c @@ -27,15 +22,15 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonExecutable (finalAttrs: { pname = "nix-store-tests"; inherit version; workDir = ./.; fileset = fileset.unions [ - ../../../build-utils-meson + ../../build-utils-meson ./build-utils-meson - ../../../.version + ../../.version ./.version ./meson.build # ./meson.options @@ -43,12 +38,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - nativeBuildInputs = [ - meson - ninja - pkg-config - ]; - buildInputs = [ nix-store nix-store-c @@ -63,7 +52,7 @@ mkMesonDerivation (finalAttrs: { # Do the meson utils, without modification. '' chmod u+w ./.version - echo ${version} > ../../../.version + echo ${version} > ../../.version ''; mesonFlags = [ @@ -73,10 +62,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - passthru = { tests = { run = let @@ -86,7 +71,7 @@ mkMesonDerivation (finalAttrs: { root = ../..; fileset = lib.fileset.unions [ ./data - ../../functional/derivation + ../../tests/functional/derivation ]; }; in runCommand "${finalAttrs.pname}-run" { @@ -95,7 +80,7 @@ mkMesonDerivation (finalAttrs: { export HOME="$PWD/home-dir" mkdir -p "$HOME" '' + '' - export _NIX_TEST_UNIT_DATA=${data + "/unit/libstore/data"} + export _NIX_TEST_UNIT_DATA=${data + "/src/libstore-tests/data"} ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out ''); diff --git a/tests/unit/libstore/path-info.cc b/src/libstore-tests/path-info.cc similarity index 100% rename from tests/unit/libstore/path-info.cc rename to src/libstore-tests/path-info.cc diff --git a/tests/unit/libstore/path.cc b/src/libstore-tests/path.cc similarity index 100% rename from tests/unit/libstore/path.cc rename to src/libstore-tests/path.cc diff --git a/tests/unit/libstore/references.cc b/src/libstore-tests/references.cc similarity index 100% rename from tests/unit/libstore/references.cc rename to src/libstore-tests/references.cc diff --git a/tests/unit/libstore/s3-binary-cache-store.cc b/src/libstore-tests/s3-binary-cache-store.cc similarity index 100% rename from tests/unit/libstore/s3-binary-cache-store.cc rename to src/libstore-tests/s3-binary-cache-store.cc diff --git a/tests/unit/libstore/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc similarity index 100% rename from tests/unit/libstore/serve-protocol.cc rename to src/libstore-tests/serve-protocol.cc diff --git a/tests/unit/libstore/ssh-store.cc b/src/libstore-tests/ssh-store.cc similarity index 100% rename from tests/unit/libstore/ssh-store.cc rename to src/libstore-tests/ssh-store.cc diff --git a/tests/unit/libstore/store-reference.cc b/src/libstore-tests/store-reference.cc similarity index 100% rename from tests/unit/libstore/store-reference.cc rename to src/libstore-tests/store-reference.cc diff --git a/tests/unit/libstore/uds-remote-store.cc b/src/libstore-tests/uds-remote-store.cc similarity index 100% rename from tests/unit/libstore/uds-remote-store.cc rename to src/libstore-tests/uds-remote-store.cc diff --git a/tests/unit/libstore/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc similarity index 100% rename from tests/unit/libstore/worker-protocol.cc rename to src/libstore-tests/worker-protocol.cc diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 6079eae7b..b921dbe2d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -90,11 +90,11 @@ struct TunnelLogger : public Logger { if (ei.level > verbosity) return; - std::stringstream oss; + std::ostringstream oss; showErrorInfo(oss, ei, false); StringSink buf; - buf << STDERR_NEXT << oss.str(); + buf << STDERR_NEXT << toView(oss); enqueueMsg(buf.s); } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 154ec6007..e9e4b2c44 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -759,12 +759,17 @@ struct curlFileTransfer : public FileTransfer S3Helper s3Helper(profile, region, scheme, endpoint); + Activity act(*logger, lvlTalkative, actFileTransfer, + fmt("downloading '%s'", request.uri), + {request.uri}, request.parentAct); + // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) throw FileTransferError(NotFound, "S3 object '%s' does not exist", request.uri); res.data = std::move(*s3Res.data); + res.urls.push_back(request.uri); callback(std::move(res)); #else throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index be922c9f7..ff3df46ba 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -1227,14 +1227,13 @@ public: Setting warnLargePathThreshold{ this, - // n.b. this is deliberately int64 max rather than uint64 max because - // this goes through the Nix language JSON parser and thus needs to be - // representable in Nix language integers. - std::numeric_limits::max(), + 0, "warn-large-path-threshold", R"( Warn when copying a path larger than this number of bytes to the Nix store (as determined by its NAR serialisation). + Default is 0, which disables the warning. + Set it to 1 to warn on all paths. )" }; }; diff --git a/src/libstore/meson.build b/src/libstore/meson.build index 6a6aabf97..c2aa5bff3 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -69,6 +69,11 @@ has_acl_support = cxx.has_header('sys/xattr.h') \ and cxx.has_function('lremovexattr') configdata.set('HAVE_ACL_SUPPORT', has_acl_support.to_int()) +if host_machine.system() == 'darwin' + sandbox = cxx.find_library('sandbox') + deps_other += [sandbox] +endif + subdir('build-utils-meson/threads') boost = dependency( diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index 86788a87e..f5ecbd74b 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -30,16 +30,15 @@ std::optional OutputsSpec::parseOpt(std::string_view s) { static std::regex regex(std::string { outputSpecRegexStr }); - std::smatch match; - std::string s2 { s }; // until some improves std::regex - if (!std::regex_match(s2, match, regex)) + std::cmatch match; + if (!std::regex_match(s.cbegin(), s.cend(), match, regex)) return std::nullopt; if (match[1].matched) return { OutputsSpec::All {} }; if (match[2].matched) - return OutputsSpec::Names { tokenizeString(match[2].str(), ",") }; + return OutputsSpec::Names { tokenizeString({match[2].first, match[2].second}, ",") }; assert(false); } diff --git a/src/libstore/package.nix b/src/libstore/package.nix index 4582ba0d2..f04e3b95f 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -1,12 +1,9 @@ { lib , stdenv -, mkMesonDerivation -, releaseTools +, mkMesonLibrary -, meson -, ninja -, pkg-config , unixtools +, darwin , nix-util , boost @@ -29,7 +26,7 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonLibrary (finalAttrs: { pname = "nix-store"; inherit version; @@ -51,13 +48,8 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "sql") ./.) ]; - outputs = [ "out" "dev" ]; - - nativeBuildInputs = [ - meson - ninja - pkg-config - ] ++ lib.optional embeddedSandboxShell unixtools.hexdump; + nativeBuildInputs = + lib.optional embeddedSandboxShell unixtools.hexdump; buildInputs = [ boost @@ -65,6 +57,7 @@ mkMesonDerivation (finalAttrs: { sqlite ] ++ lib.optional stdenv.hostPlatform.isLinux libseccomp # There have been issues building these dependencies + ++ lib.optional stdenv.hostPlatform.isDarwin darwin.apple_sdk.libs.sandbox ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) aws-sdk-cpp ; @@ -98,10 +91,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 21175b1eb..bcbf0b55e 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -9,6 +9,7 @@ #include "globals.hh" #include "compression.hh" #include "filetransfer.hh" +#include "signals.hh" #include #include @@ -117,6 +118,7 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy { bool ShouldRetry(const Aws::Client::AWSError& error, long attemptedRetries) const override { + checkInterrupt(); auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); if (retry) printError("AWS error '%s' (%s), will retry in %d ms", diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 8109ea322..10577fa2a 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -171,7 +171,7 @@ std::pair StoreDirConfig::computeStorePath( PathFilter & filter) const { auto [h, size] = hashPath(path, method.getFileIngestionMethod(), hashAlgo, filter); - if (size && *size >= settings.warnLargePathThreshold) + if (settings.warnLargePathThreshold && size && *size >= settings.warnLargePathThreshold) warn("hashed large path '%s' (%s)", path, renderSize(*size)); return { makeFixedOutputPathFromCA( @@ -214,7 +214,7 @@ StorePath Store::addToStore( auto sink = sourceToSink([&](Source & source) { LengthSource lengthSource(source); storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair); - if (lengthSource.total >= settings.warnLargePathThreshold) + if (settings.warnLargePathThreshold && lengthSource.total >= settings.warnLargePathThreshold) warn("copied large path '%s' to the store (%s)", path, renderSize(lengthSource.total)); }); dumpPath(path, *sink, fsm, filter); diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 904b48f7b..dcfaadeef 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -58,6 +58,10 @@ #if __APPLE__ #include #include +#include + +/* This definition is undocumented but depended upon by all major browsers. */ +extern "C" int sandbox_init_with_parameters(const char *profile, uint64_t flags, const char *const parameters[], char **errorbuf); #endif #include @@ -65,6 +69,7 @@ #include #include "strings.hh" +#include "signals.hh" namespace nix { @@ -433,6 +438,41 @@ static void doBind(const Path & source, const Path & target, bool optional = fal }; #endif +/** + * Rethrow the current exception as a subclass of `Error`. + */ +static void rethrowExceptionAsError() +{ + try { + throw; + } catch (Error &) { + throw; + } catch (std::exception & e) { + throw Error(e.what()); + } catch (...) { + throw Error("unknown exception"); + } +} + +/** + * Send the current exception to the parent in the format expected by + * `LocalDerivationGoal::processSandboxSetupMessages()`. + */ +static void handleChildException(bool sendException) +{ + try { + rethrowExceptionAsError(); + } catch (Error & e) { + if (sendException) { + writeFull(STDERR_FILENO, "\1\n"); + FdSink sink(STDERR_FILENO); + sink << e; + sink.flush(); + } else + std::cerr << e.msg(); + } +} + void LocalDerivationGoal::startBuilder() { if ((buildUser && buildUser->getUIDCount() != 1) @@ -949,32 +989,40 @@ void LocalDerivationGoal::startBuilder() root. */ openSlave(); - /* Drop additional groups here because we can't do it - after we've created the new user namespace. */ - if (setgroups(0, 0) == -1) { - if (errno != EPERM) - throw SysError("setgroups failed"); - if (settings.requireDropSupplementaryGroups) - throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step."); + try { + /* Drop additional groups here because we can't do it + after we've created the new user namespace. */ + if (setgroups(0, 0) == -1) { + if (errno != EPERM) + throw SysError("setgroups failed"); + if (settings.requireDropSupplementaryGroups) + throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step."); + } + + ProcessOptions options; + options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; + if (privateNetwork) + options.cloneFlags |= CLONE_NEWNET; + if (usingUserNamespace) + options.cloneFlags |= CLONE_NEWUSER; + + pid_t child = startProcess([&]() { runChild(); }, options); + + writeFull(sendPid.writeSide.get(), fmt("%d\n", child)); + _exit(0); + } catch (...) { + handleChildException(true); + _exit(1); } - - ProcessOptions options; - options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; - if (privateNetwork) - options.cloneFlags |= CLONE_NEWNET; - if (usingUserNamespace) - options.cloneFlags |= CLONE_NEWUSER; - - pid_t child = startProcess([&]() { runChild(); }, options); - - writeFull(sendPid.writeSide.get(), fmt("%d\n", child)); - _exit(0); }); sendPid.writeSide.close(); - if (helper.wait() != 0) + if (helper.wait() != 0) { + processSandboxSetupMessages(); + // Only reached if the child process didn't send an exception. throw Error("unable to start build process"); + } userNamespaceSync.readSide = -1; @@ -1050,7 +1098,12 @@ void LocalDerivationGoal::startBuilder() pid.setSeparatePG(true); worker.childStarted(shared_from_this(), {builderOut.get()}, true, true); - /* Check if setting up the build environment failed. */ + processSandboxSetupMessages(); +} + + +void LocalDerivationGoal::processSandboxSetupMessages() +{ std::vector msgs; while (true) { std::string msg = [&]() { @@ -1078,7 +1131,8 @@ void LocalDerivationGoal::startBuilder() } -void LocalDerivationGoal::initTmpDir() { +void LocalDerivationGoal::initTmpDir() +{ /* In a sandbox, for determinism, always use the same temporary directory. */ #if __linux__ @@ -1530,6 +1584,8 @@ void LocalDerivationGoal::startDaemon() FdSink(remote.get()), NotTrusted, daemon::Recursive); debug("terminated daemon connection"); + } catch (const Interrupted &) { + debug("interrupted daemon connection"); } catch (SystemError &) { ignoreExceptionExceptInterrupt(); } @@ -1959,7 +2015,7 @@ void LocalDerivationGoal::runChild() if (chdir(chrootRootDir.c_str()) == -1) throw SysError("cannot change directory to '%1%'", chrootRootDir); - if (mkdir("real-root", 0) == -1) + if (mkdir("real-root", 0500) == -1) throw SysError("cannot create real-root directory"); if (pivot_root(".", "real-root") == -1) @@ -2031,154 +2087,130 @@ void LocalDerivationGoal::runChild() throw SysError("setuid failed"); } - /* Fill in the arguments. */ - Strings args; - - std::string builder = "invalid"; - - if (drv->isBuiltin()) { - ; - } #if __APPLE__ - else { - /* This has to appear before import statements. */ - std::string sandboxProfile = "(version 1)\n"; + /* This has to appear before import statements. */ + std::string sandboxProfile = "(version 1)\n"; - if (useChroot) { + if (useChroot) { - /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ - PathSet ancestry; + /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */ + PathSet ancestry; - /* We build the ancestry before adding all inputPaths to the store because we know they'll - all have the same parents (the store), and there might be lots of inputs. This isn't - particularly efficient... I doubt it'll be a bottleneck in practice */ - for (auto & i : pathsInChroot) { - Path cur = i.first; - while (cur.compare("/") != 0) { - cur = dirOf(cur); - ancestry.insert(cur); - } - } - - /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost - path component this time, since it's typically /nix/store and we care about that. */ - Path cur = worker.store.storeDir; + /* We build the ancestry before adding all inputPaths to the store because we know they'll + all have the same parents (the store), and there might be lots of inputs. This isn't + particularly efficient... I doubt it'll be a bottleneck in practice */ + for (auto & i : pathsInChroot) { + Path cur = i.first; while (cur.compare("/") != 0) { - ancestry.insert(cur); cur = dirOf(cur); + ancestry.insert(cur); } + } - /* Add all our input paths to the chroot */ - for (auto & i : inputPaths) { - auto p = worker.store.printStorePath(i); - pathsInChroot[p] = p; - } + /* And we want the store in there regardless of how empty pathsInChroot. We include the innermost + path component this time, since it's typically /nix/store and we care about that. */ + Path cur = worker.store.storeDir; + while (cur.compare("/") != 0) { + ancestry.insert(cur); + cur = dirOf(cur); + } - /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ - if (settings.darwinLogSandboxViolations) { - sandboxProfile += "(deny default)\n"; - } else { - sandboxProfile += "(deny default (with no-log))\n"; - } + /* Add all our input paths to the chroot */ + for (auto & i : inputPaths) { + auto p = worker.store.printStorePath(i); + pathsInChroot[p] = p; + } - sandboxProfile += - #include "sandbox-defaults.sb" - ; - - if (!derivationType->isSandboxed()) - sandboxProfile += - #include "sandbox-network.sb" - ; - - /* Add the output paths we'll use at build-time to the chroot */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & [_, path] : scratchOutputs) - sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); - - sandboxProfile += ")\n"; - - /* Our inputs (transitive dependencies and any impurities computed above) - - without file-write* allowed, access() incorrectly returns EPERM - */ - sandboxProfile += "(allow file-read* file-write* process-exec\n"; - for (auto & i : pathsInChroot) { - if (i.first != i.second.source) - throw Error( - "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", - i.first, i.second.source); - - std::string path = i.first; - auto optSt = maybeLstat(path.c_str()); - if (!optSt) { - if (i.second.optional) - continue; - throw SysError("getting attributes of required path '%s", path); - } - if (S_ISDIR(optSt->st_mode)) - sandboxProfile += fmt("\t(subpath \"%s\")\n", path); - else - sandboxProfile += fmt("\t(literal \"%s\")\n", path); - } - sandboxProfile += ")\n"; - - /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ - sandboxProfile += "(allow file-read*\n"; - for (auto & i : ancestry) { - sandboxProfile += fmt("\t(literal \"%s\")\n", i); - } - sandboxProfile += ")\n"; - - sandboxProfile += additionalSandboxProfile; - } else - sandboxProfile += - #include "sandbox-minimal.sb" - ; - - debug("Generated sandbox profile:"); - debug(sandboxProfile); - - Path sandboxFile = tmpDir + "/.sandbox.sb"; - - writeFile(sandboxFile, sandboxProfile); - - bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); - - /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */ - Path globalTmpDir = canonPath(defaultTempDir(), true); - - /* They don't like trailing slashes on subpath directives */ - while (!globalTmpDir.empty() && globalTmpDir.back() == '/') - globalTmpDir.pop_back(); - - if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { - builder = "/usr/bin/sandbox-exec"; - args.push_back("sandbox-exec"); - args.push_back("-f"); - args.push_back(sandboxFile); - args.push_back("-D"); - args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir); - if (allowLocalNetworking) { - args.push_back("-D"); - args.push_back(std::string("_ALLOW_LOCAL_NETWORKING=1")); - } - args.push_back(drv->builder); + /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */ + if (settings.darwinLogSandboxViolations) { + sandboxProfile += "(deny default)\n"; } else { - builder = drv->builder; - args.push_back(std::string(baseNameOf(drv->builder))); + sandboxProfile += "(deny default (with no-log))\n"; + } + + sandboxProfile += + #include "sandbox-defaults.sb" + ; + + if (!derivationType->isSandboxed()) + sandboxProfile += + #include "sandbox-network.sb" + ; + + /* Add the output paths we'll use at build-time to the chroot */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & [_, path] : scratchOutputs) + sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path)); + + sandboxProfile += ")\n"; + + /* Our inputs (transitive dependencies and any impurities computed above) + + without file-write* allowed, access() incorrectly returns EPERM + */ + sandboxProfile += "(allow file-read* file-write* process-exec\n"; + for (auto & i : pathsInChroot) { + if (i.first != i.second.source) + throw Error( + "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin", + i.first, i.second.source); + + std::string path = i.first; + auto optSt = maybeLstat(path.c_str()); + if (!optSt) { + if (i.second.optional) + continue; + throw SysError("getting attributes of required path '%s", path); + } + if (S_ISDIR(optSt->st_mode)) + sandboxProfile += fmt("\t(subpath \"%s\")\n", path); + else + sandboxProfile += fmt("\t(literal \"%s\")\n", path); + } + sandboxProfile += ")\n"; + + /* Allow file-read* on full directory hierarchy to self. Allows realpath() */ + sandboxProfile += "(allow file-read*\n"; + for (auto & i : ancestry) { + sandboxProfile += fmt("\t(literal \"%s\")\n", i); + } + sandboxProfile += ")\n"; + + sandboxProfile += additionalSandboxProfile; + } else + sandboxProfile += + #include "sandbox-minimal.sb" + ; + + debug("Generated sandbox profile:"); + debug(sandboxProfile); + + bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); + + /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms + to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */ + Path globalTmpDir = canonPath(defaultTempDir(), true); + + /* They don't like trailing slashes on subpath directives */ + while (!globalTmpDir.empty() && globalTmpDir.back() == '/') + globalTmpDir.pop_back(); + + if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") { + Strings sandboxArgs; + sandboxArgs.push_back("_GLOBAL_TMP_DIR"); + sandboxArgs.push_back(globalTmpDir); + if (allowLocalNetworking) { + sandboxArgs.push_back("_ALLOW_LOCAL_NETWORKING"); + sandboxArgs.push_back("1"); + } + char * sandbox_errbuf = nullptr; + if (sandbox_init_with_parameters(sandboxProfile.c_str(), 0, stringsToCharPtrs(sandboxArgs).data(), &sandbox_errbuf)) { + writeFull(STDERR_FILENO, fmt("failed to configure sandbox: %s\n", sandbox_errbuf ? sandbox_errbuf : "(null)")); + _exit(1); } } -#else - else { - builder = drv->builder; - args.push_back(std::string(baseNameOf(drv->builder))); - } #endif - for (auto & i : drv->args) - args.push_back(rewriteStrings(i, inputRewrites)); - /* Indicate that we managed to set up the build environment. */ writeFull(STDERR_FILENO, std::string("\2\n")); @@ -2209,6 +2241,14 @@ void LocalDerivationGoal::runChild() } } + // Now builder is not builtin + + Strings args; + args.push_back(std::string(baseNameOf(drv->builder))); + + for (auto & i : drv->args) + args.push_back(rewriteStrings(i, inputRewrites)); + #if __APPLE__ posix_spawnattr_t attrp; @@ -2230,21 +2270,15 @@ void LocalDerivationGoal::runChild() posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); } - posix_spawn(NULL, builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); + posix_spawn(NULL, drv->builder.c_str(), NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); #else - execve(builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); + execve(drv->builder.c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); #endif throw SysError("executing '%1%'", drv->builder); - } catch (Error & e) { - if (sendException) { - writeFull(STDERR_FILENO, "\1\n"); - FdSink sink(STDERR_FILENO); - sink << e; - sink.flush(); - } else - std::cerr << e.msg(); + } catch (...) { + handleChildException(sendException); _exit(1); } } @@ -3025,7 +3059,7 @@ bool LocalDerivationGoal::isReadDesc(int fd) StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName) { // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path - // See doc/manual/src/protocols/store-path.md for details + // See doc/manual/source/protocols/store-path.md for details // TODO: We may want to separate the responsibilities of constructing the path fingerprint and of actually doing the hashing auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName); return worker.store.makeStorePath( @@ -3038,7 +3072,7 @@ StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName) StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path) { // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path - // See doc/manual/src/protocols/store-path.md for details + // See doc/manual/source/protocols/store-path.md for details auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()); return worker.store.makeStorePath( pathType, diff --git a/src/libstore/unix/build/local-derivation-goal.hh b/src/libstore/unix/build/local-derivation-goal.hh index bf25cf2a6..1ea247661 100644 --- a/src/libstore/unix/build/local-derivation-goal.hh +++ b/src/libstore/unix/build/local-derivation-goal.hh @@ -210,6 +210,11 @@ struct LocalDerivationGoal : public DerivationGoal */ void initEnv(); + /** + * Process messages send by the sandbox initialization. + */ + void processSandboxSetupMessages(); + /** * Setup tmp dir location. */ @@ -220,8 +225,15 @@ struct LocalDerivationGoal : public DerivationGoal */ void writeStructuredAttrs(); + /** + * Start an in-process nix daemon thread for recursive-nix. + */ void startDaemon(); + /** + * Stop the in-process nix daemon thread. + * @see startDaemon + */ void stopDaemon(); /** diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix index ccfafd4d3..35533f981 100644 --- a/src/libutil-c/package.nix +++ b/src/libutil-c/package.nix @@ -1,11 +1,6 @@ { lib , stdenv -, mkMesonDerivation -, releaseTools - -, meson -, ninja -, pkg-config +, mkMesonLibrary , nix-util @@ -18,7 +13,7 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonLibrary (finalAttrs: { pname = "nix-util-c"; inherit version; @@ -35,14 +30,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "h") ./.) ]; - outputs = [ "out" "dev" ]; - - nativeBuildInputs = [ - meson - ninja - pkg-config - ]; - propagatedBuildInputs = [ nix-util ]; @@ -62,10 +49,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libutil-test-support/.version b/src/libutil-test-support/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libutil-test-support/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libutil-test-support/build-utils-meson b/src/libutil-test-support/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libutil-test-support/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/tests/unit/libutil-support/local.mk b/src/libutil-test-support/local.mk similarity index 100% rename from tests/unit/libutil-support/local.mk rename to src/libutil-test-support/local.mk diff --git a/tests/unit/libutil-support/meson.build b/src/libutil-test-support/meson.build similarity index 100% rename from tests/unit/libutil-support/meson.build rename to src/libutil-test-support/meson.build diff --git a/tests/unit/libutil-support/package.nix b/src/libutil-test-support/package.nix similarity index 68% rename from tests/unit/libutil-support/package.nix rename to src/libutil-test-support/package.nix index 1665804cb..2525e1602 100644 --- a/tests/unit/libutil-support/package.nix +++ b/src/libutil-test-support/package.nix @@ -1,11 +1,6 @@ { lib , stdenv -, mkMesonDerivation -, releaseTools - -, meson -, ninja -, pkg-config +, mkMesonLibrary , nix-util @@ -20,15 +15,15 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonLibrary (finalAttrs: { pname = "nix-util-test-support"; inherit version; workDir = ./.; fileset = fileset.unions [ - ../../../build-utils-meson + ../../build-utils-meson ./build-utils-meson - ../../../.version + ../../.version ./.version ./meson.build # ./meson.options @@ -36,14 +31,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - outputs = [ "out" "dev" ]; - - nativeBuildInputs = [ - meson - ninja - pkg-config - ]; - propagatedBuildInputs = [ nix-util rapidcheck @@ -54,7 +41,7 @@ mkMesonDerivation (finalAttrs: { # Do the meson utils, without modification. '' chmod u+w ./.version - echo ${version} > ../../../.version + echo ${version} > ../../.version ''; mesonFlags = [ @@ -64,10 +51,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/tests/unit/libutil-support/tests/characterization.hh b/src/libutil-test-support/tests/characterization.hh similarity index 100% rename from tests/unit/libutil-support/tests/characterization.hh rename to src/libutil-test-support/tests/characterization.hh diff --git a/tests/unit/libutil-support/tests/gtest-with-params.hh b/src/libutil-test-support/tests/gtest-with-params.hh similarity index 100% rename from tests/unit/libutil-support/tests/gtest-with-params.hh rename to src/libutil-test-support/tests/gtest-with-params.hh diff --git a/tests/unit/libutil-support/tests/hash.cc b/src/libutil-test-support/tests/hash.cc similarity index 100% rename from tests/unit/libutil-support/tests/hash.cc rename to src/libutil-test-support/tests/hash.cc diff --git a/tests/unit/libutil-support/tests/hash.hh b/src/libutil-test-support/tests/hash.hh similarity index 100% rename from tests/unit/libutil-support/tests/hash.hh rename to src/libutil-test-support/tests/hash.hh diff --git a/tests/unit/libutil-support/tests/nix_api_util.hh b/src/libutil-test-support/tests/nix_api_util.hh similarity index 100% rename from tests/unit/libutil-support/tests/nix_api_util.hh rename to src/libutil-test-support/tests/nix_api_util.hh diff --git a/tests/unit/libutil-support/tests/string_callback.cc b/src/libutil-test-support/tests/string_callback.cc similarity index 100% rename from tests/unit/libutil-support/tests/string_callback.cc rename to src/libutil-test-support/tests/string_callback.cc diff --git a/tests/unit/libutil-support/tests/string_callback.hh b/src/libutil-test-support/tests/string_callback.hh similarity index 100% rename from tests/unit/libutil-support/tests/string_callback.hh rename to src/libutil-test-support/tests/string_callback.hh diff --git a/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc b/src/libutil-test-support/tests/tracing-file-system-object-sink.cc similarity index 100% rename from tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc rename to src/libutil-test-support/tests/tracing-file-system-object-sink.cc diff --git a/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh b/src/libutil-test-support/tests/tracing-file-system-object-sink.hh similarity index 100% rename from tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh rename to src/libutil-test-support/tests/tracing-file-system-object-sink.hh diff --git a/src/libutil-tests/.version b/src/libutil-tests/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libutil-tests/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/tests/unit/libutil/args.cc b/src/libutil-tests/args.cc similarity index 100% rename from tests/unit/libutil/args.cc rename to src/libutil-tests/args.cc diff --git a/src/libutil-tests/build-utils-meson b/src/libutil-tests/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libutil-tests/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/tests/unit/libutil/canon-path.cc b/src/libutil-tests/canon-path.cc similarity index 100% rename from tests/unit/libutil/canon-path.cc rename to src/libutil-tests/canon-path.cc diff --git a/tests/unit/libutil/checked-arithmetic.cc b/src/libutil-tests/checked-arithmetic.cc similarity index 100% rename from tests/unit/libutil/checked-arithmetic.cc rename to src/libutil-tests/checked-arithmetic.cc diff --git a/tests/unit/libutil/chunked-vector.cc b/src/libutil-tests/chunked-vector.cc similarity index 100% rename from tests/unit/libutil/chunked-vector.cc rename to src/libutil-tests/chunked-vector.cc diff --git a/tests/unit/libutil/closure.cc b/src/libutil-tests/closure.cc similarity index 100% rename from tests/unit/libutil/closure.cc rename to src/libutil-tests/closure.cc diff --git a/tests/unit/libutil/compression.cc b/src/libutil-tests/compression.cc similarity index 100% rename from tests/unit/libutil/compression.cc rename to src/libutil-tests/compression.cc diff --git a/tests/unit/libutil/config.cc b/src/libutil-tests/config.cc similarity index 100% rename from tests/unit/libutil/config.cc rename to src/libutil-tests/config.cc diff --git a/tests/unit/libutil/data/git/check-data.sh b/src/libutil-tests/data/git/check-data.sh similarity index 100% rename from tests/unit/libutil/data/git/check-data.sh rename to src/libutil-tests/data/git/check-data.sh diff --git a/tests/unit/libutil/data/git/hello-world-blob.bin b/src/libutil-tests/data/git/hello-world-blob.bin similarity index 100% rename from tests/unit/libutil/data/git/hello-world-blob.bin rename to src/libutil-tests/data/git/hello-world-blob.bin diff --git a/tests/unit/libutil/data/git/hello-world.bin b/src/libutil-tests/data/git/hello-world.bin similarity index 100% rename from tests/unit/libutil/data/git/hello-world.bin rename to src/libutil-tests/data/git/hello-world.bin diff --git a/tests/unit/libutil/data/git/tree.bin b/src/libutil-tests/data/git/tree.bin similarity index 100% rename from tests/unit/libutil/data/git/tree.bin rename to src/libutil-tests/data/git/tree.bin diff --git a/tests/unit/libutil/data/git/tree.txt b/src/libutil-tests/data/git/tree.txt similarity index 100% rename from tests/unit/libutil/data/git/tree.txt rename to src/libutil-tests/data/git/tree.txt diff --git a/tests/unit/libutil/executable-path.cc b/src/libutil-tests/executable-path.cc similarity index 100% rename from tests/unit/libutil/executable-path.cc rename to src/libutil-tests/executable-path.cc diff --git a/tests/unit/libutil/file-content-address.cc b/src/libutil-tests/file-content-address.cc similarity index 100% rename from tests/unit/libutil/file-content-address.cc rename to src/libutil-tests/file-content-address.cc diff --git a/tests/unit/libutil/file-system.cc b/src/libutil-tests/file-system.cc similarity index 100% rename from tests/unit/libutil/file-system.cc rename to src/libutil-tests/file-system.cc diff --git a/tests/unit/libutil/git.cc b/src/libutil-tests/git.cc similarity index 99% rename from tests/unit/libutil/git.cc rename to src/libutil-tests/git.cc index 9232de5b9..048956a58 100644 --- a/tests/unit/libutil/git.cc +++ b/src/libutil-tests/git.cc @@ -88,7 +88,7 @@ TEST_F(GitTest, blob_write) { /** * This data is for "shallow" tree tests. However, we use "real" hashes * so that we can check our test data in a small shell script test test - * (`tests/unit/libutil/data/git/check-data.sh`). + * (`src/libutil-tests/data/git/check-data.sh`). */ const static Tree tree = { { diff --git a/tests/unit/libutil/hash.cc b/src/libutil-tests/hash.cc similarity index 100% rename from tests/unit/libutil/hash.cc rename to src/libutil-tests/hash.cc diff --git a/tests/unit/libutil/hilite.cc b/src/libutil-tests/hilite.cc similarity index 100% rename from tests/unit/libutil/hilite.cc rename to src/libutil-tests/hilite.cc diff --git a/tests/unit/libutil/json-utils.cc b/src/libutil-tests/json-utils.cc similarity index 100% rename from tests/unit/libutil/json-utils.cc rename to src/libutil-tests/json-utils.cc diff --git a/tests/unit/libutil/local.mk b/src/libutil-tests/local.mk similarity index 96% rename from tests/unit/libutil/local.mk rename to src/libutil-tests/local.mk index 404f35cf1..c747863a4 100644 --- a/tests/unit/libutil/local.mk +++ b/src/libutil-tests/local.mk @@ -17,7 +17,7 @@ endif libutil-tests_SOURCES := $(wildcard $(d)/*.cc) libutil-tests_EXTRA_INCLUDES = \ - -I tests/unit/libutil-support \ + -I src/libutil-test-support \ $(INCLUDE_libutil) \ $(INCLUDE_libutilc) diff --git a/tests/unit/libutil/logging.cc b/src/libutil-tests/logging.cc similarity index 100% rename from tests/unit/libutil/logging.cc rename to src/libutil-tests/logging.cc diff --git a/tests/unit/libutil/lru-cache.cc b/src/libutil-tests/lru-cache.cc similarity index 100% rename from tests/unit/libutil/lru-cache.cc rename to src/libutil-tests/lru-cache.cc diff --git a/tests/unit/libutil/meson.build b/src/libutil-tests/meson.build similarity index 100% rename from tests/unit/libutil/meson.build rename to src/libutil-tests/meson.build diff --git a/tests/unit/libutil/nix_api_util.cc b/src/libutil-tests/nix_api_util.cc similarity index 100% rename from tests/unit/libutil/nix_api_util.cc rename to src/libutil-tests/nix_api_util.cc diff --git a/tests/unit/libutil/package.nix b/src/libutil-tests/package.nix similarity index 80% rename from tests/unit/libutil/package.nix rename to src/libutil-tests/package.nix index 2fce5bfa8..b099037ee 100644 --- a/tests/unit/libutil/package.nix +++ b/src/libutil-tests/package.nix @@ -1,12 +1,7 @@ { lib , buildPackages , stdenv -, mkMesonDerivation -, releaseTools - -, meson -, ninja -, pkg-config +, mkMesonExecutable , nix-util , nix-util-c @@ -25,15 +20,15 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonExecutable (finalAttrs: { pname = "nix-util-tests"; inherit version; workDir = ./.; fileset = fileset.unions [ - ../../../build-utils-meson + ../../build-utils-meson ./build-utils-meson - ../../../.version + ../../.version ./.version ./meson.build # ./meson.options @@ -41,12 +36,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - nativeBuildInputs = [ - meson - ninja - pkg-config - ]; - buildInputs = [ nix-util nix-util-c @@ -60,7 +49,7 @@ mkMesonDerivation (finalAttrs: { # Do the meson utils, without modification. '' chmod u+w ./.version - echo ${version} > ../../../.version + echo ${version} > ../../.version ''; mesonFlags = [ @@ -70,10 +59,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - passthru = { tests = { run = runCommand "${finalAttrs.pname}-run" { diff --git a/tests/unit/libutil/pool.cc b/src/libutil-tests/pool.cc similarity index 100% rename from tests/unit/libutil/pool.cc rename to src/libutil-tests/pool.cc diff --git a/tests/unit/libutil/position.cc b/src/libutil-tests/position.cc similarity index 100% rename from tests/unit/libutil/position.cc rename to src/libutil-tests/position.cc diff --git a/tests/unit/libutil/processes.cc b/src/libutil-tests/processes.cc similarity index 100% rename from tests/unit/libutil/processes.cc rename to src/libutil-tests/processes.cc diff --git a/tests/unit/libutil/references.cc b/src/libutil-tests/references.cc similarity index 100% rename from tests/unit/libutil/references.cc rename to src/libutil-tests/references.cc diff --git a/tests/unit/libutil/spawn.cc b/src/libutil-tests/spawn.cc similarity index 100% rename from tests/unit/libutil/spawn.cc rename to src/libutil-tests/spawn.cc diff --git a/tests/unit/libutil/strings.cc b/src/libutil-tests/strings.cc similarity index 100% rename from tests/unit/libutil/strings.cc rename to src/libutil-tests/strings.cc diff --git a/tests/unit/libutil/suggestions.cc b/src/libutil-tests/suggestions.cc similarity index 100% rename from tests/unit/libutil/suggestions.cc rename to src/libutil-tests/suggestions.cc diff --git a/tests/unit/libutil/terminal.cc b/src/libutil-tests/terminal.cc similarity index 100% rename from tests/unit/libutil/terminal.cc rename to src/libutil-tests/terminal.cc diff --git a/tests/unit/libutil/url.cc b/src/libutil-tests/url.cc similarity index 100% rename from tests/unit/libutil/url.cc rename to src/libutil-tests/url.cc diff --git a/tests/unit/libutil/util.cc b/src/libutil-tests/util.cc similarity index 100% rename from tests/unit/libutil/util.cc rename to src/libutil-tests/util.cc diff --git a/tests/unit/libutil/xml-writer.cc b/src/libutil-tests/xml-writer.cc similarity index 100% rename from tests/unit/libutil/xml-writer.cc rename to src/libutil-tests/xml-writer.cc diff --git a/src/libutil/args.cc b/src/libutil/args.cc index d58f4b4ae..4e87389d6 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -293,7 +293,7 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) // We match one space after `nix` so that we preserve indentation. // No space is necessary for an empty line. An empty line has basically no effect. if (std::regex_match(line, match, std::regex("^#!\\s*nix(:? |$)(.*)$"))) - shebangContent += match[2].str() + "\n"; + shebangContent += std::string_view{match[2].first, match[2].second} + "\n"; } for (const auto & word : parseShebangContent(shebangContent)) { cmdline.push_back(word); diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 224b78b23..fd51d7d3c 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -630,7 +630,28 @@ void setWriteTime( time_t modificationTime, std::optional optIsSymlink) { -#ifndef _WIN32 +#ifdef _WIN32 + // FIXME use `fs::last_write_time`. + // + // Would be nice to use std::filesystem unconditionally, but + // doesn't support access time just modification time. + // + // System clock vs File clock issues also make that annoying. + warn("Changing file times is not yet implemented on Windows, path is '%s'", path); +#elif HAVE_UTIMENSAT && HAVE_DECL_AT_SYMLINK_NOFOLLOW + struct timespec times[2] = { + { + .tv_sec = accessedTime, + .tv_nsec = 0, + }, + { + .tv_sec = modificationTime, + .tv_nsec = 0, + }, + }; + if (utimensat(AT_FDCWD, path.c_str(), times, AT_SYMLINK_NOFOLLOW) == -1) + throw SysError("changing modification time of '%s' (using `utimensat`)", path); +#else struct timeval times[2] = { { .tv_sec = accessedTime, @@ -641,42 +662,21 @@ void setWriteTime( .tv_usec = 0, }, }; -#endif - - auto nonSymlink = [&]{ - bool isSymlink = optIsSymlink - ? *optIsSymlink - : fs::is_symlink(path); - - if (!isSymlink) { -#ifdef _WIN32 - // FIXME use `fs::last_write_time`. - // - // Would be nice to use std::filesystem unconditionally, but - // doesn't support access time just modification time. - // - // System clock vs File clock issues also make that annoying. - warn("Changing file times is not yet implemented on Windows, path is '%s'", path); -#else - if (utimes(path.c_str(), times) == -1) { - - throw SysError("changing modification time of '%s' (not a symlink)", path); - } -#endif - } else { - throw Error("Cannot modification time of symlink '%s'", path); - } - }; - #if HAVE_LUTIMES - if (lutimes(path.c_str(), times) == -1) { - if (errno == ENOSYS) - nonSymlink(); - else - throw SysError("changing modification time of '%s'", path); - } + if (lutimes(path.c_str(), times) == -1) + throw SysError("changing modification time of '%s'", path); #else - nonSymlink(); + bool isSymlink = optIsSymlink + ? *optIsSymlink + : fs::is_symlink(path); + + if (!isSymlink) { + if (utimes(path.c_str(), times) == -1) + throw SysError("changing modification time of '%s' (not a symlink)", path); + } else { + throw Error("Cannot modification time of symlink '%s'", path); + } +#endif #endif } diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 3ef71a716..3d7371457 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -85,10 +85,10 @@ public: void logEI(const ErrorInfo & ei) override { - std::stringstream oss; + std::ostringstream oss; showErrorInfo(oss, ei, loggerSettings.showTrace.get()); - log(ei.level, oss.str()); + log(ei.level, toView(oss)); } void startActivity(ActivityId act, Verbosity lvl, ActivityType type, diff --git a/src/libutil/meson.build b/src/libutil/meson.build index 57b741a50..08413783d 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -42,6 +42,8 @@ check_funcs = [ # Optionally used to try to close more file descriptors (e.g. before # forking) on Unix. 'sysconf', + # Optionally used for changing the mtime of files and symlinks. + 'utimensat', ] foreach funcspec : check_funcs define_name = 'HAVE_' + funcspec.underscorify().to_upper() @@ -49,6 +51,8 @@ foreach funcspec : check_funcs configdata.set(define_name, define_value) endforeach +configdata.set('HAVE_DECL_AT_SYMLINK_NOFOLLOW', cxx.has_header_symbol('fcntl.h', 'AT_SYMLINK_NOFOLLOW').to_int()) + subdir('build-utils-meson/threads') # Check if -latomic is needed diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 4ce1a75b0..17a156740 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -1,11 +1,6 @@ { lib , stdenv -, mkMesonDerivation -, releaseTools - -, meson -, ninja -, pkg-config +, mkMesonLibrary , boost , brotli @@ -24,7 +19,7 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonLibrary (finalAttrs: { pname = "nix-util"; inherit version; @@ -43,14 +38,6 @@ mkMesonDerivation (finalAttrs: { (fileset.fileFilter (file: file.hasExt "hh") ./.) ]; - outputs = [ "out" "dev" ]; - - nativeBuildInputs = [ - meson - ninja - pkg-config - ]; - buildInputs = [ brotli libsodium @@ -88,10 +75,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index 5cad95758..c221a43c6 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -1,11 +1,27 @@ #include #include +#include #include "strings-inline.hh" #include "os-string.hh" namespace nix { +struct view_stringbuf : public std::stringbuf +{ + inline std::string_view toView() + { + auto begin = pbase(); + return {begin, begin + pubseekoff(0, std::ios_base::cur, std::ios_base::out)}; + } +}; + +std::string_view toView(const std::ostringstream & os) +{ + auto buf = static_cast(os.rdbuf()); + return buf->toView(); +} + template std::list tokenizeString(std::string_view s, std::string_view separators); template std::set tokenizeString(std::string_view s, std::string_view separators); template std::vector tokenizeString(std::string_view s, std::string_view separators); diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh index 88b48d770..533126be1 100644 --- a/src/libutil/strings.hh +++ b/src/libutil/strings.hh @@ -8,6 +8,11 @@ namespace nix { +/* + * workaround for unavailable view() method (C++20) of std::ostringstream under MacOS with clang-16 + */ +std::string_view toView(const std::ostringstream & os); + /** * String tokenizer. * diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index 0355e1f07..0725c1926 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -110,10 +110,15 @@ void ThreadPool::doWork(bool mainThread) propagate it. */ try { std::rethrow_exception(exc); + } catch (const Interrupted &) { + // The interrupted state may be picked up by multiple + // workers, which is expected, so we should ignore + // it silently and let the first one bubble up, + // rethrown via the original state->exception. + } catch (const ThreadPoolShutDown &) { + // Similarly expected. } catch (std::exception & e) { - if (!dynamic_cast(&e)) - ignoreExceptionExceptInterrupt(); - } catch (...) { + ignoreExceptionExceptInterrupt(); } } } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index a5b9e1e54..c394836da 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -36,7 +36,7 @@ extern char * * environ __attribute__((weak)); /* Recreate the effect of the perl shellwords function, breaking up a * string into arguments like a shell word, including escapes */ -static std::vector shellwords(const std::string & s) +static std::vector shellwords(std::string_view s) { std::regex whitespace("^\\s+"); auto begin = s.cbegin(); @@ -51,7 +51,7 @@ static std::vector shellwords(const std::string & s) auto it = begin; for (; it != s.cend(); ++it) { if (st == sBegin) { - std::smatch match; + std::cmatch match; if (regex_search(it, s.cend(), match, whitespace)) { cur.append(begin, it); res.push_back(cur); @@ -173,7 +173,7 @@ static void main_nix_build(int argc, char * * argv) line = chomp(line); std::smatch match; if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell\\s+(.*)$"))) - for (const auto & word : shellwords(match[1].str())) + for (const auto & word : shellwords({match[1].first, match[1].second})) args.push_back(word); } } @@ -260,9 +260,9 @@ static void main_nix_build(int argc, char * * argv) // read the shebang to understand which packages to read from. Since // this is handled via nix-shell -p, we wrap our ruby script execution // in ruby -e 'load' which ignores the shebangs. - envCommand = fmt("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%", execArgs, interpreter, shellEscape(script), joined.str()); + envCommand = fmt("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%", execArgs, interpreter, shellEscape(script), toView(joined)); } else { - envCommand = fmt("exec %1% %2% %3% %4%", execArgs, interpreter, shellEscape(script), joined.str()); + envCommand = fmt("exec %1% %2% %3% %4%", execArgs, interpreter, shellEscape(script), toView(joined)); } } @@ -526,8 +526,6 @@ static void main_nix_build(int argc, char * * argv) // Set the environment. auto env = getEnv(); - auto tmp = getEnvNonEmpty("TMPDIR").value_or("/tmp"); - if (pure) { decltype(env) newEnv; for (auto & i : env) @@ -538,18 +536,16 @@ static void main_nix_build(int argc, char * * argv) env["__ETC_PROFILE_SOURCED"] = "1"; } - env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmp; + env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDir.path().string(); env["NIX_STORE"] = store->storeDir; env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores); auto passAsFile = tokenizeString(getOr(drv.env, "passAsFile", "")); - bool keepTmp = false; int fileNr = 0; for (auto & var : drv.env) if (passAsFile.count(var.first)) { - keepTmp = true; auto fn = ".attr-" + std::to_string(fileNr++); Path p = (tmpDir.path() / fn).string(); writeFile(p, var.second); @@ -591,7 +587,6 @@ static void main_nix_build(int argc, char * * argv) env["NIX_ATTRS_SH_FILE"] = attrsSH; env["NIX_ATTRS_JSON_FILE"] = attrsJSON; - keepTmp = true; } } @@ -601,12 +596,10 @@ static void main_nix_build(int argc, char * * argv) lose the current $PATH directories. */ auto rcfile = (tmpDir.path() / "rc").string(); std::string rc = fmt( - R"(_nix_shell_clean_tmpdir() { command rm -rf %1%; }; )"s + - (keepTmp ? - "trap _nix_shell_clean_tmpdir EXIT; " - "exitHooks+=(_nix_shell_clean_tmpdir); " - "failureHooks+=(_nix_shell_clean_tmpdir); ": - "_nix_shell_clean_tmpdir; ") + + (R"(_nix_shell_clean_tmpdir() { command rm -rf %1%; };)"s + "trap _nix_shell_clean_tmpdir EXIT; " + "exitHooks+=(_nix_shell_clean_tmpdir); " + "failureHooks+=(_nix_shell_clean_tmpdir); ") + (pure ? "" : "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;") + "%2%" // always clear PATH. diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index a24dd11d6..ee62077c0 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -111,9 +111,7 @@ bool createUserEnv(EvalState & state, PackageInfos & elems, auto manifestFile = ({ std::ostringstream str; printAmbiguous(manifest, state.symbols, str, nullptr, std::numeric_limits::max()); - // TODO with C++20 we can use str.view() instead and avoid copy. - std::string str2 = str.str(); - StringSource source { str2 }; + StringSource source { toView(str) }; state.store->addToStoreFromDump( source, "env-manifest.nix", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references); }); diff --git a/src/nix-expr-test-support b/src/nix-expr-test-support deleted file mode 120000 index 427b80dff..000000000 --- a/src/nix-expr-test-support +++ /dev/null @@ -1 +0,0 @@ -../tests/unit/libexpr-support \ No newline at end of file diff --git a/src/nix-expr-tests b/src/nix-expr-tests deleted file mode 120000 index 3af7110d3..000000000 --- a/src/nix-expr-tests +++ /dev/null @@ -1 +0,0 @@ -../tests/unit/libexpr \ No newline at end of file diff --git a/src/nix-fetchers-tests b/src/nix-fetchers-tests deleted file mode 120000 index 80e4b68ae..000000000 --- a/src/nix-fetchers-tests +++ /dev/null @@ -1 +0,0 @@ -../tests/unit/libfetchers \ No newline at end of file diff --git a/src/nix-flake-tests b/src/nix-flake-tests deleted file mode 120000 index bb2d49400..000000000 --- a/src/nix-flake-tests +++ /dev/null @@ -1 +0,0 @@ -../tests/unit/libflake \ No newline at end of file diff --git a/src/nix-manual b/src/nix-manual new file mode 120000 index 000000000..492c97408 --- /dev/null +++ b/src/nix-manual @@ -0,0 +1 @@ +../doc/manual/ \ No newline at end of file diff --git a/src/nix-store-test-support b/src/nix-store-test-support deleted file mode 120000 index af4befd90..000000000 --- a/src/nix-store-test-support +++ /dev/null @@ -1 +0,0 @@ -../tests/unit/libstore-support \ No newline at end of file diff --git a/src/nix-store-tests b/src/nix-store-tests deleted file mode 120000 index fc9b910af..000000000 --- a/src/nix-store-tests +++ /dev/null @@ -1 +0,0 @@ -../tests/unit/libstore \ No newline at end of file diff --git a/src/nix-util-test-support b/src/nix-util-test-support deleted file mode 120000 index 4b25930eb..000000000 --- a/src/nix-util-test-support +++ /dev/null @@ -1 +0,0 @@ -../tests/unit/libutil-support \ No newline at end of file diff --git a/src/nix-util-tests b/src/nix-util-tests deleted file mode 120000 index e1138411a..000000000 --- a/src/nix-util-tests +++ /dev/null @@ -1 +0,0 @@ -../tests/unit/libutil \ No newline at end of file diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index 6cf73785e..a72b06542 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -26,17 +26,17 @@ std::string formatProtocol(unsigned int proto) return "unknown"; } -bool checkPass(const std::string & msg) { +bool checkPass(std::string_view msg) { notice(ANSI_GREEN "[PASS] " ANSI_NORMAL + msg); return true; } -bool checkFail(const std::string & msg) { +bool checkFail(std::string_view msg) { notice(ANSI_RED "[FAIL] " ANSI_NORMAL + msg); return false; } -void checkInfo(const std::string & msg) { +void checkInfo(std::string_view msg) { notice(ANSI_BLUE "[INFO] " ANSI_NORMAL + msg); } @@ -87,11 +87,11 @@ struct CmdConfigCheck : StoreCommand } if (dirs.size() != 1) { - std::stringstream ss; + std::ostringstream ss; ss << "Multiple versions of nix found in PATH:\n"; for (auto & dir : dirs) ss << " " << dir << "\n"; - return checkFail(ss.str()); + return checkFail(toView(ss)); } return checkPass("PATH contains only one nix version."); @@ -125,14 +125,14 @@ struct CmdConfigCheck : StoreCommand } if (!dirs.empty()) { - std::stringstream ss; + std::ostringstream ss; ss << "Found profiles outside of " << settings.nixStateDir << "/profiles.\n" << "The generation this profile points to might not have a gcroot and could be\n" << "garbage collected, resulting in broken symlinks.\n\n"; for (auto & dir : dirs) ss << " " << dir << "\n"; ss << "\n"; - return checkFail(ss.str()); + return checkFail(toView(ss)); } return checkPass("All profiles are gcroots."); @@ -145,13 +145,13 @@ struct CmdConfigCheck : StoreCommand : PROTOCOL_VERSION; if (clientProto != storeProto) { - std::stringstream ss; + std::ostringstream ss; ss << "Warning: protocol version of this client does not match the store.\n" << "While this is not necessarily a problem it's recommended to keep the client in\n" << "sync with the daemon.\n\n" << "Client protocol: " << formatProtocol(clientProto) << "\n" << "Store protocol: " << formatProtocol(storeProto) << "\n\n"; - return checkFail(ss.str()); + return checkFail(toView(ss)); } return checkPass("Client protocol matches store protocol."); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 512e68711..babf2ca32 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -11,11 +11,13 @@ using namespace nix; +namespace nix::fs { using namespace std::filesystem; } + struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption { bool raw = false; std::optional apply; - std::optional writeTo; + std::optional writeTo; CmdEval() : InstallableValueCommand() { @@ -75,20 +77,21 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption if (writeTo) { stopProgressBar(); - if (pathExists(*writeTo)) - throw Error("path '%s' already exists", *writeTo); + if (fs::symlink_exists(*writeTo)) + throw Error("path '%s' already exists", writeTo->string()); - std::function recurse; + std::function recurse; - recurse = [&](Value & v, const PosIdx pos, const std::filesystem::path & path) + recurse = [&](Value & v, const PosIdx pos, const fs::path & path) { state->forceValue(v, pos); if (v.type() == nString) // FIXME: disallow strings with contexts? writeFile(path.string(), v.string_view()); else if (v.type() == nAttrs) { - // TODO abstract mkdir perms for Windows - createDir(path.string(), 0777); + [[maybe_unused]] bool directoryCreated = fs::create_directory(path); + // Directory should not already exist + assert(directoryCreated); for (auto & attr : *v.attrs()) { std::string_view name = state->symbols[attr.name]; try { diff --git a/src/nix/eval.md b/src/nix/eval.md index 48d5aa597..bd5b035e1 100644 --- a/src/nix/eval.md +++ b/src/nix/eval.md @@ -50,8 +50,9 @@ R""( # Description -This command evaluates the given Nix expression and prints the -result on standard output. +This command evaluates the given Nix expression, and prints the result on standard output. + +It also evaluates any nested attribute values and list items. # Output format diff --git a/src/nix/help-stores.md b/src/nix/help-stores.md index 5c5624f5e..ff9d39c50 120000 --- a/src/nix/help-stores.md +++ b/src/nix/help-stores.md @@ -1 +1 @@ -../../doc/manual/src/store/types/index.md.in \ No newline at end of file +../../doc/manual/source/store/types/index.md.in \ No newline at end of file diff --git a/src/nix/meson.build b/src/nix/meson.build index 6edb768e3..55089d821 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -212,18 +212,23 @@ nix_symlinks = [ 'nix-store', ] +executable_suffix = '' +if host_machine.system() == 'windows' + executable_suffix = '.exe' +endif + foreach linkname : nix_symlinks install_symlink( - linkname, + linkname + executable_suffix, # TODO(Qyriad): should these continue to be relative symlinks? - pointing_to : 'nix', + pointing_to : fs.name(this_exe), install_dir : get_option('bindir'), # The 'runtime' tag is what executables default to, which we want to emulate here. install_tag : 'runtime' ) t = custom_target( command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'], - output: linkname, + output: linkname + executable_suffix, # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working) build_by_default: true ) @@ -233,15 +238,15 @@ endforeach install_symlink( 'build-remote', - pointing_to : '..' / '..'/ get_option('bindir') / 'nix', - install_dir : get_option('libexecdir') / 'nix', + pointing_to : '..' / '..'/ get_option('bindir') / fs.name(this_exe), + install_dir : get_option('libexecdir') / fs.name(this_exe), # The 'runtime' tag is what executables default to, which we want to emulate here. install_tag : 'runtime' ) custom_target( command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'], - output: 'build-remote', + output: 'build-remote' + executable_suffix, # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working) build_by_default: true ) diff --git a/src/nix/package.nix b/src/nix/package.nix index 3e19c6dca..c7b24efce 100644 --- a/src/nix/package.nix +++ b/src/nix/package.nix @@ -1,21 +1,12 @@ { lib , stdenv -, mkMesonDerivation -, releaseTools - -, meson -, ninja -, pkg-config +, mkMesonExecutable , nix-store , nix-expr , nix-main , nix-cmd -, rapidcheck -, gtest -, runCommand - # Configuration Options , version @@ -25,7 +16,7 @@ let inherit (lib) fileset; in -mkMesonDerivation (finalAttrs: { +mkMesonExecutable (finalAttrs: { pname = "nix"; inherit version; @@ -66,9 +57,9 @@ mkMesonDerivation (finalAttrs: { ../nix-env/buildenv.nix ./get-env.sh ./help-stores.md - ../../doc/manual/src/store/types/index.md.in + ../../doc/manual/source/store/types/index.md.in ./profiles.md - ../../doc/manual/src/command-ref/files/profiles.md + ../../doc/manual/source/command-ref/files/profiles.md # Files ] ++ lib.concatMap @@ -90,12 +81,6 @@ mkMesonDerivation (finalAttrs: { ] ); - nativeBuildInputs = [ - meson - ninja - pkg-config - ]; - buildInputs = [ nix-store nix-expr @@ -118,10 +103,6 @@ mkMesonDerivation (finalAttrs: { LDFLAGS = "-fuse-ld=gold"; }; - separateDebugInfo = !stdenv.hostPlatform.isStatic; - - hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; - meta = { platforms = lib.platforms.unix ++ lib.platforms.windows; }; diff --git a/src/nix/profiles.md b/src/nix/profiles.md index c67a86194..f65e71065 120000 --- a/src/nix/profiles.md +++ b/src/nix/profiles.md @@ -1 +1 @@ -../../doc/manual/src/command-ref/files/profiles.md \ No newline at end of file +../../doc/manual/source/command-ref/files/profiles.md \ No newline at end of file diff --git a/src/nix/run.cc b/src/nix/run.cc index 956563591..c9857e13e 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -167,10 +167,9 @@ void chrootHelper(int argc, char * * argv) /* Bind-mount realStoreDir on /nix/store. If the latter mount point doesn't already exists, we have to create a chroot environment containing the mount point and bind mounts for the - children of /. Would be nice if we could use overlayfs here, - but that doesn't work in a user namespace yet (Ubuntu has a - patch for this: - https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1478578). */ + children of /. + Overlayfs for user namespaces is fixed in Linux since ac519625ed + (v5.11, 14 February 2021) */ if (!pathExists(storeDir)) { // FIXME: Use overlayfs? @@ -206,8 +205,9 @@ void chrootHelper(int argc, char * * argv) if (chdir(cwd) == -1) throw SysError("chdir to '%s' in chroot", cwd); } else - if (mount(realStoreDir.c_str(), storeDir.c_str(), "", MS_BIND, 0) == -1) - throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); + if (mount("overlay", storeDir.c_str(), "overlay", MS_MGC_VAL, fmt("lowerdir=%s:%s", storeDir, realStoreDir).c_str()) == -1) + if (mount(realStoreDir.c_str(), storeDir.c_str(), "", MS_BIND, 0) == -1) + throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); writeFile(fs::path{"/proc/self/setgroups"}, "deny"); writeFile(fs::path{"/proc/self/uid_map"}, fmt("%d %d %d", uid, uid, 1)); diff --git a/src/nix/self-exe.cc b/src/nix/self-exe.cc index 81a117e60..77d20a835 100644 --- a/src/nix/self-exe.cc +++ b/src/nix/self-exe.cc @@ -13,11 +13,12 @@ fs::path getNixBin(std::optional binaryNameOpt) { auto getBinaryName = [&] { return binaryNameOpt ? *binaryNameOpt : "nix"; }; - // If the environment variable is set, use it unconditionally + // If the environment variable is set, use it unconditionally. if (auto envOpt = getEnvNonEmpty("NIX_BIN_DIR")) return fs::path{*envOpt} / std::string{getBinaryName()}; - // Use some-times avaiable OS tricks to get to the path of this Nix, and try that + // Try OS tricks, if available, to get to the path of this Nix, and + // see if we can find the right executable next to that. if (auto selfOpt = getSelfExe()) { fs::path path{*selfOpt}; if (binaryNameOpt) diff --git a/src/nix/self-exe.hh b/src/nix/self-exe.hh index 0772afa67..3161553ec 100644 --- a/src/nix/self-exe.hh +++ b/src/nix/self-exe.hh @@ -17,9 +17,9 @@ namespace nix { * Instead, we'll query the OS for the path to the current executable, * using `getSelfExe()`. * - * As a last resort, we resort to `PATH`. Hopefully we find a `nix` - * there that's compatible. If you're porting Nix to a new platform, - * that might be good enough for a while, but you'll want to improve + * As a last resort, we rely on `PATH`. Hopefully we find a `nix` there + * that's compatible. If you're porting Nix to a new platform, that + * might be good enough for a while, but you'll want to improve * `getSelfExe()` to work on your platform. * * @param binary_name the exact binary name we're looking up. Might be diff --git a/src/perl/package.nix b/src/perl/package.nix index 0b9343fba..fe617fd47 100644 --- a/src/perl/package.nix +++ b/src/perl/package.nix @@ -1,13 +1,10 @@ { lib , stdenv , mkMesonDerivation +, pkg-config , perl , perlPackages -, meson -, ninja -, pkg-config , nix-store -, darwin , version , curl , bzip2 @@ -36,8 +33,6 @@ perl.pkgs.toPerlModule (mkMesonDerivation (finalAttrs: { ]); nativeBuildInputs = [ - meson - ninja pkg-config perl curl diff --git a/subprojects b/subprojects deleted file mode 120000 index e8310385c..000000000 --- a/subprojects +++ /dev/null @@ -1 +0,0 @@ -src \ No newline at end of file diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 6a177b657..ff39ab3b7 100755 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -238,7 +238,7 @@ clearCache # preserve quotes variables in the single-quoted string # shellcheck disable=SC2016 outPath=$(nix-build --no-out-link -E ' - with import ./config.nix; + with import '"${config_nix}"'; mkDerivation { name = "nar-listing"; buildCommand = "mkdir $out; echo foo > $out/bar; ln -s xyzzy $out/link"; @@ -258,7 +258,7 @@ clearCache # preserve quotes variables in the single-quoted string # shellcheck disable=SC2016 outPath=$(nix-build --no-out-link -E ' - with import ./config.nix; + with import '"${config_nix}"'; mkDerivation { name = "debug-info"; buildCommand = "mkdir -p $out/lib/debug/.build-id/02; echo foo > $out/lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"; @@ -276,7 +276,7 @@ diff -u \ # preserve quotes variables in the single-quoted string # shellcheck disable=SC2016 expr=' - with import ./config.nix; + with import '"${config_nix}"'; mkDerivation { name = "multi-output"; buildCommand = "mkdir -p $out; echo foo > $doc; echo $doc > $out/docref"; diff --git a/tests/functional/build-hook-ca-fixed.nix b/tests/functional/build-hook-ca-fixed.nix index 0ce6d9b12..427ec2c31 100644 --- a/tests/functional/build-hook-ca-fixed.nix +++ b/tests/functional/build-hook-ca-fixed.nix @@ -1,6 +1,6 @@ { busybox }: -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; let diff --git a/tests/functional/build-hook.nix b/tests/functional/build-hook.nix index 99a13aee4..1f0e17a3b 100644 --- a/tests/functional/build-hook.nix +++ b/tests/functional/build-hook.nix @@ -1,6 +1,6 @@ { busybox, contentAddressed ? false }: -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; let diff --git a/tests/functional/build.sh b/tests/functional/build.sh index 5396a465f..3f65a7c2c 100755 --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -84,6 +84,7 @@ expectStderr 1 nix build --expr '""' --no-link \ | grepQuiet "has 0 entries in its context. It should only have exactly one entry" # Too much string context +# shellcheck disable=SC2016 # The ${} in this is Nix, not shell expectStderr 1 nix build --impure --expr 'with (import ./multiple-outputs.nix).e.a_a; "${drvPath}${outPath}"' --no-link \ | grepQuiet "has 2 entries in its context. It should only have exactly one entry" @@ -160,7 +161,7 @@ printf "%s\n" "$drv^*" | nix build --no-link --stdin --json | jq --exit-status ' out="$(nix build -f fod-failing.nix -L 2>&1)" && status=0 || status=$? test "$status" = 1 # one "hash mismatch" error, one "build of ... failed" -test "$(<<<"$out" grep -E '^error:' | wc -l)" = 2 +test "$(<<<"$out" grep -cE '^error:')" = 2 <<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x1\\.drv'" <<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x3\\.drv'" <<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x2\\.drv'" @@ -169,7 +170,7 @@ test "$(<<<"$out" grep -E '^error:' | wc -l)" = 2 out="$(nix build -f fod-failing.nix -L x1 x2 x3 --keep-going 2>&1)" && status=0 || status=$? test "$status" = 1 # three "hash mismatch" errors - for each failing fod, one "build of ... failed" -test "$(<<<"$out" grep -E '^error:' | wc -l)" = 4 +test "$(<<<"$out" grep -cE '^error:')" = 4 <<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x1\\.drv'" <<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x3\\.drv'" <<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x2\\.drv'" @@ -177,13 +178,13 @@ test "$(<<<"$out" grep -E '^error:' | wc -l)" = 4 out="$(nix build -f fod-failing.nix -L x4 2>&1)" && status=0 || status=$? test "$status" = 1 -test "$(<<<"$out" grep -E '^error:' | wc -l)" = 2 +test "$(<<<"$out" grep -cE '^error:')" = 2 <<<"$out" grepQuiet -E "error: 1 dependencies of derivation '.*-x4\\.drv' failed to build" <<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x2\\.drv'" out="$(nix build -f fod-failing.nix -L x4 --keep-going 2>&1)" && status=0 || status=$? test "$status" = 1 -test "$(<<<"$out" grep -E '^error:' | wc -l)" = 3 +test "$(<<<"$out" grep -cE '^error:')" = 3 <<<"$out" grepQuiet -E "error: 2 dependencies of derivation '.*-x4\\.drv' failed to build" <<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x3\\.drv'" <<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x2\\.drv'" diff --git a/tests/functional/ca/content-addressed.nix b/tests/functional/ca/content-addressed.nix index 2559c562f..411ebb86b 100644 --- a/tests/functional/ca/content-addressed.nix +++ b/tests/functional/ca/content-addressed.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/ca/config.nix"; let mkCADerivation = args: mkDerivation ({ __contentAddressed = true; diff --git a/tests/functional/ca/derivation-json.sh b/tests/functional/ca/derivation-json.sh index 1e2a8fe35..bd6dd7177 100644 --- a/tests/functional/ca/derivation-json.sh +++ b/tests/functional/ca/derivation-json.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# + source common.sh export NIX_TESTS_CA_BY_DEFAULT=1 diff --git a/tests/functional/ca/duplicate-realisation-in-closure.sh b/tests/functional/ca/duplicate-realisation-in-closure.sh index 0baf15cc2..4a5e8c042 100644 --- a/tests/functional/ca/duplicate-realisation-in-closure.sh +++ b/tests/functional/ca/duplicate-realisation-in-closure.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -source ./common.sh +source common.sh requireDaemonNewerThan "2.4pre20210625" diff --git a/tests/functional/ca/import-from-derivation.sh b/tests/functional/ca/import-from-derivation.sh index 0713619a6..708d2fc78 100644 --- a/tests/functional/ca/import-from-derivation.sh +++ b/tests/functional/ca/import-from-derivation.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh export NIX_TESTS_CA_BY_DEFAULT=1 diff --git a/tests/functional/ca/meson.build b/tests/functional/ca/meson.build index 00cf8b35f..7a7fcc5cf 100644 --- a/tests/functional/ca/meson.build +++ b/tests/functional/ca/meson.build @@ -29,5 +29,5 @@ suites += { 'substitute.sh', 'why-depends.sh', ], - 'workdir': meson.current_build_dir(), + 'workdir': meson.current_source_dir(), } diff --git a/tests/functional/ca/new-build-cmd.sh b/tests/functional/ca/new-build-cmd.sh index 432d4d132..408bfb0f6 100644 --- a/tests/functional/ca/new-build-cmd.sh +++ b/tests/functional/ca/new-build-cmd.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh export NIX_TESTS_CA_BY_DEFAULT=1 diff --git a/tests/functional/ca/nix-run.sh b/tests/functional/ca/nix-run.sh index 920950c11..21c09117e 100755 --- a/tests/functional/ca/nix-run.sh +++ b/tests/functional/ca/nix-run.sh @@ -2,6 +2,11 @@ source common.sh -FLAKE_PATH=path:$PWD +flakeDir="$TEST_HOME/flake" +mkdir -p "${flakeDir}" +cp flake.nix "${_NIX_TEST_BUILD_DIR}/ca/config.nix" content-addressed.nix "${flakeDir}" -nix run --no-write-lock-file "$FLAKE_PATH#runnable" +# `config.nix` cannot be gotten via build dir / env var (runs afoul pure eval). Instead get from flake. +removeBuildDirRef "$flakeDir"/*.nix + +nix run --no-write-lock-file "path:${flakeDir}#runnable" diff --git a/tests/functional/ca/nix-shell.sh b/tests/functional/ca/nix-shell.sh index 1c5a6639f..d1fbe54d1 100755 --- a/tests/functional/ca/nix-shell.sh +++ b/tests/functional/ca/nix-shell.sh @@ -5,4 +5,3 @@ source common.sh CONTENT_ADDRESSED=true cd .. source ./nix-shell.sh - diff --git a/tests/functional/ca/nondeterministic.nix b/tests/functional/ca/nondeterministic.nix index d6d099a3e..740be4bd2 100644 --- a/tests/functional/ca/nondeterministic.nix +++ b/tests/functional/ca/nondeterministic.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/ca/config.nix"; let mkCADerivation = args: mkDerivation ({ __contentAddressed = true; diff --git a/tests/functional/ca/racy.nix b/tests/functional/ca/racy.nix index 555a15484..cadd98675 100644 --- a/tests/functional/ca/racy.nix +++ b/tests/functional/ca/racy.nix @@ -2,7 +2,7 @@ # build it at once. -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/ca/config.nix"; mkDerivation { name = "simple"; diff --git a/tests/functional/ca/repl.sh b/tests/functional/ca/repl.sh index 3808c7cb2..0bbbebd85 100644 --- a/tests/functional/ca/repl.sh +++ b/tests/functional/ca/repl.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh export NIX_TESTS_CA_BY_DEFAULT=1 diff --git a/tests/functional/ca/why-depends.sh b/tests/functional/ca/why-depends.sh index 0c079f63b..0af8a5440 100644 --- a/tests/functional/ca/why-depends.sh +++ b/tests/functional/ca/why-depends.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh export NIX_TESTS_CA_BY_DEFAULT=1 diff --git a/tests/functional/check-refs.nix b/tests/functional/check-refs.nix index 89690e456..54957f635 100644 --- a/tests/functional/check-refs.nix +++ b/tests/functional/check-refs.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; rec { diff --git a/tests/functional/check-reqs.nix b/tests/functional/check-reqs.nix index 41436cb48..4e059f5a4 100644 --- a/tests/functional/check-reqs.nix +++ b/tests/functional/check-reqs.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; rec { dep1 = mkDerivation { diff --git a/tests/functional/check.nix b/tests/functional/check.nix index ddab8eea9..13638eae8 100644 --- a/tests/functional/check.nix +++ b/tests/functional/check.nix @@ -1,6 +1,6 @@ {checkBuildId ? 0}: -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; { nondeterministic = mkDerivation { diff --git a/tests/functional/check.sh b/tests/functional/check.sh index 9b15dccb6..b21349288 100755 --- a/tests/functional/check.sh +++ b/tests/functional/check.sh @@ -7,9 +7,9 @@ buggyNeedLocalStore "see #4813" checkBuildTempDirRemoved () { - buildDir=$(sed -n 's/CHECK_TMPDIR=//p' $1 | head -1) + buildDir=$(sed -n 's/CHECK_TMPDIR=//p' "$1" | head -1) checkBuildIdFile=${buildDir}/checkBuildId - [[ ! -f $checkBuildIdFile ]] || ! grep $checkBuildId $checkBuildIdFile + [[ ! -f $checkBuildIdFile ]] || ! grep "$checkBuildId" "$checkBuildIdFile" } # written to build temp directories to verify created by this instance @@ -23,20 +23,20 @@ nix-build dependencies.nix --no-out-link nix-build dependencies.nix --no-out-link --check # Build failure exit codes (100, 104, etc.) are from -# doc/manual/src/command-ref/status-build-failure.md +# doc/manual/source/command-ref/status-build-failure.md # check for dangling temporary build directories # only retain if build fails and --keep-failed is specified, or... # ...build is non-deterministic and --check and --keep-failed are both specified -nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \ - --no-out-link 2> $TEST_ROOT/log || status=$? +nix-build check.nix -A failed --argstr checkBuildId "$checkBuildId" \ + --no-out-link 2> "$TEST_ROOT/log" || status=$? [ "$status" = "100" ] -checkBuildTempDirRemoved $TEST_ROOT/log +checkBuildTempDirRemoved "$TEST_ROOT/log" -nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \ - --no-out-link --keep-failed 2> $TEST_ROOT/log || status=$? +nix-build check.nix -A failed --argstr checkBuildId "$checkBuildId" \ + --no-out-link --keep-failed 2> "$TEST_ROOT/log" || status=$? [ "$status" = "100" ] -if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi +if checkBuildTempDirRemoved "$TEST_ROOT/log"; then false; fi test_custom_build_dir() { local customBuildDir="$TEST_ROOT/custom-build-dir" @@ -44,42 +44,46 @@ test_custom_build_dir() { # Nix does not create the parent directories, and perhaps it shouldn't try to # decide the permissions of build-dir. mkdir "$customBuildDir" - nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \ - --no-out-link --keep-failed --option build-dir "$TEST_ROOT/custom-build-dir" 2> $TEST_ROOT/log || status=$? + nix-build check.nix -A failed --argstr checkBuildId "$checkBuildId" \ + --no-out-link --keep-failed --option build-dir "$TEST_ROOT/custom-build-dir" 2> "$TEST_ROOT/log" || status=$? [ "$status" = "100" ] [[ 1 == "$(count "$customBuildDir/nix-build-"*)" ]] - local buildDir="$customBuildDir/nix-build-"*"" - if [[ -e $buildDir/build ]]; then - buildDir=$buildDir/build + local buildDir=("$customBuildDir/nix-build-"*) + if [[ "${#buildDir[@]}" -ne 1 ]]; then + echo "expected one nix-build-* directory, got: ${buildDir[*]}" >&2 + exit 1 fi - grep $checkBuildId $buildDir/checkBuildId + if [[ -e ${buildDir[*]}/build ]]; then + buildDir[0]="${buildDir[*]}/build" + fi + grep "$checkBuildId" "${buildDir[*]}/checkBuildId" } test_custom_build_dir -nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ - --no-out-link 2> $TEST_ROOT/log -checkBuildTempDirRemoved $TEST_ROOT/log +nix-build check.nix -A deterministic --argstr checkBuildId "$checkBuildId" \ + --no-out-link 2> "$TEST_ROOT/log" +checkBuildTempDirRemoved "$TEST_ROOT/log" -nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ - --no-out-link --check --keep-failed 2> $TEST_ROOT/log -if grepQuiet 'may not be deterministic' $TEST_ROOT/log; then false; fi -checkBuildTempDirRemoved $TEST_ROOT/log +nix-build check.nix -A deterministic --argstr checkBuildId "$checkBuildId" \ + --no-out-link --check --keep-failed 2> "$TEST_ROOT/log" +if grepQuiet 'may not be deterministic' "$TEST_ROOT/log"; then false; fi +checkBuildTempDirRemoved "$TEST_ROOT/log" -nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ - --no-out-link 2> $TEST_ROOT/log -checkBuildTempDirRemoved $TEST_ROOT/log +nix-build check.nix -A nondeterministic --argstr checkBuildId "$checkBuildId" \ + --no-out-link 2> "$TEST_ROOT/log" +checkBuildTempDirRemoved "$TEST_ROOT/log" -nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ - --no-out-link --check 2> $TEST_ROOT/log || status=$? -grep 'may not be deterministic' $TEST_ROOT/log +nix-build check.nix -A nondeterministic --argstr checkBuildId "$checkBuildId" \ + --no-out-link --check 2> "$TEST_ROOT/log" || status=$? +grep 'may not be deterministic' "$TEST_ROOT/log" [ "$status" = "104" ] -checkBuildTempDirRemoved $TEST_ROOT/log +checkBuildTempDirRemoved "$TEST_ROOT/log" -nix-build check.nix -A nondeterministic --argstr checkBuildId $checkBuildId \ - --no-out-link --check --keep-failed 2> $TEST_ROOT/log || status=$? -grep 'may not be deterministic' $TEST_ROOT/log +nix-build check.nix -A nondeterministic --argstr checkBuildId "$checkBuildId" \ + --no-out-link --check --keep-failed 2> "$TEST_ROOT/log" || status=$? +grep 'may not be deterministic' "$TEST_ROOT/log" [ "$status" = "104" ] -if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi +if checkBuildTempDirRemoved "$TEST_ROOT/log"; then false; fi TODO_NixOS @@ -87,24 +91,24 @@ clearStore path=$(nix-build check.nix -A fetchurl --no-out-link) -chmod +w $path -echo foo > $path -chmod -w $path +chmod +w "$path" +echo foo > "$path" +chmod -w "$path" nix-build check.nix -A fetchurl --no-out-link --check # Note: "check" doesn't repair anything, it just compares to the hash stored in the database. -[[ $(cat $path) = foo ]] +[[ $(cat "$path") = foo ]] nix-build check.nix -A fetchurl --no-out-link --repair -[[ $(cat $path) != foo ]] +[[ $(cat "$path") != foo ]] -echo 'Hello World' > $TEST_ROOT/dummy +echo 'Hello World' > "$TEST_ROOT/dummy" nix-build check.nix -A hashmismatch --no-out-link || status=$? [ "$status" = "102" ] -echo -n > $TEST_ROOT/dummy +echo -n > "$TEST_ROOT/dummy" nix-build check.nix -A hashmismatch --no-out-link -echo 'Hello World' > $TEST_ROOT/dummy +echo 'Hello World' > "$TEST_ROOT/dummy" nix-build check.nix -A hashmismatch --no-out-link --check || status=$? [ "$status" = "102" ] diff --git a/tests/functional/chroot-store.sh b/tests/functional/chroot-store.sh index 03803a2b9..8c2a969d3 100755 --- a/tests/functional/chroot-store.sh +++ b/tests/functional/chroot-store.sh @@ -37,7 +37,10 @@ if canUseSandbox; then } EOF - cp simple.nix shell.nix simple.builder.sh config.nix "$flakeDir/" + cp simple.nix shell.nix simple.builder.sh "${config_nix}" "$flakeDir/" + + # `config.nix` cannot be gotten via build dir / env var (runs afoul pure eval). Instead get from flake. + removeBuildDirRef "$flakeDir"/*.nix TODO_NixOS diff --git a/tests/functional/common/functions.sh b/tests/functional/common/functions.sh index d05fac4e7..286bb58e8 100644 --- a/tests/functional/common/functions.sh +++ b/tests/functional/common/functions.sh @@ -28,7 +28,7 @@ clearProfiles() { # Clear the store, but do not fail if we're in an environment where we can't. # This allows the test to run in a NixOS test environment, where we use the system store. -# See doc/manual/src/contributing/testing.md / Running functional tests on NixOS. +# See doc/manual/source/contributing/testing.md / Running functional tests on NixOS. clearStoreIfPossible() { if isTestOnNixOS; then echo "clearStoreIfPossible: Not clearing store, because we're on NixOS. Moving on." @@ -343,6 +343,15 @@ count() { echo $# } +# Sometimes, e.g. due to pure eval, restricted eval, or sandboxing, we +# cannot look up `config.nix` in the build dir, and have to instead get +# it from the current directory. (In this case, the current directly +# will be somewhere in `$TEST_ROOT`.) +removeBuildDirRef() { + # shellcheck disable=SC2016 # The ${} in this is Nix, not shell + sed -i -e 's,"${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/[^ ]*config.nix",./config.nix,' "$@" +} + trap onError ERR fi # COMMON_FUNCTIONS_SH_SOURCED diff --git a/tests/functional/common/paths.sh b/tests/functional/common/paths.sh index 70bdca796..dec3592af 100644 --- a/tests/functional/common/paths.sh +++ b/tests/functional/common/paths.sh @@ -8,11 +8,10 @@ COMMON_PATHS_SH_SOURCED=1 commonDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")" -# Since these are generated files -# shellcheck disable=SC1091 +# Just for `isTestOnNixOS` source "$commonDir/functions.sh" # shellcheck disable=SC1091 -source "$commonDir/subst-vars.sh" +source "${_NIX_TEST_BUILD_DIR}/common/subst-vars.sh" # Make sure shellcheck knows this will be defined by the above generated snippet : "${bash?}" "${bindir?}" diff --git a/tests/functional/common/vars.sh b/tests/functional/common/vars.sh index c99a6a5c2..4b88e8526 100644 --- a/tests/functional/common/vars.sh +++ b/tests/functional/common/vars.sh @@ -6,11 +6,14 @@ if [[ -z "${COMMON_VARS_SH_SOURCED-}" ]]; then COMMON_VARS_SH_SOURCED=1 +_NIX_TEST_SOURCE_DIR=$(realpath "${_NIX_TEST_SOURCE_DIR}") +_NIX_TEST_BUILD_DIR=$(realpath "${_NIX_TEST_BUILD_DIR}") + commonDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")" # Since this is a generated file # shellcheck disable=SC1091 -source "$commonDir/subst-vars.sh" +source "${_NIX_TEST_BUILD_DIR}/common/subst-vars.sh" # Make sure shellcheck knows all these will be defined by the above generated snippet : "${bindir?} ${coreutils?} ${dot?} ${SHELL?} ${busybox?} ${version?} ${system?}" export coreutils dot busybox version system @@ -69,4 +72,9 @@ if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true _canUseSandbox=1 fi +# Very common, shorthand helps +# Used in other files +# shellcheck disable=SC2034 +config_nix="${_NIX_TEST_BUILD_DIR}/config.nix" + fi # COMMON_VARS_SH_SOURCED diff --git a/tests/functional/dependencies.nix b/tests/functional/dependencies.nix index be1a7ae9a..db06321da 100644 --- a/tests/functional/dependencies.nix +++ b/tests/functional/dependencies.nix @@ -1,5 +1,5 @@ { hashInvalidator ? "" }: -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; let { diff --git a/tests/functional/dyn-drv/meson.build b/tests/functional/dyn-drv/meson.build index 3c671d013..5b60a4698 100644 --- a/tests/functional/dyn-drv/meson.build +++ b/tests/functional/dyn-drv/meson.build @@ -15,5 +15,5 @@ suites += { 'dep-built-drv.sh', 'old-daemon-error-hack.sh', ], - 'workdir': meson.current_build_dir(), + 'workdir': meson.current_source_dir(), } diff --git a/tests/functional/dyn-drv/old-daemon-error-hack.nix b/tests/functional/dyn-drv/old-daemon-error-hack.nix index c9d4a62d4..7d3ccf7e4 100644 --- a/tests/functional/dyn-drv/old-daemon-error-hack.nix +++ b/tests/functional/dyn-drv/old-daemon-error-hack.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/dyn-drv/config.nix"; # A simple content-addressed derivation. # The derivation can be arbitrarily modified by passing a different `seed`, diff --git a/tests/functional/dyn-drv/recursive-mod-json.nix b/tests/functional/dyn-drv/recursive-mod-json.nix index c6a24ca4f..0e778aa7f 100644 --- a/tests/functional/dyn-drv/recursive-mod-json.nix +++ b/tests/functional/dyn-drv/recursive-mod-json.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/dyn-drv/config.nix"; let innerName = "foo"; in diff --git a/tests/functional/dyn-drv/text-hashed-output.nix b/tests/functional/dyn-drv/text-hashed-output.nix index 99203b518..aa46fff61 100644 --- a/tests/functional/dyn-drv/text-hashed-output.nix +++ b/tests/functional/dyn-drv/text-hashed-output.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/dyn-drv/config.nix"; # A simple content-addressed derivation. # The derivation can be arbitrarily modified by passing a different `seed`, diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh index 22d2d02a2..7af49d7fd 100755 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -35,17 +35,17 @@ nix-instantiate --eval -E 'assert 1 + 2 == 3; true' [[ "$(nix-instantiate --eval -E '{"assert"=1;bar=2;}')" == '{ "assert" = 1; bar = 2; }' ]] # Check that symlink cycles don't cause a hang. -ln -sfn cycle.nix $TEST_ROOT/cycle.nix -(! nix eval --file $TEST_ROOT/cycle.nix) +ln -sfn cycle.nix "$TEST_ROOT/cycle.nix" +(! nix eval --file "$TEST_ROOT/cycle.nix") # Check that relative symlinks are resolved correctly. -mkdir -p $TEST_ROOT/xyzzy $TEST_ROOT/foo -ln -sfn ../xyzzy $TEST_ROOT/foo/bar -printf 123 > $TEST_ROOT/xyzzy/default.nix +mkdir -p "$TEST_ROOT/xyzzy" "$TEST_ROOT/foo" +ln -sfn ../xyzzy "$TEST_ROOT/foo/bar" +printf 123 > "$TEST_ROOT/xyzzy/default.nix" [[ $(nix eval --impure --expr "import $TEST_ROOT/foo/bar") = 123 ]] # Test --arg-from-file. -[[ "$(nix eval --raw --arg-from-file foo config.nix --expr '{ foo }: { inherit foo; }' foo)" = "$(cat config.nix)" ]] +[[ "$(nix eval --raw --arg-from-file foo "${config_nix}" --expr '{ foo }: { inherit foo; }' foo)" = "$(cat "${config_nix}")" ]] # Check that special(-ish) files are drained. if [[ -e /proc/version ]]; then @@ -57,7 +57,7 @@ fi # Test that unknown settings are warned about out="$(expectStderr 0 nix eval --option foobar baz --expr '""' --raw)" -[[ "$(echo "$out" | grep foobar | wc -l)" = 1 ]] +[[ "$(echo "$out" | grep -c foobar)" = 1 ]] # Test flag alias out="$(nix eval --expr '{}' --build-cores 1)" diff --git a/tests/functional/export-graph.nix b/tests/functional/export-graph.nix index 64fe36bd1..97ffe73a9 100644 --- a/tests/functional/export-graph.nix +++ b/tests/functional/export-graph.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; rec { diff --git a/tests/functional/extra-sandbox-profile.nix b/tests/functional/extra-sandbox-profile.nix index aa680b918..5f0e4753f 100644 --- a/tests/functional/extra-sandbox-profile.nix +++ b/tests/functional/extra-sandbox-profile.nix @@ -1,6 +1,6 @@ { destFile, seed }: -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; mkDerivation { name = "simple"; diff --git a/tests/functional/failing.nix b/tests/functional/failing.nix index d25e2d6b6..8b7990679 100644 --- a/tests/functional/failing.nix +++ b/tests/functional/failing.nix @@ -1,5 +1,5 @@ { busybox }: -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; let mkDerivation = args: diff --git a/tests/functional/fetchPath.sh b/tests/functional/fetchPath.sh index 560a270c1..1df895b61 100755 --- a/tests/functional/fetchPath.sh +++ b/tests/functional/fetchPath.sh @@ -6,3 +6,6 @@ touch "$TEST_ROOT/foo" -t 202211111111 # We only check whether 2022-11-1* **:**:** is the last modified date since # `lastModified` is transformed into UTC in `builtins.fetchTarball`. [[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$TEST_ROOT/foo\").lastModifiedDate")" =~ 2022111.* ]] + +# Check that we can override lastModified for "path:" inputs. +[[ "$(nix eval --impure --expr "(builtins.fetchTree { type = \"path\"; path = \"$TEST_ROOT/foo\"; lastModified = 123; }).lastModified")" = 123 ]] diff --git a/tests/functional/fetchurl.sh b/tests/functional/fetchurl.sh index 5af44fcf2..c25ac3216 100755 --- a/tests/functional/fetchurl.sh +++ b/tests/functional/fetchurl.sh @@ -9,12 +9,12 @@ clearStore # Test fetching a flat file. hash=$(nix-hash --flat --type sha256 ./fetchurl.sh) -outPath=$(nix-build -vvvvv --expr 'import ' --argstr url file://$(pwd)/fetchurl.sh --argstr sha256 $hash --no-out-link) +outPath=$(nix-build -vvvvv --expr 'import ' --argstr url "file://$(pwd)/fetchurl.sh" --argstr sha256 "$hash" --no-out-link) -cmp $outPath fetchurl.sh +cmp "$outPath" fetchurl.sh # Do not re-fetch paths already present. -outPath2=$(nix-build -vvvvv --expr 'import ' --argstr url file:///does-not-exist/must-remain-unused/fetchurl.sh --argstr sha256 $hash --no-out-link) +outPath2=$(nix-build -vvvvv --expr 'import ' --argstr url file:///does-not-exist/must-remain-unused/fetchurl.sh --argstr sha256 "$hash" --no-out-link) test "$outPath" == "$outPath2" # Now using a base-64 hash. @@ -22,9 +22,9 @@ clearStore hash=$(nix hash file --type sha512 --base64 ./fetchurl.sh) -outPath=$(nix-build -vvvvv --expr 'import ' --argstr url file://$(pwd)/fetchurl.sh --argstr sha512 $hash --no-out-link) +outPath=$(nix-build -vvvvv --expr 'import ' --argstr url "file://$(pwd)/fetchurl.sh" --argstr sha512 "$hash" --no-out-link) -cmp $outPath fetchurl.sh +cmp "$outPath" fetchurl.sh # Now using an SRI hash. clearStore @@ -33,58 +33,58 @@ hash=$(nix hash file ./fetchurl.sh) [[ $hash =~ ^sha256- ]] -outPath=$(nix-build -vvvvv --expr 'import ' --argstr url file://$(pwd)/fetchurl.sh --argstr hash $hash --no-out-link) +outPath=$(nix-build -vvvvv --expr 'import ' --argstr url "file://$(pwd)/fetchurl.sh" --argstr hash "$hash" --no-out-link) -cmp $outPath fetchurl.sh +cmp "$outPath" fetchurl.sh # Test that we can substitute from a different store dir. clearStore -other_store=file://$TEST_ROOT/other_store?store=/fnord/store +other_store="file://$TEST_ROOT/other_store?store=/fnord/store" hash=$(nix hash file --type sha256 --base16 ./fetchurl.sh) -storePath=$(nix --store $other_store store add-file ./fetchurl.sh) +nix --store "$other_store" store add-file ./fetchurl.sh -outPath=$(nix-build -vvvvv --expr 'import ' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 $hash --no-out-link --substituters $other_store) +outPath=$(nix-build -vvvvv --expr 'import ' --argstr url file:///no-such-dir/fetchurl.sh --argstr sha256 "$hash" --no-out-link --substituters "$other_store") # Test hashed mirrors with an SRI hash. -nix-build -vvvvv --expr 'import ' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash $(nix hash to-sri --type sha256 $hash) \ - --no-out-link --substituters $other_store +nix-build -vvvvv --expr 'import ' --argstr url file:///no-such-dir/fetchurl.sh --argstr hash "$(nix hash to-sri --type sha256 "$hash")" \ + --no-out-link --substituters "$other_store" # Test unpacking a NAR. -rm -rf $TEST_ROOT/archive -mkdir -p $TEST_ROOT/archive -cp ./fetchurl.sh $TEST_ROOT/archive -chmod +x $TEST_ROOT/archive/fetchurl.sh -ln -s foo $TEST_ROOT/archive/symlink -nar=$TEST_ROOT/archive.nar -nix-store --dump $TEST_ROOT/archive > $nar +rm -rf "$TEST_ROOT/archive" +mkdir -p "$TEST_ROOT/archive" +cp ./fetchurl.sh "$TEST_ROOT/archive" +chmod +x "$TEST_ROOT/archive/fetchurl.sh" +ln -s foo "$TEST_ROOT/archive/symlink" +nar="$TEST_ROOT/archive.nar" +nix-store --dump "$TEST_ROOT/archive" > "$nar" -hash=$(nix-hash --flat --type sha256 $nar) +hash=$(nix-hash --flat --type sha256 "$nar") -outPath=$(nix-build -vvvvv --expr 'import ' --argstr url file://$nar --argstr sha256 $hash \ +outPath=$(nix-build -vvvvv --expr 'import ' --argstr url "file://$nar" --argstr sha256 "$hash" \ --arg unpack true --argstr name xyzzy --no-out-link) -echo $outPath | grepQuiet 'xyzzy' +echo "$outPath" | grepQuiet 'xyzzy' -test -x $outPath/fetchurl.sh -test -L $outPath/symlink +test -x "$outPath/fetchurl.sh" +test -L "$outPath/symlink" -nix-store --delete $outPath +nix-store --delete "$outPath" # Test unpacking a compressed NAR. -narxz=$TEST_ROOT/archive.nar.xz -rm -f $narxz -xz --keep $nar -outPath=$(nix-build -vvvvv --expr 'import ' --argstr url file://$narxz --argstr sha256 $hash \ +narxz="$TEST_ROOT/archive.nar.xz" +rm -f "$narxz" +xz --keep "$nar" +outPath=$(nix-build -vvvvv --expr 'import ' --argstr url "file://$narxz" --argstr sha256 "$hash" \ --arg unpack true --argstr name xyzzy --no-out-link) -test -x $outPath/fetchurl.sh -test -L $outPath/symlink +test -x "$outPath/fetchurl.sh" +test -L "$outPath/symlink" # Make sure that *not* passing a outputHash fails. requireDaemonNewerThan "2.20" expected=100 if [[ -v NIX_DAEMON_PACKAGE ]]; then expected=1; fi # work around the daemon not returning a 100 status correctly -expectStderr $expected nix-build --expr '{ url }: builtins.derivation { name = "nix-cache-info"; system = "x86_64-linux"; builder = "builtin:fetchurl"; inherit url; outputHashMode = "flat"; }' --argstr url file://$narxz 2>&1 | grep 'must be a fixed-output or impure derivation' +expectStderr $expected nix-build --expr '{ url }: builtins.derivation { name = "nix-cache-info"; system = "x86_64-linux"; builder = "builtin:fetchurl"; inherit url; outputHashMode = "flat"; }' --argstr url "file://$narxz" 2>&1 | grep 'must be a fixed-output or impure derivation' diff --git a/tests/functional/filter-source.nix b/tests/functional/filter-source.nix index 907163639..dcef9c4e2 100644 --- a/tests/functional/filter-source.nix +++ b/tests/functional/filter-source.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; mkDerivation { name = "filter"; diff --git a/tests/functional/fixed.nix b/tests/functional/fixed.nix index a920a2167..f70b89091 100644 --- a/tests/functional/fixed.nix +++ b/tests/functional/fixed.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; rec { diff --git a/tests/functional/flakes/build-paths.sh b/tests/functional/flakes/build-paths.sh index f8486528b..cbae6a4e6 100755 --- a/tests/functional/flakes/build-paths.sh +++ b/tests/functional/flakes/build-paths.sh @@ -79,7 +79,7 @@ cat > "$flake1Dir"/flake.nix < "$flake1Dir/foo" diff --git a/tests/functional/flakes/bundle.sh b/tests/functional/flakes/bundle.sh index 5e185cbf6..61aa040e7 100755 --- a/tests/functional/flakes/bundle.sh +++ b/tests/functional/flakes/bundle.sh @@ -2,7 +2,10 @@ source common.sh -cp ../simple.nix ../simple.builder.sh ../config.nix "$TEST_HOME" +cp ../simple.nix ../simple.builder.sh "${config_nix}" "$TEST_HOME" + +# `config.nix` cannot be gotten via build dir / env var (runs afoul pure eval). Instead get from flake. +removeBuildDirRef "$TEST_HOME"/*.nix cd "$TEST_HOME" diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index f83a02aba..8af72f2ad 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -1,10 +1,13 @@ +# shellcheck shell=bash + source ../common.sh +# shellcheck disable=SC2034 # this variable is used by tests that source this file registry=$TEST_ROOT/registry.json writeSimpleFlake() { local flakeDir="$1" - cat > $flakeDir/flake.nix < "$flakeDir/flake.nix" < $flakeDir/flake.nix < "$flakeDir/flake.nix" < $flakeDir/flake.nix < "$flakeDir/flake.nix" <$TEST_HOME/flake.nix { inputs.nixpkgs.url = "$TEST_HOME/nixpkgs"; @@ -25,7 +25,11 @@ EOF # Create fake nixpkgs flake. mkdir -p $TEST_HOME/nixpkgs -cp ../config.nix ../shell.nix $TEST_HOME/nixpkgs +cp "${config_nix}" ../shell.nix $TEST_HOME/nixpkgs + +# `config.nix` cannot be gotten via build dir / env var (runs afoul pure eval). Instead get from flake. +removeBuildDirRef "$TEST_HOME/nixpkgs"/*.nix + cat <$TEST_HOME/nixpkgs/flake.nix { outputs = {self}: { diff --git a/tests/functional/flakes/eval-cache.sh b/tests/functional/flakes/eval-cache.sh index e3fd0bbfe..40a0db618 100755 --- a/tests/functional/flakes/eval-cache.sh +++ b/tests/functional/flakes/eval-cache.sh @@ -7,7 +7,7 @@ requireGit flake1Dir="$TEST_ROOT/eval-cache-flake" createGitRepo "$flake1Dir" "" -cp ../simple.nix ../simple.builder.sh ../config.nix "$flake1Dir/" +cp ../simple.nix ../simple.builder.sh "${config_nix}" "$flake1Dir/" git -C "$flake1Dir" add simple.nix simple.builder.sh config.nix git -C "$flake1Dir" commit -m "config.nix" diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index aa4cb1e18..5e901c5d1 100755 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -390,7 +390,7 @@ cat > "$flake3Dir/flake.nix" < flake.nix diff --git a/tests/functional/fmt.sh b/tests/functional/fmt.sh index 4e0fd57a5..e9bff50d5 100755 --- a/tests/functional/fmt.sh +++ b/tests/functional/fmt.sh @@ -7,7 +7,7 @@ TODO_NixOS # Provide a `shell` variable. Try not to `export` it, perhaps. clearStoreIfPossible rm -rf "$TEST_HOME"/.cache "$TEST_HOME"/.config "$TEST_HOME"/.local -cp ./simple.nix ./simple.builder.sh ./fmt.simple.sh ./config.nix "$TEST_HOME" +cp ./simple.nix ./simple.builder.sh ./fmt.simple.sh "${config_nix}" "$TEST_HOME" cd "$TEST_HOME" diff --git a/tests/functional/fod-failing.nix b/tests/functional/fod-failing.nix index 37c04fe12..7881a3fbf 100644 --- a/tests/functional/fod-failing.nix +++ b/tests/functional/fod-failing.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; rec { x1 = mkDerivation { name = "x1"; diff --git a/tests/functional/gc-auto.sh b/tests/functional/gc-auto.sh index 8f25be3e9..efe3e4b2b 100755 --- a/tests/functional/gc-auto.sh +++ b/tests/functional/gc-auto.sh @@ -23,7 +23,7 @@ fifoLock=$TEST_ROOT/fifoLock mkfifo "$fifoLock" expr=$(cat < $fifo2\"; diff --git a/tests/functional/gc-runtime.nix b/tests/functional/gc-runtime.nix index ee5980bdf..2603fafdf 100644 --- a/tests/functional/gc-runtime.nix +++ b/tests/functional/gc-runtime.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; mkDerivation { name = "gc-runtime"; diff --git a/tests/functional/gc.sh b/tests/functional/gc.sh index 7707a7e27..c58f47021 100755 --- a/tests/functional/gc.sh +++ b/tests/functional/gc.sh @@ -11,46 +11,46 @@ outPath=$(nix-store -rvv "$drvPath") # Set a GC root. rm -f "$NIX_STATE_DIR/gcroots/foo" -ln -sf $outPath "$NIX_STATE_DIR/gcroots/foo" +ln -sf "$outPath" "$NIX_STATE_DIR/gcroots/foo" -[ "$(nix-store -q --roots $outPath)" = "$NIX_STATE_DIR/gcroots/foo -> $outPath" ] +[ "$(nix-store -q --roots "$outPath")" = "$NIX_STATE_DIR/gcroots/foo -> $outPath" ] -nix-store --gc --print-roots | grep $outPath -nix-store --gc --print-live | grep $outPath -nix-store --gc --print-dead | grep $drvPath -if nix-store --gc --print-dead | grep -E $outPath$; then false; fi +nix-store --gc --print-roots | grep "$outPath" +nix-store --gc --print-live | grep "$outPath" +nix-store --gc --print-dead | grep "$drvPath" +if nix-store --gc --print-dead | grep -E "$outPath"$; then false; fi nix-store --gc --print-dead -inUse=$(readLink $outPath/reference-to-input-2) -if nix-store --delete $inUse; then false; fi -test -e $inUse +inUse=$(readLink "$outPath/reference-to-input-2") +if nix-store --delete "$inUse"; then false; fi +test -e "$inUse" -if nix-store --delete $outPath; then false; fi -test -e $outPath +if nix-store --delete "$outPath"; then false; fi +test -e "$outPath" -for i in $NIX_STORE_DIR/*; do +for i in "$NIX_STORE_DIR"/*; do if [[ $i =~ /trash ]]; then continue; fi # compat with old daemon - touch $i.lock - touch $i.chroot + touch "$i.lock" + touch "$i.chroot" done nix-collect-garbage # Check that the root and its dependencies haven't been deleted. -cat $outPath/foobar -cat $outPath/reference-to-input-2/bar +cat "$outPath/foobar" +cat "$outPath/reference-to-input-2/bar" # Check that the derivation has been GC'd. -if test -e $drvPath; then false; fi +if test -e "$drvPath"; then false; fi rm "$NIX_STATE_DIR/gcroots/foo" nix-collect-garbage # Check that the output has been GC'd. -if test -e $outPath/foobar; then false; fi +if test -e "$outPath/foobar"; then false; fi # Check that the store is empty. -rmdir $NIX_STORE_DIR/.links -rmdir $NIX_STORE_DIR +rmdir "$NIX_STORE_DIR/.links" +rmdir "$NIX_STORE_DIR" diff --git a/tests/functional/git-hashing/meson.build b/tests/functional/git-hashing/meson.build index 7486bfb8f..470c53fc5 100644 --- a/tests/functional/git-hashing/meson.build +++ b/tests/functional/git-hashing/meson.build @@ -4,5 +4,5 @@ suites += { 'tests': [ 'simple.sh', ], - 'workdir': meson.current_build_dir(), + 'workdir': meson.current_source_dir(), } diff --git a/tests/functional/help.sh b/tests/functional/help.sh index 61efc1cb2..127cc455b 100755 --- a/tests/functional/help.sh +++ b/tests/functional/help.sh @@ -65,5 +65,6 @@ def recurse($prefix): } nix __dump-cli | subcommands | while IFS= read -r cmd; do + # shellcheck disable=SC2086 # word splitting of cmd is intended nix $cmd --help done diff --git a/tests/functional/hermetic.nix b/tests/functional/hermetic.nix index d1dccdff3..dafe8ad9f 100644 --- a/tests/functional/hermetic.nix +++ b/tests/functional/hermetic.nix @@ -5,7 +5,7 @@ , withFinalRefs ? false }: -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; let contentAddressedByDefault = builtins.getEnv "NIX_TESTS_CA_BY_DEFAULT" == "1"; diff --git a/tests/functional/ifd.nix b/tests/functional/ifd.nix index d0b9b54ad..c84ffbc66 100644 --- a/tests/functional/ifd.nix +++ b/tests/functional/ifd.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; import ( mkDerivation { name = "foo"; diff --git a/tests/functional/import-from-derivation.nix b/tests/functional/import-from-derivation.nix index cc53451cf..8864fb30a 100644 --- a/tests/functional/import-from-derivation.nix +++ b/tests/functional/import-from-derivation.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; rec { bar = mkDerivation { diff --git a/tests/functional/impure-derivations.nix b/tests/functional/impure-derivations.nix index 98547e6c1..04710323f 100644 --- a/tests/functional/impure-derivations.nix +++ b/tests/functional/impure-derivations.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; rec { diff --git a/tests/functional/impure-env.nix b/tests/functional/impure-env.nix index 2b0380ed7..6b9e5a825 100644 --- a/tests/functional/impure-env.nix +++ b/tests/functional/impure-env.nix @@ -1,6 +1,6 @@ { var, value }: -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; mkDerivation { name = "test"; diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index 46cf3f1fe..e64663d30 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -37,12 +37,14 @@ nix-instantiate --eval -E 'let x = { repeating = x; tracing = builtins.trace x t 2>&1 | grepQuiet -F 'trace: { repeating = «repeated»; tracing = «potential infinite recursion»; }' nix-instantiate --eval -E 'builtins.warn "Hello" 123' 2>&1 | grepQuiet 'warning: Hello' +# shellcheck disable=SC2016 # The ${} in this is Nix, not shell nix-instantiate --eval -E 'builtins.addErrorContext "while doing ${"something"} interesting" (builtins.warn "Hello" 123)' 2>/dev/null | grepQuiet 123 # warn does not accept non-strings for now expectStderr 1 nix-instantiate --eval -E 'let x = builtins.warn { x = x; } true; in x' \ | grepQuiet "expected a string but found a set" expectStderr 1 nix-instantiate --eval --abort-on-warn -E 'builtins.warn "Hello" 123' | grepQuiet Hello +# shellcheck disable=SC2016 # The ${} in this is Nix, not shell NIX_ABORT_ON_WARN=1 expectStderr 1 nix-instantiate --eval -E 'builtins.addErrorContext "while doing ${"something"} interesting" (builtins.warn "Hello" 123)' | grepQuiet "while doing something interesting" set +x @@ -106,6 +108,7 @@ for i in lang/eval-fail-*.nix; do fi )" if + # shellcheck disable=SC2086 # word splitting of flags is intended expectStderr 1 nix-instantiate $flags "lang/$i.nix" \ | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" then diff --git a/tests/functional/linux-sandbox-cert-test.nix b/tests/functional/linux-sandbox-cert-test.nix index 2fc083ea9..e506b6a0f 100644 --- a/tests/functional/linux-sandbox-cert-test.nix +++ b/tests/functional/linux-sandbox-cert-test.nix @@ -1,6 +1,6 @@ { mode }: -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; mkDerivation ( { diff --git a/tests/functional/linux-sandbox.sh b/tests/functional/linux-sandbox.sh index 653a3873f..81ef36237 100755 --- a/tests/functional/linux-sandbox.sh +++ b/tests/functional/linux-sandbox.sh @@ -40,14 +40,14 @@ nix-sandbox-build dependencies.nix --check # Test that sandboxed builds with --check and -K can move .check directory to store nix-sandbox-build check.nix -A nondeterministic -# `100 + 4` means non-determinstic, see doc/manual/src/command-ref/status-build-failure.md +# `100 + 4` means non-determinstic, see doc/manual/source/command-ref/status-build-failure.md expectStderr 104 nix-sandbox-build check.nix -A nondeterministic --check -K > $TEST_ROOT/log grepQuietInverse 'error: renaming' $TEST_ROOT/log grepQuiet 'may not be deterministic' $TEST_ROOT/log # Test that sandboxed builds cannot write to /etc easily -# `100` means build failure without extra info, see doc/manual/src/command-ref/status-build-failure.md -expectStderr 100 nix-sandbox-build -E 'with import ./config.nix; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' | +# `100` means build failure without extra info, see doc/manual/source/command-ref/status-build-failure.md +expectStderr 100 nix-sandbox-build -E 'with import '"${config_nix}"'; mkDerivation { name = "etc-write"; buildCommand = "echo > /etc/test"; }' | grepQuiet "/etc/test: Permission denied" @@ -56,7 +56,7 @@ testCert () { expectation=$1 # "missing" | "present" mode=$2 # "normal" | "fixed-output" certFile=$3 # a string that can be the path to a cert file - # `100` means build failure without extra info, see doc/manual/src/command-ref/status-build-failure.md + # `100` means build failure without extra info, see doc/manual/source/command-ref/status-build-failure.md [ "$mode" == fixed-output ] && ret=1 || ret=100 expectStderr $ret nix-sandbox-build linux-sandbox-cert-test.nix --argstr mode "$mode" --option ssl-cert-file "$certFile" | grepQuiet "CERT_${expectation}_IN_SANDBOX" diff --git a/tests/functional/local-overlay-store/meson.build b/tests/functional/local-overlay-store/meson.build index 6ff5d3169..b7ba5a323 100644 --- a/tests/functional/local-overlay-store/meson.build +++ b/tests/functional/local-overlay-store/meson.build @@ -14,5 +14,5 @@ suites += { 'optimise.sh', 'stale-file-handle.sh', ], - 'workdir': meson.current_build_dir(), + 'workdir': meson.current_source_dir(), } diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 3f796291a..e50b5eaf1 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -114,7 +114,6 @@ nix_tests = \ impure-env.sh \ debugger.sh \ extra-sandbox-profile.sh \ - help.sh ifeq ($(HAVE_LIBCPUID), 1) nix_tests += compute-levels.sh @@ -128,6 +127,10 @@ ifeq ($(ENABLE_BUILD), yes) endif endif +ifeq ($(ENABLE_DOC_GEN), yes) + nix_tests += help.sh +endif + $(d)/test-libstoreconsumer.sh.test $(d)/test-libstoreconsumer.sh.test-debug: \ $(buildprefix)$(d)/test-libstoreconsumer/test-libstoreconsumer $(d)/plugins.sh.test $(d)/plugins.sh.test-debug: \ diff --git a/tests/functional/logging.sh b/tests/functional/logging.sh index bd80a9163..c026ac9c2 100755 --- a/tests/functional/logging.sh +++ b/tests/functional/logging.sh @@ -22,7 +22,7 @@ nix-build dependencies.nix --no-out-link --compress-build-log builder="$(realpath "$(mktemp)")" echo -e "#!/bin/sh\nmkdir \$out" > "$builder" outp="$(nix-build -E \ - 'with import ./config.nix; mkDerivation { name = "fnord"; builder = '"$builder"'; }' \ + 'with import '"${config_nix}"'; mkDerivation { name = "fnord"; builder = '"$builder"'; }' \ --out-link "$(mktemp -d)/result")" test -d "$outp" diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 69b6d3194..b3ac1560c 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -14,27 +14,6 @@ project('nix-functional-tests', 'cpp', fs = import('fs') -# Need to combine source and build trees -run_command( - 'rsync', - '-a', - '--copy-unsafe-links', - meson.current_source_dir() / '', - meson.current_build_dir() / '', -) -# This current-source-escaping relative is no good because we don't know -# where the build directory will be, therefore we fix it up. Once the -# Make build system is gone, we should think about doing this better. -scripts_dir = fs.relative_to( - meson.current_source_dir() / '..' / '..' / 'scripts', - meson.current_build_dir(), -) -run_command( - 'sed', - '-i', meson.current_build_dir() / 'bash-profile.sh', - '-e', 's^../../scripts^@0@^'.format(scripts_dir), -) - nix = find_program('nix') bash = find_program('bash', native : true) busybox = find_program('busybox', native : true, required : false) @@ -185,7 +164,7 @@ suites = [ 'extra-sandbox-profile.sh', 'help.sh', ], - 'workdir': meson.current_build_dir(), + 'workdir': meson.current_source_dir(), }, ] @@ -200,7 +179,7 @@ if nix_store.found() 'tests': [ 'test-libstoreconsumer.sh', ], - 'workdir': meson.current_build_dir(), + 'workdir': meson.current_source_dir(), } endif @@ -217,7 +196,7 @@ if nix_expr.found() and get_option('default_library') != 'static' 'tests': [ 'plugins.sh', ], - 'workdir': meson.current_build_dir(), + 'workdir': meson.current_source_dir(), } endif @@ -228,14 +207,12 @@ subdir('git-hashing') subdir('local-overlay-store') foreach suite : suites + workdir = suite['workdir'] + suite_name = suite['name'] foreach script : suite['tests'] - workdir = suite['workdir'] - prefix = fs.relative_to(workdir, meson.project_build_root()) - - script = script # Turns, e.g., `tests/functional/flakes/show.sh` into a Meson test target called # `functional-flakes-show`. - name = fs.replace_suffix(prefix / script, '') + name = fs.replace_suffix(script, '') test( name, @@ -247,14 +224,17 @@ foreach suite : suites '-o', 'pipefail', script, ], - suite : suite['name'], + suite : suite_name, env : { - 'TEST_NAME': name, + '_NIX_TEST_SOURCE_DIR': meson.current_source_dir(), + '_NIX_TEST_BUILD_DIR': meson.current_build_dir(), + 'TEST_NAME': suite_name / name, 'NIX_REMOTE': '', 'PS4': '+(${BASH_SOURCE[0]-$0}:$LINENO) ', }, - # some tests take 15+ seconds even on an otherwise idle machine, on a loaded machine - # this can easily drive them to failure. give them more time than default of 30sec + # Some tests take 15+ seconds even on an otherwise idle machine; + # on a loaded machine this can easily drive them to failure. Give + # them more time than the default of 30 seconds. timeout : 300, # Used for target dependency/ordering tracking, not adding compiler flags or anything. depends : suite['deps'], diff --git a/tests/functional/multiple-outputs.nix b/tests/functional/multiple-outputs.nix index 6ba7c523d..19ae2a45d 100644 --- a/tests/functional/multiple-outputs.nix +++ b/tests/functional/multiple-outputs.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; rec { diff --git a/tests/functional/nar-access.nix b/tests/functional/nar-access.nix index 0e2a7f721..78972bd36 100644 --- a/tests/functional/nar-access.nix +++ b/tests/functional/nar-access.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; rec { a = mkDerivation { @@ -20,4 +20,4 @@ asdom 12398 EOF ''; }; -} \ No newline at end of file +} diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh index b254081cf..2b0a6a329 100755 --- a/tests/functional/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -9,57 +9,57 @@ cd "$TEST_ROOT" # Dump path to nar. narFile="$TEST_ROOT/path.nar" -nix-store --dump $storePath > $narFile +nix-store --dump "$storePath" > "$narFile" # Check that find and nar ls match. -( cd $storePath; find . | sort ) > files.find -nix nar ls -R -d $narFile "" | sort > files.ls-nar +( cd "$storePath"; find . | sort ) > files.find +nix nar ls -R -d "$narFile" "" | sort > files.ls-nar diff -u files.find files.ls-nar # Check that file contents of data match. -nix nar cat $narFile /foo/data > data.cat-nar -diff -u data.cat-nar $storePath/foo/data +nix nar cat "$narFile" /foo/data > data.cat-nar +diff -u data.cat-nar "$storePath/foo/data" # Check that file contents of baz match. -nix nar cat $narFile /foo/baz > baz.cat-nar -diff -u baz.cat-nar $storePath/foo/baz +nix nar cat "$narFile" /foo/baz > baz.cat-nar +diff -u baz.cat-nar "$storePath/foo/baz" -nix store cat $storePath/foo/baz > baz.cat-nar -diff -u baz.cat-nar $storePath/foo/baz +nix store cat "$storePath/foo/baz" > baz.cat-nar +diff -u baz.cat-nar "$storePath/foo/baz" TODO_NixOS # Check that 'nix store cat' fails on invalid store paths. -invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo" -cp -r $storePath $invalidPath -expect 1 nix store cat $invalidPath/foo/baz +invalidPath="$(dirname "$storePath")/99999999999999999999999999999999-foo" +cp -r "$storePath" "$invalidPath" +expect 1 nix store cat "$invalidPath/foo/baz" # Test --json. diff -u \ - <(nix nar ls --json $narFile / | jq -S) \ + <(nix nar ls --json "$narFile" / | jq -S) \ <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S) diff -u \ - <(nix nar ls --json -R $narFile /foo | jq -S) \ + <(nix nar ls --json -R "$narFile" /foo | jq -S) \ <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0,"narOffset":368},"baz":{"type":"regular","size":0,"narOffset":552},"data":{"type":"regular","size":58,"narOffset":736}}}' | jq -S) diff -u \ - <(nix nar ls --json -R $narFile /foo/bar | jq -S) \ + <(nix nar ls --json -R "$narFile" /foo/bar | jq -S) \ <(echo '{"type":"regular","size":0,"narOffset":368}' | jq -S) diff -u \ - <(nix store ls --json $storePath | jq -S) \ + <(nix store ls --json "$storePath" | jq -S) \ <(echo '{"type":"directory","entries":{"foo":{},"foo-x":{},"qux":{},"zyx":{}}}' | jq -S) diff -u \ - <(nix store ls --json -R $storePath/foo | jq -S) \ + <(nix store ls --json -R "$storePath/foo" | jq -S) \ <(echo '{"type":"directory","entries":{"bar":{"type":"regular","size":0},"baz":{"type":"regular","size":0},"data":{"type":"regular","size":58}}}' | jq -S) diff -u \ - <(nix store ls --json -R $storePath/foo/bar| jq -S) \ + <(nix store ls --json -R "$storePath/foo/bar"| jq -S) \ <(echo '{"type":"regular","size":0}' | jq -S) # Test missing files. -expect 1 nix store ls --json -R $storePath/xyzzy 2>&1 | grep 'does not exist' -expect 1 nix store ls $storePath/xyzzy 2>&1 | grep 'does not exist' +expect 1 nix store ls --json -R "$storePath/xyzzy" 2>&1 | grep 'does not exist' +expect 1 nix store ls "$storePath/xyzzy" 2>&1 | grep 'does not exist' # Test failure to dump. -if nix-store --dump $storePath >/dev/full ; then +if nix-store --dump "$storePath" >/dev/full ; then echo "dumping to /dev/full should fail" - exit -1 + exit 1 fi diff --git a/tests/functional/nars.sh b/tests/functional/nars.sh index 39d9389db..dd90345a6 100755 --- a/tests/functional/nars.sh +++ b/tests/functional/nars.sh @@ -134,7 +134,7 @@ rm -f "$TEST_ROOT/unicode-*" # Unpacking a NAR with a NUL character in a file name should fail. rm -rf "$TEST_ROOT/out" -expectStderr 1 nix-store --restore "$TEST_ROOT/out" < nul.nar | grepQuiet "NAR contains invalid file name 'f" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < nul-character.nar | grepQuiet "NAR contains invalid file name 'f" # Likewise for a '.' filename. rm -rf "$TEST_ROOT/out" diff --git a/tests/functional/nested-sandboxing.sh b/tests/functional/nested-sandboxing.sh index ae0256de2..7462d2968 100755 --- a/tests/functional/nested-sandboxing.sh +++ b/tests/functional/nested-sandboxing.sh @@ -8,6 +8,15 @@ TODO_NixOS requireSandboxSupport +start="$TEST_ROOT/start" +mkdir -p "$start" +cp -r common common.sh ${config_nix} ./nested-sandboxing "$start" +cp "${_NIX_TEST_BUILD_DIR}/common/subst-vars.sh" "$start/common" +# N.B. redefine +_NIX_TEST_SOURCE_DIR="$start" +_NIX_TEST_BUILD_DIR="$start" +cd "$start" + source ./nested-sandboxing/command.sh expectStderr 100 runNixBuild badStoreUrl 2 | grepQuiet '`sandbox-build-dir` must not contain' diff --git a/tests/functional/nested-sandboxing/runner.nix b/tests/functional/nested-sandboxing/runner.nix index 1e79d5065..48ad512b3 100644 --- a/tests/functional/nested-sandboxing/runner.nix +++ b/tests/functional/nested-sandboxing/runner.nix @@ -19,6 +19,9 @@ mkDerivation { export PATH=${builtins.getEnv "NIX_BIN_DIR"}:$PATH + export _NIX_TEST_SOURCE_DIR=$PWD + export _NIX_TEST_BUILD_DIR=$PWD + source common.sh source ./nested-sandboxing/command.sh diff --git a/tests/functional/nix-build-examples.nix b/tests/functional/nix-build-examples.nix index e54dbbf62..aaea8fc07 100644 --- a/tests/functional/nix-build-examples.nix +++ b/tests/functional/nix-build-examples.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; rec { diff --git a/tests/functional/nix-channel.sh b/tests/functional/nix-channel.sh index a4870e7a8..16d6a1355 100755 --- a/tests/functional/nix-channel.sh +++ b/tests/functional/nix-channel.sh @@ -35,7 +35,7 @@ drvPath=$(nix-instantiate dependencies.nix) nix copy --to file://$TEST_ROOT/foo?compression="bzip2" $(nix-store -r "$drvPath") rm -rf $TEST_ROOT/nixexprs mkdir -p $TEST_ROOT/nixexprs -cp config.nix dependencies.nix dependencies.builder*.sh $TEST_ROOT/nixexprs/ +cp "${config_nix}" dependencies.nix dependencies.builder*.sh $TEST_ROOT/nixexprs/ ln -s dependencies.nix $TEST_ROOT/nixexprs/default.nix (cd $TEST_ROOT && tar cvf - nixexprs) | bzip2 > $TEST_ROOT/foo/nixexprs.tar.bz2 diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index e2f19b99e..7cf5fcb74 100755 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -47,7 +47,7 @@ printf World > $flake1Dir/who printf 1.0 > $flake1Dir/version printf false > $flake1Dir/ca.nix -cp ./config.nix $flake1Dir/ +cp "${config_nix}" $flake1Dir/ # Test upgrading from nix-env. nix-env -f ./user-envs.nix -i foo-1.0 @@ -140,7 +140,7 @@ nix profile install $(nix-build --no-out-link ./simple.nix) # Test packages with same name from different sources mkdir $TEST_ROOT/simple-too -cp ./simple.nix ./config.nix simple.builder.sh $TEST_ROOT/simple-too +cp ./simple.nix "${config_nix}" simple.builder.sh $TEST_ROOT/simple-too nix profile install --file $TEST_ROOT/simple-too/simple.nix '' nix profile list | grep -A4 'Name:.*simple' | grep 'Name:.*simple-1' nix profile remove simple 2>&1 | grep 'removed 1 packages' diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh index b9625eb66..2b78216f4 100755 --- a/tests/functional/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -31,6 +31,15 @@ output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR "$shellDotNix" -A shellDrv [ "$output" = " - foo - bar - baz" ] +# test NIX_BUILD_TOP +testTmpDir=$(pwd)/nix-shell +mkdir -p "$testTmpDir" +output=$(TMPDIR="$testTmpDir" nix-shell --pure "$shellDotNix" -A shellDrv --run 'echo $NIX_BUILD_TOP') +[[ "$output" =~ ${testTmpDir}.* ]] || { + echo "expected $output =~ ${testTmpDir}.*" >&2 + exit 1 +} + # Test nix-shell on a .drv [[ $(nix-shell --pure $(nix-instantiate "$shellDotNix" -A shellDrv) --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]] @@ -70,7 +79,7 @@ sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.expr > $TEST_ROOT/shell.sheb chmod a+rx $TEST_ROOT/shell.shebang.expr # Should fail due to expressions using relative path ! $TEST_ROOT/shell.shebang.expr bar -cp shell.nix config.nix $TEST_ROOT +cp shell.nix "${config_nix}" $TEST_ROOT # Should succeed echo "cwd: $PWD" output=$($TEST_ROOT/shell.shebang.expr bar) @@ -117,7 +126,7 @@ $TEST_ROOT/shell.shebang.nix mkdir $TEST_ROOT/lookup-test $TEST_ROOT/empty echo "import $shellDotNix" > $TEST_ROOT/lookup-test/shell.nix -cp config.nix $TEST_ROOT/lookup-test/ +cp "${config_nix}" $TEST_ROOT/lookup-test/ echo 'abort "do not load default.nix!"' > $TEST_ROOT/lookup-test/default.nix nix-shell $TEST_ROOT/lookup-test -A shellDrv --run 'echo "it works"' | grepQuiet "it works" diff --git a/tests/functional/nul.nar b/tests/functional/nul-character.nar similarity index 100% rename from tests/functional/nul.nar rename to tests/functional/nul-character.nar diff --git a/tests/functional/optimise-store.sh b/tests/functional/optimise-store.sh index 0bedafc43..05c4c41e4 100755 --- a/tests/functional/optimise-store.sh +++ b/tests/functional/optimise-store.sh @@ -4,8 +4,8 @@ source common.sh clearStoreIfPossible -outPath1=$(echo 'with import ./config.nix; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) -outPath2=$(echo 'with import ./config.nix; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) +outPath1=$(echo 'with import '"${config_nix}"'; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) +outPath2=$(echo 'with import '"${config_nix}"'; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) TODO_NixOS # ignoring the client-specified setting 'auto-optimise-store', because it is a restricted setting and you are not a trusted user # TODO: only continue when trusted user or root @@ -23,7 +23,7 @@ if [ "$nlink" != 3 ]; then exit 1 fi -outPath3=$(echo 'with import ./config.nix; mkDerivation { name = "foo3"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link) +outPath3=$(echo 'with import '"${config_nix}"'; mkDerivation { name = "foo3"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link) inode3="$(stat --format=%i $outPath3/foo)" if [ "$inode1" = "$inode3" ]; then diff --git a/tests/functional/package.nix b/tests/functional/package.nix index 675cefa64..9cf6b62e1 100644 --- a/tests/functional/package.nix +++ b/tests/functional/package.nix @@ -6,7 +6,6 @@ , meson , ninja , pkg-config -, rsync , jq , git @@ -48,11 +47,10 @@ mkMesonDerivation (finalAttrs: { ]; # Hack for sake of the dev shell - passthru.baseNativeBuildInputs = [ + passthru.externalNativeBuildInputs = [ meson ninja pkg-config - rsync jq git @@ -66,7 +64,7 @@ mkMesonDerivation (finalAttrs: { util-linux ]; - nativeBuildInputs = finalAttrs.passthru.baseNativeBuildInputs ++ [ + nativeBuildInputs = finalAttrs.passthru.externalNativeBuildInputs ++ [ nix-cli ]; @@ -75,7 +73,6 @@ mkMesonDerivation (finalAttrs: { nix-expr ]; - preConfigure = # "Inline" .version so it's not a symlink, and includes the suffix. # Do the meson utils, without modification. diff --git a/tests/functional/parallel.nix b/tests/functional/parallel.nix index 23f142059..1f2411c92 100644 --- a/tests/functional/parallel.nix +++ b/tests/functional/parallel.nix @@ -1,6 +1,6 @@ {sleepTime ? 3}: -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; let diff --git a/tests/functional/pass-as-file.sh b/tests/functional/pass-as-file.sh index 6487bfffd..66a8e588e 100755 --- a/tests/functional/pass-as-file.sh +++ b/tests/functional/pass-as-file.sh @@ -5,7 +5,7 @@ source common.sh clearStoreIfPossible outPath=$(nix-build --no-out-link -E " -with import ./config.nix; +with import ${config_nix}; mkDerivation { name = \"pass-as-file\"; diff --git a/tests/functional/path.nix b/tests/functional/path.nix index 883c3c41b..b23300f90 100644 --- a/tests/functional/path.nix +++ b/tests/functional/path.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; mkDerivation { name = "filter"; diff --git a/tests/functional/placeholders.sh b/tests/functional/placeholders.sh index 33ec0c2b7..374203af8 100755 --- a/tests/functional/placeholders.sh +++ b/tests/functional/placeholders.sh @@ -5,7 +5,7 @@ source common.sh clearStoreIfPossible nix-build --no-out-link -E ' - with import ./config.nix; + with import '"${config_nix}"'; mkDerivation { name = "placeholders"; diff --git a/tests/functional/plugins.sh b/tests/functional/plugins.sh index fc2d1907c..b685a8878 100755 --- a/tests/functional/plugins.sh +++ b/tests/functional/plugins.sh @@ -3,7 +3,7 @@ source common.sh for ext in so dylib; do - plugin="$PWD/plugins/libplugintest.$ext" + plugin="${_NIX_TEST_BUILD_DIR}/plugins/libplugintest.$ext" [[ -f "$plugin" ]] && break done diff --git a/tests/functional/readfile-context.nix b/tests/functional/readfile-context.nix index 54cd1afd9..b8f4a4c27 100644 --- a/tests/functional/readfile-context.nix +++ b/tests/functional/readfile-context.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; let diff --git a/tests/functional/recursive.nix b/tests/functional/recursive.nix index fa8cc04db..fe438f0ba 100644 --- a/tests/functional/recursive.nix +++ b/tests/functional/recursive.nix @@ -1,4 +1,5 @@ -with import ./config.nix; +let config_nix = /. + "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; in +with import config_nix; mkDerivation rec { name = "recursive"; @@ -41,7 +42,7 @@ mkDerivation rec { # Build a derivation. nix $opts build -L --impure --expr ' - with import ${./config.nix}; + with import ${config_nix}; mkDerivation { name = "inner1"; buildCommand = "echo $fnord blaat > $out"; diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index e5fe9c136..63bf56cd7 100755 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -7,8 +7,19 @@ clearStoreIfPossible nix-instantiate --restrict-eval --eval -E '1 + 2' (! nix-instantiate --eval --restrict-eval ./restricted.nix) (! nix-instantiate --eval --restrict-eval <(echo '1 + 2')) + +mkdir -p "$TEST_ROOT/nix" +cp ./simple.nix "$TEST_ROOT/nix" +cp ./simple.builder.sh "$TEST_ROOT/nix" +cp "${config_nix}" "$TEST_ROOT/nix" +simple_nix="$TEST_ROOT/nix/simple.nix" +# N.B. redefine +config_nix="$TEST_ROOT/nix/config.nix" +removeBuildDirRef "${simple_nix}" +cd "$TEST_ROOT/nix" + nix-instantiate --restrict-eval ./simple.nix -I src=. -nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh +nix-instantiate --restrict-eval ./simple.nix -I src1=./simple.nix -I src2=./config.nix -I src3=./simple.builder.sh # no default NIX_PATH (unset NIX_PATH; ! nix-instantiate --restrict-eval --find-file .) @@ -19,25 +30,25 @@ nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I sr expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' | grepQuiet "forbidden in restricted mode" nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' -I src=. -p=$(nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)") -cmp $p restricted.sh +p=$(nix eval --raw --expr "builtins.fetchurl file://${_NIX_TEST_SOURCE_DIR}/restricted.sh" --impure --restrict-eval --allowed-uris "file://${_NIX_TEST_SOURCE_DIR}") +cmp "$p" "${_NIX_TEST_SOURCE_DIR}/restricted.sh" -(! nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval) +(! nix eval --raw --expr "builtins.fetchurl file://${_NIX_TEST_SOURCE_DIR}/restricted.sh" --impure --restrict-eval) -(! nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh/") +(! nix eval --raw --expr "builtins.fetchurl file://${_NIX_TEST_SOURCE_DIR}/restricted.sh" --impure --restrict-eval --allowed-uris "file://${_NIX_TEST_SOURCE_DIR}/restricted.sh/") -nix eval --raw --expr "builtins.fetchurl file://$(pwd)/restricted.sh" --impure --restrict-eval --allowed-uris "file://$(pwd)/restricted.sh" +nix eval --raw --expr "builtins.fetchurl file://${_NIX_TEST_SOURCE_DIR}/restricted.sh" --impure --restrict-eval --allowed-uris "file://${_NIX_TEST_SOURCE_DIR}/restricted.sh" (! nix eval --raw --expr "builtins.fetchurl https://github.com/NixOS/patchelf/archive/master.tar.gz" --impure --restrict-eval) (! nix eval --raw --expr "builtins.fetchTarball https://github.com/NixOS/patchelf/archive/master.tar.gz" --impure --restrict-eval) (! nix eval --raw --expr "fetchGit git://github.com/NixOS/patchelf.git" --impure --restrict-eval) -ln -sfn $(pwd)/restricted.nix $TEST_ROOT/restricted.nix +ln -sfn "${_NIX_TEST_SOURCE_DIR}/restricted.nix" "$TEST_ROOT/restricted.nix" [[ $(nix-instantiate --eval $TEST_ROOT/restricted.nix) == 3 ]] (! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix) (! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT) (! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I .) -nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT -I . +nix-instantiate --eval --restrict-eval "$TEST_ROOT/restricted.nix" -I "$TEST_ROOT" -I "${_NIX_TEST_SOURCE_DIR}" [[ $(nix eval --raw --impure --restrict-eval -I . --expr 'builtins.readFile "${import ./simple.nix}/hello"') == 'Hello World!' ]] @@ -54,12 +65,12 @@ expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { pr [[ $(nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d) == '{ "tunnel.d" = "directory"; }' ]] # Check whether we can leak symlink information through directory traversal. -traverseDir="$(pwd)/restricted-traverse-me" -ln -sfn "$(pwd)/restricted-secret" "$(pwd)/restricted-innocent" +traverseDir="${_NIX_TEST_SOURCE_DIR}/restricted-traverse-me" +ln -sfn "${_NIX_TEST_SOURCE_DIR}/restricted-secret" "${_NIX_TEST_SOURCE_DIR}/restricted-innocent" mkdir -p "$traverseDir" goUp="..$(echo "$traverseDir" | sed -e 's,[^/]\+,..,g')" output="$(nix eval --raw --restrict-eval -I "$traverseDir" \ - --expr "builtins.readFile \"$traverseDir/$goUp$(pwd)/restricted-innocent\"" \ + --expr "builtins.readFile \"$traverseDir/$goUp${_NIX_TEST_SOURCE_DIR}/restricted-innocent\"" \ 2>&1 || :)" echo "$output" | grep "is forbidden" echo "$output" | grepInverse -F restricted-secret diff --git a/tests/functional/search.nix b/tests/functional/search.nix index fea6e7a7a..3c3564bda 100644 --- a/tests/functional/search.nix +++ b/tests/functional/search.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; { hello = mkDerivation rec { diff --git a/tests/functional/secure-drv-outputs.nix b/tests/functional/secure-drv-outputs.nix index b4ac8ff53..cd111c315 100644 --- a/tests/functional/secure-drv-outputs.nix +++ b/tests/functional/secure-drv-outputs.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; { diff --git a/tests/functional/selfref-gc.sh b/tests/functional/selfref-gc.sh index 518aea66b..dc4f14cc1 100755 --- a/tests/functional/selfref-gc.sh +++ b/tests/functional/selfref-gc.sh @@ -7,7 +7,7 @@ requireDaemonNewerThan "2.6.0pre20211215" clearStoreIfPossible nix-build --no-out-link -E ' - with import ./config.nix; + with import '"${config_nix}"'; let d1 = mkDerivation { name = "selfref-gc"; diff --git a/tests/functional/shell-hello.nix b/tests/functional/shell-hello.nix index c920d7cb4..fa02e2bb4 100644 --- a/tests/functional/shell-hello.nix +++ b/tests/functional/shell-hello.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; rec { hello = mkDerivation { diff --git a/tests/functional/shell.nix b/tests/functional/shell.nix index 9cae14b78..f6622a487 100644 --- a/tests/functional/shell.nix +++ b/tests/functional/shell.nix @@ -1,6 +1,6 @@ { inNixShell ? false, contentAddressed ? false, fooContents ? "foo" }: -let cfg = import ./config.nix; in +let cfg = import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; in with cfg; let diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh index 04a03eef5..cfc8e4102 100755 --- a/tests/functional/shell.sh +++ b/tests/functional/shell.sh @@ -27,8 +27,8 @@ expect 1 nix shell -f shell-hello.nix forbidden-symlink -c hello 2>&1 | grepQuie # For instance, we might set an environment variable temporarily to affect some # initialization or whatnot, but this must not leak into the environment of the # command being run. -env > $TEST_ROOT/expected-env -nix shell -f shell-hello.nix hello -c env > $TEST_ROOT/actual-env +env > "$TEST_ROOT/expected-env" +nix shell -f shell-hello.nix hello -c env > "$TEST_ROOT/actual-env" # Remove/reset variables we expect to be different. # - PATH is modified by nix shell # - we unset TMPDIR on macOS if it contains /var/folders @@ -39,10 +39,10 @@ sed -i \ -e 's/_=.*/_=.../' \ -e '/^TMPDIR=\/var\/folders\/.*/d' \ -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \ - $TEST_ROOT/expected-env $TEST_ROOT/actual-env -sort $TEST_ROOT/expected-env > $TEST_ROOT/expected-env.sorted -sort $TEST_ROOT/actual-env > $TEST_ROOT/actual-env.sorted -diff $TEST_ROOT/expected-env.sorted $TEST_ROOT/actual-env.sorted + "$TEST_ROOT/expected-env" "$TEST_ROOT/actual-env" +sort "$TEST_ROOT/expected-env" > "$TEST_ROOT/expected-env.sorted" +sort "$TEST_ROOT/actual-env" > "$TEST_ROOT/actual-env.sorted" +diff "$TEST_ROOT/expected-env.sorted" "$TEST_ROOT/actual-env.sorted" if isDaemonNewer "2.20.0pre20231220"; then # Test that command line attribute ordering is reflected in the PATH @@ -53,8 +53,8 @@ fi requireSandboxSupport -chmod -R u+w $TEST_ROOT/store0 || true -rm -rf $TEST_ROOT/store0 +chmod -R u+w "$TEST_ROOT/store0" || true +rm -rf "$TEST_ROOT/store0" clearStore @@ -64,10 +64,10 @@ path=$(nix eval --raw -f shell-hello.nix hello) # visible in the sandbox. nix shell --sandbox-build-dir /build-tmp \ --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' \ - --store $TEST_ROOT/store0 -f shell-hello.nix hello -c hello | grep 'Hello World' + --store "$TEST_ROOT/store0" -f shell-hello.nix hello -c hello | grep 'Hello World' -path2=$(nix shell --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store $TEST_ROOT/store0 -f shell-hello.nix hello -c $SHELL -c 'type -p hello') +path2=$(nix shell --sandbox-paths '/nix? /bin? /lib? /lib64? /usr?' --store "$TEST_ROOT/store0" -f shell-hello.nix hello -c "$SHELL" -c 'type -p hello') -[[ $path/bin/hello = $path2 ]] +[[ "$path/bin/hello" = "$path2" ]] -[[ -e $TEST_ROOT/store0/nix/store/$(basename $path)/bin/hello ]] +[[ -e $TEST_ROOT/store0/nix/store/$(basename "$path")/bin/hello ]] diff --git a/tests/functional/simple-failing.nix b/tests/functional/simple-failing.nix index d176c9c51..228971734 100644 --- a/tests/functional/simple-failing.nix +++ b/tests/functional/simple-failing.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; mkDerivation { name = "simple-failing"; diff --git a/tests/functional/simple.nix b/tests/functional/simple.nix index 2035ca294..96237695c 100644 --- a/tests/functional/simple.nix +++ b/tests/functional/simple.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; mkDerivation { name = "simple"; diff --git a/tests/functional/structured-attrs-shell.nix b/tests/functional/structured-attrs-shell.nix index 57c1e6bd2..7ed28c03f 100644 --- a/tests/functional/structured-attrs-shell.nix +++ b/tests/functional/structured-attrs-shell.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; let dep = mkDerivation { name = "dep"; diff --git a/tests/functional/structured-attrs.nix b/tests/functional/structured-attrs.nix index e93139a44..ae461c21a 100644 --- a/tests/functional/structured-attrs.nix +++ b/tests/functional/structured-attrs.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; let diff --git a/tests/functional/supplementary-groups.sh b/tests/functional/supplementary-groups.sh index 5d329efc9..50259a3e1 100755 --- a/tests/functional/supplementary-groups.sh +++ b/tests/functional/supplementary-groups.sh @@ -9,7 +9,7 @@ needLocalStore "The test uses --store always so we would just be bypassing the d TODO_NixOS -unshare --mount --map-root-user bash <; 1 + 2' -I fnord=file:///no-such-tarball"$ext" (! nix-instantiate --eval -E ' 1' -I fnord=file:///no-such-tarball"$ext") - nix-instantiate --eval -E '' -I fnord=file:///no-such-tarball"$ext" -I fnord=. + nix-instantiate --eval -E '' -I fnord=file:///no-such-tarball"$ext" -I fnord="${_NIX_TEST_BUILD_DIR}" # Ensure that the `name` attribute isn’t accepted as that would mess # with the content-addressing @@ -97,3 +97,17 @@ chmod +x "$TEST_ROOT/tar_root/foo" tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" [[ $(cat "$path/foo") = bar ]] + +# Test a tarball with non-contiguous directory entries. +rm -rf "$TEST_ROOT/tar_root" +mkdir -p "$TEST_ROOT/tar_root/a/b" +echo foo > "$TEST_ROOT/tar_root/a/b/foo" +echo bla > "$TEST_ROOT/tar_root/bla" +tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . +echo abc > "$TEST_ROOT/tar_root/bla" +echo xyzzy > "$TEST_ROOT/tar_root/a/b/xyzzy" +tar rvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" ./a/b/xyzzy ./bla +path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" +[[ $(cat "$path/a/b/xyzzy") = xyzzy ]] +[[ $(cat "$path/a/b/foo") = foo ]] +[[ $(cat "$path/bla") = abc ]] diff --git a/tests/functional/test-libstoreconsumer.sh b/tests/functional/test-libstoreconsumer.sh index 2adead1c0..5b019a1d0 100755 --- a/tests/functional/test-libstoreconsumer.sh +++ b/tests/functional/test-libstoreconsumer.sh @@ -4,5 +4,5 @@ source common.sh drv="$(nix-instantiate simple.nix)" cat "$drv" -out="$(./test-libstoreconsumer/test-libstoreconsumer "$drv")" +out="$("${_NIX_TEST_BUILD_DIR}/test-libstoreconsumer/test-libstoreconsumer" "$drv")" grep -F "Hello World!" < "$out/hello" diff --git a/tests/functional/timeout.nix b/tests/functional/timeout.nix index d0e949e31..ad71e61e2 100644 --- a/tests/functional/timeout.nix +++ b/tests/functional/timeout.nix @@ -1,4 +1,4 @@ -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; { diff --git a/tests/functional/user-envs.nix b/tests/functional/user-envs.nix index 46f8b51dd..c8e846d4b 100644 --- a/tests/functional/user-envs.nix +++ b/tests/functional/user-envs.nix @@ -2,7 +2,7 @@ { foo ? "foo" }: -with import ./config.nix; +with import "${builtins.getEnv "_NIX_TEST_BUILD_DIR"}/config.nix"; assert foo == "foo"; diff --git a/tests/functional/why-depends.sh b/tests/functional/why-depends.sh index ce53546d8..45d1f2f0b 100755 --- a/tests/functional/why-depends.sh +++ b/tests/functional/why-depends.sh @@ -4,7 +4,7 @@ source common.sh clearStoreIfPossible -cp ./dependencies.nix ./dependencies.builder0.sh ./config.nix $TEST_HOME +cp ./dependencies.nix ./dependencies.builder0.sh "${config_nix}" $TEST_HOME cd $TEST_HOME diff --git a/tests/functional/zstd.sh b/tests/functional/zstd.sh index 1eb1b5343..226d5eb68 100755 --- a/tests/functional/zstd.sh +++ b/tests/functional/zstd.sh @@ -11,25 +11,25 @@ cacheURI="file://$cacheDir?compression=zstd" outPath=$(nix-build dependencies.nix --no-out-link) -nix copy --to $cacheURI $outPath +nix copy --to "$cacheURI" "$outPath" -HASH=$(nix hash path $outPath) +HASH=$(nix hash path "$outPath") clearStore clearCacheCache -nix copy --from $cacheURI $outPath --no-check-sigs --profile $TEST_ROOT/profile --out-link $TEST_ROOT/result +nix copy --from "$cacheURI" "$outPath" --no-check-sigs --profile "$TEST_ROOT/profile" --out-link "$TEST_ROOT/result" [[ -e $TEST_ROOT/profile ]] [[ -e $TEST_ROOT/result ]] -if ls $cacheDir/nar/*.zst &> /dev/null; then +if ls "$cacheDir/nar/"*.zst &> /dev/null; then echo "files do exist" else echo "nars do not exist" exit 1 fi -HASH2=$(nix hash path $outPath) +HASH2=$(nix hash path "$outPath") -[[ $HASH = $HASH2 ]] +[[ "$HASH" = "$HASH2" ]] diff --git a/tests/nixos/chroot-store.nix b/tests/nixos/chroot-store.nix new file mode 100644 index 000000000..4b167fc38 --- /dev/null +++ b/tests/nixos/chroot-store.nix @@ -0,0 +1,31 @@ +{ lib, config, nixpkgs, ... }: + +let + pkgs = config.nodes.machine.nixpkgs.pkgs; + pkgA = pkgs.hello; + pkgB = pkgs.cowsay; +in { + name = "chroot-store"; + + nodes = + { machine = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + virtualisation.additionalPaths = [ pkgA ]; + environment.systemPackages = [ pkgB ]; + nix.extraOptions = "experimental-features = nix-command"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + start_all() + + machine.succeed("nix copy --no-check-sigs --to /tmp/nix ${pkgA}") + + machine.succeed("nix shell --store /tmp/nix ${pkgA} --command hello >&2") + + # Test that /nix/store is available via an overlayfs mount. + machine.succeed("nix shell --store /tmp/nix ${pkgA} --command cowsay foo >&2") + ''; +} diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index c61a2888f..49e2603e1 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -161,4 +161,6 @@ in cgroups = runNixOSTestFor "x86_64-linux" ./cgroups; fetchurl = runNixOSTestFor "x86_64-linux" ./fetchurl.nix; + + chrootStore = runNixOSTestFor "x86_64-linux" ./chroot-store.nix; } diff --git a/tests/nixos/functional/common.nix b/tests/nixos/functional/common.nix index 51fd76884..86d55d0b6 100644 --- a/tests/nixos/functional/common.nix +++ b/tests/nixos/functional/common.nix @@ -55,6 +55,10 @@ in -e 's!nix_tests += test-libstoreconsumer\.sh!!' \ ; + _NIX_TEST_SOURCE_DIR="$(realpath tests/functional)" + export _NIX_TEST_SOURCE_DIR + export _NIX_TEST_BUILD_DIR="''${_NIX_TEST_SOURCE_DIR}" + export isTestOnNixOS=1 export version=${config.nix.package.version} export NIX_REMOTE_=daemon diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index 015457968..6c51fcba5 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -12,7 +12,7 @@ let storeUrl = "s3://my-cache?endpoint=http://server:9000®ion=eu-west-1"; in { - name = "nix-copy-closure"; + name = "s3-binary-cache-store"; nodes = { server = @@ -51,6 +51,9 @@ in { server.succeed("${env} nix copy --to '${storeUrl}' ${pkgA}") + # Test fetchurl on s3:// URLs while we're at it. + client.succeed("${env} nix eval --impure --expr 'builtins.fetchurl { name = \"foo\"; url = \"s3://my-cache/nix-cache-info?endpoint=http://server:9000®ion=eu-west-1\"; }'") + # Copy a package from the binary cache. client.fail("nix path-info ${pkgA}") diff --git a/tests/unit/libexpr-support/.version b/tests/unit/libexpr-support/.version deleted file mode 120000 index 0df9915bf..000000000 --- a/tests/unit/libexpr-support/.version +++ /dev/null @@ -1 +0,0 @@ -../../../.version \ No newline at end of file diff --git a/tests/unit/libexpr-support/build-utils-meson b/tests/unit/libexpr-support/build-utils-meson deleted file mode 120000 index f2d8e8a50..000000000 --- a/tests/unit/libexpr-support/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libexpr/.version b/tests/unit/libexpr/.version deleted file mode 120000 index 0df9915bf..000000000 --- a/tests/unit/libexpr/.version +++ /dev/null @@ -1 +0,0 @@ -../../../.version \ No newline at end of file diff --git a/tests/unit/libexpr/build-utils-meson b/tests/unit/libexpr/build-utils-meson deleted file mode 120000 index f2d8e8a50..000000000 --- a/tests/unit/libexpr/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libfetchers/.version b/tests/unit/libfetchers/.version deleted file mode 120000 index 0df9915bf..000000000 --- a/tests/unit/libfetchers/.version +++ /dev/null @@ -1 +0,0 @@ -../../../.version \ No newline at end of file diff --git a/tests/unit/libfetchers/build-utils-meson b/tests/unit/libfetchers/build-utils-meson deleted file mode 120000 index f2d8e8a50..000000000 --- a/tests/unit/libfetchers/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libflake/.version b/tests/unit/libflake/.version deleted file mode 120000 index 0df9915bf..000000000 --- a/tests/unit/libflake/.version +++ /dev/null @@ -1 +0,0 @@ -../../../.version \ No newline at end of file diff --git a/tests/unit/libflake/build-utils-meson b/tests/unit/libflake/build-utils-meson deleted file mode 120000 index f2d8e8a50..000000000 --- a/tests/unit/libflake/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libstore-support/.version b/tests/unit/libstore-support/.version deleted file mode 120000 index 0df9915bf..000000000 --- a/tests/unit/libstore-support/.version +++ /dev/null @@ -1 +0,0 @@ -../../../.version \ No newline at end of file diff --git a/tests/unit/libstore-support/build-utils-meson b/tests/unit/libstore-support/build-utils-meson deleted file mode 120000 index f2d8e8a50..000000000 --- a/tests/unit/libstore-support/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libstore/.version b/tests/unit/libstore/.version deleted file mode 120000 index 0df9915bf..000000000 --- a/tests/unit/libstore/.version +++ /dev/null @@ -1 +0,0 @@ -../../../.version \ No newline at end of file diff --git a/tests/unit/libstore/build-utils-meson b/tests/unit/libstore/build-utils-meson deleted file mode 120000 index f2d8e8a50..000000000 --- a/tests/unit/libstore/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv deleted file mode 120000 index 353090ad8..000000000 --- a/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv +++ /dev/null @@ -1 +0,0 @@ -../../../../functional/derivation/advanced-attributes-defaults.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv deleted file mode 120000 index 11713da12..000000000 --- a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv +++ /dev/null @@ -1 +0,0 @@ -../../../../functional/derivation/advanced-attributes-structured-attrs-defaults.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv deleted file mode 120000 index 962f8ea3f..000000000 --- a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv +++ /dev/null @@ -1 +0,0 @@ -../../../../functional/derivation/advanced-attributes-structured-attrs.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes.drv b/tests/unit/libstore/data/derivation/advanced-attributes.drv deleted file mode 120000 index 2a53a05ca..000000000 --- a/tests/unit/libstore/data/derivation/advanced-attributes.drv +++ /dev/null @@ -1 +0,0 @@ -../../../../functional/derivation/advanced-attributes.drv \ No newline at end of file diff --git a/tests/unit/libutil-support/.version b/tests/unit/libutil-support/.version deleted file mode 120000 index 0df9915bf..000000000 --- a/tests/unit/libutil-support/.version +++ /dev/null @@ -1 +0,0 @@ -../../../.version \ No newline at end of file diff --git a/tests/unit/libutil-support/build-utils-meson b/tests/unit/libutil-support/build-utils-meson deleted file mode 120000 index f2d8e8a50..000000000 --- a/tests/unit/libutil-support/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libutil/.version b/tests/unit/libutil/.version deleted file mode 120000 index 0df9915bf..000000000 --- a/tests/unit/libutil/.version +++ /dev/null @@ -1 +0,0 @@ -../../../.version \ No newline at end of file diff --git a/tests/unit/libutil/build-utils-meson b/tests/unit/libutil/build-utils-meson deleted file mode 120000 index f2d8e8a50..000000000 --- a/tests/unit/libutil/build-utils-meson +++ /dev/null @@ -1 +0,0 @@ -../../../build-utils-meson/ \ No newline at end of file