diff --git a/.clang-format b/.clang-format index 73eac7ef6..4f191fc18 100644 --- a/.clang-format +++ b/.clang-format @@ -15,7 +15,7 @@ SpaceAfterCStyleCast: true SpaceAfterTemplateKeyword: false AccessModifierOffset: -4 AlignAfterOpenBracket: AlwaysBreak -AlignEscapedNewlines: DontAlign +AlignEscapedNewlines: Left ColumnLimit: 120 BreakStringLiterals: false BitFieldColonSpacing: None @@ -30,3 +30,5 @@ BreakBeforeBinaryOperators: NonAssignment AlwaysBreakBeforeMultilineStrings: true IndentPPDirectives: AfterHash PPIndentWidth: 2 +BinPackArguments: false +BreakBeforeTernaryOperators: true diff --git a/.editorconfig b/.editorconfig index 86360e658..e1c8bae39 100644 --- a/.editorconfig +++ b/.editorconfig @@ -4,20 +4,20 @@ # Top-most EditorConfig file root = true -# Unix-style newlines with a newline ending every file, utf-8 charset +# Unix-style newlines with a newline ending every file, UTF-8 charset [*] end_of_line = lf insert_final_newline = true trim_trailing_whitespace = true charset = utf-8 -# Match nix files, set indent to spaces with width of two +# Match Nix files, set indent to spaces with width of two [*.nix] indent_style = space indent_size = 2 -# Match c++/shell/perl, set indent to spaces with width of four -[*.{hpp,cc,hh,sh,pl,xs}] +# Match C++/C/shell/Perl, set indent to spaces with width of four +[*.{hpp,cc,hh,c,h,sh,pl,xs}] indent_style = space indent_size = 4 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 59db217d9..a9ca74c17 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -11,7 +11,16 @@ .github/CODEOWNERS @edolstra # Documentation of built-in functions -src/libexpr/primops.cc @roberth +src/libexpr/primops.cc @roberth @fricklerhandwerk + +# Documentation of settings +src/libexpr/eval-settings.hh @fricklerhandwerk +src/libstore/globals.hh @fricklerhandwerk + +# Documentation +doc/manual @fricklerhandwerk +maintainers/*.md @fricklerhandwerk +src/**/*.md @fricklerhandwerk # Libstore layer -/src/libstore @thufschmitt @ericson2314 +/src/libstore @ericson2314 diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index d12a4d36c..69da87db7 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,3 +1,22 @@ + + # Motivation diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml deleted file mode 100644 index 8f83b913c..000000000 --- a/.github/workflows/backport.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Backport -on: - pull_request_target: - types: [closed, labeled] -permissions: - contents: read -jobs: - backport: - name: Backport Pull Request - permissions: - # for zeebe-io/backport-action - contents: write - pull-requests: write - if: github.repository_owner == 'NixOS' && github.event.pull_request.merged == true && (github.event_name != 'labeled' || startsWith('backport', github.event.label.name)) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - # required to find all branches - fetch-depth: 0 - - name: Create backport PRs - # should be kept in sync with `version` - uses: zeebe-io/backport-action@v2.5.0 - with: - # Config README: https://github.com/zeebe-io/backport-action#backport-action - github_token: ${{ secrets.GITHUB_TOKEN }} - github_workspace: ${{ github.workspace }} - pull_description: |- - Automatic backport to `${target_branch}`, triggered by a label in #${pull_number}. - # should be kept in sync with `uses` - version: v0.0.5 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2b8eac49d..0e2e07da2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,19 +20,46 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v26 + - uses: cachix/install-nix-action@v30 with: # The sandbox would otherwise be disabled by default on Darwin extra_nix_config: "sandbox = true" - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v14 + - uses: cachix/cachix-action@v15 if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + - if: matrix.os == 'ubuntu-latest' + run: | + free -h + swapon --show + swap=$(swapon --show --noheadings | head -n 1 | awk '{print $1}') + echo "Found swap: $swap" + sudo swapoff $swap + # resize it (fallocate) + sudo fallocate -l 10G $swap + sudo mkswap $swap + sudo swapon $swap + free -h + ( + while sleep 60; do + free -h + done + ) & - run: nix --experimental-features 'nix-command flakes' flake check -L + - run: nix --experimental-features 'nix-command flakes' flake show --all-systems --json + # Steps to test CI automation in your own fork. + # Cachix: + # 1. Sign-up for https://www.cachix.org/ + # 2. Create a cache for $githubuser-nix-install-tests + # 3. Create a cachix auth token and save it in https://github.com/$githubuser/nix/settings/secrets/actions in "Repository secrets" as CACHIX_AUTH_TOKEN + # Dockerhub: + # 1. Sign-up for https://hub.docker.com/ + # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions) + # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork check_secrets: permissions: contents: none @@ -62,14 +89,15 @@ jobs: with: fetch-depth: 0 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v26 + - uses: cachix/install-nix-action@v30 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install - - uses: cachix/cachix-action@v14 + - uses: cachix/cachix-action@v15 with: name: '${{ env.CACHIX_NAME }}' signingKey: '${{ secrets.CACHIX_SIGNING_KEY }}' authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' + cachixArgs: '-v' - id: prepare-installer run: scripts/prepare-installer-for-github-actions @@ -84,7 +112,7 @@ jobs: steps: - uses: actions/checkout@v4 - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - uses: cachix/install-nix-action@v26 + - uses: cachix/install-nix-action@v30 with: install_url: '${{needs.installer.outputs.installerURL}}' install_options: "--tarball-url-prefix https://${{ env.CACHIX_NAME }}.cachix.org/serve" @@ -114,12 +142,12 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: cachix/install-nix-action@v26 + - uses: cachix/install-nix-action@v30 with: install_url: https://releases.nixos.org/nix/nix-2.20.3/install - run: echo CACHIX_NAME="$(echo $GITHUB_REPOSITORY-install-tests | tr "[A-Z]/" "[a-z]-")" >> $GITHUB_ENV - - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#default.version | tr -d \")" >> $GITHUB_ENV - - uses: cachix/cachix-action@v14 + - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV + - uses: cachix/cachix-action@v15 if: needs.check_secrets.outputs.cachix == 'true' with: name: '${{ env.CACHIX_NAME }}' @@ -127,8 +155,8 @@ jobs: authToken: '${{ secrets.CACHIX_AUTH_TOKEN }}' - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L - run: docker load -i ./result/image.tar.gz - - run: docker tag nix:$NIX_VERSION nixos/nix:$NIX_VERSION - - run: docker tag nix:$NIX_VERSION nixos/nix:master + - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION + - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:master # We'll deploy the newly built image to both Docker Hub and Github Container Registry. # # Push to Docker Hub first @@ -137,8 +165,8 @@ jobs: with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - run: docker push nixos/nix:$NIX_VERSION - - run: docker push nixos/nix:master + - run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION + - run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:master # Push to GitHub Container Registry as well - name: Login to GitHub Container Registry uses: docker/login-action@v3 @@ -166,4 +194,24 @@ jobs: - uses: actions/checkout@v4 - uses: DeterminateSystems/nix-installer-action@main - uses: DeterminateSystems/magic-nix-cache-action@main - - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes + - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes .#hydraJobs.tests.functional_user + + flake_regressions: + needs: vm_tests + runs-on: ubuntu-22.04 + steps: + - name: Checkout nix + uses: actions/checkout@v4 + - name: Checkout flake-regressions + uses: actions/checkout@v4 + with: + repository: NixOS/flake-regressions + path: flake-regressions + - name: Checkout flake-regressions-data + uses: actions/checkout@v4 + with: + repository: NixOS/flake-regressions-data + path: flake-regressions/tests + - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main + - run: nix build --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH scripts/flake-regressions.sh diff --git a/.github/workflows/hydra_status.yml b/.github/workflows/hydra_status.yml deleted file mode 100644 index 2a7574747..000000000 --- a/.github/workflows/hydra_status.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: Hydra status - -permissions: read-all - -on: - schedule: - - cron: "12,42 * * * *" - workflow_dispatch: - -jobs: - check_hydra_status: - name: Check Hydra status - if: github.repository_owner == 'NixOS' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - run: bash scripts/check-hydra-status.sh - diff --git a/.gitignore b/.gitignore index 52aaec23f..a17b627f4 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,8 @@ perl/Makefile.config /svn-revision /libtool /config/config.* +# Default meson build dir +/build # /doc/manual/ /doc/manual/*.1 @@ -52,6 +54,9 @@ perl/Makefile.config # /src/libfetchers /tests/unit/libfetchers/libnixfetchers-tests +# /src/libflake +/tests/unit/libflake/libnixflake-tests + # /src/libstore/ *.gen.* /src/libstore/tests @@ -92,7 +97,7 @@ perl/Makefile.config # /tests/functional/ /tests/functional/test-tmp -/tests/functional/common/vars-and-functions.sh +/tests/functional/common/subst-vars.sh /tests/functional/result* /tests/functional/restricted-innocent /tests/functional/shell diff --git a/.mergify.yml b/.mergify.yml new file mode 100644 index 000000000..c297d3d5e --- /dev/null +++ b/.mergify.yml @@ -0,0 +1,92 @@ +queue_rules: + - name: default + # all required tests need to go here + merge_conditions: + - check-success=installer + - check-success=installer_test (macos-latest) + - check-success=installer_test (ubuntu-latest) + - check-success=tests (macos-latest) + - check-success=tests (ubuntu-latest) + - check-success=vm_tests + merge_method: rebase + batch_size: 5 + +pull_request_rules: + - name: merge using the merge queue + conditions: + - base~=master|.+-maintenance + - label~=merge-queue|dependencies + actions: + queue: {} + +# The rules below will first create backport pull requests and put those in a merge queue. + + - name: backport patches to 2.18 + conditions: + - label=backport 2.18-maintenance + actions: + backport: + branches: + - 2.18-maintenance + labels: + - merge-queue + + - name: backport patches to 2.19 + conditions: + - label=backport 2.19-maintenance + actions: + backport: + branches: + - 2.19-maintenance + labels: + - merge-queue + + - name: backport patches to 2.20 + conditions: + - label=backport 2.20-maintenance + actions: + backport: + branches: + - 2.20-maintenance + labels: + - merge-queue + + - name: backport patches to 2.21 + conditions: + - label=backport 2.21-maintenance + actions: + backport: + branches: + - 2.21-maintenance + labels: + - merge-queue + + - name: backport patches to 2.22 + conditions: + - label=backport 2.22-maintenance + actions: + backport: + branches: + - 2.22-maintenance + labels: + - merge-queue + + - name: backport patches to 2.23 + conditions: + - label=backport 2.23-maintenance + actions: + backport: + branches: + - 2.23-maintenance + labels: + - merge-queue + + - name: backport patches to 2.24 + conditions: + - label=backport 2.24-maintenance + actions: + backport: + branches: + - "2.24-maintenance" + labels: + - merge-queue diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 000000000..de98055f7 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,4 @@ +external-sources=true +source-path=SCRIPTDIR +# Hack for scripts in e.g. tests/functional/ca +source-path=SCRIPTDIR/.. diff --git a/.version b/.version index e9763f6bf..5c18f9195 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.23.0 +2.25.0 diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 000000000..0105fb823 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,42 @@ +cff-version: 1.2.0 +title: Nix +message: >- + If you use this software, please cite it using the + metadata from this file. +type: software +authors: + - given-names: Eelco + family-names: Dolstra + email: edolstra@gmail.com + - name: The Nix contributors + website: 'https://github.com/NixOS/nix' +references: + - title: The Purely Functional Software Deployment Model + authors: + - family-names: Dolstra + given-names: Eelco + year: 2006 + type: thesis + thesis-type: PhD thesis + isbn: 90-393-4130-3 + url: https://dspace.library.uu.nl/handle/1874/7540 + database-provider: Utrecht University Repository + institution: + name: Utrecht University + keywords: + - configuration management + - software deployment + - purely functional + - component-based software engineering +repository-code: 'https://github.com/NixOS/nix' +url: 'https://nixos.org/' +abstract: >- + Nix, a purely functional package manager, is a powerful + package manager for Linux and other Unix systems that + makes package management reliable and reproducible. +keywords: + - reproducibility + - open-source + - c++ + - functional +license: LGPL-2.1 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 38f5d43b7..56508df34 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,9 +41,9 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). There are many open pull requests that might already do what you intend to work on. You can use [labels](https://github.com/NixOS/nix/labels) to filter for relevant topics. -3. Check the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html) for information on building Nix and running its tests. +3. Check the [Nix reference manual](https://nix.dev/manual/nix/development/development/building.html) for information on building Nix and running its tests. - For contributions to the command line interface, please check the [CLI guidelines](https://nixos.org/manual/nix/unstable/contributing/cli-guideline.html). + For contributions to the command line interface, please check the [CLI guidelines](https://nix.dev/manual/nix/development/development/cli-guideline.html). 4. Make your change! @@ -52,6 +52,20 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). Link related issues to inform interested parties and future contributors about your change. If your pull request closes one or multiple issues, mention that in the description using `Closes: #`, as it will then happen automatically when your change is merged. + * Credit original authors when you're reusing or building on their work. + * Link to relevant changes in other projects, so that others can understand the full context of the change in the future when you or someone else will change or troubleshoot the code. + This is especially important when your change is based on work done in other repositories. + + Example: + ``` + This is based on the work of @user in . + This solution took inspiration from . + + Co-authored-by: User Name + ``` + + When cherry-picking from a different repository, use the `-x` flag, and then amend the commits to turn the hashes into URLs. + * Make sure to have [a clean history of commits on your branch by using rebase](https://www.digitalocean.com/community/tutorials/how-to-rebase-and-update-a-pull-request). * [Mark the pull request as draft](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/changing-the-stage-of-a-pull-request) if you're not done with the changes. @@ -69,7 +83,7 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). - [ ] API documentation in header files - [ ] Code and comments are self-explanatory - [ ] Commit message explains **why** the change was made - - [ ] New feature or incompatible change: [add a release note](https://nixos.org/manual/nix/stable/contributing/hacking#add-a-release-note) + - [ ] New feature or incompatible change: [add a release note](https://nix.dev/manual/nix/development/development/contributing.html#add-a-release-note) 7. If you need additional feedback or help to getting pull request into shape, ask other contributors using [@mentions](https://docs.github.com/en/get-started/writing-on-github/getting-started-with-writing-and-formatting-on-github/basic-writing-and-formatting-syntax#mentioning-people-and-teams). @@ -78,9 +92,9 @@ Check out the [security policy](https://github.com/NixOS/nix/security/policy). The Nix reference manual is hosted on https://nixos.org/manual/nix. The underlying source files are located in [`doc/manual/src`](./doc/manual/src). For small changes you can [use GitHub to edit these files](https://docs.github.com/en/repositories/working-with-files/managing-files/editing-files) -For larger changes see the [Nix reference manual](https://nixos.org/manual/nix/unstable/contributing/hacking.html). +For larger changes see the [Nix reference manual](https://nix.dev/manual/nix/development/development/contributing.html). ## Getting help Whenever you're stuck or do not know how to proceed, you can always ask for help. -The appropriate channels to do so can be found on the [NixOS Community](https://nixos.org/community/) page. +We invite you to use our [Matrix room](https://matrix.to/#/#nix-dev:nixos.org) to ask questions. diff --git a/HACKING.md b/HACKING.md new file mode 120000 index 000000000..d3576d60d --- /dev/null +++ b/HACKING.md @@ -0,0 +1 @@ +doc/manual/src/development/building.md \ No newline at end of file diff --git a/Makefile b/Makefile index ea0754fa5..dbf510a3e 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,7 @@ makefiles = \ src/libfetchers/local.mk \ src/libmain/local.mk \ src/libexpr/local.mk \ + src/libflake/local.mk \ src/libcmd/local.mk \ src/nix/local.mk \ src/libutil-c/local.mk \ @@ -45,13 +46,15 @@ makefiles += \ tests/unit/libstore-support/local.mk \ tests/unit/libfetchers/local.mk \ tests/unit/libexpr/local.mk \ - tests/unit/libexpr-support/local.mk + tests/unit/libexpr-support/local.mk \ + tests/unit/libflake/local.mk endif ifeq ($(ENABLE_FUNCTIONAL_TESTS), yes) ifdef HOST_UNIX makefiles += \ tests/functional/local.mk \ + tests/functional/flakes/local.mk \ tests/functional/ca/local.mk \ tests/functional/git-hashing/local.mk \ tests/functional/dyn-drv/local.mk \ @@ -68,14 +71,6 @@ ifeq ($(ENABLE_DOC_GEN), yes) makefiles-late += doc/manual/local.mk endif -ifeq ($(ENABLE_INTERNAL_API_DOCS), yes) -makefiles-late += doc/internal-api/local.mk -endif - -ifeq ($(ENABLE_EXTERNAL_API_DOCS), yes) -makefiles-late += doc/external-api/local.mk -endif - # Miscellaneous global Flags OPTIMIZE = 1 @@ -98,7 +93,7 @@ ifdef HOST_WINDOWS GLOBAL_LDFLAGS += -Wl,--export-all-symbols endif -GLOBAL_CXXFLAGS += -g -Wall -Wimplicit-fallthrough -include $(buildprefix)config.h -std=c++2a -I src +GLOBAL_CXXFLAGS += -g -Wall -Wdeprecated-copy -Wignored-qualifiers -Wimplicit-fallthrough -Werror=unused-result -Werror=suggest-override -include $(buildprefix)config.h -std=c++2a -I src # Include the main lib, causing rules to be defined @@ -131,17 +126,3 @@ manual-html manpages: @echo "Generated docs are disabled. Configure without '--disable-doc-gen', or avoid calling 'make manpages' and 'make manual-html'." @exit 1 endif - -ifneq ($(ENABLE_INTERNAL_API_DOCS), yes) -.PHONY: internal-api-html -internal-api-html: - @echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'." - @exit 1 -endif - -ifneq ($(ENABLE_EXTERNAL_API_DOCS), yes) -.PHONY: external-api-html -external-api-html: - @echo "External API docs are disabled. Configure with '--enable-external-api-docs', or avoid calling 'make external-api-html'." - @exit 1 -endif diff --git a/Makefile.config.in b/Makefile.config.in index 7f517898c..3100d2073 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -11,8 +11,6 @@ EDITLINE_LIBS = @EDITLINE_LIBS@ ENABLE_BUILD = @ENABLE_BUILD@ ENABLE_DOC_GEN = @ENABLE_DOC_GEN@ ENABLE_FUNCTIONAL_TESTS = @ENABLE_FUNCTIONAL_TESTS@ -ENABLE_INTERNAL_API_DOCS = @ENABLE_INTERNAL_API_DOCS@ -ENABLE_EXTERNAL_API_DOCS = @ENABLE_EXTERNAL_API_DOCS@ ENABLE_S3 = @ENABLE_S3@ ENABLE_UNIT_TESTS = @ENABLE_UNIT_TESTS@ GTEST_LIBS = @GTEST_LIBS@ diff --git a/README.md b/README.md index e1cace3b4..54a6fcc39 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,37 @@ # Nix [![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos) -[![Test](https://github.com/NixOS/nix/workflows/Test/badge.svg)](https://github.com/NixOS/nix/actions) +[![CI](https://github.com/NixOS/nix/workflows/CI/badge.svg)](https://github.com/NixOS/nix/actions/workflows/ci.yml) Nix is a powerful package manager for Linux and other Unix systems that makes package -management reliable and reproducible. Please refer to the [Nix manual](https://nixos.org/nix/manual) +management reliable and reproducible. Please refer to the [Nix manual](https://nix.dev/reference/nix-manual) for more details. ## Installation and first steps Visit [nix.dev](https://nix.dev) for [installation instructions](https://nix.dev/tutorials/install-nix) and [beginner tutorials](https://nix.dev/tutorials/first-steps). -Full reference documentation can be found in the [Nix manual](https://nixos.org/nix/manual). +Full reference documentation can be found in the [Nix manual](https://nix.dev/reference/nix-manual). -## Building And Developing +## Building and developing -See our [Hacking guide](https://nixos.org/manual/nix/unstable/contributing/hacking.html) in our manual for instruction on how to - set up a development environment and build Nix from source. +Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/development/building.html). ## Contributing Check the [contributing guide](./CONTRIBUTING.md) if you want to get involved with developing Nix. -## Additional Resources +## Additional resources -- [Nix manual](https://nixos.org/nix/manual) -- [Nix jobsets on hydra.nixos.org](https://hydra.nixos.org/project/nix) -- [NixOS Discourse](https://discourse.nixos.org/) -- [Matrix - #nix:nixos.org](https://matrix.to/#/#nix:nixos.org) +Nix was created by Eelco Dolstra and developed as the subject of his PhD thesis [The Purely Functional Software Deployment Model](https://edolstra.github.io/pubs/phd-thesis.pdf), published 2006. +Today, a world-wide developer community contributes to Nix and the ecosystem that has grown around it. + +- [The Nix, Nixpkgs, NixOS Community on nixos.org](https://nixos.org/) +- [Official documentation on nix.dev](https://nix.dev) +- [Nixpkgs](https://github.com/NixOS/nixpkgs) is [the largest, most up-to-date free software repository in the world](https://repology.org/repositories/graphs) +- [NixOS](https://github.com/NixOS/nixpkgs/tree/master/nixos) is a Linux distribution that can be configured fully declaratively +- [Discourse](https://discourse.nixos.org/) +- [Matrix](https://matrix.to/#/#nix:nixos.org) ## License diff --git a/build-utils-meson/deps-lists/meson.build b/build-utils-meson/deps-lists/meson.build new file mode 100644 index 000000000..237eac545 --- /dev/null +++ b/build-utils-meson/deps-lists/meson.build @@ -0,0 +1,36 @@ +# These are private dependencies with pkg-config files. What private +# means is that the dependencies are used by the library but they are +# *not* used (e.g. `#include`-ed) in any installed header file, and only +# in regular source code (`*.cc`) or private, uninstalled headers. They +# are thus part of the *implementation* of the library, but not its +# *interface*. +# +# See `man pkg-config` for some details. +deps_private = [ ] + +# These are public dependencies with pkg-config files. Public is the +# opposite of private: these dependencies are used in installed header +# files. They are part of the interface (and implementation) of the +# library. +# +# N.B. This concept is mostly unrelated to our own concept of a public +# (stable) API, for consumption outside of the Nix repository. +# `libnixutil` is an unstable C++ library, whose public interface is +# likewise unstable. `libutilc` conversely is a hopefully-soon stable +# C library, whose public interface --- including public but not private +# dependencies --- will also likewise soon be stable. +# +# N.B. For distributions that care about "ABI" stability and not just +# "API" stability, the private dependencies also matter as they can +# potentially affect the public ABI. +deps_public = [ ] + +# These are subproject deps (type == "internal"). They are other +# packages in `/src` in this repo. The private vs public distinction is +# the same as above. +deps_private_subproject = [ ] +deps_public_subproject = [ ] + +# These are dependencencies without pkg-config files. Ideally they are +# just private, but they may also be public (e.g. boost). +deps_other = [ ] diff --git a/build-utils-meson/diagnostics/meson.build b/build-utils-meson/diagnostics/meson.build new file mode 100644 index 000000000..30eedfc13 --- /dev/null +++ b/build-utils-meson/diagnostics/meson.build @@ -0,0 +1,11 @@ +add_project_arguments( + '-Wdeprecated-copy', + '-Werror=suggest-override', + '-Werror=switch', + '-Werror=switch-enum', + '-Werror=unused-result', + '-Wignored-qualifiers', + '-Wimplicit-fallthrough', + '-Wno-deprecated-declarations', + language : 'cpp', +) diff --git a/build-utils-meson/export-all-symbols/meson.build b/build-utils-meson/export-all-symbols/meson.build new file mode 100644 index 000000000..d7c086749 --- /dev/null +++ b/build-utils-meson/export-all-symbols/meson.build @@ -0,0 +1,11 @@ +if host_machine.system() == 'cygwin' or host_machine.system() == 'windows' + # Windows DLLs are stricter about symbol visibility than Unix shared + # objects --- see https://gcc.gnu.org/wiki/Visibility for details. + # This is a temporary sledgehammer to export everything like on Unix, + # and not detail with this yet. + # + # TODO do not do this, and instead do fine-grained export annotations. + linker_export_flags = ['-Wl,--export-all-symbols'] +else + linker_export_flags = [] +endif diff --git a/build-utils-meson/export/meson.build b/build-utils-meson/export/meson.build new file mode 100644 index 000000000..9f5950572 --- /dev/null +++ b/build-utils-meson/export/meson.build @@ -0,0 +1,33 @@ +requires_private = [] +foreach dep : deps_private_subproject + requires_private += dep.name() +endforeach +requires_private += deps_private + +requires_public = [] +foreach dep : deps_public_subproject + requires_public += dep.name() +endforeach +requires_public += deps_public + +extra_pkg_config_variables = get_variable('extra_pkg_config_variables', {}) +import('pkgconfig').generate( + this_library, + filebase : meson.project_name(), + name : 'Nix', + description : 'Nix Package Manager', + subdirs : ['nix'], + extra_cflags : ['-std=c++2a'], + requires : requires_public, + requires_private : requires_private, + libraries_private : libraries_private, + variables : extra_pkg_config_variables, +) + +meson.override_dependency(meson.project_name(), declare_dependency( + include_directories : include_dirs, + link_with : this_library, + compile_args : ['-std=c++2a'], + dependencies : deps_public_subproject + deps_public, + variables : extra_pkg_config_variables, +)) diff --git a/build-utils-meson/generate-header/meson.build b/build-utils-meson/generate-header/meson.build new file mode 100644 index 000000000..dfbe1375f --- /dev/null +++ b/build-utils-meson/generate-header/meson.build @@ -0,0 +1,7 @@ +bash = find_program('bash', native: true) + +gen_header = generator( + bash, + arguments : [ '-c', '{ echo \'R"__NIX_STR(\' && cat @INPUT@ && echo \')__NIX_STR"\'; } > "$1"', '_ignored_argv0', '@OUTPUT@' ], + output : '@PLAINNAME@.gen.hh', +) diff --git a/build-utils-meson/subprojects/meson.build b/build-utils-meson/subprojects/meson.build new file mode 100644 index 000000000..30a54ed91 --- /dev/null +++ b/build-utils-meson/subprojects/meson.build @@ -0,0 +1,19 @@ +foreach maybe_subproject_dep : deps_private_maybe_subproject + if maybe_subproject_dep.type_name() == 'internal' + deps_private_subproject += maybe_subproject_dep + # subproject sadly no good for pkg-config module + deps_other += maybe_subproject_dep + else + deps_private += maybe_subproject_dep + endif +endforeach + +foreach maybe_subproject_dep : deps_public_maybe_subproject + if maybe_subproject_dep.type_name() == 'internal' + deps_public_subproject += maybe_subproject_dep + # subproject sadly no good for pkg-config module + deps_other += maybe_subproject_dep + else + deps_public += maybe_subproject_dep + endif +endforeach diff --git a/build-utils-meson/threads/meson.build b/build-utils-meson/threads/meson.build new file mode 100644 index 000000000..294160de1 --- /dev/null +++ b/build-utils-meson/threads/meson.build @@ -0,0 +1,6 @@ +# This is only conditional to work around +# https://github.com/mesonbuild/meson/issues/13293. It should be +# unconditional. +if not (host_machine.system() == 'windows' and cxx.get_id() == 'gcc') + deps_private += dependency('threads') +endif diff --git a/configure.ac b/configure.ac index b2a5794b5..198198dea 100644 --- a/configure.ac +++ b/configure.ac @@ -86,7 +86,7 @@ static char buf[1024];]], AC_LANG_POP(C++) -AC_CHECK_FUNCS([statvfs pipe2]) +AC_CHECK_FUNCS([statvfs pipe2 close_range]) # Check for lutimes, optionally used for changing the mtime of @@ -149,11 +149,6 @@ AC_ARG_ENABLE(unit-tests, AS_HELP_STRING([--disable-unit-tests],[Do not build th ENABLE_UNIT_TESTS=$enableval, ENABLE_UNIT_TESTS=$ENABLE_BUILD) AC_SUBST(ENABLE_UNIT_TESTS) -# Build external API docs by default -AC_ARG_ENABLE(external_api_docs, AS_HELP_STRING([--enable-external-api-docs],[Build API docs for Nix's C interface]), - external_api_docs=$enableval, external_api_docs=yes) -AC_SUBST(external_api_docs) - AS_IF( [test "$ENABLE_BUILD" == "no" && test "$ENABLE_UNIT_TESTS" == "yes"], [AC_MSG_ERROR([Cannot enable unit tests when building overall is disabled. Please do not pass '--enable-unit-tests' or do not pass '--disable-build'.])]) @@ -171,15 +166,6 @@ AS_IF( [test "$ENABLE_BUILD" == "no" && test "$ENABLE_DOC_GEN" == "yes"], [AC_MSG_ERROR([Cannot enable generated docs when building overall is disabled. Please do not pass '--enable-doc-gen' or do not pass '--disable-build'.])]) -# Building without API docs is the default as Nix' C++ interfaces are internal and unstable. -AC_ARG_ENABLE(internal-api-docs, AS_HELP_STRING([--enable-internal-api-docs],[Build API docs for Nix's internal unstable C++ interfaces]), - ENABLE_INTERNAL_API_DOCS=$enableval, ENABLE_INTERNAL_API_DOCS=no) -AC_SUBST(ENABLE_INTERNAL_API_DOCS) - -AC_ARG_ENABLE(external-api-docs, AS_HELP_STRING([--enable-external-api-docs],[Build API docs for Nix's external unstable C interfaces]), - ENABLE_EXTERNAL_API_DOCS=$enableval, ENABLE_EXTERNAL_API_DOCS=no) -AC_SUBST(ENABLE_EXTERNAL_API_DOCS) - AS_IF( [test "$ENABLE_FUNCTIONAL_TESTS" == "yes" || test "$ENABLE_DOC_GEN" == "yes"], [NEED_PROG(jq, jq)]) @@ -313,7 +299,7 @@ case "$host_os" in ])) if test "x$enable_seccomp_sandboxing" != "xno"; then PKG_CHECK_MODULES([LIBSECCOMP], [libseccomp], - [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS"]) + [CXXFLAGS="$LIBSECCOMP_CFLAGS $CXXFLAGS" CFLAGS="$LIBSECCOMP_CFLAGS $CFLAGS"]) have_seccomp=1 AC_DEFINE([HAVE_SECCOMP], [1], [Whether seccomp is available and should be used for sandboxing.]) AC_COMPILE_IFELSE([ @@ -354,13 +340,6 @@ AC_CHECK_HEADERS([aws/s3/S3Client.h], AC_SUBST(ENABLE_S3, [$enable_s3]) AC_LANG_POP(C++) -if test -n "$enable_s3"; then - declare -a aws_version_tokens=($(printf '#include \nAWS_SDK_VERSION_STRING' | $CPP $CPPFLAGS - | grep -v '^#.*' | sed 's/"//g' | tr '.' ' ')) - AC_DEFINE_UNQUOTED([AWS_VERSION_MAJOR], ${aws_version_tokens@<:@0@:>@}, [Major version of aws-sdk-cpp.]) - AC_DEFINE_UNQUOTED([AWS_VERSION_MINOR], ${aws_version_tokens@<:@1@:>@}, [Minor version of aws-sdk-cpp.]) - AC_DEFINE_UNQUOTED([AWS_VERSION_PATCH], ${aws_version_tokens@<:@2@:>@}, [Patch version of aws-sdk-cpp.]) -fi - # Whether to use the Boehm garbage collector. AC_ARG_ENABLE(gc, AS_HELP_STRING([--enable-gc],[enable garbage collection in the Nix expression evaluator (requires Boehm GC) [default=yes]]), @@ -369,6 +348,14 @@ if test "$gc" = yes; then PKG_CHECK_MODULES([BDW_GC], [bdw-gc]) CXXFLAGS="$BDW_GC_CFLAGS $CXXFLAGS" AC_DEFINE(HAVE_BOEHMGC, 1, [Whether to use the Boehm garbage collector.]) + + # See `fixupBoehmStackPointer`, for the integration between Boehm GC + # and Boost coroutines. + old_CFLAGS="$CFLAGS" + # Temporary set `-pthread` just for the next check + CFLAGS="$CFLAGS -pthread" + AC_CHECK_FUNCS([pthread_attr_get_np pthread_getattr_np]) + CFLAGS="$old_CFLAGS" fi AS_IF([test "$ENABLE_UNIT_TESTS" == "yes"],[ @@ -406,6 +393,11 @@ AS_CASE(["$enable_markdown"], PKG_CHECK_MODULES([LIBGIT2], [libgit2]) +# Look for toml11, a required dependency. +AC_LANG_PUSH(C++) +AC_CHECK_HEADER([toml.hpp], [], [AC_MSG_ERROR([toml11 is not found.])]) +AC_LANG_POP(C++) + # Setuid installations. AC_CHECK_FUNCS([setresuid setreuid lchown]) diff --git a/dep-patches/boehmgc-coroutine-sp-fallback.diff b/dep-patches/boehmgc-coroutine-sp-fallback.diff deleted file mode 100644 index 2afbe9671..000000000 --- a/dep-patches/boehmgc-coroutine-sp-fallback.diff +++ /dev/null @@ -1,99 +0,0 @@ -diff --git a/darwin_stop_world.c b/darwin_stop_world.c -index 0468aaec..b348d869 100644 ---- a/darwin_stop_world.c -+++ b/darwin_stop_world.c -@@ -356,6 +356,7 @@ GC_INNER void GC_push_all_stacks(void) - int nthreads = 0; - word total_size = 0; - mach_msg_type_number_t listcount = (mach_msg_type_number_t)THREAD_TABLE_SZ; -+ size_t stack_limit; - if (!EXPECT(GC_thr_initialized, TRUE)) - GC_thr_init(); - -@@ -411,6 +412,19 @@ GC_INNER void GC_push_all_stacks(void) - GC_push_all_stack_sections(lo, hi, p->traced_stack_sect); - } - if (altstack_lo) { -+ // When a thread goes into a coroutine, we lose its original sp until -+ // control flow returns to the thread. -+ // While in the coroutine, the sp points outside the thread stack, -+ // so we can detect this and push the entire thread stack instead, -+ // as an approximation. -+ // We assume that the coroutine has similarly added its entire stack. -+ // This could be made accurate by cooperating with the application -+ // via new functions and/or callbacks. -+ stack_limit = pthread_get_stacksize_np(p->id); -+ if (altstack_lo >= altstack_hi || altstack_lo < altstack_hi - stack_limit) { // sp outside stack -+ altstack_lo = altstack_hi - stack_limit; -+ } -+ - total_size += altstack_hi - altstack_lo; - GC_push_all_stack(altstack_lo, altstack_hi); - } -diff --git a/include/gc.h b/include/gc.h -index edab6c22..f2c61282 100644 ---- a/include/gc.h -+++ b/include/gc.h -@@ -2172,6 +2172,11 @@ GC_API void GC_CALL GC_win32_free_heap(void); - (*GC_amiga_allocwrapper_do)(a,GC_malloc_atomic_ignore_off_page) - #endif /* _AMIGA && !GC_AMIGA_MAKINGLIB */ - -+#if !__APPLE__ -+/* Patch doesn't work on apple */ -+#define NIX_BOEHM_PATCH_VERSION 1 -+#endif -+ - #ifdef __cplusplus - } /* extern "C" */ - #endif -diff --git a/pthread_stop_world.c b/pthread_stop_world.c -index b5d71e62..aed7b0bf 100644 ---- a/pthread_stop_world.c -+++ b/pthread_stop_world.c -@@ -768,6 +768,8 @@ STATIC void GC_restart_handler(int sig) - /* world is stopped. Should not fail if it isn't. */ - GC_INNER void GC_push_all_stacks(void) - { -+ size_t stack_limit; -+ pthread_attr_t pattr; - GC_bool found_me = FALSE; - size_t nthreads = 0; - int i; -@@ -851,6 +853,37 @@ GC_INNER void GC_push_all_stacks(void) - hi = p->altstack + p->altstack_size; - /* FIXME: Need to scan the normal stack too, but how ? */ - /* FIXME: Assume stack grows down */ -+ } else { -+#ifdef HAVE_PTHREAD_ATTR_GET_NP -+ if (!pthread_attr_init(&pattr) -+ || !pthread_attr_get_np(p->id, &pattr)) -+#else /* HAVE_PTHREAD_GETATTR_NP */ -+ if (pthread_getattr_np(p->id, &pattr)) -+#endif -+ { -+ ABORT("GC_push_all_stacks: pthread_getattr_np failed!"); -+ } -+ if (pthread_attr_getstacksize(&pattr, &stack_limit)) { -+ ABORT("GC_push_all_stacks: pthread_attr_getstacksize failed!"); -+ } -+ if (pthread_attr_destroy(&pattr)) { -+ ABORT("GC_push_all_stacks: pthread_attr_destroy failed!"); -+ } -+ // When a thread goes into a coroutine, we lose its original sp until -+ // control flow returns to the thread. -+ // While in the coroutine, the sp points outside the thread stack, -+ // so we can detect this and push the entire thread stack instead, -+ // as an approximation. -+ // We assume that the coroutine has similarly added its entire stack. -+ // This could be made accurate by cooperating with the application -+ // via new functions and/or callbacks. -+ #ifndef STACK_GROWS_UP -+ if (lo >= hi || lo < hi - stack_limit) { // sp outside stack -+ lo = hi - stack_limit; -+ } -+ #else -+ #error "STACK_GROWS_UP not supported in boost_coroutine2 (as of june 2021), so we don't support it in Nix." -+ #endif - } - GC_push_all_stack_sections(lo, hi, traced_stack_sect); - # ifdef STACK_GROWS_UP diff --git a/dep-patches/boehmgc-traceable_allocator-public.diff b/dep-patches/boehmgc-traceable_allocator-public.diff deleted file mode 100644 index 903c707a6..000000000 --- a/dep-patches/boehmgc-traceable_allocator-public.diff +++ /dev/null @@ -1,12 +0,0 @@ -diff --git a/include/gc_allocator.h b/include/gc_allocator.h -index 597c7f13..587286be 100644 ---- a/include/gc_allocator.h -+++ b/include/gc_allocator.h -@@ -312,6 +312,7 @@ public: - - template<> - class traceable_allocator { -+public: - typedef size_t size_type; - typedef ptrdiff_t difference_type; - typedef void* pointer; diff --git a/doc/external-api/local.mk b/doc/external-api/local.mk deleted file mode 100644 index ae2b44db8..000000000 --- a/doc/external-api/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -$(docdir)/external-api/html/index.html $(docdir)/external-api/latex: $(d)/doxygen.cfg src/lib*-c/*.h - mkdir -p $(docdir)/external-api - { cat $< ; echo "OUTPUT_DIRECTORY=$(docdir)/external-api" ; } | doxygen - - -# Generate the HTML API docs for Nix's unstable C bindings -.PHONY: external-api-html -external-api-html: $(docdir)/external-api/html/index.html diff --git a/doc/internal-api/local.mk b/doc/internal-api/local.mk deleted file mode 100644 index be9b7bb55..000000000 --- a/doc/internal-api/local.mk +++ /dev/null @@ -1,7 +0,0 @@ -$(docdir)/internal-api/html/index.html $(docdir)/internal-api/latex: $(d)/doxygen.cfg src/**/*.hh - mkdir -p $(docdir)/internal-api - { cat $< ; echo "OUTPUT_DIRECTORY=$(docdir)/internal-api" ; } | doxygen - - -# Generate the HTML API docs for Nix's unstable internal interfaces. -.PHONY: internal-api-html -internal-api-html: $(docdir)/internal-api/html/index.html diff --git a/doc/manual/.version b/doc/manual/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/doc/manual/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/doc/manual/book.toml b/doc/manual/book.toml index 73fb7e75e..acae7aec7 100644 --- a/doc/manual/book.toml +++ b/doc/manual/book.toml @@ -7,9 +7,21 @@ additional-js = ["redirects.js"] edit-url-template = "https://github.com/NixOS/nix/tree/master/doc/manual/{path}" git-repository-url = "https://github.com/NixOS/nix" +# Handles replacing @docroot@ with a path to ./src relative to that markdown file, +# {{#include handlebars}}, and the @generated@ syntax used within these. it mostly +# but not entirely replaces the links preprocessor (which we cannot simply use due +# to @generated@ files living in a different directory to make meson happy). we do +# not want to disable the links preprocessor entirely though because that requires +# disabling *all* built-in preprocessors and selectively reenabling those we want. +[preprocessor.substitute] +command = "python3 ./substitute.py" +before = ["anchors", "links"] + [preprocessor.anchors] renderers = ["html"] -command = "jq --from-file doc/manual/anchors.jq" +command = "jq --from-file ./anchors.jq" + +[output.markdown] [output.linkcheck] # no Internet during the build (in the sandbox) diff --git a/doc/manual/custom.css b/doc/manual/custom.css index 9e8e3886f..7af150be3 100644 --- a/doc/manual/custom.css +++ b/doc/manual/custom.css @@ -12,8 +12,8 @@ h1.menu-title::before { } -h1.menu-title { - padding: 0.5em; +.menu-bar { + padding: 0.5em 0em; } .sidebar .sidebar-scrollbox { diff --git a/doc/manual/generate-builtin-constants.nix b/doc/manual/generate-builtin-constants.nix deleted file mode 100644 index cccd1e279..000000000 --- a/doc/manual/generate-builtin-constants.nix +++ /dev/null @@ -1,31 +0,0 @@ -let - inherit (builtins) concatStringsSep attrValues mapAttrs; - inherit (import ) optionalString squash; -in - -builtinsInfo: -let - showBuiltin = name: { doc, type, impure-only }: - let - type' = optionalString (type != null) " (${type})"; - - impureNotice = optionalString impure-only '' - > **Note** - > - > Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). - ''; - in - squash '' -
- ${name}${type'} -
-
- - ${doc} - - ${impureNotice} - -
- ''; -in -concatStringsSep "\n" (attrValues (mapAttrs showBuiltin builtinsInfo)) diff --git a/doc/manual/generate-builtins.nix b/doc/manual/generate-builtins.nix index 007b698f1..37ed12a43 100644 --- a/doc/manual/generate-builtins.nix +++ b/doc/manual/generate-builtins.nix @@ -5,12 +5,14 @@ in builtinsInfo: let - showBuiltin = name: { doc, args, arity, experimental-feature }: + showBuiltin = name: { doc, type ? null, args ? [ ], experimental-feature ? null, impure-only ? false }: let + type' = optionalString (type != null) " (${type})"; + experimentalNotice = optionalString (experimental-feature != null) '' > **Note** > - > This function is only available if the [`${experimental-feature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimental-feature}) is enabled. + > This function is only available if the [`${experimental-feature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimental-feature}) is enabled. > > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > @@ -18,18 +20,26 @@ let > extra-experimental-features = ${experimental-feature} > ``` ''; + + impureNotice = optionalString impure-only '' + > **Note** + > + > Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). + ''; in squash ''
- ${name} ${listArgs args} + ${name}${listArgs args}${type'}
${experimentalNotice} ${doc} + + ${impureNotice}
''; - listArgs = args: concatStringsSep " " (map (s: "${s}") args); + listArgs = args: concatStringsSep "" (map (s: " ${s}") args); in concatStringsSep "\n" (attrValues (mapAttrs showBuiltin builtinsInfo)) diff --git a/doc/manual/generate-deps.py b/doc/manual/generate-deps.py new file mode 100755 index 000000000..297bd3939 --- /dev/null +++ b/doc/manual/generate-deps.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python3 + +import glob +import sys + +# meson expects makefile-style dependency declarations, i.e. +# +# target: dependency... +# +# meson seems to pass depfiles straight on to ninja even though +# it also parses the file itself (or at least has code to do so +# in its tree), so we must live by ninja's rules: only slashes, +# spaces and octothorpes can be escaped, anything else is taken +# literally. since the rules for these aren't even the same for +# all three we will just fail when we encounter any of them (if +# asserts are off for some reason the depfile will likely point +# to nonexistant paths, making everything phony and thus fine.) +for path in glob.glob(sys.argv[1] + '/**', recursive=True): + assert '\\' not in path + assert ' ' not in path + assert '#' not in path + print("ignored:", path) diff --git a/doc/manual/generate-manpage.nix b/doc/manual/generate-manpage.nix index ba5667a43..791bfd2c7 100644 --- a/doc/manual/generate-manpage.nix +++ b/doc/manual/generate-manpage.nix @@ -38,7 +38,7 @@ let result = '' > **Warning** \ > This program is - > [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) + > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and its interface is subject to change. # Name @@ -116,9 +116,12 @@ let storeInfo = commandInfo.stores; inherit inlineHTML; }; + hasInfix = infix: content: + builtins.stringLength content != builtins.stringLength (replaceStrings [ infix ] [ "" ] content); in optionalString (details ? doc) ( - if match ".*@store-types@.*" details.doc != null + # An alternate implementation with builtins.match stack overflowed on some systems. + if hasInfix "@store-types@" details.doc then help-stores else details.doc ); diff --git a/doc/manual/generate-settings.nix b/doc/manual/generate-settings.nix index 504cda362..93a8e093e 100644 --- a/doc/manual/generate-settings.nix +++ b/doc/manual/generate-settings.nix @@ -33,10 +33,10 @@ let > **Warning** > > This setting is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). + > [experimental feature](@docroot@/development/experimental-features.md). > > To change this setting, make sure the - > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > [`${experimentalFeature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimentalFeature}) > is enabled. > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > diff --git a/doc/manual/generate-store-info.nix b/doc/manual/generate-store-info.nix index c311c3c39..cc3704124 100644 --- a/doc/manual/generate-store-info.nix +++ b/doc/manual/generate-store-info.nix @@ -32,10 +32,10 @@ let > **Warning** > > This store is part of an - > [experimental feature](@docroot@/contributing/experimental-features.md). + > [experimental feature](@docroot@/development/experimental-features.md). > > To use this store, make sure the - > [`${experimentalFeature}` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-${experimentalFeature}) + > [`${experimentalFeature}` experimental feature](@docroot@/development/experimental-features.md#xp-feature-${experimentalFeature}) > is enabled. > For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): > diff --git a/doc/manual/generate-xp-features-shortlist.nix b/doc/manual/generate-xp-features-shortlist.nix index ec09f4b75..eb735ba5f 100644 --- a/doc/manual/generate-xp-features-shortlist.nix +++ b/doc/manual/generate-xp-features-shortlist.nix @@ -4,6 +4,6 @@ with import ; let showExperimentalFeature = name: doc: '' - - [`${name}`](@docroot@/contributing/experimental-features.md#xp-feature-${name}) + - [`${name}`](@docroot@/development/experimental-features.md#xp-feature-${name}) ''; in xps: indent " " (concatStrings (attrValues (mapAttrs showExperimentalFeature xps))) diff --git a/doc/manual/local.mk b/doc/manual/local.mk index 71ad5c8e6..3c777efc3 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -95,7 +95,7 @@ $(d)/nix-profiles.5: $(d)/src/command-ref/files/profiles.md $(trace-gen) lowdown -sT man --nroff-nolinks -M section=5 $^.tmp -o $@ @rm $^.tmp -$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md +$(d)/src/SUMMARY.md: $(d)/src/SUMMARY.md.in $(d)/src/SUMMARY-rl-next.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md @cp $< $@ @$(call process-includes,$@,$@) @@ -124,7 +124,7 @@ $(d)/conf-file.json: $(doc_nix) $(trace-gen) $(dummy-env) $(doc_nix) config show --json --experimental-features nix-command > $@.tmp @mv $@.tmp $@ -$(d)/src/contributing/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) +$(d)/src/development/experimental-feature-descriptions.md: $(d)/xp-features.json $(d)/utils.nix $(d)/generate-xp-features.nix $(doc_nix) @rm -rf $@ $@.tmp $(trace-gen) $(nix-eval) --write-to $@.tmp --expr 'import doc/manual/generate-xp-features.nix (builtins.fromJSON (builtins.readFile $<))' @mv $@.tmp $@ @@ -140,16 +140,10 @@ $(d)/xp-features.json: $(doc_nix) $(d)/src/language/builtins.md: $(d)/language.json $(d)/generate-builtins.nix $(d)/src/language/builtins-prefix.md $(doc_nix) @cat doc/manual/src/language/builtins-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<)).builtins' >> $@.tmp; + $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtins.nix (builtins.fromJSON (builtins.readFile $<))' >> $@.tmp; @cat doc/manual/src/language/builtins-suffix.md >> $@.tmp @mv $@.tmp $@ -$(d)/src/language/builtin-constants.md: $(d)/language.json $(d)/generate-builtin-constants.nix $(d)/src/language/builtin-constants-prefix.md $(doc_nix) - @cat doc/manual/src/language/builtin-constants-prefix.md > $@.tmp - $(trace-gen) $(nix-eval) --expr 'import doc/manual/generate-builtin-constants.nix (builtins.fromJSON (builtins.readFile $<)).constants' >> $@.tmp; - @cat doc/manual/src/language/builtin-constants-suffix.md >> $@.tmp - @mv $@.tmp $@ - $(d)/language.json: $(doc_nix) $(trace-gen) $(dummy-env) $(doc_nix) __dump-language > $@.tmp @mv $@.tmp $@ @@ -213,11 +207,11 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli done @touch $@ -# the `! -name 'contributing.md'` filter excludes the one place where +# the `! -name 'documentation.md'` filter excludes the one place where # `@docroot@` is to be preserved for documenting the mechanism # FIXME: maybe contributing guides should live right next to the code # instead of in the manual -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/development/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ @@ -229,8 +223,13 @@ $(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/ sed -i "s,@docroot@,$$docroot,g" "$$file"; \ done; \ set -euo pipefail; \ - RUST_LOG=warn mdbook build "$$tmp/manual" -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \ - | { grep -Fv "because fragment resolution isn't implemented" || :; }; \ + ( \ + cd "$$tmp/manual"; \ + RUST_LOG=warn \ + MDBOOK_SUBSTITUTE_SEARCH=$(d)/src \ + mdbook build -d $(DESTDIR)$(docdir)/manual.tmp 2>&1 \ + | { grep -Fv "because fragment resolution isn't implemented" || :; } \ + ); \ rm -rf "$$tmp/manual" @rm -rf $(DESTDIR)$(docdir)/manual @mv $(DESTDIR)$(docdir)/manual.tmp/html $(DESTDIR)$(docdir)/manual diff --git a/doc/manual/meson.build b/doc/manual/meson.build new file mode 100644 index 000000000..31d1814d7 --- /dev/null +++ b/doc/manual/meson.build @@ -0,0 +1,353 @@ +project('nix-manual', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +nix = find_program('nix', native : true) + +mdbook = find_program('mdbook', native : true) +bash = find_program('bash', native : true) + +pymod = import('python') +python = pymod.find_installation('python3') + +nix_env_for_docs = { + 'HOME': '/dummy', + 'NIX_CONF_DIR': '/dummy', + 'NIX_SSL_CERT_FILE': '/dummy/no-ca-bundle.crt', + 'NIX_STATE_DIR': '/dummy', + 'NIX_CONFIG': 'cores = 0', +} + +nix_for_docs = [nix, '--experimental-features', 'nix-command'] +nix_eval_for_docs_common = nix_for_docs + [ + 'eval', + '-I', 'nix=' + meson.current_source_dir(), + '--store', 'dummy://', + '--impure', +] +nix_eval_for_docs = nix_eval_for_docs_common + '--raw' + +conf_file_json = custom_target( + command : nix_for_docs + ['config', 'show', '--json'], + capture : true, + output : 'conf-file.json', + env : nix_env_for_docs, +) + +language_json = custom_target( + command: [nix, '__dump-language'], + output : 'language.json', + capture : true, + env : nix_env_for_docs, +) + +nix3_cli_json = custom_target( + command : [nix, '__dump-cli'], + capture : true, + output : 'nix.json', + env : nix_env_for_docs, +) + +generate_manual_deps = files( + 'generate-deps.py', +) + +# Generates types +subdir('src/store') +# Generates builtins.md and builtin-constants.md. +subdir('src/language') +# Generates new-cli pages, experimental-features-shortlist.md, and conf-file.md. +subdir('src/command-ref') +# Generates experimental-feature-descriptions.md. +subdir('src/development') +# Generates rl-next-generated.md. +subdir('src/release-notes') +subdir('src') + +# Hacky way to figure out if `nix` is an `ExternalProgram` or +# `Exectuable`. Only the latter can occur in custom target input lists. +if nix.full_path().startswith(meson.build_root()) + nix_input = nix +else + nix_input = [] +endif + +manual = custom_target( + 'manual', + command : [ + bash, + '-euo', 'pipefail', + '-c', + ''' + @0@ @INPUT0@ @CURRENT_SOURCE_DIR@ > @DEPFILE@ + @0@ @INPUT1@ summary @2@ < @CURRENT_SOURCE_DIR@/src/SUMMARY.md.in > @2@/src/SUMMARY.md + rsync -r --include='*.md' @CURRENT_SOURCE_DIR@/ @2@/ + (cd @2@; RUST_LOG=warn @1@ build -d @2@ 3>&2 2>&1 1>&3) | { grep -Fv "because fragment resolution isn't implemented" || :; } 3>&2 2>&1 1>&3 + rm -rf @2@/manual + mv @2@/html @2@/manual + find @2@/manual -iname meson.build -delete + '''.format( + python.full_path(), + mdbook.full_path(), + meson.current_build_dir(), + ), + ], + input : [ + generate_manual_deps, + 'substitute.py', + 'book.toml', + 'anchors.jq', + 'custom.css', + nix3_cli_files, + experimental_features_shortlist_md, + experimental_feature_descriptions_md, + types_dir, + conf_file_md, + builtins_md, + rl_next_generated, + summary_rl_next, + nix_input, + ], + output : [ + 'manual', + 'markdown', + ], + depfile : 'manual.d', + env : { + 'RUST_LOG': 'info', + 'MDBOOK_SUBSTITUTE_SEARCH': meson.current_build_dir() / 'src', + }, +) +manual_html = manual[0] +manual_md = manual[1] + +install_subdir( + manual_html.full_path(), + install_dir : get_option('datadir') / 'doc/nix', +) + +nix_nested_manpages = [ + [ 'nix-env', + [ + 'delete-generations', + 'install', + 'list-generations', + 'query', + 'rollback', + 'set-flag', + 'set', + 'switch-generation', + 'switch-profile', + 'uninstall', + 'upgrade', + ], + ], + [ 'nix-store', + [ + 'add-fixed', + 'add', + 'delete', + 'dump-db', + 'dump', + 'export', + 'gc', + 'generate-binary-cache-key', + 'import', + 'load-db', + 'optimise', + 'print-env', + 'query', + 'read-log', + 'realise', + 'repair-path', + 'restore', + 'serve', + 'verify', + 'verify-path', + ], + ], +] + +foreach command : nix_nested_manpages + foreach page : command[1] + title = command[0] + ' --' + page + section = '1' + custom_target( + command : [ + bash, + files('./render-manpage.sh'), + '--out-no-smarty', + title, + section, + '@INPUT0@/command-ref' / command[0] / (page + '.md'), + '@OUTPUT0@', + ], + input : [ + manual_md, + nix_input, + ], + output : command[0] + '-' + page + '.1', + install : true, + install_dir : get_option('mandir') / 'man1', + ) + endforeach +endforeach + +nix3_manpages = [ + 'nix3-build', + 'nix3-bundle', + 'nix3-config', + 'nix3-config-show', + 'nix3-copy', + 'nix3-daemon', + 'nix3-derivation-add', + 'nix3-derivation', + 'nix3-derivation-show', + 'nix3-develop', + #'nix3-doctor', + 'nix3-edit', + 'nix3-eval', + 'nix3-flake-archive', + 'nix3-flake-check', + 'nix3-flake-clone', + 'nix3-flake-info', + 'nix3-flake-init', + 'nix3-flake-lock', + 'nix3-flake', + 'nix3-flake-metadata', + 'nix3-flake-new', + 'nix3-flake-prefetch', + 'nix3-flake-show', + 'nix3-flake-update', + 'nix3-fmt', + 'nix3-hash-file', + 'nix3-hash', + 'nix3-hash-path', + 'nix3-hash-to-base16', + 'nix3-hash-to-base32', + 'nix3-hash-to-base64', + 'nix3-hash-to-sri', + 'nix3-help', + 'nix3-help-stores', + 'nix3-key-convert-secret-to-public', + 'nix3-key-generate-secret', + 'nix3-key', + 'nix3-log', + 'nix3-nar-cat', + 'nix3-nar-dump-path', + 'nix3-nar-ls', + 'nix3-nar', + 'nix3-path-info', + 'nix3-print-dev-env', + 'nix3-profile-diff-closures', + 'nix3-profile-history', + 'nix3-profile-install', + 'nix3-profile-list', + 'nix3-profile', + 'nix3-profile-remove', + 'nix3-profile-rollback', + 'nix3-profile-upgrade', + 'nix3-profile-wipe-history', + 'nix3-realisation-info', + 'nix3-realisation', + 'nix3-registry-add', + 'nix3-registry-list', + 'nix3-registry', + 'nix3-registry-pin', + 'nix3-registry-remove', + 'nix3-repl', + 'nix3-run', + 'nix3-search', + #'nix3-shell', + 'nix3-store-add-file', + 'nix3-store-add-path', + 'nix3-store-cat', + 'nix3-store-copy-log', + 'nix3-store-copy-sigs', + 'nix3-store-delete', + 'nix3-store-diff-closures', + 'nix3-store-dump-path', + 'nix3-store-gc', + 'nix3-store-ls', + 'nix3-store-make-content-addressed', + 'nix3-store', + 'nix3-store-optimise', + 'nix3-store-path-from-hash-part', + 'nix3-store-ping', + 'nix3-store-prefetch-file', + 'nix3-store-repair', + 'nix3-store-sign', + 'nix3-store-verify', + 'nix3-upgrade-nix', + 'nix3-why-depends', + 'nix', +] + +foreach page : nix3_manpages + section = '1' + custom_target( + command : [ + bash, + '@INPUT0@', + page, + section, + '@INPUT1@/command-ref/new-cli/@0@.md'.format(page), + '@OUTPUT@', + ], + input : [ + files('./render-manpage.sh'), + manual_md, + nix_input, + ], + output : page + '.1', + install : true, + install_dir : get_option('mandir') / 'man1', + ) +endforeach + +nix_manpages = [ + [ 'nix-env', 1 ], + [ 'nix-store', 1 ], + [ 'nix-build', 1 ], + [ 'nix-shell', 1 ], + [ 'nix-instantiate', 1 ], + [ 'nix-collect-garbage', 1 ], + [ 'nix-prefetch-url', 1 ], + [ 'nix-channel', 1 ], + [ 'nix-hash', 1 ], + [ 'nix-copy-closure', 1 ], + [ 'nix.conf', 5, conf_file_md.full_path() ], + [ 'nix-daemon', 8 ], + [ 'nix-profiles', 5, 'files/profiles.md' ], +] + +foreach entry : nix_manpages + title = entry[0] + # nix.conf.5 and nix-profiles.5 are based off of conf-file.md and files/profiles.md, + # rather than a stem identical to its mdbook source. + # Therefore we use an optional third element of this array to override the name pattern + md_file = entry.get(2, title + '.md') + section = entry[1].to_string() + md_file_resolved = join_paths('@INPUT1@/command-ref/', md_file) + custom_target( + command : [ + bash, + '@INPUT0@', + title, + section, + md_file_resolved, + '@OUTPUT@', + ], + input : [ + files('./render-manpage.sh'), + manual_md, + entry.get(3, []), + nix_input, + ], + output : '@0@.@1@'.format(entry[0], entry[1]), + install : true, + install_dir : get_option('mandir') / 'man@0@'.format(entry[1]), + ) +endforeach diff --git a/doc/manual/package.nix b/doc/manual/package.nix new file mode 100644 index 000000000..2e6fcede3 --- /dev/null +++ b/doc/manual/package.nix @@ -0,0 +1,71 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, lowdown +, mdbook +, mdbook-linkcheck +, jq +, python3 +, rsync +, nix-cli + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-manual"; + inherit version; + + workDir = ./.; + fileset = fileset.difference + (fileset.unions [ + ../../.version + # Too many different types of files to filter for now + ../../doc/manual + ./. + ]) + # Do a blacklist instead + ../../doc/manual/package.nix; + + # TODO the man pages should probably be separate + outputs = [ "out" "man" ]; + + # Hack for sake of the dev shell + passthru.externalNativeBuildInputs = [ + meson + ninja + (lib.getBin lowdown) + mdbook + mdbook-linkcheck + jq + python3 + rsync + ]; + + nativeBuildInputs = finalAttrs.passthru.externalNativeBuildInputs ++ [ + nix-cli + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''$out/nix-support + echo "doc manual ''$out/share/doc/nix/manual" >> ''$out/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index ec5645ea7..cb8cd18fa 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -1,7 +1,7 @@ // redirect rules for URL fragments (client-side) to prevent link rot. // this must be done on the client side, as web servers do not see the fragment part of the URL. // it will only work with JavaScript enabled in the browser, but this is the best we can do here. -// see ./_redirects for path redirects (client-side) +// see src/_redirects for path redirects (server-side) // redirects are declared as follows: // each entry has as its key a path matching the requested URL path, relative to the mdBook document root. @@ -143,7 +143,7 @@ const redirects = { "opt-timeout": "command-ref/opt-common.html#opt-timeout", "sec-common-options": "command-ref/opt-common.html", "ch-utilities": "command-ref/utilities.html", - "chap-hacking": "contributing/hacking.html", + "chap-hacking": "development/building.html", "adv-attr-allowSubstitutes": "language/advanced-attributes.html#adv-attr-allowSubstitutes", "adv-attr-allowedReferences": "language/advanced-attributes.html#adv-attr-allowedReferences", "adv-attr-allowedRequisites": "language/advanced-attributes.html#adv-attr-allowedRequisites", @@ -238,12 +238,12 @@ const redirects = { "attr-system": "language/derivations.html#attr-system", "ssec-derivation": "language/derivations.html", "ch-expression-language": "language/index.html", - "sec-constructs": "language/constructs.html", - "sect-let-language": "language/constructs.html#let-language", - "ss-functions": "language/constructs.html#functions", + "sec-constructs": "language/syntax.html", + "sect-let-language": "language/syntax.html#let-expressions", + "ss-functions": "language/syntax.html#functions", "sec-language-operators": "language/operators.html", "table-operators": "language/operators.html", - "ssec-values": "language/values.html", + "ssec-values": "language/types.html", "gloss-closure": "glossary.html#gloss-closure", "gloss-derivation": "glossary.html#gloss-derivation", "gloss-deriver": "glossary.html#gloss-deriver", @@ -285,7 +285,7 @@ const redirects = { "ch-basic-package-mgmt": "package-management/basic-package-mgmt.html", "ssec-binary-cache-substituter": "package-management/binary-cache-substituter.html", "sec-channels": "command-ref/nix-channel.html", - "ssec-copy-closure": "package-management/copy-closure.html", + "ssec-copy-closure": "command-ref/nix-copy-closure.html", "sec-garbage-collection": "package-management/garbage-collection.html", "ssec-gc-roots": "package-management/garbage-collector-roots.html", "chap-package-management": "package-management/index.html", @@ -335,18 +335,23 @@ const redirects = { "ssec-relnotes-2.2": "release-notes/rl-2.2.html", "ssec-relnotes-2.3": "release-notes/rl-2.3.html", }, - "language/values.html": { + "language/types.html": { "simple-values": "#primitives", "lists": "#list", "strings": "#string", "attribute-sets": "#attribute-set", + "type-number": "#type-int", + }, + "language/syntax.html": { + "scoping-rules": "scoping.html", + "string-literal": "string-literals.html", }, "installation/installing-binary.html": { "linux": "uninstall.html#linux", "macos": "uninstall.html#macos", "uninstalling": "uninstall.html", }, - "contributing/hacking.html": { + "development/building.html": { "nix-with-flakes": "#building-nix-with-flakes", "classic-nix": "#building-nix", "running-tests": "testing.html#running-tests", @@ -357,7 +362,12 @@ const redirects = { "installer-tests": "testing.html#installer-tests", "one-time-setup": "testing.html#one-time-setup", "using-the-ci-generated-installer-for-manual-testing": "testing.html#using-the-ci-generated-installer-for-manual-testing", - "characterization-testing": "#characterisation-testing-unit", + "characterization-testing": "testing.html#characterisation-testing-unit", + "add-a-release-note": "contributing.html#add-a-release-note", + "add-an-entry": "contributing.html#add-an-entry", + "build-process": "contributing.html#build-process", + "reverting": "contributing.html#reverting", + "branches": "contributing.html#branches", }, "glossary.html": { "gloss-local-store": "store/types/local-store.html", diff --git a/doc/manual/remove_before_wrapper.py b/doc/manual/remove_before_wrapper.py new file mode 100644 index 000000000..6da4c19b0 --- /dev/null +++ b/doc/manual/remove_before_wrapper.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import sys +import shutil +import typing as t + +def main(): + if len(sys.argv) < 4 or '--' not in sys.argv: + print("Usage: remove-before-wrapper -- ") + sys.exit(1) + + # Extract the parts + output: str = sys.argv[1] + nix_command_idx: int = sys.argv.index('--') + 1 + nix_command: t.List[str] = sys.argv[nix_command_idx:] + + output_temp: str = output + '.tmp' + + # Remove the output and temp output in case they exist + shutil.rmtree(output, ignore_errors=True) + shutil.rmtree(output_temp, ignore_errors=True) + + # Execute nix command with `--write-to` tempary output + nix_command_write_to = nix_command + ['--write-to', output_temp] + subprocess.run(nix_command_write_to, check=True) + + # Move the temporary output to the intended location + os.rename(output_temp, output) + +if __name__ == "__main__": + main() diff --git a/doc/manual/render-manpage.sh b/doc/manual/render-manpage.sh new file mode 100755 index 000000000..65a9c124e --- /dev/null +++ b/doc/manual/render-manpage.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -euo pipefail + +lowdown_args= + +if [ "$1" = --out-no-smarty ]; then + lowdown_args=--out-no-smarty + shift +fi + +[ "$#" = 4 ] || { + echo "wrong number of args passed" >&2 + exit 1 +} + +title="$1" +section="$2" +infile="$3" +outfile="$4" + +( + printf "Title: %s\n\n" "$title" + cat "$infile" +) | lowdown -sT man --nroff-nolinks $lowdown_args -M section="$section" -o "$outfile" diff --git a/doc/manual/rl-next/add-nix-state-home.md b/doc/manual/rl-next/add-nix-state-home.md new file mode 100644 index 000000000..bbfdd5d38 --- /dev/null +++ b/doc/manual/rl-next/add-nix-state-home.md @@ -0,0 +1,14 @@ +--- +synopsis: Use envvars NIX_CACHE_HOME, NIX_CONFIG_HOME, NIX_DATA_HOME, NIX_STATE_HOME if defined +prs: [11351] +--- + +Added new environment variables: + +- `NIX_CACHE_HOME` +- `NIX_CONFIG_HOME` +- `NIX_DATA_HOME` +- `NIX_STATE_HOME` + +Each, if defined, takes precedence over the corresponding [XDG environment variable](@docroot@/command-ref/env-common.md#xdg-base-directories). +This provides more fine-grained control over where Nix looks for files, and allows to have a stand-alone Nix environment, which only uses files in a specific directory, and doesn't interfere with the user environment. diff --git a/doc/manual/rl-next/ban-integer-overflow.md b/doc/manual/rl-next/ban-integer-overflow.md new file mode 100644 index 000000000..0e553af76 --- /dev/null +++ b/doc/manual/rl-next/ban-integer-overflow.md @@ -0,0 +1,21 @@ +--- +synopsis: Define integer overflow in the Nix language as an error +issues: [10968] +prs: [11188] +--- + +Previously, integer overflow in the Nix language invoked C++ level signed overflow, which was undefined behaviour, but *usually* manifested as wrapping around on overflow. + +Since prior to the public release of Lix, Lix had C++ signed overflow defined to crash the process and nobody noticed this having accidentally removed overflow from the Nix language for three months until it was caught by fiddling around. +Given the significant body of actual Nix code that has been evaluated by Lix in that time, it does not appear that nixpkgs or much of importance depends on integer overflow, so it appears safe to turn into an error. + +Some other overflows were fixed: +- `builtins.fromJSON` of values greater than the maximum representable value in a signed 64-bit integer will generate an error. +- `nixConfig` in flakes will no longer accept negative values for configuration options. + +Integer overflow now looks like the following: + +``` +$ nix eval --expr '9223372036854775807 + 1' +error: integer overflow in adding 9223372036854775807 + 1 +``` diff --git a/doc/manual/rl-next/build-hook-default.md b/doc/manual/rl-next/build-hook-default.md new file mode 100644 index 000000000..f13537983 --- /dev/null +++ b/doc/manual/rl-next/build-hook-default.md @@ -0,0 +1,22 @@ +--- +synopsis: |- + The `build-hook` setting's default is less useful when using `libnixstore` as a library +prs: +- 11178 +--- + +*This is an obscure issue that only affects usage of the `libnixstore` library outside of the Nix executable.* + +As part the ongoing [rewrite of the build system](https://github.com/NixOS/nix/issues/2503) to use [Meson](https://mesonbuild.com/), we are also switching to packaging individual Nix components separately (and building them in separate derivations). +This means that when building `libnixstore` we do not know where the Nix binaries will be installed --- `libnixstore` doesn't know about downstream consumers like the Nix binaries at all. + +*This is also unrelated to the _`post`_-`build-hook`*, which is often used for pushing to a cache.* + +This has a small adverse affect on remote building --- the `build-remote` executable that is specified from the [`build-hook`](@docroot@/command-ref/conf-file.md#conf-build-hook) setting will not be gotten from the (presumed) installation location, but instead looked up on the `PATH`. +This means that other applications linking `libnixstore` that wish to use remote building must arrange for the `nix` command to be on the PATH (or manually overriding `build-hook`) in order for that to work. + +Long term we don't envision this being a downside, because we plan to [get rid of `build-remote` and the build hook setting entirely](https://github.com/NixOS/nix/issues/1221). +There should simply be no need to have an extra, intermediate layer of remote-procedure-calling when we want to connect to a remote builder. +The build hook protocol did in principle support custom ways of remote building, but that can also be accomplished with a custom service for the ssh or daemon/ssh-ng protocols, or with a custom [store type](@docroot@/store/types/index.md) i.e. `Store` subclass. + +The Perl bindings no longer expose `getBinDir` either, since the underlying C++ libraries those bindings wrap no longer know the location of installed binaries as described above. diff --git a/doc/manual/rl-next/filesystem-errors.md b/doc/manual/rl-next/filesystem-errors.md new file mode 100644 index 000000000..2d5b26228 --- /dev/null +++ b/doc/manual/rl-next/filesystem-errors.md @@ -0,0 +1,14 @@ +--- +synopsis: wrap filesystem exceptions more correctly +issues: [] +prs: [11378] +--- + + +With the switch to `std::filesystem` in different places, Nix started to throw `std::filesystem::filesystem_error` in many places instead of its own exceptions. + +This lead to no longer generating error traces, for example when listing a non-existing directory, and can also lead to crashes inside the Nix REPL. + +This version catches these types of exception correctly and wrap them into Nix's own exeception type. + +Author: [**@Mic92**](https://github.com/Mic92) diff --git a/doc/manual/rl-next/fsync-store-paths.md b/doc/manual/rl-next/fsync-store-paths.md new file mode 100644 index 000000000..0e9e7f7f2 --- /dev/null +++ b/doc/manual/rl-next/fsync-store-paths.md @@ -0,0 +1,9 @@ +--- +synopsis: Add setting `fsync-store-paths` +issues: [1218] +prs: [7126] +--- + +Nix now has a setting `fsync-store-paths` that ensures that new store paths are durably written to disk before they are registered as "valid" in Nix's database. This can prevent Nix store corruption if the system crashes or there is a power loss. This setting defaults to `false`. + +Author: [**@squalus**](https://github.com/squalus) diff --git a/doc/manual/rl-next/nix-flake-show-description.md b/doc/manual/rl-next/nix-flake-show-description.md new file mode 100644 index 000000000..7feb08483 --- /dev/null +++ b/doc/manual/rl-next/nix-flake-show-description.md @@ -0,0 +1,25 @@ +--- +synopsis: Show package descriptions with `nix flake show` +issues: [10977] +prs: [10980] +--- + +`nix flake show` will now display a package's `meta.description` if it exists. If the description does not fit in the terminal it will be truncated to fit the terminal width. If the size of the terminal width is unknown the description will be capped at 80 characters. + +``` +$ nix flake show +└───packages + └───x86_64-linux + ├───builderImage: package 'docker-image-ara-builder-image.tar.gz' - 'Docker image hosting the nix build environment' + └───runnerImage: package 'docker-image-gitlab-runner.tar.gz' - 'Docker image hosting the gitlab-runner executable' +``` + +In a narrower terminal: + +``` +$ nix flake show +└───packages + └───x86_64-linux + ├───builderImage: package 'docker-image-ara-builder-image.tar.gz' - 'Docker image hosting the nix b... + └───runnerImage: package 'docker-image-gitlab-runner.tar.gz' - 'Docker image hosting the gitlab-run... +``` diff --git a/doc/manual/rl-next/nix-fmt-default-argument.md b/doc/manual/rl-next/nix-fmt-default-argument.md new file mode 100644 index 000000000..54161ab30 --- /dev/null +++ b/doc/manual/rl-next/nix-fmt-default-argument.md @@ -0,0 +1,17 @@ +--- +synopsis: Removing the default argument passed to the `nix fmt` formatter +issues: [] +prs: [11438] +--- + +The underlying formatter no longer receives the ". " default argument when `nix fmt` is called with no arguments. + +This change was necessary as the formatter wasn't able to distinguish between +a user wanting to format the current folder with `nix fmt .` or the generic +`nix fmt`. + +The default behaviour is now the responsibility of the formatter itself, and +allows tools such as treefmt to format the whole tree instead of only the +current directory and below. + +Author: [**@zimbatm**](https://github.com/zimbatm) diff --git a/doc/manual/rl-next/no-flake-substitution.md b/doc/manual/rl-next/no-flake-substitution.md new file mode 100644 index 000000000..67ec58750 --- /dev/null +++ b/doc/manual/rl-next/no-flake-substitution.md @@ -0,0 +1,8 @@ +--- +synopsis: Flakes are no longer substituted +prs: [10612] +--- + +Nix will no longer attempt to substitute the source code of flakes from a binary cache. This functionality was broken because it could lead to different evaluation results depending on whether the flake was available in the binary cache, or even depending on whether the flake was already in the local store. + +Author: [**@edolstra**](https://github.com/edolstra) diff --git a/doc/manual/rl-next/verify-tls.md b/doc/manual/rl-next/verify-tls.md new file mode 100644 index 000000000..afc689f46 --- /dev/null +++ b/doc/manual/rl-next/verify-tls.md @@ -0,0 +1,8 @@ +--- +synopsis: "`` uses TLS verification" +prs: [11585] +--- + +Previously `` did not do TLS verification. This was because the Nix sandbox in the past did not have access to TLS certificates, and Nix checks the hash of the fetched file anyway. However, this can expose authentication data from `netrc` and URLs to man-in-the-middle attackers. In addition, Nix now in some cases (such as when using impure derivations) does *not* check the hash. Therefore we have now enabled TLS verification. This means that downloads by `` will now fail if you're fetching from a HTTPS server that does not have a valid certificate. + +`` is also known as the builtin derivation builder `builtin:fetchurl`. It's not to be confused with the evaluation-time function `builtins.fetchurl`, which was not affected by this issue. diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index cf40877d2..a6b9c1746 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -24,8 +24,9 @@ - [Abstract Building](store/abstract/building.md) - [The concrete model for Unix](store/concrete/index.md) - [File System Object](store/concrete/file-system-object.md) - - [Content-Addressing File System Objects](store/concrete/file-system-object/content-addressing.md) + - [Content-Addressing File System Objects](store/concrete/file-system-object/content-address.md) - [Store Object](store/concrete/object.md) + - [Content-Addressing Store Objects](store/concrete/store-object/content-address.md) - [Store Path](store/concrete/path.md) - [Content-Addressing Store Objects](store/concrete/ca-object.md) - [Reference Scanning & Rewriting](store/concrete/reference-scanning.md) @@ -40,17 +41,20 @@ {{#include ./store/types/SUMMARY.md}} - [Advanced Topic: Related work](store/related-work.md) - [Nix Language](language/index.md) - - [Data Types](language/values.md) - - [Language Constructs](language/constructs.md) + - [Data Types](language/types.md) + - [String context](language/string-context.md) + - [Syntax and semantics](language/syntax.md) + - [Variables](language/variables.md) + - [String literals](language/string-literals.md) + - [Identifiers](language/identifiers.md) + - [Scoping rules](language/scope.md) - [String interpolation](language/string-interpolation.md) - [Lookup path](language/constructs/lookup-path.md) - - [String context](language/string-context.md) - [Operators](language/operators.md) - - [Derivations](language/derivations.md) - - [Advanced Attributes](language/advanced-attributes.md) - - [Import From Derivation](language/import-from-derivation.md) - - [Built-in Constants](language/builtin-constants.md) - - [Built-in Functions](language/builtins.md) + - [Built-ins](language/builtins.md) + - [Derivations](language/derivations.md) + - [Advanced Attributes](language/advanced-attributes.md) + - [Import From Derivation](language/import-from-derivation.md) - [Package Management](package-management/index.md) - [Profiles](package-management/profiles.md) - [Garbage Collection](package-management/garbage-collection.md) @@ -58,7 +62,6 @@ - [Advanced Topics](advanced-topics/index.md) - [Sharing Packages Between Machines](package-management/sharing-packages.md) - [Serving a Nix store via HTTP](package-management/binary-cache-substituter.md) - - [Copying Closures via SSH](package-management/copy-closure.md) - [Serving a Nix store via SSH](package-management/ssh-substituter.md) - [Remote Builds](advanced-topics/distributed-builds.md) - [Tuning Cores and Jobs](advanced-topics/cores-vs-jobs.md) @@ -131,15 +134,19 @@ - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [C API](c-api.md) - [Glossary](glossary.md) -- [Contributing](contributing/index.md) - - [Hacking](contributing/hacking.md) - - [Testing](contributing/testing.md) - - [Documentation](contributing/documentation.md) - - [Experimental Features](contributing/experimental-features.md) - - [CLI guideline](contributing/cli-guideline.md) - - [C++ style guide](contributing/cxx.md) +- [Development](development/index.md) + - [Building](development/building.md) + - [Testing](development/testing.md) + - [Documentation](development/documentation.md) + - [CLI guideline](development/cli-guideline.md) + - [JSON guideline](development/json-guideline.md) + - [C++ style guide](development/cxx.md) + - [Experimental Features](development/experimental-features.md) + - [Contributing](development/contributing.md) - [Releases](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} + - [Release 2.24 (2024-07-31)](release-notes/rl-2.24.md) + - [Release 2.23 (2024-06-03)](release-notes/rl-2.23.md) - [Release 2.22 (2024-04-23)](release-notes/rl-2.22.md) - [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md) - [Release 2.20 (2024-01-29)](release-notes/rl-2.20.md) diff --git a/doc/manual/src/_redirects b/doc/manual/src/_redirects index 8bf0e854b..07b3130f9 100644 --- a/doc/manual/src/_redirects +++ b/doc/manual/src/_redirects @@ -1,5 +1,5 @@ # redirect rules for paths (server-side) to prevent link rot. -# see ./redirects.js for redirects based on URL fragments (client-side) +# see ../redirects.js for redirects based on URL fragments (client-side) # # concrete user story this supports: # - user finds URL to the manual for Nix x.y @@ -20,13 +20,24 @@ /command-ref/command-ref /command-ref 301! -/contributing/contributing /contributing 301! +/contributing/contributing /development 301! +/contributing /development 301! +/contributing/hacking /development/building 301! +/contributing/testing /development/testing 301! +/contributing/documentation /development/documentation 301! +/contributing/experimental-features /development/experimental-features 301! +/contributing/cli-guideline /development/cli-guideline 301! +/contributing/json-guideline /development/json-guideline 301! +/contributing/cxx /development/cxx 301! /expressions/expression-language /language/ 301! /expressions/language-constructs /language/constructs 301! /expressions/language-operators /language/operators 301! /expressions/language-values /language/values 301! /expressions/* /language/:splat 301! +/language/values /language/types 301! +/language/constructs /language/syntax 301! +/language/builtin-constants /language/builtins 301! /installation/installation /installation 301! @@ -39,3 +50,5 @@ /json/* /protocols/json/:splat 301! /release-notes/release-notes /release-notes 301! + +/package-management/copy-closure /command-ref/nix-copy-closure 301! diff --git a/doc/manual/src/advanced-topics/distributed-builds.md b/doc/manual/src/advanced-topics/distributed-builds.md index ddabaeb4d..52acd039c 100644 --- a/doc/manual/src/advanced-topics/distributed-builds.md +++ b/doc/manual/src/advanced-topics/distributed-builds.md @@ -12,14 +12,14 @@ machine is accessible via SSH and that it has Nix installed. You can test whether connecting to the remote Nix instance works, e.g. ```console -$ nix store ping --store ssh://mac +$ nix store info --store ssh://mac ``` will try to connect to the machine named `mac`. It is possible to specify an SSH identity file as part of the remote store URI, e.g. ```console -$ nix store ping --store ssh://mac?ssh-key=/home/alice/my-key +$ nix store info --store ssh://mac?ssh-key=/home/alice/my-key ``` Since builds should be non-interactive, the key should not have a diff --git a/doc/manual/src/c-api.md b/doc/manual/src/c-api.md index 29df0b644..0cdd83832 100644 --- a/doc/manual/src/c-api.md +++ b/doc/manual/src/c-api.md @@ -10,7 +10,7 @@ See: - [Matrix Room *Nix Bindings*](https://matrix.to/#/#nix-bindings:nixos.org) for discussion and questions. - [Stabilisation Milestone](https://github.com/NixOS/nix/milestone/52) - [Other C API PRs and issues](https://github.com/NixOS/nix/labels/c%20api) -- [Contributing C API Documentation](contributing/documentation.md#c-api-documentation), including how to build it locally. +- [Contributing C API Documentation](development/documentation.md#c-api-documentation), including how to build it locally. [Getting Started]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs [Index]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs/globals.html diff --git a/doc/manual/src/command-ref/env-common.md b/doc/manual/src/command-ref/env-common.md index 34e0dbfbd..ee3995111 100644 --- a/doc/manual/src/command-ref/env-common.md +++ b/doc/manual/src/command-ref/env-common.md @@ -9,22 +9,26 @@ Most Nix commands interpret the following environment variables: - [`NIX_PATH`](#env-NIX_PATH) - A colon-separated list of directories used to look up the location of Nix - expressions using [paths](@docroot@/language/values.md#type-path) - enclosed in angle brackets (i.e., ``), - e.g. `/home/eelco/Dev:/etc/nixos`. It can be extended using the - [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + A colon-separated list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). - If `NIX_PATH` is not set at all, Nix will fall back to the following list in [impure](@docroot@/command-ref/conf-file.md#conf-pure-eval) and [unrestricted](@docroot@/command-ref/conf-file.md#conf-restrict-eval) evaluation mode: + This environment variable overrides the value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path). - 1. `$HOME/.nix-defexpr/channels` - 2. `nixpkgs=/nix/var/nix/profiles/per-user/root/channels/nixpkgs` - 3. `/nix/var/nix/profiles/per-user/root/channels` + It can be extended using the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I). + + > **Example** + > + > ```bash + > $ export NIX_PATH=`/home/eelco/Dev:nixos-config=/etc/nixos + > ``` If `NIX_PATH` is set to an empty string, resolving search paths will always fail. - For example, attempting to use `` will produce: - error: file 'nixpkgs' was not found in the Nix search path + > **Example** + > + > ```bash + > $ NIX_PATH= nix-instantiate --eval '' + > error: file 'nixpkgs' was not found in the Nix search path (add it using $NIX_PATH or -I) + > ``` - [`NIX_IGNORE_SYMLINK_STORE`](#env-NIX_IGNORE_SYMLINK_STORE) @@ -134,6 +138,19 @@ The following environment variables are used to determine locations of various s - [`XDG_STATE_HOME`]{#env-XDG_STATE_HOME} (default `~/.local/state`) - [`XDG_CACHE_HOME`]{#env-XDG_CACHE_HOME} (default `~/.cache`) - [XDG Base Directory Specification]: https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html [`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories + +In addition, setting the following environment variables overrides the XDG base directories: + +- [`NIX_CONFIG_HOME`]{#env-NIX_CONFIG_HOME} (default `$XDG_CONFIG_HOME/nix`) +- [`NIX_STATE_HOME`]{#env-NIX_STATE_HOME} (default `$XDG_STATE_HOME/nix`) +- [`NIX_CACHE_HOME`]{#env-NIX_CACHE_HOME} (default `$XDG_CACHE_HOME/nix`) + +When [`use-xdg-base-directories`] is enabled, the configuration directory is: + +1. `$NIX_CONFIG_HOME`, if it is defined +2. Otherwise, `$XDG_CONFIG_HOME/nix`, if `XDG_CONFIG_HOME` is defined +3. Otherwise, `~/.config/nix`. + +Likewise for the state and cache directories. diff --git a/doc/manual/src/command-ref/experimental-commands.md b/doc/manual/src/command-ref/experimental-commands.md index 286ddc6d6..1190729a2 100644 --- a/doc/manual/src/command-ref/experimental-commands.md +++ b/doc/manual/src/command-ref/experimental-commands.md @@ -1,6 +1,6 @@ # Experimental Commands -This section lists [experimental commands](@docroot@/contributing/experimental-features.md#xp-feature-nix-command). +This section lists [experimental commands](@docroot@/development/experimental-features.md#xp-feature-nix-command). > **Warning** > diff --git a/doc/manual/src/command-ref/files/default-nix-expression.md b/doc/manual/src/command-ref/files/default-nix-expression.md index 620f7035c..2bd45ff5d 100644 --- a/doc/manual/src/command-ref/files/default-nix-expression.md +++ b/doc/manual/src/command-ref/files/default-nix-expression.md @@ -1,6 +1,6 @@ ## Default Nix expression -The source for the default [Nix expressions](@docroot@/language/index.md) used by [`nix-env`]: +The source for the [Nix expressions](@docroot@/glossary.md#gloss-nix-expression) used by [`nix-env`] by default: - `~/.nix-defexpr` - `$XDG_STATE_HOME/nix/defexpr` if [`use-xdg-base-directories`] is set to `true`. @@ -18,24 +18,25 @@ Then, the resulting expression is interpreted like this: - If the expression is an attribute set, it is used as the default Nix expression. - If the expression is a function, an empty set is passed as argument and the return value is used as the default Nix expression. - -For example, if the default expression contains two files, `foo.nix` and `bar.nix`, then the default Nix expression will be equivalent to - -```nix -{ - foo = import ~/.nix-defexpr/foo.nix; - bar = import ~/.nix-defexpr/bar.nix; -} -``` +> **Example** +> +> If the default expression contains two files, `foo.nix` and `bar.nix`, then the default Nix expression will be equivalent to +> +> ```nix +> { +> foo = import ~/.nix-defexpr/foo.nix; +> bar = import ~/.nix-defexpr/bar.nix; +> } +> ``` The file [`manifest.nix`](@docroot@/command-ref/files/manifest.nix.md) is always ignored. -The command [`nix-channel`] places a symlink to the user's current [channels profile](@docroot@/command-ref/files/channels.md) in this directory. +The command [`nix-channel`] places a symlink to the current user's [channels] in this directory, the [user channel link](#user-channel-link). This makes all subscribed channels available as attributes in the default expression. ## User channel link -A symlink that ensures that [`nix-env`] can find your channels: +A symlink that ensures that [`nix-env`] can find the current user's [channels]: - `~/.nix-defexpr/channels` - `$XDG_STATE_HOME/defexpr/channels` if [`use-xdg-base-directories`] is set to `true`. @@ -45,8 +46,9 @@ This symlink points to: - `$XDG_STATE_HOME/profiles/channels` for regular users - `$NIX_STATE_DIR/profiles/per-user/root/channels` for `root` -In a multi-user installation, you may also have `~/.nix-defexpr/channels_root`, which links to the channels of the root user.[`nix-env`]: ../nix-env.md +In a multi-user installation, you may also have `~/.nix-defexpr/channels_root`, which links to the channels of the root user. -[`nix-env`]: @docroot@/command-ref/nix-env.md [`nix-channel`]: @docroot@/command-ref/nix-channel.md +[`nix-env`]: @docroot@/command-ref/nix-env.md [`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories +[channels]: @docroot@/command-ref/files/channels.md diff --git a/doc/manual/src/command-ref/meson.build b/doc/manual/src/command-ref/meson.build new file mode 100644 index 000000000..2976f69ff --- /dev/null +++ b/doc/manual/src/command-ref/meson.build @@ -0,0 +1,63 @@ +xp_features_json = custom_target( + command : [nix, '__dump-xp-features'], + capture : true, + output : 'xp-features.json', +) + +experimental_features_shortlist_md = custom_target( + command : nix_eval_for_docs + [ + '--expr', + 'import @INPUT0@ (builtins.fromJSON (builtins.readFile ./@INPUT1@))', + ], + input : [ + '../../generate-xp-features-shortlist.nix', + xp_features_json, + ], + output : 'experimental-features-shortlist.md', + capture : true, + env : nix_env_for_docs, +) + +nix3_cli_files = custom_target( + command : [ + python.full_path(), + '@INPUT0@', + '@OUTPUT@', + '--' + ] + nix_eval_for_docs + [ + '--expr', + 'import @INPUT1@ true (builtins.readFile ./@INPUT2@)', + ], + input : [ + '../../remove_before_wrapper.py', + '../../generate-manpage.nix', + nix3_cli_json, + ], + output : 'new-cli', + env : nix_env_for_docs, +) + +conf_file_md_body = custom_target( + command : [ + nix_eval_for_docs, + '--expr', + 'import @INPUT0@ { prefix = "conf"; } (builtins.fromJSON (builtins.readFile ./@INPUT1@))', + ], + capture : true, + input : [ + '../../generate-settings.nix', + conf_file_json, + ], + output : 'conf-file.body.md', + env : nix_env_for_docs, +) + +conf_file_md = custom_target( + command : [ 'cat', '@INPUT0@', '@INPUT1@' ], + capture : true, + input : [ + 'conf-file-prefix.md', + conf_file_md_body, + ], + output : 'conf-file.md', +) diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md index e4223b542..3bb59cbed 100644 --- a/doc/manual/src/command-ref/nix-build.md +++ b/doc/manual/src/command-ref/nix-build.md @@ -55,20 +55,20 @@ All options not listed here are passed to [`nix-store --realise`](nix-store/realise.md), except for `--arg` and `--attr` / `-A` which are passed to [`nix-instantiate`](nix-instantiate.md). - - [`--no-out-link`](#opt-no-out-link) +- [`--no-out-link`](#opt-no-out-link) - Do not create a symlink to the output path. Note that as a result - the output does not become a root of the garbage collector, and so - might be deleted by `nix-store --gc`. + Do not create a symlink to the output path. Note that as a result + the output does not become a root of the garbage collector, and so + might be deleted by `nix-store --gc`. - - [`--dry-run`](#opt-dry-run) +- [`--dry-run`](#opt-dry-run) - Show what store paths would be built or downloaded. + Show what store paths would be built or downloaded. - - [`--out-link`](#opt-out-link) / `-o` *outlink* +- [`--out-link`](#opt-out-link) / `-o` *outlink* - Change the name of the symlink to the output path created from - `result` to *outlink*. + Change the name of the symlink to the output path created from + `result` to *outlink*. {{#include ./status-build-failure.md}} diff --git a/doc/manual/src/command-ref/nix-channel.md b/doc/manual/src/command-ref/nix-channel.md index cebbc7b00..8b58392b7 100644 --- a/doc/manual/src/command-ref/nix-channel.md +++ b/doc/manual/src/command-ref/nix-channel.md @@ -27,40 +27,46 @@ The moving parts of channels are: This command has the following operations: - - `--add` *url* \[*name*\]\ - Add a channel *name* located at *url* to the list of subscribed channels. - If *name* is omitted, default to the last component of *url*, with the suffixes `-stable` or `-unstable` removed. +- `--add` *url* \[*name*\] - > **Note** - > - > `--add` does not automatically perform an update. - > Use `--update` explicitly. + Add a channel *name* located at *url* to the list of subscribed channels. + If *name* is omitted, default to the last component of *url*, with the suffixes `-stable` or `-unstable` removed. - A channel URL must point to a directory containing a file `nixexprs.tar.gz`. - At the top level, that tarball must contain a single directory with a `default.nix` file that serves as the channel’s entry point. + > **Note** + > + > `--add` does not automatically perform an update. + > Use `--update` explicitly. - - `--remove` *name*\ - Remove the channel *name* from the list of subscribed channels. + A channel URL must point to a directory containing a file `nixexprs.tar.gz`. + At the top level, that tarball must contain a single directory with a `default.nix` file that serves as the channel’s entry point. - - `--list`\ - Print the names and URLs of all subscribed channels on standard output. +- `--remove` *name* - - `--update` \[*names*…\]\ - Download the Nix expressions of subscribed channels and create a new generation. - Update all channels if none is specified, and only those included in *names* otherwise. + Remove the channel *name* from the list of subscribed channels. - - `--list-generations`\ - Prints a list of all the current existing generations for the - channel profile. +- `--list` - Works the same way as - ``` - nix-env --profile /nix/var/nix/profiles/per-user/$USER/channels --list-generations - ``` + Print the names and URLs of all subscribed channels on standard output. - - `--rollback` \[*generation*\]\ - Revert channels to the state before the last call to `nix-channel --update`. - Optionally, you can specify a specific channel *generation* number to restore. +- `--update` \[*names*…\] + + Download the Nix expressions of subscribed channels and create a new generation. + Update all channels if none is specified, and only those included in *names* otherwise. + +- `--list-generations` + + Prints a list of all the current existing generations for the + channel profile. + + Works the same way as + ``` + nix-env --profile /nix/var/nix/profiles/per-user/$USER/channels --list-generations + ``` + +- `--rollback` \[*generation*\] + + Revert channels to the state before the last call to `nix-channel --update`. + Optionally, you can specify a specific channel *generation* number to restore. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-collect-garbage.md b/doc/manual/src/command-ref/nix-collect-garbage.md index 8e1307c48..2136d28e9 100644 --- a/doc/manual/src/command-ref/nix-collect-garbage.md +++ b/doc/manual/src/command-ref/nix-collect-garbage.md @@ -48,12 +48,14 @@ Instead, it looks in a few locations, and acts on all profiles it finds there: These options are for deleting old [profiles] prior to deleting unreachable [store objects]. -- [`--delete-old`](#opt-delete-old) / `-d`\ +- [`--delete-old`](#opt-delete-old) / `-d` + Delete all old generations of profiles. This is the equivalent of invoking [`nix-env --delete-generations old`](@docroot@/command-ref/nix-env/delete-generations.md#generations-old) on each found profile. -- [`--delete-older-than`](#opt-delete-older-than) *period*\ +- [`--delete-older-than`](#opt-delete-older-than) *period* + Delete all generations of profiles older than the specified amount (except for the generations that were active at that point in time). *period* is a value such as `30d`, which would mean 30 days. diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md index 46d381f5d..8cfd6ebad 100644 --- a/doc/manual/src/command-ref/nix-copy-closure.md +++ b/doc/manual/src/command-ref/nix-copy-closure.md @@ -1,91 +1,91 @@ # Name -`nix-copy-closure` - copy a closure to or from a remote machine via SSH +`nix-copy-closure` - copy store objects to or from a remote machine via SSH # Synopsis `nix-copy-closure` - [`--to` | `--from`] + [`--to` | `--from` ] [`--gzip`] [`--include-outputs`] [`--use-substitutes` | `-s`] [`-v`] - _user@machine_ _paths_ + [_user_@]_machine_[:_port_] _paths_ # Description -`nix-copy-closure` gives you an easy and efficient way to exchange -software between machines. Given one or more Nix store _paths_ on the -local machine, `nix-copy-closure` computes the closure of those paths -(i.e. all their dependencies in the Nix store), and copies all paths -in the closure to the remote machine via the `ssh` (Secure Shell) -command. With the `--from` option, the direction is reversed: the -closure of _paths_ on a remote machine is copied to the Nix store on -the local machine. +Given _paths_ from one machine, `nix-copy-closure` computes the [closure](@docroot@/glossary.md#gloss-closure) of those paths (i.e. all their dependencies in the Nix store), and copies [store objects](@docroot@/glossary.md#gloss-store-object) in that closure to another machine via SSH. +It doesn’t copy store objects that are already present on the other machine. -This command is efficient because it only sends the store paths -that are missing on the target machine. +> **Note** +> +> While the Nix store to use on the local machine can be specified on the command line with the [`--store`](@docroot@/command-ref/conf-file.md#conf-store) option, the Nix store to be accessed on the remote machine can only be [configured statically](@docroot@/command-ref/conf-file.md#configuration-file) on that remote machine. -Since `nix-copy-closure` calls `ssh`, you may be asked to type in the -appropriate password or passphrase. In fact, you may be asked _twice_ -because `nix-copy-closure` currently connects twice to the remote -machine, first to get the set of paths missing on the target machine, -and second to send the dump of those paths. When using public key -authentication, you can avoid typing the passphrase with `ssh-agent`. +Since `nix-copy-closure` calls `ssh`, you may need to authenticate with the remote machine. +In fact, you may be asked for authentication _twice_ because `nix-copy-closure` currently connects twice to the remote machine: first to get the set of paths missing on the target machine, and second to send the dump of those paths. +When using public key authentication, you can avoid typing the passphrase with `ssh-agent`. # Options - - `--to`\ - Copy the closure of _paths_ from the local Nix store to the Nix - store on _machine_. This is the default. +- `--to` - - `--from`\ - Copy the closure of _paths_ from the Nix store on _machine_ to the - local Nix store. + Copy the closure of _paths_ from a Nix store accessible from the local machine to the Nix store on the remote _machine_. + This is the default behavior. - - `--gzip`\ - Enable compression of the SSH connection. +- `--from` - - `--include-outputs`\ - Also copy the outputs of [store derivation]s included in the closure. + Copy the closure of _paths_ from the Nix store on the remote _machine_ to the local machine's specified Nix store. - [store derivation]: @docroot@/glossary.md#gloss-store-derivation +- `--gzip` - - `--use-substitutes` / `-s`\ - Attempt to download missing paths on the target machine using Nix’s - substitute mechanism. Any paths that cannot be substituted on the - target are still copied normally from the source. This is useful, - for instance, if the connection between the source and target - machine is slow, but the connection between the target machine and - `nixos.org` (the default binary cache server) is - fast. + Enable compression of the SSH connection. - - `-v`\ - Show verbose output. +- `--include-outputs` + + Also copy the outputs of [store derivation]s included in the closure. + + [store derivation]: @docroot@/glossary.md#gloss-store-derivation + +- `--use-substitutes` / `-s` + + Attempt to download missing store objects on the target from [substituters](@docroot@/command-ref/conf-file.md#conf-substituters). + Any store objects that cannot be substituted on the target are still copied normally from the source. + This is useful, for instance, if the connection between the source and target machine is slow, but the connection between the target machine and `cache.nixos.org` (the default binary cache server) is fast. {{#include ./opt-common.md}} # Environment variables - - `NIX_SSHOPTS`\ - Additional options to be passed to `ssh` on the command - line. +- `NIX_SSHOPTS` + + Additional options to be passed to `ssh` on the command line. {{#include ./env-common.md}} # Examples -Copy Firefox with all its dependencies to a remote machine: +> **Example** +> +> Copy GNU Hello with all its dependencies to a remote machine: +> +> ```shell-session +> $ storePath="$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link)" +> $ nix-copy-closure --to alice@itchy.example.org "$storePath" +> copying 5 paths... +> copying path '/nix/store/nrwkk6ak3rgkrxbqhsscb01jpzmslf2r-xgcc-13.2.0-libgcc' to 'ssh://alice@itchy.example.org'... +> copying path '/nix/store/gm61h1y42pqyl6178g90x8zm22n6pyy5-libunistring-1.1' to 'ssh://alice@itchy.example.org'... +> copying path '/nix/store/ddfzjdykw67s20c35i7a6624by3iz5jv-libidn2-2.3.7' to 'ssh://alice@itchy.example.org'... +> copying path '/nix/store/apab5i73dqa09wx0q27b6fbhd1r18ihl-glibc-2.39-31' to 'ssh://alice@itchy.example.org'... +> copying path '/nix/store/g1n2vryg06amvcc1avb2mcq36faly0mh-hello-2.12.1' to 'ssh://alice@itchy.example.org'... +> ``` -```console -$ nix-copy-closure --to alice@itchy.example.org $(type -P firefox) -``` - -Copy Subversion from a remote machine and then install it into a user -environment: - -```console -$ nix-copy-closure --from alice@itchy.example.org \ - /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4 -$ nix-env --install /nix/store/0dj0503hjxy5mbwlafv1rsbdiyx1gkdy-subversion-1.4.4 -``` +> **Example** +> +> Copy GNU Hello from a remote machine using a known store path, and run it: +> +> ```shell-session +> $ storePath="$(nix-instantiate --eval '' -I nixpkgs=channel:nixpkgs-unstable -A hello.outPath | tr -d '"')" +> $ nix-copy-closure --from alice@itchy.example.org "$storePath" +> $ "$storePath"/bin/hello +> Hello, world! +> ``` diff --git a/doc/manual/src/command-ref/nix-env.md b/doc/manual/src/command-ref/nix-env.md index c6f627365..bda02149e 100644 --- a/doc/manual/src/command-ref/nix-env.md +++ b/doc/manual/src/command-ref/nix-env.md @@ -62,7 +62,7 @@ These pages can be viewed offline: Several operations, such as [`nix-env --query`](./nix-env/query.md) and [`nix-env --install`](./nix-env/install.md), take a list of *arguments* that specify the packages on which to operate. -Packages are identified based on a `name` part and a `version` part of a [symbolic derivation name](@docroot@/language/derivations.md#attr-names): +Packages are identified based on a `name` part and a `version` part of a [symbolic derivation name](@docroot@/language/derivations.md#attr-name): - `name`: Everything up to but not including the first dash (`-`) that is *not* followed by a letter. - `version`: The rest, excluding the separating dash. diff --git a/doc/manual/src/command-ref/nix-env/delete-generations.md b/doc/manual/src/command-ref/nix-env/delete-generations.md index ae618b2c6..b1ff0bb69 100644 --- a/doc/manual/src/command-ref/nix-env/delete-generations.md +++ b/doc/manual/src/command-ref/nix-env/delete-generations.md @@ -12,7 +12,8 @@ This operation deletes the specified generations of the current profile. *generations* can be a one of the following: -- [`...`](#generations-list):\ +- [`...`](#generations-list) + A list of generation numbers, each one a separate command-line argument. Delete exactly the profile generations given by their generation number. @@ -30,7 +31,8 @@ This operation deletes the specified generations of the current profile. > Because one can roll back to a previous generation, it is possible to have generations newer than the current one. > They will also be deleted. -- [`d`](#generations-time):\ +- [`d`](#generations-time) + The last *number* days *Example*: `30d` @@ -38,7 +40,8 @@ This operation deletes the specified generations of the current profile. Delete all generations created more than *number* days ago, except the most recent one of them. This allows rolling back to generations that were available within the specified period. -- [`+`](#generations-count):\ +- [`+`](#generations-count) + The last *number* generations up to the present *Example*: `+5` diff --git a/doc/manual/src/command-ref/nix-env/env-common.md b/doc/manual/src/command-ref/nix-env/env-common.md index 735817959..200da7219 100644 --- a/doc/manual/src/command-ref/nix-env/env-common.md +++ b/doc/manual/src/command-ref/nix-env/env-common.md @@ -1,6 +1,7 @@ # Environment variables -- `NIX_PROFILE`\ +- `NIX_PROFILE` + Location of the Nix profile. Defaults to the target of the symlink `~/.nix-profile`, if it exists, or `/nix/var/nix/profiles/default` otherwise. diff --git a/doc/manual/src/command-ref/nix-env/install.md b/doc/manual/src/command-ref/nix-env/install.md index 85f37904f..748dd1e7a 100644 --- a/doc/manual/src/command-ref/nix-env/install.md +++ b/doc/manual/src/command-ref/nix-env/install.md @@ -21,125 +21,125 @@ It is based on the current generation of the active [profile](@docroot@/command- The arguments *args* map to store paths in a number of possible ways: - - By default, *args* is a set of [derivation] names denoting derivations in the [default Nix expression]. - These are [realised], and the resulting output paths are installed. - Currently installed derivations with a name equal to the name of a derivation being added are removed unless the option `--preserve-installed` is specified. +- By default, *args* is a set of [derivation] names denoting derivations in the [default Nix expression]. + These are [realised], and the resulting output paths are installed. + Currently installed derivations with a name equal to the name of a derivation being added are removed unless the option `--preserve-installed` is specified. - [derivation]: @docroot@/glossary.md#gloss-derivation - [default Nix expression]: @docroot@/command-ref/files/default-nix-expression.md - [realised]: @docroot@/glossary.md#gloss-realise + [derivation]: @docroot@/glossary.md#gloss-derivation + [default Nix expression]: @docroot@/command-ref/files/default-nix-expression.md + [realised]: @docroot@/glossary.md#gloss-realise - If there are multiple derivations matching a name in *args* that - have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the - derivation with the highest *priority* is used. A derivation can - define a priority by declaring the `meta.priority` attribute. This - attribute should be a number, with a higher value denoting a lower - priority. The default priority is `5`. + If there are multiple derivations matching a name in *args* that + have the same name (e.g., `gcc-3.3.6` and `gcc-4.1.1`), then the + derivation with the highest *priority* is used. A derivation can + define a priority by declaring the `meta.priority` attribute. This + attribute should be a number, with a higher value denoting a lower + priority. The default priority is `5`. - If there are multiple matching derivations with the same priority, - then the derivation with the highest version will be installed. + If there are multiple matching derivations with the same priority, + then the derivation with the highest version will be installed. - You can force the installation of multiple derivations with the same - name by being specific about the versions. For instance, `nix-env --install - gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will - probably cause a user environment conflict\!). + You can force the installation of multiple derivations with the same + name by being specific about the versions. For instance, `nix-env --install + gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will + probably cause a user environment conflict\!). - - If [`--attr`](#opt-attr) / `-A` is specified, the arguments are *attribute paths* that select attributes from the [default Nix expression]. - This is faster than using derivation names and unambiguous. - Show the attribute paths of available packages with [`nix-env --query`](./query.md): +- If [`--attr`](#opt-attr) / `-A` is specified, the arguments are *attribute paths* that select attributes from the [default Nix expression]. + This is faster than using derivation names and unambiguous. + Show the attribute paths of available packages with [`nix-env --query`](./query.md): - ```console - nix-env --query --available --attr-path - ``` + ```console + nix-env --query --available --attr-path + ``` - - If `--from-profile` *path* is given, *args* is a set of names - denoting installed [store paths] in the profile *path*. This is an - easy way to copy user environment elements from one profile to - another. +- If `--from-profile` *path* is given, *args* is a set of names + denoting installed [store paths] in the profile *path*. This is an + easy way to copy user environment elements from one profile to + another. - - If `--from-expression` is given, *args* are [Nix language functions](@docroot@/language/constructs.md#functions) that are called with the [default Nix expression] as their single argument. - The derivations returned by those function calls are installed. - This allows derivations to be specified in an unambiguous way, which is necessary if there are multiple derivations with the same name. +- If `--from-expression` is given, *args* are [Nix language functions](@docroot@/language/syntax.md#functions) that are called with the [default Nix expression] as their single argument. + The derivations returned by those function calls are installed. + This allows derivations to be specified in an unambiguous way, which is necessary if there are multiple derivations with the same name. - - If *args* are [store derivations](@docroot@/glossary.md#gloss-store-derivation), then these are [realised], and the resulting output paths are installed. +- If *args* are [store derivations](@docroot@/glossary.md#gloss-store-derivation), then these are [realised], and the resulting output paths are installed. - - If *args* are [store paths] that are not store derivations, then these are [realised] and installed. +- If *args* are [store paths] that are not store derivations, then these are [realised] and installed. - - By default all [outputs](@docroot@/language/derivations.md#attr-outputs) are installed for each [derivation]. - This can be overridden by adding a `meta.outputsToInstall` attribute on the derivation listing a subset of the output names. +- By default all [outputs](@docroot@/language/derivations.md#attr-outputs) are installed for each [derivation]. + This can be overridden by adding a `meta.outputsToInstall` attribute on the derivation listing a subset of the output names. - Example: + Example: - The file `example.nix` defines a derivation with two outputs `foo` and `bar`, each containing a file. + The file `example.nix` defines a derivation with two outputs `foo` and `bar`, each containing a file. - ```nix - # example.nix - let - pkgs = import {}; - command = '' - ${pkgs.coreutils}/bin/mkdir -p $foo $bar - echo foo > $foo/foo-file - echo bar > $bar/bar-file - ''; - in - derivation { - name = "example"; - builder = "${pkgs.bash}/bin/bash"; - args = [ "-c" command ]; - outputs = [ "foo" "bar" ]; - system = builtins.currentSystem; - } - ``` + ```nix + # example.nix + let + pkgs = import {}; + command = '' + ${pkgs.coreutils}/bin/mkdir -p $foo $bar + echo foo > $foo/foo-file + echo bar > $bar/bar-file + ''; + in + derivation { + name = "example"; + builder = "${pkgs.bash}/bin/bash"; + args = [ "-c" command ]; + outputs = [ "foo" "bar" ]; + system = builtins.currentSystem; + } + ``` - Installing from this Nix expression will make files from both outputs appear in the current profile. + Installing from this Nix expression will make files from both outputs appear in the current profile. - ```console - $ nix-env --install --file example.nix - installing 'example' - $ ls ~/.nix-profile - foo-file - bar-file - manifest.nix - ``` + ```console + $ nix-env --install --file example.nix + installing 'example' + $ ls ~/.nix-profile + foo-file + bar-file + manifest.nix + ``` - Adding `meta.outputsToInstall` to that derivation will make `nix-env` only install files from the specified outputs. + Adding `meta.outputsToInstall` to that derivation will make `nix-env` only install files from the specified outputs. - ```nix - # example-outputs.nix - import ./example.nix // { meta.outputsToInstall = [ "bar" ]; } - ``` + ```nix + # example-outputs.nix + import ./example.nix // { meta.outputsToInstall = [ "bar" ]; } + ``` - ```console - $ nix-env --install --file example-outputs.nix - installing 'example' - $ ls ~/.nix-profile - bar-file - manifest.nix - ``` + ```console + $ nix-env --install --file example-outputs.nix + installing 'example' + $ ls ~/.nix-profile + bar-file + manifest.nix + ``` # Options - - `--prebuilt-only` / `-b` +- `--prebuilt-only` / `-b` - Use only derivations for which a substitute is registered, i.e., - there is a pre-built binary available that can be downloaded in lieu - of building the derivation. Thus, no packages will be built from - source. + Use only derivations for which a substitute is registered, i.e., + there is a pre-built binary available that can be downloaded in lieu + of building the derivation. Thus, no packages will be built from + source. - - `--preserve-installed` / `-P` +- `--preserve-installed` / `-P` - Do not remove derivations with a name matching one of the - derivations being installed. Usually, trying to have two versions of - the same package installed in the same generation of a profile will - lead to an error in building the generation, due to file name - clashes between the two versions. However, this is not the case for - all packages. + Do not remove derivations with a name matching one of the + derivations being installed. Usually, trying to have two versions of + the same package installed in the same generation of a profile will + lead to an error in building the generation, due to file name + clashes between the two versions. However, this is not the case for + all packages. - - `--remove-all` / `-r` +- `--remove-all` / `-r` - Remove all previously installed packages first. This is equivalent - to running `nix-env --uninstall '.*'` first, except that everything happens - in a single transaction. + Remove all previously installed packages first. This is equivalent + to running `nix-env --uninstall '.*'` first, except that everything happens + in a single transaction. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-env/opt-common.md b/doc/manual/src/command-ref/nix-env/opt-common.md index 636281b6d..1479ca0bd 100644 --- a/doc/manual/src/command-ref/nix-env/opt-common.md +++ b/doc/manual/src/command-ref/nix-env/opt-common.md @@ -2,34 +2,37 @@ The following options are allowed for all `nix-env` operations, but may not always have an effect. - - `--file` / `-f` *path*\ - Specifies the Nix expression (designated below as the *active Nix - expression*) used by the `--install`, `--upgrade`, and `--query - --available` operations to obtain derivations. The default is - `~/.nix-defexpr`. +- `--file` / `-f` *path* - If the argument starts with `http://` or `https://`, it is - interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must include a single - top-level directory containing at least a file named `default.nix`. + Specifies the Nix expression (designated below as the *active Nix + expression*) used by the `--install`, `--upgrade`, and `--query + --available` operations to obtain derivations. The default is + `~/.nix-defexpr`. - - `--profile` / `-p` *path*\ - Specifies the profile to be used by those operations that operate on - a profile (designated below as the *active profile*). A profile is a - sequence of user environments called *generations*, one of which is - the *current generation*. + If the argument starts with `http://` or `https://`, it is + interpreted as the URL of a tarball that will be downloaded and + unpacked to a temporary location. The tarball must include a single + top-level directory containing at least a file named `default.nix`. - - `--dry-run`\ - For the `--install`, `--upgrade`, `--uninstall`, - `--switch-generation`, `--delete-generations` and `--rollback` - operations, this flag will cause `nix-env` to print what *would* be - done if this flag had not been specified, without actually doing it. +- `--profile` / `-p` *path* - `--dry-run` also prints out which paths will be - [substituted](@docroot@/glossary.md) (i.e., downloaded) and which paths - will be built from source (because no substitute is available). + Specifies the profile to be used by those operations that operate on + a profile (designated below as the *active profile*). A profile is a + sequence of user environments called *generations*, one of which is + the *current generation*. - - `--system-filter` *system*\ - By default, operations such as `--query - --available` show derivations matching any platform. This option - allows you to use derivations for the specified platform *system*. +- `--dry-run` + + For the `--install`, `--upgrade`, `--uninstall`, + `--switch-generation`, `--delete-generations` and `--rollback` + operations, this flag will cause `nix-env` to print what *would* be + done if this flag had not been specified, without actually doing it. + + `--dry-run` also prints out which paths will be + [substituted](@docroot@/glossary.md) (i.e., downloaded) and which paths + will be built from source (because no substitute is available). + +- `--system-filter` *system* + + By default, operations such as `--query --available` show derivations matching any platform. This option + allows you to use derivations for the specified platform *system*. diff --git a/doc/manual/src/command-ref/nix-env/query.md b/doc/manual/src/command-ref/nix-env/query.md index c9b4d8513..c67794ed5 100644 --- a/doc/manual/src/command-ref/nix-env/query.md +++ b/doc/manual/src/command-ref/nix-env/query.md @@ -35,11 +35,13 @@ The derivations are sorted by their `name` attributes. The following flags specify the set of things on which the query operates. - - `--installed`\ + - `--installed` + The query operates on the store paths that are installed in the current generation of the active profile. This is the default. - - `--available`; `-a`\ + - `--available` / `-a` + The query operates on the derivations that are available in the active Nix expression. @@ -50,24 +52,28 @@ selected derivations. Multiple flags may be specified, in which case the information is shown in the order given here. Note that the name of the derivation is shown unless `--no-name` is specified. - - `--xml`\ + - `--xml` + Print the result in an XML representation suitable for automatic processing by other tools. The root element is called `items`, which contains a `item` element for each available or installed derivation. The fields discussed below are all stored in attributes of the `item` elements. - - `--json`\ + - `--json` + Print the result in a JSON representation suitable for automatic processing by other tools. - - `--prebuilt-only` / `-b`\ + - `--prebuilt-only` / `-b` + Show only derivations for which a substitute is registered, i.e., there is a pre-built binary available that can be downloaded in lieu of building the derivation. Thus, this shows all packages that probably can be installed quickly. - - `--status`; `-s`\ + - `--status` / `-s` + Print the *status* of the derivation. The status consists of three characters. The first is `I` or `-`, indicating whether the derivation is currently installed in the current generation of the @@ -78,49 +84,61 @@ derivation is shown unless `--no-name` is specified. derivation to be built. The third is `S` or `-`, indicating whether a substitute is available for the derivation. - - `--attr-path`; `-P`\ + - `--attr-path` / `-P` + Print the *attribute path* of the derivation, which can be used to unambiguously select it using the `--attr` option available in commands that install derivations like `nix-env --install`. This option only works together with `--available` - - `--no-name`\ + - `--no-name` + Suppress printing of the `name` attribute of each derivation. - - `--compare-versions` / `-c`\ + - `--compare-versions` / `-c` + Compare installed versions to available versions, or vice versa (if `--available` is given). This is useful for quickly seeing whether upgrades for installed packages are available in a Nix expression. A column is added with the following meaning: - - `<` *version*\ + - `<` *version* + A newer version of the package is available or installed. - - `=` *version*\ + - `=` *version* + At most the same version of the package is available or installed. - - `>` *version*\ + - `>` *version* + Only older versions of the package are available or installed. - - `- ?`\ + - `- ?` + No version of the package is available or installed. - - `--system`\ + - `--system` + Print the `system` attribute of the derivation. - - `--drv-path`\ + - `--drv-path` + Print the path of the [store derivation](@docroot@/glossary.md#gloss-store-derivation). - - `--out-path`\ + - `--out-path` + Print the output path of the derivation. - - `--description`\ + - `--description` + Print a short (one-line) description of the derivation, if available. The description is taken from the `meta.description` attribute of the derivation. - - `--meta`\ + - `--meta` + Print all of the meta-attributes of the derivation. This option is only available with `--xml` or `--json`. diff --git a/doc/manual/src/command-ref/nix-env/set-flag.md b/doc/manual/src/command-ref/nix-env/set-flag.md index e04b22a91..58a0248bb 100644 --- a/doc/manual/src/command-ref/nix-env/set-flag.md +++ b/doc/manual/src/command-ref/nix-env/set-flag.md @@ -13,24 +13,24 @@ to be modified. There are several attributes that can be usefully modified, because they affect the behaviour of `nix-env` or the user environment build script: - - `priority` can be changed to resolve filename clashes. The user - environment build script uses the `meta.priority` attribute of - derivations to resolve filename collisions between packages. Lower - priority values denote a higher priority. For instance, the GCC - wrapper package and the Binutils package in Nixpkgs both have a file - `bin/ld`, so previously if you tried to install both you would get a - collision. Now, on the other hand, the GCC wrapper declares a higher - priority than Binutils, so the former’s `bin/ld` is symlinked in the - user environment. +- `priority` can be changed to resolve filename clashes. The user + environment build script uses the `meta.priority` attribute of + derivations to resolve filename collisions between packages. Lower + priority values denote a higher priority. For instance, the GCC + wrapper package and the Binutils package in Nixpkgs both have a file + `bin/ld`, so previously if you tried to install both you would get a + collision. Now, on the other hand, the GCC wrapper declares a higher + priority than Binutils, so the former’s `bin/ld` is symlinked in the + user environment. - - `keep` can be set to `true` to prevent the package from being - upgraded or replaced. This is useful if you want to hang on to an - older version of a package. +- `keep` can be set to `true` to prevent the package from being + upgraded or replaced. This is useful if you want to hang on to an + older version of a package. - - `active` can be set to `false` to “disable” the package. That is, no - symlinks will be generated to the files of the package, but it - remains part of the profile (so it won’t be garbage-collected). It - can be set back to `true` to re-enable the package. +- `active` can be set to `false` to “disable” the package. That is, no + symlinks will be generated to the files of the package, but it + remains part of the profile (so it won’t be garbage-collected). It + can be set back to `true` to re-enable the package. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-env/upgrade.md b/doc/manual/src/command-ref/nix-env/upgrade.md index 322dfbda2..2779363c3 100644 --- a/doc/manual/src/command-ref/nix-env/upgrade.md +++ b/doc/manual/src/command-ref/nix-env/upgrade.md @@ -28,42 +28,48 @@ version is installed. # Flags - - `--lt`\ - Only upgrade a derivation to newer versions. This is the default. +- `--lt` - - `--leq`\ - In addition to upgrading to newer versions, also “upgrade” to - derivations that have the same version. Version are not a unique - identification of a derivation, so there may be many derivations - that have the same version. This flag may be useful to force - “synchronisation” between the installed and available derivations. + Only upgrade a derivation to newer versions. This is the default. - - `--eq`\ - *Only* “upgrade” to derivations that have the same version. This may - not seem very useful, but it actually is, e.g., when there is a new - release of Nixpkgs and you want to replace installed applications - with the same versions built against newer dependencies (to reduce - the number of dependencies floating around on your system). +- `--leq` - - `--always`\ - In addition to upgrading to newer versions, also “upgrade” to - derivations that have the same or a lower version. I.e., derivations - may actually be downgraded depending on what is available in the - active Nix expression. + In addition to upgrading to newer versions, also “upgrade” to + derivations that have the same version. Version are not a unique + identification of a derivation, so there may be many derivations + that have the same version. This flag may be useful to force + “synchronisation” between the installed and available derivations. - - `--prebuilt-only` / `-b`\ - Use only derivations for which a substitute is registered, i.e., - there is a pre-built binary available that can be downloaded in lieu - of building the derivation. Thus, no packages will be built from - source. +- `--eq` - - `--preserve-installed` / `-P`\ - Do not remove derivations with a name matching one of the - derivations being installed. Usually, trying to have two versions of - the same package installed in the same generation of a profile will - lead to an error in building the generation, due to file name - clashes between the two versions. However, this is not the case for - all packages. + *Only* “upgrade” to derivations that have the same version. This may + not seem very useful, but it actually is, e.g., when there is a new + release of Nixpkgs and you want to replace installed applications + with the same versions built against newer dependencies (to reduce + the number of dependencies floating around on your system). + +- `--always` + + In addition to upgrading to newer versions, also “upgrade” to + derivations that have the same or a lower version. I.e., derivations + may actually be downgraded depending on what is available in the + active Nix expression. + +- `--prebuilt-only` / `-b` + + Use only derivations for which a substitute is registered, i.e., + there is a pre-built binary available that can be downloaded in lieu + of building the derivation. Thus, no packages will be built from + source. + +- `--preserve-installed` / `-P` + + Do not remove derivations with a name matching one of the + derivations being installed. Usually, trying to have two versions of + the same package installed in the same generation of a profile will + lead to an error in building the generation, due to file name + clashes between the two versions. However, this is not the case for + all packages. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-hash.md b/doc/manual/src/command-ref/nix-hash.md index 24e91df12..f249c2b84 100644 --- a/doc/manual/src/command-ref/nix-hash.md +++ b/doc/manual/src/command-ref/nix-hash.md @@ -29,54 +29,65 @@ md5sum`. # Options - - `--flat`\ - Print the cryptographic hash of the contents of each regular file *path*. - That is, instead of computing - the hash of the [Nix Archive (NAR)](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) of *path*, - just [directly hash]((@docroot@/store/file-system-object/content-address.md#serial-flat) *path* as is. - This requires *path* to resolve to a regular file rather than directory. - The result is identical to that produced by the GNU commands - `md5sum` and `sha1sum`. +- `--flat` - - `--base16`\ - Print the hash in a hexadecimal representation (default). + Print the cryptographic hash of the contents of each regular file *path*. + That is, instead of computing + the hash of the [Nix Archive (NAR)](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) of *path*, + just [directly hash]((@docroot@/store/file-system-object/content-address.md#serial-flat) *path* as is. + This requires *path* to resolve to a regular file rather than directory. + The result is identical to that produced by the GNU commands + `md5sum` and `sha1sum`. - - `--base32`\ - Print the hash in a base-32 representation rather than hexadecimal. - This base-32 representation is more compact and can be used in Nix - expressions (such as in calls to `fetchurl`). +- `--base16` - - `--base64`\ - Similar to --base32, but print the hash in a base-64 representation, - which is more compact than the base-32 one. + Print the hash in a hexadecimal representation (default). - - `--sri`\ - Print the hash in SRI format with base-64 encoding. - The type of hash algorithm will be prepended to the hash string, - followed by a hyphen (-) and the base-64 hash body. +- `--base32` - - `--truncate`\ - Truncate hashes longer than 160 bits (such as SHA-256) to 160 bits. + Print the hash in a base-32 representation rather than hexadecimal. + This base-32 representation is more compact and can be used in Nix + expressions (such as in calls to `fetchurl`). - - `--type` *hashAlgo*\ - Use the specified cryptographic hash algorithm, which can be one of - `md5`, `sha1`, `sha256`, and `sha512`. +- `--base64` - - `--to-base16`\ - Don’t hash anything, but convert the base-32 hash representation - *hash* to hexadecimal. + Similar to --base32, but print the hash in a base-64 representation, + which is more compact than the base-32 one. - - `--to-base32`\ - Don’t hash anything, but convert the hexadecimal hash representation - *hash* to base-32. +- `--sri` - - `--to-base64`\ - Don’t hash anything, but convert the hexadecimal hash representation - *hash* to base-64. + Print the hash in SRI format with base-64 encoding. + The type of hash algorithm will be prepended to the hash string, + followed by a hyphen (-) and the base-64 hash body. - - `--to-sri`\ - Don’t hash anything, but convert the hexadecimal hash representation - *hash* to SRI. +- `--truncate` + + Truncate hashes longer than 160 bits (such as SHA-256) to 160 bits. + +- `--type` *hashAlgo* + + Use the specified cryptographic hash algorithm, which can be one of + `md5`, `sha1`, `sha256`, and `sha512`. + +- `--to-base16` + + Don’t hash anything, but convert the base-32 hash representation + *hash* to hexadecimal. + +- `--to-base32` + + Don’t hash anything, but convert the hexadecimal hash representation + *hash* to base-32. + +- `--to-base64` + + Don’t hash anything, but convert the hexadecimal hash representation + *hash* to base-64. + +- `--to-sri` + + Don’t hash anything, but convert the hexadecimal hash representation + *hash* to SRI. # Examples diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md index dffbb2d70..6f6fcdc1f 100644 --- a/doc/manual/src/command-ref/nix-instantiate.md +++ b/doc/manual/src/command-ref/nix-instantiate.md @@ -30,89 +30,97 @@ standard input. # Options - - `--add-root` *path*\ - See the [corresponding option](nix-store.md) in `nix-store`. +- `--add-root` *path* - - `--parse`\ - Just parse the input files, and print their abstract syntax trees on - standard output as a Nix expression. + See the [corresponding option](nix-store.md) in `nix-store`. - - `--eval`\ - Just parse and evaluate the input files, and print the resulting - values on standard output. No instantiation of store derivations - takes place. +- `--parse` - > **Warning** - > - > This option produces output which can be parsed as a Nix expression which - > will produce a different result than the input expression when evaluated. - > For example, these two Nix expressions print the same result despite - > having different meaning: - > - > ```console - > $ nix-instantiate --eval --expr '{ a = {}; }' - > { a = ; } - > $ nix-instantiate --eval --expr '{ a = ; }' - > { a = ; } - > ``` - > - > For human-readable output, `nix eval` (experimental) is more informative: - > - > ```console - > $ nix-instantiate --eval --expr 'a: a' - > - > $ nix eval --expr 'a: a' - > «lambda @ «string»:1:1» - > ``` - > - > For machine-readable output, the `--xml` option produces unambiguous - > output: - > - > ```console - > $ nix-instantiate --eval --xml --expr '{ foo = ; }' - > - > - > - > - > - > - > - > - > ``` + Just parse the input files, and print their abstract syntax trees on + standard output as a Nix expression. - - `--find-file`\ - Look up the given files in Nix’s search path (as specified by the - `NIX_PATH` environment variable). If found, print the corresponding - absolute paths on standard output. For instance, if `NIX_PATH` is - `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file - nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. +- `--eval` - - `--strict`\ - When used with `--eval`, recursively evaluate list elements and - attributes. Normally, such sub-expressions are left unevaluated - (since the Nix language is lazy). + Just parse and evaluate the input files, and print the resulting + values on standard output. No instantiation of store derivations + takes place. - > **Warning** - > - > This option can cause non-termination, because lazy data - > structures can be infinitely large. + > **Warning** + > + > This option produces output which can be parsed as a Nix expression which + > will produce a different result than the input expression when evaluated. + > For example, these two Nix expressions print the same result despite + > having different meaning: + > + > ```console + > $ nix-instantiate --eval --expr '{ a = {}; }' + > { a = ; } + > $ nix-instantiate --eval --expr '{ a = ; }' + > { a = ; } + > ``` + > + > For human-readable output, `nix eval` (experimental) is more informative: + > + > ```console + > $ nix-instantiate --eval --expr 'a: a' + > + > $ nix eval --expr 'a: a' + > «lambda @ «string»:1:1» + > ``` + > + > For machine-readable output, the `--xml` option produces unambiguous + > output: + > + > ```console + > $ nix-instantiate --eval --xml --expr '{ foo = ; }' + > + > + > + > + > + > + > + > + > ``` - - `--json`\ - When used with `--eval`, print the resulting value as an JSON - representation of the abstract syntax tree rather than as a Nix expression. +- `--find-file` - - `--xml`\ - When used with `--eval`, print the resulting value as an XML - representation of the abstract syntax tree rather than as a Nix expression. - The schema is the same as that used by the [`toXML` - built-in](../language/builtins.md). + Look up the given files in Nix’s search path (as specified by the + `NIX_PATH` environment variable). If found, print the corresponding + absolute paths on standard output. For instance, if `NIX_PATH` is + `nixpkgs=/home/alice/nixpkgs`, then `nix-instantiate --find-file + nixpkgs/default.nix` will print `/home/alice/nixpkgs/default.nix`. - - `--read-write-mode`\ - When used with `--eval`, perform evaluation in read/write mode so - nix language features that require it will still work (at the cost - of needing to do instantiation of every evaluated derivation). If - this option is not enabled, there may be uninstantiated store paths - in the final output. +- `--strict` + + When used with `--eval`, recursively evaluate list elements and + attributes. Normally, such sub-expressions are left unevaluated + (since the Nix language is lazy). + + > **Warning** + > + > This option can cause non-termination, because lazy data + > structures can be infinitely large. + +- `--json` + + When used with `--eval`, print the resulting value as an JSON + representation of the abstract syntax tree rather than as a Nix expression. + +- `--xml` + + When used with `--eval`, print the resulting value as an XML + representation of the abstract syntax tree rather than as a Nix expression. + The schema is the same as that used by the [`toXML` + built-in](../language/builtins.md). + +- `--read-write-mode` + + When used with `--eval`, perform evaluation in read/write mode so + nix language features that require it will still work (at the cost + of needing to do instantiation of every evaluated derivation). If + this option is not enabled, there may be uninstantiated store paths + in the final output. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-prefetch-url.md b/doc/manual/src/command-ref/nix-prefetch-url.md index 45ef01e02..ffab94b8a 100644 --- a/doc/manual/src/command-ref/nix-prefetch-url.md +++ b/doc/manual/src/command-ref/nix-prefetch-url.md @@ -39,27 +39,32 @@ the path of the downloaded file in the Nix store is also printed. # Options - - `--type` *hashAlgo*\ - Use the specified cryptographic hash algorithm, - which can be one of `md5`, `sha1`, `sha256`, and `sha512`. - The default is `sha256`. +- `--type` *hashAlgo* - - `--print-path`\ - Print the store path of the downloaded file on standard output. + Use the specified cryptographic hash algorithm, + which can be one of `md5`, `sha1`, `sha256`, and `sha512`. + The default is `sha256`. - - `--unpack`\ - Unpack the archive (which must be a tarball or zip file) and add the - result to the Nix store. The resulting hash can be used with - functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. +- `--print-path` - - `--executable`\ - Set the executable bit on the downloaded file. + Print the store path of the downloaded file on standard output. - - `--name` *name*\ - Override the name of the file in the Nix store. By default, this is - `hash-basename`, where *basename* is the last component of *url*. - Overriding the name is necessary when *basename* contains characters - that are not allowed in Nix store paths. +- `--unpack` + + Unpack the archive (which must be a tarball or zip file) and add the + result to the Nix store. The resulting hash can be used with + functions such as Nixpkgs’s `fetchzip` or `fetchFromGitHub`. + +- `--executable` + + Set the executable bit on the downloaded file. + +- `--name` *name* + + Override the name of the file in the Nix store. By default, this is + `hash-basename`, where *basename* is the last component of *url*. + Overriding the name is necessary when *basename* contains characters + that are not allowed in Nix store paths. # Examples diff --git a/doc/manual/src/command-ref/nix-shell.md b/doc/manual/src/command-ref/nix-shell.md index 1eaf3c36a..69a711bd5 100644 --- a/doc/manual/src/command-ref/nix-shell.md +++ b/doc/manual/src/command-ref/nix-shell.md @@ -60,55 +60,63 @@ All options not listed here are passed to `nix-store --realise`, except for `--arg` and `--attr` / `-A` which are passed to `nix-instantiate`. - - `--command` *cmd*\ - In the environment of the derivation, run the shell command *cmd*. - This command is executed in an interactive shell. (Use `--run` to - use a non-interactive shell instead.) However, a call to `exit` is - implicitly added to the command, so the shell will exit after - running the command. To prevent this, add `return` at the end; - e.g. `--command "echo Hello; return"` will print `Hello` and then - drop you into the interactive shell. This can be useful for doing - any additional initialisation. +- `--command` *cmd* - - `--run` *cmd*\ - Like `--command`, but executes the command in a non-interactive - shell. This means (among other things) that if you hit Ctrl-C while - the command is running, the shell exits. + In the environment of the derivation, run the shell command *cmd*. + This command is executed in an interactive shell. (Use `--run` to + use a non-interactive shell instead.) However, a call to `exit` is + implicitly added to the command, so the shell will exit after + running the command. To prevent this, add `return` at the end; + e.g. `--command "echo Hello; return"` will print `Hello` and then + drop you into the interactive shell. This can be useful for doing + any additional initialisation. - - `--exclude` *regexp*\ - Do not build any dependencies whose store path matches the regular - expression *regexp*. This option may be specified multiple times. +- `--run` *cmd* - - `--pure`\ - If this flag is specified, the environment is almost entirely - cleared before the interactive shell is started, so you get an - environment that more closely corresponds to the “real” Nix build. A - few variables, in particular `HOME`, `USER` and `DISPLAY`, are - retained. + Like `--command`, but executes the command in a non-interactive + shell. This means (among other things) that if you hit Ctrl-C while + the command is running, the shell exits. - - `--packages` / `-p` *packages*…\ - Set up an environment in which the specified packages are present. - The command line arguments are interpreted as attribute names inside - the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` - will start a shell in which the packages denoted by the attribute - names `libjpeg` and `openjdk` are present. +- `--exclude` *regexp* - - `-i` *interpreter*\ - The chained script interpreter to be invoked by `nix-shell`. Only - applicable in `#!`-scripts (described below). + Do not build any dependencies whose store path matches the regular + expression *regexp*. This option may be specified multiple times. - - `--keep` *name*\ - When a `--pure` shell is started, keep the listed environment - variables. +- `--pure` + + If this flag is specified, the environment is almost entirely + cleared before the interactive shell is started, so you get an + environment that more closely corresponds to the “real” Nix build. A + few variables, in particular `HOME`, `USER` and `DISPLAY`, are + retained. + +- `--packages` / `-p` *packages*… + + Set up an environment in which the specified packages are present. + The command line arguments are interpreted as attribute names inside + the Nix Packages collection. Thus, `nix-shell --packages libjpeg openjdk` + will start a shell in which the packages denoted by the attribute + names `libjpeg` and `openjdk` are present. + +- `-i` *interpreter* + + The chained script interpreter to be invoked by `nix-shell`. Only + applicable in `#!`-scripts (described below). + +- `--keep` *name* + + When a `--pure` shell is started, keep the listed environment + variables. {{#include ./opt-common.md}} # Environment variables - - `NIX_BUILD_SHELL`\ - Shell used to start the interactive environment. Defaults to the - `bash` found in ``, falling back to the `bash` found in - `PATH` if not found. +- `NIX_BUILD_SHELL` + + Shell used to start the interactive environment. Defaults to the + `bash` found in ``, falling back to the `bash` found in + `PATH` if not found. {{#include ./env-common.md}} @@ -202,14 +210,14 @@ For example, here is a Python script that depends on Python and the ```python #! /usr/bin/env nix-shell -#! nix-shell -i python --packages python pythonPackages.prettytable +#! nix-shell -i python3 --packages python3 python3Packages.prettytable import prettytable # Print a simple table. t = prettytable.PrettyTable(["N", "N^2"]) for n in range(1, 10): t.add_row([n, n * n]) -print t +print(t) ``` Similarly, the following is a Perl script that specifies that it @@ -289,3 +297,8 @@ with import {}; runCommand "dummy" { buildInputs = [ python pythonPackages.prettytable ]; } "" ``` + +The script's file name is passed as the first argument to the interpreter specified by the `-i` flag. + +Aside from the very first line, which is a directive to the operating system, the additional `#! nix-shell` lines do not need to be at the beginning of the file. +This allows wrapping them in block comments for languages where `#` does not start a comment, such as ECMAScript, Erlang, PHP, or Ruby. diff --git a/doc/manual/src/command-ref/nix-store/add-fixed.md b/doc/manual/src/command-ref/nix-store/add-fixed.md index d25db091c..bebf15026 100644 --- a/doc/manual/src/command-ref/nix-store/add-fixed.md +++ b/doc/manual/src/command-ref/nix-store/add-fixed.md @@ -16,9 +16,10 @@ public url or broke since the download expression was written. This operation has the following options: - - `--recursive`\ - Use recursive instead of flat hashing mode, used when adding - directories to the store. +- `--recursive` + + Use recursive instead of flat hashing mode, used when adding + directories to the store. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/dump.md b/doc/manual/src/command-ref/nix-store/dump.md index b1066fd4c..3de0e27b0 100644 --- a/doc/manual/src/command-ref/nix-store/dump.md +++ b/doc/manual/src/command-ref/nix-store/dump.md @@ -8,7 +8,7 @@ ## Description -The operation `--dump` produces a [NAR (Nix ARchive)][Nix Archive] file containing the +The operation `--dump` produces a [Nix archive](@docroot@/glossary.md#gloss-nar) (NAR) file containing the contents of the file system tree rooted at *path*. The archive is written to standard output. @@ -30,8 +30,7 @@ NAR archives support filenames of unlimited length and 64-bit file sizes. They can contain regular files, directories, and symbolic links, but not other types of files (such as device nodes). -A Nix archive can be unpacked using `nix-store ---restore`. +A Nix archive can be unpacked using [`nix-store --restore`](@docroot@/command-ref/nix-store/restore.md). [Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/command-ref/nix-store/export.md b/doc/manual/src/command-ref/nix-store/export.md index 09f876865..ba772eb43 100644 --- a/doc/manual/src/command-ref/nix-store/export.md +++ b/doc/manual/src/command-ref/nix-store/export.md @@ -8,16 +8,20 @@ ## Description -The operation `--export` writes a serialisation of the specified store -paths to standard output in a format that can be imported into another -Nix store with `nix-store --import`. This is like `nix-store ---dump`, except that the [Nix Archive (NAR)][Nix Archive] produced by that command doesn’t -contain the necessary meta-information to allow it to be imported into -another Nix store (namely, the set of references of the path). +The operation `--export` writes a serialisation of the given [store objects](@docroot@/glossary.md#gloss-store-object) to standard output in a format that can be imported into another [Nix store](@docroot@/store/index.md) with [`nix-store --import`](./import.md). -This command does not produce a *closure* of the specified paths, so if -a store path references other store paths that are missing in the target -Nix store, the import will fail. +> **Warning** +> +> This command *does not* produce a [closure](@docroot@/glossary.md#gloss-closure) of the specified store paths. +> Trying to import a store object that refers to store paths not available in the target Nix store will fail. +> +> Use [`nix-store --query`](@docroot@/command-ref/nix-store/query.md) to obtain the closure of a store path. + +This command is different from [`nix-store --dump`](./dump.md), which produces a [Nix archive](@docroot@/glossary.md#gloss-nar) that *does not* contain the set of [references](@docroot@/glossary.md#gloss-reference) of a given store path. + +> **Note** +> +> For efficient transfer of closures to remote machines over SSH, use [`nix-copy-closure`](@docroot@/command-ref/nix-copy-closure.md). [Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive @@ -29,15 +33,21 @@ Nix store, the import will fail. # Examples -To copy a whole closure, do something -like: - -```console -$ nix-store --export $(nix-store --query --requisites paths) > out -``` - -To import the whole closure again, run: - -```console -$ nix-store --import < out -``` +> **Example** +> +> Deploy GNU Hello to an airgapped machine via USB stick. +> +> Write the closure to the block device on a machine with internet connection: +> +> ```shell-session +> [alice@itchy]$ storePath=$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link) +> [alice@itchy]$ nix-store --export $(nix-store --query --requisites $storePath) | sudo dd of=/dev/usb +> ``` +> +> Read the closure from the block device on the machine without internet connection: +> +> ```shell-session +> [bob@scratchy]$ hello=$(sudo dd if=/dev/usb | nix-store --import | tail -1) +> [bob@scratchy]$ $hello/bin/hello +> Hello, world! +> ``` diff --git a/doc/manual/src/command-ref/nix-store/gc.md b/doc/manual/src/command-ref/nix-store/gc.md index 7be0d559a..f432e00eb 100644 --- a/doc/manual/src/command-ref/nix-store/gc.md +++ b/doc/manual/src/command-ref/nix-store/gc.md @@ -14,30 +14,34 @@ reachable via file system references from a set of “roots”, are deleted. The following suboperations may be specified: - - `--print-roots`\ - This operation prints on standard output the set of roots used by - the garbage collector. +- `--print-roots` - - `--print-live`\ - This operation prints on standard output the set of “live” store - paths, which are all the store paths reachable from the roots. Live - paths should never be deleted, since that would break consistency — - it would become possible that applications are installed that - reference things that are no longer present in the store. + This operation prints on standard output the set of roots used by + the garbage collector. - - `--print-dead`\ - This operation prints out on standard output the set of “dead” store - paths, which is just the opposite of the set of live paths: any path - in the store that is not live (with respect to the roots) is dead. +- `--print-live` + + This operation prints on standard output the set of “live” store + paths, which are all the store paths reachable from the roots. Live + paths should never be deleted, since that would break consistency — + it would become possible that applications are installed that + reference things that are no longer present in the store. + +- `--print-dead` + + This operation prints out on standard output the set of “dead” store + paths, which is just the opposite of the set of live paths: any path + in the store that is not live (with respect to the roots) is dead. By default, all unreachable paths are deleted. The following options control what gets deleted and in what order: - - `--max-freed` *bytes*\ - Keep deleting paths until at least *bytes* bytes have been deleted, - then stop. The argument *bytes* can be followed by the - multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB - or TiB units. +- `--max-freed` *bytes* + + Keep deleting paths until at least *bytes* bytes have been deleted, + then stop. The argument *bytes* can be followed by the + multiplicative suffix `K`, `M`, `G` or `T`, denoting KiB, MiB, GiB + or TiB units. The behaviour of the collector is also influenced by the `keep-outputs` and `keep-derivations` settings in the Nix diff --git a/doc/manual/src/command-ref/nix-store/import.md b/doc/manual/src/command-ref/nix-store/import.md index 7c3ff9fc8..3f6b3d076 100644 --- a/doc/manual/src/command-ref/nix-store/import.md +++ b/doc/manual/src/command-ref/nix-store/import.md @@ -23,3 +23,21 @@ If a path [refers](@docroot@/glossary.md#gloss-reference) to another path that d {{#include ../opt-common.md}} {{#include ../env-common.md}} + +# Examples + +> **Example** +> +> Given a closure of GNU Hello as a file: +> +> ```shell-session +> $ storePath="$(nix-build '' -I nixpkgs=channel:nixpkgs-unstable -A hello --no-out-link)" +> $ nix-store --export $(nix-store --query --requisites $storePath) > hello.closure +> ``` +> +> Import the closure into a [remote SSH store](@docroot@/store/types/ssh-store.md) using the [`--store`](@docroot@/command-ref/conf-file.md#conf-store) option: +> +> ```console +> $ nix-store --import --store ssh://alice@itchy.example.org < hello.closure +> ``` + diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md index 0bcacfe0c..601f46af6 100644 --- a/doc/manual/src/command-ref/nix-store/query.md +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -24,122 +24,138 @@ symlink. # Common query options - - `--use-output`; `-u`\ - For each argument to the query that is a [store derivation], apply the - query to the output path of the derivation instead. +- `--use-output` / `-u` - - `--force-realise`; `-f`\ - Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). + For each argument to the query that is a [store derivation], apply the + query to the output path of the derivation instead. + +- `--force-realise` / `-f` + + Realise each argument to the query first (see [`nix-store --realise`](./realise.md)). [store derivation]: @docroot@/glossary.md#gloss-store-derivation # Queries - - `--outputs`\ - Prints out the [output paths] of the store - derivations *paths*. These are the paths that will be produced when - the derivation is built. +- `--outputs` - [output paths]: @docroot@/glossary.md#gloss-output-path + Prints out the [output paths] of the store + derivations *paths*. These are the paths that will be produced when + the derivation is built. - - `--requisites`; `-R`\ - Prints out the [closure] of the store path *paths*. + [output paths]: @docroot@/glossary.md#gloss-output-path - [closure]: @docroot@/glossary.md#gloss-closure +- `--requisites` / `-R` - This query has one option: + Prints out the [closure] of the store path *paths*. - - `--include-outputs` - Also include the existing output paths of [store derivation]s, - and their closures. + [closure]: @docroot@/glossary.md#gloss-closure - This query can be used to implement various kinds of deployment. A - *source deployment* is obtained by distributing the closure of a - store derivation. A *binary deployment* is obtained by distributing - the closure of an output path. A *cache deployment* (combined - source/binary deployment, including binaries of build-time-only - dependencies) is obtained by distributing the closure of a store - derivation and specifying the option `--include-outputs`. + This query has one option: - - `--references`\ - Prints the set of [references] of the store paths - *paths*, that is, their immediate dependencies. (For *all* - dependencies, use `--requisites`.) + - `--include-outputs` + Also include the existing output paths of [store derivation]s, + and their closures. - [references]: @docroot@/glossary.md#gloss-reference + This query can be used to implement various kinds of deployment. A + *source deployment* is obtained by distributing the closure of a + store derivation. A *binary deployment* is obtained by distributing + the closure of an output path. A *cache deployment* (combined + source/binary deployment, including binaries of build-time-only + dependencies) is obtained by distributing the closure of a store + derivation and specifying the option `--include-outputs`. - - `--referrers`\ - Prints the set of *referrers* of the store paths *paths*, that is, - the store paths currently existing in the Nix store that refer to - one of *paths*. Note that contrary to the references, the set of - referrers is not constant; it can change as store paths are added or - removed. +- `--references` - - `--referrers-closure`\ - Prints the closure of the set of store paths *paths* under the - referrers relation; that is, all store paths that directly or - indirectly refer to one of *paths*. These are all the path currently - in the Nix store that are dependent on *paths*. + Prints the set of [references] of the store paths + *paths*, that is, their immediate dependencies. (For *all* + dependencies, use `--requisites`.) - - `--deriver`; `-d`\ - Prints the [deriver] that was used to build the store paths *paths*. If - the path has no deriver (e.g., if it is a source file), or if the - deriver is not known (e.g., in the case of a binary-only - deployment), the string `unknown-deriver` is printed. - The returned deriver is not guaranteed to exist in the local store, for - example when *paths* were substituted from a binary cache. - Use `--valid-derivers` instead to obtain valid paths only. + [references]: @docroot@/glossary.md#gloss-reference - [deriver]: @docroot@/glossary.md#gloss-deriver +- `--referrers` - - `--valid-derivers`\ - Prints a set of derivation files (`.drv`) which are supposed produce - said paths when realized. Might print nothing, for example for source paths - or paths subsituted from a binary cache. + Prints the set of *referrers* of the store paths *paths*, that is, + the store paths currently existing in the Nix store that refer to + one of *paths*. Note that contrary to the references, the set of + referrers is not constant; it can change as store paths are added or + removed. - - `--graph`\ - Prints the references graph of the store paths *paths* in the format - of the `dot` tool of AT\&T's [Graphviz - package](http://www.graphviz.org/). This can be used to visualise - dependency graphs. To obtain a build-time dependency graph, apply - this to a store derivation. To obtain a runtime dependency graph, - apply it to an output path. +- `--referrers-closure` - - `--tree`\ - Prints the references graph of the store paths *paths* as a nested - ASCII tree. References are ordered by descending closure size; this - tends to flatten the tree, making it more readable. The query only - recurses into a store path when it is first encountered; this - prevents a blowup of the tree representation of the graph. + Prints the closure of the set of store paths *paths* under the + referrers relation; that is, all store paths that directly or + indirectly refer to one of *paths*. These are all the path currently + in the Nix store that are dependent on *paths*. - - `--graphml`\ - Prints the references graph of the store paths *paths* in the - [GraphML](http://graphml.graphdrawing.org/) file format. This can be - used to visualise dependency graphs. To obtain a build-time - dependency graph, apply this to a [store derivation]. To obtain a - runtime dependency graph, apply it to an output path. +- `--deriver` / `-d` - - `--binding` *name*; `-b` *name*\ - Prints the value of the attribute *name* (i.e., environment - variable) of the [store derivation]s *paths*. It is an error for a - derivation to not have the specified attribute. + Prints the [deriver] that was used to build the store paths *paths*. If + the path has no deriver (e.g., if it is a source file), or if the + deriver is not known (e.g., in the case of a binary-only + deployment), the string `unknown-deriver` is printed. + The returned deriver is not guaranteed to exist in the local store, for + example when *paths* were substituted from a binary cache. + Use `--valid-derivers` instead to obtain valid paths only. - - `--hash`\ - Prints the SHA-256 hash of the contents of the store paths *paths* - (that is, the hash of the output of `nix-store --dump` on the given - paths). Since the hash is stored in the Nix database, this is a fast - operation. + [deriver]: @docroot@/glossary.md#gloss-deriver - - `--size`\ - Prints the size in bytes of the contents of the store paths *paths* - — to be precise, the size of the output of `nix-store --dump` on - the given paths. Note that the actual disk space required by the - store paths may be higher, especially on filesystems with large - cluster sizes. +- `--valid-derivers` - - `--roots`\ - Prints the garbage collector roots that point, directly or - indirectly, at the store paths *paths*. + Prints a set of derivation files (`.drv`) which are supposed produce + said paths when realized. Might print nothing, for example for source paths + or paths substituted from a binary cache. + +- `--graph` + + Prints the references graph of the store paths *paths* in the format + of the `dot` tool of AT\&T's [Graphviz + package](http://www.graphviz.org/). This can be used to visualise + dependency graphs. To obtain a build-time dependency graph, apply + this to a store derivation. To obtain a runtime dependency graph, + apply it to an output path. + +- `--tree` + + Prints the references graph of the store paths *paths* as a nested + ASCII tree. References are ordered by descending closure size; this + tends to flatten the tree, making it more readable. The query only + recurses into a store path when it is first encountered; this + prevents a blowup of the tree representation of the graph. + +- `--graphml` + + Prints the references graph of the store paths *paths* in the + [GraphML](http://graphml.graphdrawing.org/) file format. This can be + used to visualise dependency graphs. To obtain a build-time + dependency graph, apply this to a [store derivation]. To obtain a + runtime dependency graph, apply it to an output path. + +- `--binding` *name* / `-b` *name* + + Prints the value of the attribute *name* (i.e., environment + variable) of the [store derivation]s *paths*. It is an error for a + derivation to not have the specified attribute. + +- `--hash` + + Prints the SHA-256 hash of the contents of the store paths *paths* + (that is, the hash of the output of `nix-store --dump` on the given + paths). Since the hash is stored in the Nix database, this is a fast + operation. + +- `--size` + + Prints the size in bytes of the contents of the store paths *paths* + — to be precise, the size of the output of `nix-store --dump` on + the given paths. Note that the actual disk space required by the + store paths may be higher, especially on filesystems with large + cluster sizes. + +- `--roots` + + Prints the garbage collector roots that point, directly or + indirectly, at the store paths *paths*. {{#include ./opt-common.md}} @@ -225,4 +241,3 @@ $ nix-store --query --roots $(which svn) /nix/var/nix/profiles/default-82-link /home/eelco/.local/state/nix/profiles/profile-97-link ``` - diff --git a/doc/manual/src/command-ref/nix-store/realise.md b/doc/manual/src/command-ref/nix-store/realise.md index 6e56387eb..a899758df 100644 --- a/doc/manual/src/command-ref/nix-store/realise.md +++ b/doc/manual/src/command-ref/nix-store/realise.md @@ -32,7 +32,7 @@ If no substitutes are available and no store derivation is given, realisation fa [store objects]: @docroot@/store/store-object.md [closure]: @docroot@/glossary.md#gloss-closure [substituters]: @docroot@/command-ref/conf-file.md#conf-substituters -[content-addressed derivations]: @docroot@/contributing/experimental-features.md#xp-feature-ca-derivations +[content-addressed derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations [Nix database]: @docroot@/glossary.md#gloss-nix-database The resulting paths are printed on standard output. @@ -42,23 +42,26 @@ For non-derivation arguments, the argument itself is printed. # Options - - `--dry-run`\ - Print on standard error a description of what packages would be - built or downloaded, without actually performing the operation. +- `--dry-run` - - `--ignore-unknown`\ - If a non-derivation path does not have a substitute, then silently - ignore it. + Print on standard error a description of what packages would be + built or downloaded, without actually performing the operation. - - `--check`\ - This option allows you to check whether a derivation is - deterministic. It rebuilds the specified derivation and checks - whether the result is bitwise-identical with the existing outputs, - printing an error if that’s not the case. The outputs of the - specified derivation must already exist. When used with `-K`, if an - output path is not identical to the corresponding output from the - previous build, the new output path is left in - `/nix/store/name.check.` +- `--ignore-unknown` + + If a non-derivation path does not have a substitute, then silently + ignore it. + +- `--check` + + This option allows you to check whether a derivation is + deterministic. It rebuilds the specified derivation and checks + whether the result is bitwise-identical with the existing outputs, + printing an error if that’s not the case. The outputs of the + specified derivation must already exist. When used with `-K`, if an + output path is not identical to the corresponding output from the + previous build, the new output path is left in + `/nix/store/name.check.` {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/serve.md b/doc/manual/src/command-ref/nix-store/serve.md index 0f90f65ae..9a4cf5216 100644 --- a/doc/manual/src/command-ref/nix-store/serve.md +++ b/doc/manual/src/command-ref/nix-store/serve.md @@ -14,10 +14,11 @@ access to a restricted ssh user. The following flags are available: - - `--write`\ - Allow the connected client to request the realization of - derivations. In effect, this can be used to make the host act as a - remote builder. +- `--write` + + Allow the connected client to request the realization of + derivations. In effect, this can be used to make the host act as a + remote builder. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/nix-store/verify.md b/doc/manual/src/command-ref/nix-store/verify.md index 2695b3361..40c9180db 100644 --- a/doc/manual/src/command-ref/nix-store/verify.md +++ b/doc/manual/src/command-ref/nix-store/verify.md @@ -16,18 +16,20 @@ being modified by non-Nix tools, or of bugs in Nix itself. This operation has the following options: - - `--check-contents`\ - Checks that the contents of every valid store path has not been - altered by computing a SHA-256 hash of the contents and comparing it - with the hash stored in the Nix database at build time. Paths that - have been modified are printed out. For large stores, - `--check-contents` is obviously quite slow. +- `--check-contents` - - `--repair`\ - If any valid path is missing from the store, or (if - `--check-contents` is given) the contents of a valid path has been - modified, then try to repair the path by redownloading it. See - `nix-store --repair-path` for details. + Checks that the contents of every valid store path has not been + altered by computing a SHA-256 hash of the contents and comparing it + with the hash stored in the Nix database at build time. Paths that + have been modified are printed out. For large stores, + `--check-contents` is obviously quite slow. + +- `--repair` + + If any valid path is missing from the store, or (if + `--check-contents` is given) the contents of a valid path has been + modified, then try to repair the path by redownloading it. See + `nix-store --repair-path` for details. {{#include ./opt-common.md}} diff --git a/doc/manual/src/command-ref/opt-common.md b/doc/manual/src/command-ref/opt-common.md index 114b292f9..70ae03959 100644 --- a/doc/manual/src/command-ref/opt-common.md +++ b/doc/manual/src/command-ref/opt-common.md @@ -1,3 +1,7 @@ + + # Common Options Most Nix commands accept the following command-line options: @@ -37,7 +41,7 @@ Most Nix commands accept the following command-line options: Print even more informational messages. - `4` “Debug” - + Print debug information. - `5` “Vomit” @@ -143,7 +147,7 @@ Most Nix commands accept the following command-line options: This option is accepted by `nix-env`, `nix-instantiate`, `nix-shell` and `nix-build`. When evaluating Nix expressions, the expression evaluator will automatically try to call functions that it encounters. - It can automatically call functions for which every argument has a [default value](@docroot@/language/constructs.md#functions) (e.g., `{ argName ? defaultValue }: ...`). + It can automatically call functions for which every argument has a [default value](@docroot@/language/syntax.md#functions) (e.g., `{ argName ? defaultValue }: ...`). With `--arg`, you can also call functions that have arguments without a default value (or override a default value). That is, if the evaluator encounters a function with an argument named *name*, it will call it with value *value*. @@ -161,6 +165,14 @@ Most Nix commands accept the following command-line options: You can override this using `--arg`, e.g., `nix-env --install --attr pkgname --arg system \"i686-freebsd\"`. (Note that since the argument is a Nix string literal, you have to escape the quotes.) +- [`--arg-from-file`](#opt-arg-from-file) *name* *path* + + Pass the contents of file *path* as the argument *name* to Nix functions. + +- [`--arg-from-stdin`](#opt-arg-from-stdin) *name* + + Pass the contents of stdin as the argument *name* to Nix functions. + - [`--argstr`](#opt-argstr) *name* *value* This option is like `--arg`, only the value is not a Nix expression but a string. @@ -179,6 +191,10 @@ Most Nix commands accept the following command-line options: attribute of the fourth element of the array in the `foo` attribute of the top-level expression. +- [`--eval-store`](#opt-eval-store) *store-url* + + The [URL to the Nix store](@docroot@/store/types/index.md#store-url-format) to use for evaluation, i.e. where to store derivations (`.drv` files) and inputs referenced by them. + - [`--expr`](#opt-expr) / `-E` Interpret the command line arguments as a list of Nix expressions to be parsed and evaluated, rather than as a list of file names of Nix expressions. @@ -187,11 +203,16 @@ Most Nix commands accept the following command-line options: For `nix-shell`, this option is commonly used to give you a shell in which you can build the packages returned by the expression. If you want to get a shell which contain the *built* packages ready for use, give your expression to the `nix-shell --packages ` convenience flag instead. -- [`-I`](#opt-I) *path* +- [`-I` / `--include`](#opt-I) *path* - Add an entry to the [Nix expression search path](@docroot@/command-ref/conf-file.md#conf-nix-path). + Add an entry to the list of search paths used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). This option may be given multiple times. - Paths added through `-I` take precedence over [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH). + + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). + +- [`--impure`](#opt-impure) + + Allow access to mutable paths and repositories. - [`--option`](#opt-option) *name* *value* diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/development/building.md similarity index 57% rename from doc/manual/src/contributing/hacking.md rename to doc/manual/src/development/building.md index 08ba84faa..dbf080296 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/development/building.md @@ -1,24 +1,72 @@ -# Hacking +# Building Nix -This section provides some notes on how to hack on Nix. To get the -latest version of Nix from GitHub: +This section provides some notes on how to start hacking on Nix. +To get the latest version of Nix from GitHub: ```console $ git clone https://github.com/NixOS/nix.git $ cd nix ``` -The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. If you don't have it installed, follow the [installation instructions]. +> **Note** +> +> The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. +> If you don't have it installed, follow the [installation instructions](../installation/index.md). -[installation instructions]: ../installation/index.md + +To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: + +```console +$ nix-shell +``` + +To get a shell with one of the other [supported compilation environments](#compilation-environments): + +```console +$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages +``` + +> **Note** +> +> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. +> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. + +To build Nix itself in this shell: + +```console +[nix-shell]$ mesonFlags+=" --prefix=$(pwd)/outputs/out" +[nix-shell]$ dontAddPrefix=1 mesonConfigurePhase +[nix-shell]$ ninjaBuildPhase +``` + +To test it: + +```console +[nix-shell]$ mesonCheckPhase +``` + +To install it in `$(pwd)/outputs`: + +```console +[nix-shell]$ ninjaInstallPhase +[nix-shell]$ ./outputs/out/bin/nix --version +nix (Nix) 2.12 +``` + +To build a release version of Nix for the current operating system and CPU architecture: + +```console +$ nix-build +``` + +You can also build Nix for one of the [supported platforms](#platforms). ## Building Nix with flakes This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. -See the [Building Nix](#building-nix) section for equivalent instructions using stable Nix interfaces. -[`flakes`]: @docroot@/contributing/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/contributing/experimental-features.md#xp-nix-command +[`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes +[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: @@ -42,16 +90,20 @@ $ nix develop .#native-clangStdenvPackages To build Nix itself in this shell: ```console -[nix-shell]$ autoreconfPhase -[nix-shell]$ configurePhase -[nix-shell]$ make -j $NIX_BUILD_CORES OPTIMIZE=0 +[nix-shell]$ mesonConfigurePhase +[nix-shell]$ ninjaBuildPhase ``` -To install it in `$(pwd)/outputs` and test it: +To test it: ```console -[nix-shell]$ make install OPTIMIZE=0 -[nix-shell]$ make installcheck check -j $NIX_BUILD_CORES +[nix-shell]$ mesonCheckPhase +``` + +To install it in `$(pwd)/outputs`: + +```console +[nix-shell]$ ninjaInstallPhase [nix-shell]$ nix --version nix (Nix) 2.12 ``` @@ -67,70 +119,6 @@ $ nix build You can also build Nix for one of the [supported platforms](#platforms). -## Building Nix - -To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -To get a shell with one of the other [supported compilation environments](#compilation-environments): - -```console -$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages -``` - -> **Note** -> -> You can use `native-ccacheStdenvPackages` to drastically improve rebuild time. -> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. - -To build Nix itself in this shell: - -```console -[nix-shell]$ autoreconfPhase -[nix-shell]$ ./configure $configureFlags --prefix=$(pwd)/outputs/out -[nix-shell]$ make -j $NIX_BUILD_CORES -``` - -To install it in `$(pwd)/outputs` and test it: - -```console -[nix-shell]$ make install -[nix-shell]$ make installcheck -j $NIX_BUILD_CORES -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 2.12 -``` - -To build a release version of Nix for the current operating system and CPU architecture: - -```console -$ nix-build -``` - -You can also build Nix for one of the [supported platforms](#platforms). - -## Makefile variables - -You may need `profiledir=$out/etc/profile.d` and `sysconfdir=$out/etc` to run `make install`. - -Run `make` with [`-e` / `--environment-overrides`](https://www.gnu.org/software/make/manual/make.html#index-_002de) to allow environment variables to override `Makefile` variables: - -- `ENABLE_BUILD=yes` to enable building the C++ code. -- `ENABLE_DOC_GEN=yes` to enable building the documentation (manual, man pages, etc.). - - The docs can take a while to build, so you may want to disable this for local development. -- `ENABLE_FUNCTIONAL_TESTS=yes` to enable building the functional tests. -- `ENABLE_UNIT_TESTS=yes` to enable building the unit tests. -- `OPTIMIZE=1` to enable optimizations. -- `libraries=libutil programs=` to only build a specific library. - - This will fail in the linking phase if the other libraries haven't been built, but is useful for checking types. -- `libraries= programs=nix` to only build a specific program. - - This will not work in general, because the programs need the libraries. - ## Platforms Nix can be built for various platforms, as specified in [`flake.nix`]: @@ -177,27 +165,38 @@ Add more [system types](#system-type) to `crossSystems` in `flake.nix` to bootst It is useful to perform multiple cross and native builds on the same source tree, for example to ensure that better support for one platform doesn't break the build for another. -In order to facilitate this, Nix has some support for being built out of tree – that is, placing build artefacts in a different directory than the source code: +Meson thankfully makes this very easy by confining all build products to the build directory --- one simple shares the source directory between multiple build directories, each of which contains the build for Nix to a different platform. -1. Create a directory for the build, e.g. +Nixpkgs's `mesonConfigurePhase` always chooses `build` in the current directory as the name and location of the build. +This makes having multiple build directories slightly more inconvenient. +The good news is that Meson/Ninja seem to cope well with relocating the build directory after it is created. + +Here's how to do that + +1. Configure as usual ```bash - mkdir build + mesonConfigurePhase ``` -2. Run the configure script from that directory, e.g. +2. Rename the build directory ```bash - cd build - ../configure + cd .. # since `mesonConfigurePhase` cd'd inside + mv build build-linux # or whatever name we want + cd build-linux ``` -3. Run make from the source directory, but with the build directory specified, e.g. +3. Build as usual ```bash - make builddir=build + ninjaBuildPhase ``` +> **N.B.** +> [`nixpkgs#335818`](https://github.com/NixOS/nixpkgs/issues/335818) tracks giving `mesonConfigurePhase` proper support for custom build directories. +> When it is fixed, we can simplify these instructions and then remove this notice. + ## System type Nix uses a string with the following format to identify the *system type* or *platform* it runs on: @@ -259,11 +258,8 @@ You can use any of the other supported environments in place of `nix-ccacheStden The `clangd` LSP server is installed by default on the `clang`-based `devShell`s. See [supported compilation environments](#compilation-environments) and instructions how to set up a shell [with flakes](#nix-with-flakes) or in [classic Nix](#classic-nix). -To use the LSP with your editor, you first need to [set up `clangd`](https://clangd.llvm.org/installation#project-setup) by running: - -```console -make compile_commands.json -``` +To use the LSP with your editor, you will want a `compile_commands.json` file telling `clangd` how we are compiling the code. +Meson's configure always produces this inside the build directory. Configure your editor to use the `clangd` from the `.#native-clangStdenvPackages` shell. You can do that either by running it inside the development shell, or by using [nix-direnv](https://github.com/nix-community/nix-direnv) and [the appropriate editor plugin](https://github.com/direnv/direnv/wiki#editor-integration). @@ -278,7 +274,7 @@ Configure your editor to use the `clangd` from the `.#native-clangStdenvPackages You may run the formatters as a one-off using: ```console -make format +./maintainers/format.sh ``` If you'd like to run the formatters before every commit, install the hooks: @@ -295,81 +291,3 @@ If it fails, run `git add --patch` to approve the suggestions _and commit again_ To refresh pre-commit hook's config file, do the following: 1. Exit the development shell and start it again by running `nix develop`. 2. If you also use the pre-commit hook, also run `pre-commit-hooks-install` again. - -## Add a release note - -`doc/manual/rl-next` contains release notes entries for all unreleased changes. - -User-visible changes should come with a release note. - -### Add an entry - -Here's what a complete entry looks like. The file name is not incorporated in the document. - -``` ---- -synopsis: Basically a title -issues: 1234 -prs: 1238 ---- - -Here's one or more paragraphs that describe the change. - -- It's markdown -- Add references to the manual using @docroot@ -``` - -Significant changes should add the following header, which moves them to the top. - -``` -significance: significant -``` - - -See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). - -### Build process - -Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. - -## Branches - -- [`master`](https://github.com/NixOS/nix/commits/master) - - The main development branch. All changes are approved and merged here. - When developing a change, create a branch based on the latest `master`. - - Maintainers try to [keep it in a release-worthy state](#reverting). - -- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) - - These branches are the subject of backports only, and are - also [kept](#reverting) in a release-worthy state. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) - - The latest patch release of the latest minor version. - - See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) - -- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) - - Generally branches created by the backport action. - - See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) - -- [_other_](https://github.com/NixOS/nix/branches/all) - - Branches that do not conform to the above patterns should be feature branches. - -## Reverting - -If a change turns out to be merged by mistake, or contain a regression, it may be reverted. -A revert is not a rejection of the contribution, but merely part of an effective development process. -It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. -If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. -By embracing reverts as a good part of the development process, everyone wins. - -However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cli-guideline.md b/doc/manual/src/development/cli-guideline.md similarity index 88% rename from doc/manual/src/contributing/cli-guideline.md rename to doc/manual/src/development/cli-guideline.md index e90d6de8d..23df844ec 100644 --- a/doc/manual/src/contributing/cli-guideline.md +++ b/doc/manual/src/development/cli-guideline.md @@ -389,88 +389,6 @@ colors, no emojis and using ASCII instead of Unicode symbols). The same should happen when TTY is not detected on STDERR. We should not display progress / status section, but only print warnings and errors. -## Returning future proof JSON - -The schema of JSON output should allow for backwards compatible extension. This section explains how to achieve this. - -Two definitions are helpful here, because while JSON only defines one "key-value" -object type, we use it to cover two use cases: - - - **dictionary**: a map from names to value that all have the same type. In - C++ this would be a `std::map` with string keys. - - **record**: a fixed set of attributes each with their own type. In C++, this - would be represented by a `struct`. - -It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type. - -This leads to the following guidelines: - - - The top-level (root) value must be a record. - - Otherwise, one can not change the structure of a command's output. - - - The value of a dictionary item must be a record. - - Otherwise, the item type can not be extended. - - - List items should be records. - - Otherwise, one can not change the structure of the list items. - - If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. If the order of the items needs to be preserved, return a list of records. - - - Streaming JSON should return records. - - An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. These JSON values can be considered top-level values or list items, and they must be records. - -### Examples - - -This is bad, because all keys must be assumed to be store types: - -```json -{ - "local": { ... }, - "remote": { ... }, - "http": { ... } -} -``` - -This is good, because the it is extensible at the root, and is somewhat self-documenting: - -```json -{ - "storeTypes": { "local": { ... }, ... }, - "pluginSupport": true -} -``` - -While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. -For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. - - - -The following representation is bad because it is not extensible: - -```json -{ "outputs": [ "out" "bin" ] } -``` - -However, simply converting everything to records is not enough, because the order of outputs must be preserved: - -```json -{ "outputs": { "bin": {}, "out": {} } } -``` - -The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. -While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. - -This representation is extensible and preserves the ordering: - -```json -{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } -``` - ## Dialog with the user CLIs don't always make it clear when an action has taken place. For every diff --git a/doc/manual/src/development/contributing.md b/doc/manual/src/development/contributing.md new file mode 100644 index 000000000..7de7489dc --- /dev/null +++ b/doc/manual/src/development/contributing.md @@ -0,0 +1,79 @@ +# Contributing + +## Add a release note + +`doc/manual/rl-next` contains release notes entries for all unreleased changes. + +User-visible changes should come with a release note. + +### Add an entry + +Here's what a complete entry looks like. The file name is not incorporated in the document. + +``` +--- +synopsis: Basically a title +issues: 1234 +prs: 1238 +--- + +Here's one or more paragraphs that describe the change. + +- It's markdown +- Add references to the manual using @docroot@ +``` + +Significant changes should add the following header, which moves them to the top. + +``` +significance: significant +``` + + +See also the [format documentation](https://github.com/haskell/cabal/blob/master/CONTRIBUTING.md#changelog). + +### Build process + +Releases have a precomputed `rl-MAJOR.MINOR.md`, and no `rl-next.md`. + +## Branches + +- [`master`](https://github.com/NixOS/nix/commits/master) + + The main development branch. All changes are approved and merged here. + When developing a change, create a branch based on the latest `master`. + + Maintainers try to [keep it in a release-worthy state](#reverting). + +- [`maintenance-*.*`](https://github.com/NixOS/nix/branches/all?query=maintenance) + + These branches are the subject of backports only, and are + also [kept](#reverting) in a release-worthy state. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [`latest-release`](https://github.com/NixOS/nix/tree/latest-release) + + The latest patch release of the latest minor version. + + See [`maintainers/release-process.md`](https://github.com/NixOS/nix/blob/master/maintainers/release-process.md) + +- [`backport-*-to-*`](https://github.com/NixOS/nix/branches/all?query=backport) + + Generally branches created by the backport action. + + See [`maintainers/backporting.md`](https://github.com/NixOS/nix/blob/master/maintainers/backporting.md) + +- [_other_](https://github.com/NixOS/nix/branches/all) + + Branches that do not conform to the above patterns should be feature branches. + +## Reverting + +If a change turns out to be merged by mistake, or contain a regression, it may be reverted. +A revert is not a rejection of the contribution, but merely part of an effective development process. +It makes sure that development keeps running smoothly, with minimal uncertainty, and less overhead. +If maintainers have to worry too much about avoiding reverts, they would not be able to merge as much. +By embracing reverts as a good part of the development process, everyone wins. + +However, taking a step back may be frustrating, so maintainers will be extra supportive on the next try. diff --git a/doc/manual/src/contributing/cxx.md b/doc/manual/src/development/cxx.md similarity index 100% rename from doc/manual/src/contributing/cxx.md rename to doc/manual/src/development/cxx.md diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/development/documentation.md similarity index 93% rename from doc/manual/src/contributing/documentation.md rename to doc/manual/src/development/documentation.md index 6e7c0a967..d5a95e0c1 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/development/documentation.md @@ -13,18 +13,18 @@ Incremental refactorings of the documentation build setup to make it faster or e Build the manual from scratch: ```console -nix-build $(nix-instantiate)'!doc' +nix-build -E '(import ./.).packages.${builtins.currentSystem}.nix.doc' ``` or ```console -nix build .#^doc +nix build .#nix^doc ``` and open `./result-doc/share/doc/nix/manual/index.html`. -To build the manual incrementally, [enter the development shell](./hacking.md) and run: +To build the manual incrementally, [enter the development shell](./building.md) and run: ```console make manual-html-open -j $NIX_BUILD_CORES @@ -196,15 +196,16 @@ You can also build and view it yourself: [Doxygen API documentation]: https://hydra.nixos.org/job/nix/master/internal-api-docs/latest/download-by-type/doc/internal-api-docs ```console -# nix build .#hydraJobs.internal-api-docs -# xdg-open ./result/share/doc/nix/internal-api/html/index.html +$ nix build .#hydraJobs.internal-api-docs +$ xdg-open ./result/share/doc/nix/internal-api/html/index.html ``` or inside `nix-shell` or `nix develop`: -``` -# make internal-api-html -# xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html +```console +$ mesonConfigurePhase +$ ninja src/internal-api-docs/html +$ xdg-open src/internal-api-docs/html/index.html ``` ## C API documentation @@ -216,13 +217,14 @@ You can also build and view it yourself: [C API documentation]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs ```console -# nix build .#hydraJobs.external-api-docs -# xdg-open ./result/share/doc/nix/external-api/html/index.html +$ nix build .#hydraJobs.external-api-docs +$ xdg-open ./result/share/doc/nix/external-api/html/index.html ``` or inside `nix-shell` or `nix develop`: ``` -# make external-api-html -# xdg-open ./outputs/doc/share/doc/nix/external-api/html/index.html +$ mesonConfigurePhase +$ ninja src/external-api-docs/html +$ xdg-open src/external-api-docs/html/index.html ``` diff --git a/doc/manual/src/contributing/experimental-features.md b/doc/manual/src/development/experimental-features.md similarity index 100% rename from doc/manual/src/contributing/experimental-features.md rename to doc/manual/src/development/experimental-features.md diff --git a/doc/manual/src/contributing/index.md b/doc/manual/src/development/index.md similarity index 77% rename from doc/manual/src/contributing/index.md rename to doc/manual/src/development/index.md index 4d55c17a4..6403c3e66 100644 --- a/doc/manual/src/contributing/index.md +++ b/doc/manual/src/development/index.md @@ -5,4 +5,4 @@ Check the [contributing guide](https://github.com/NixOS/nix/blob/master/CONTRIBU This chapter is a collection of guides for making changes to the code and documentation. -If you're not sure where to start, try to [compile Nix from source](./hacking.md) and consider [making improvements to documentation](./documentation.md). +If you're not sure where to start, try to [compile Nix from source](./building.md) and consider [making improvements to documentation](./documentation.md). diff --git a/doc/manual/src/development/json-guideline.md b/doc/manual/src/development/json-guideline.md new file mode 100644 index 000000000..309b4b3a0 --- /dev/null +++ b/doc/manual/src/development/json-guideline.md @@ -0,0 +1,128 @@ +# JSON guideline + +Nix consumes and produces JSON in a variety of contexts. +These guidelines ensure consistent practices for all our JSON interfaces, for ease of use, and so that experience in one part carries over to another. + +## Extensibility + +The schema of JSON input and output should allow for backwards compatible extension. +This section explains how to achieve this. + +Two definitions are helpful here, because while JSON only defines one "key-value" object type, we use it to cover two use cases: + + - **dictionary**: a map from names to value that all have the same type. + In C++ this would be a `std::map` with string keys. + + - **record**: a fixed set of attributes each with their own type. + In C++, this would be represented by a `struct`. + +It is best not to mix these use cases, as that may lead to incompatibilities when the schema changes. +For example, adding a record field to a dictionary breaks consumers that assume all JSON object fields to have the same meaning and type, and dictionary items with a colliding name can not be represented anymore. + +This leads to the following guidelines: + + - The top-level (root) value must be a record. + + Otherwise, one can not change the structure of a command's output. + + - The value of a dictionary item must be a record. + + Otherwise, the item type can not be extended. + + - List items should be records. + + Otherwise, one can not change the structure of the list items. + + If the order of the items does not matter, and each item has a unique key that is a string, consider representing the list as a dictionary instead. + If the order of the items needs to be preserved, return a list of records. + + - Streaming JSON should return records. + + An example of a streaming JSON format is [JSON lines](https://jsonlines.org/), where each line represents a JSON value. + These JSON values can be considered top-level values or list items, and they must be records. + +### Examples + +This is bad, because all keys must be assumed to be store types: + +```json +{ + "local": { ... }, + "remote": { ... }, + "http": { ... } +} +``` + +This is good, because the it is extensible at the root, and is somewhat self-documenting: + +```json +{ + "storeTypes": { "local": { ... }, ... }, + "pluginSupport": true +} +``` + +While the dictionary of store types seems like a very complete response at first, a use case may arise that warrants returning additional information. +For example, the presence of plugin support may be crucial information for a client to proceed when their desired store type is missing. + + + +The following representation is bad because it is not extensible: + +```json +{ "outputs": [ "out" "bin" ] } +``` + +However, simply converting everything to records is not enough, because the order of outputs must be preserved: + +```json +{ "outputs": { "bin": {}, "out": {} } } +``` + +The first item is the default output. Deriving this information from the outputs ordering is not great, but this is how Nix currently happens to work. +While it is possible for a JSON parser to preserve the order of fields, we can not rely on this capability to be present in all JSON libraries. + +This representation is extensible and preserves the ordering: + +```json +{ "outputs": [ { "outputName": "out" }, { "outputName": "bin" } ] } +``` + +## Self-describing values + +As described in the previous section, it's crucial that schemas can be extended with new fields without breaking compatibility. +However, that should *not* mean we use the presence/absence of fields to indicate optional information *within* a version of the schema. +Instead, always include the field, and use `null` to indicate the "nothing" case. + +### Examples + +Here are two JSON objects: + +```json +{ + "foo": {} +} +``` +```json +{ + "foo": {}, + "bar": {} +} +``` + +Since they differ in which fields they contain, they should *not* both be valid values of the same schema. +At most, they can match two different schemas where the second (with `foo` and `bar`) is considered a newer version of the first (with just `foo`). +Within each version, all fields are mandatory (always `foo`, and always `foo` and `bar`). +Only *between* each version, `bar` gets added as a new mandatory field. + +Here are another two JSON objects: + +```json +{ "foo": null } +``` +```json +{ "foo": { "bar": 1 } } +``` + +Since they both contain a `foo` field, they could be valid values of the same schema. +The schema would have `foo` has an optional field, which is either `null` or an object where `bar` is an integer. diff --git a/doc/manual/src/development/meson.build b/doc/manual/src/development/meson.build new file mode 100644 index 000000000..5ffbfe394 --- /dev/null +++ b/doc/manual/src/development/meson.build @@ -0,0 +1,12 @@ +experimental_feature_descriptions_md = custom_target( + command : nix_eval_for_docs + [ + '--expr', + 'import @INPUT0@ (builtins.fromJSON (builtins.readFile @INPUT1@))', + ], + input : [ + '../../generate-xp-features.nix', + xp_features_json, + ], + capture : true, + output : 'experimental-feature-descriptions.md', +) diff --git a/doc/manual/src/contributing/testing.md b/doc/manual/src/development/testing.md similarity index 76% rename from doc/manual/src/contributing/testing.md rename to doc/manual/src/development/testing.md index 607914ba3..0df72cc38 100644 --- a/doc/manual/src/contributing/testing.md +++ b/doc/manual/src/development/testing.md @@ -59,15 +59,15 @@ The unit tests are defined using the [googletest] and [rapidcheck] frameworks. > … > ``` -The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `tests/unit/${library_name_without-nix}`. -Given a interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `tests/unit/libexpr/tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `tests/unit/libexpr-support/tests/value/context.{hh,cc}`. +The tests for each Nix library (`libnixexpr`, `libnixstore`, etc..) live inside a directory `src/${library_name_without-nix}-test`. +Given an interface (header) and implementation pair in the original library, say, `src/libexpr/value/context.{hh,cc}`, we write tests for it in `src/nix-expr-tests/value/context.cc`, and (possibly) declare/define additional interfaces for testing purposes in `src/nix-expr-test-support/tests/value/context.{hh,cc}`. Data for unit tests is stored in a `data` subdir of the directory for each unit test executable. -For example, `libnixstore` code is in `src/libstore`, and its test data is in `tests/unit/libstore/data`. -The path to the `tests/unit/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. +For example, `libnixstore` code is in `src/libstore`, and its test data is in `src/nix-store-tests/data`. +The path to the `src/${library_name_without-nix}-test/data` directory is passed to the unit test executable with the environment variable `_NIX_TEST_UNIT_DATA`. Note that each executable only gets the data for its tests. -The unit test libraries are in `tests/unit/${library_name_without-nix}-lib`. +The unit test libraries are in `src/${library_name_without-nix}-test-support`. All headers are in a `tests` subdirectory so they are included with `#include "tests/"`. The use of all these separate directories for the unit tests might seem inconvenient, as for example the tests are not "right next to" the part of the code they are testing. @@ -76,8 +76,25 @@ there is no risk of any build-system wildcards for the library accidentally pick ### Running tests -You can run the whole testsuite with `make check`, or the tests for a specific component with `make libfoo-tests_RUN`. -Finer-grained filtering is also possible using the [--gtest_filter](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) command-line option, or the `GTEST_FILTER` environment variable, e.g. `GTEST_FILTER='ErrorTraceTest.*' make check`. +You can run the whole testsuite with `meson test` from the Meson build directory, or the tests for a specific component with `meson test nix-store-tests`. +A environment variables that Google Test accepts are also worth knowing: + +1. [`GTEST_FILTER`](https://google.github.io/googletest/advanced.html#running-a-subset-of-the-tests) + + This is used for finer-grained filtering of which tests to run. + + +2. [`GTEST_BRIEF`](https://google.github.io/googletest/advanced.html#suppressing-test-passes) + + This is used to avoid logging passing tests. + +Putting the two together, one might run + +```bash +GTEST_BRIEF=1 GTEST_FILTER='ErrorTraceTest.*' meson test nix-expr-tests -v +``` + +for short but comprensive output. ### Characterisation testing { #characaterisation-testing-unit } @@ -86,7 +103,7 @@ See [functional characterisation testing](#characterisation-testing-functional) Like with the functional characterisation, `_NIX_TEST_ACCEPT=1` is also used. For example: ```shell-session -$ _NIX_TEST_ACCEPT=1 make libstore-tests_RUN +$ _NIX_TEST_ACCEPT=1 meson test nix-store-tests -v ... [ SKIPPED ] WorkerProtoTest.string_read [ SKIPPED ] WorkerProtoTest.string_write @@ -114,62 +131,60 @@ On other platforms they wouldn't be run at all. The functional tests reside under the `tests/functional` directory and are listed in `tests/functional/local.mk`. Each test is a bash script. +Functional tests are run during `installCheck` in the `nix` package build, as well as separately from the build, in VM tests. + ### Running the whole test suite -The whole test suite can be run with: +The whole test suite (functional and unit tests) can be run with: ```shell-session -$ make install && make installcheck -ran test tests/functional/foo.sh... [PASS] -ran test tests/functional/bar.sh... [PASS] -... +$ mesonCheckPhase ``` ### Grouping tests Sometimes it is useful to group related tests so they can be easily run together without running the entire test suite. Each test group is in a subdirectory of `tests`. -For example, `tests/functional/ca/local.mk` defines a `ca` test group for content-addressed derivation outputs. +For example, `tests/functional/ca/meson.build` defines a `ca` test group for content-addressed derivation outputs. That test group can be run like this: ```shell-session -$ make ca.test-group -j50 -ran test tests/functional/ca/nix-run.sh... [PASS] -ran test tests/functional/ca/import-derivation.sh... [PASS] -... -``` - -The test group is defined in Make like this: -```makefile -$(test-group-name)-tests := \ - $(d)/test0.sh \ - $(d)/test1.sh \ - ... - -install-tests-groups += $(test-group-name) +$ meson test --suite ca +ninja: Entering directory `/home/jcericson/src/nix/master/build' +ninja: no work to do. +[1-20/20] 🌑 nix-functional-tests:ca / ca/why-depends 1/20 nix-functional-tests:ca / ca/nix-run OK 0.16s +[2-20/20] 🌒 nix-functional-tests:ca / ca/why-depends 2/20 nix-functional-tests:ca / ca/import-derivation OK 0.17s ``` ### Running individual tests -Individual tests can be run with `make`: +Individual tests can be run with `meson`: ```shell-session -$ make tests/functional/${testName}.sh.test -ran test tests/functional/${testName}.sh... [PASS] +$ meson test --verbose ${testName} +ninja: Entering directory `/home/jcericson/src/nix/master/build' +ninja: no work to do. +1/1 nix-functional-tests:main / ${testName} OK 0.41s + +Ok: 1 +Expected Fail: 0 +Fail: 0 +Unexpected Pass: 0 +Skipped: 0 +Timeout: 0 + +Full log written to /home/jcericson/src/nix/master/build/meson-logs/testlog.txt ``` -or without `make`: +The `--verbose` flag will make Meson also show the console output of each test for easier debugging. +The test script will then be traced with `set -x` and the output displayed as it happens, +regardless of whether the test succeeds or fails. + +Tests can be also run directly without `meson`: ```shell-session -$ ./mk/run-test.sh tests/functional/${testName}.sh -ran test tests/functional/${testName}.sh... [PASS] -``` - -To see the complete output, one can also run: - -```shell-session -$ ./mk/debug-test.sh tests/functional/${testName}.sh +$ TEST_NAME=${testName} NIX_REMOTE='' PS4='+(${BASH_SOURCE[0]-$0}:$LINENO) tests/functional/${testName}.sh +(${testName}.sh:1) foo output from foo +(${testName}.sh:2) bar @@ -177,8 +192,6 @@ output from bar ... ``` -The test script will then be traced with `set -x` and the output displayed as it happens, regardless of whether the test succeeds or fails. - ### Debugging failing functional tests When a functional test fails, it usually does so somewhere in the middle of the script. @@ -235,7 +248,7 @@ It is frequently useful to regenerate the expected output. To do that, rerun the failed test(s) with `_NIX_TEST_ACCEPT=1`. For example: ```bash -_NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test +_NIX_TEST_ACCEPT=1 meson test lang ``` This convention is shared with the [characterisation unit tests](#characterisation-testing-unit) too. @@ -252,13 +265,26 @@ Regressions are caught, and improvements always show up in code review. To ensure that characterisation testing doesn't make it harder to intentionally change these interfaces, there always must be an easy way to regenerate the expected output, as we do with `_NIX_TEST_ACCEPT=1`. +### Running functional tests on NixOS + +We run the functional tests not just in the build, but also in VM tests. +This helps us ensure that Nix works correctly on NixOS, and environments that have similar characteristics that are hard to reproduce in a build environment. + +These can be run with: + +```shell +nix build .#hydraJobs.tests.functional_user +``` + +Generally, this build is sufficient, but in nightly or CI we also test the attributes `functional_root` and `functional_trusted`, in which the test suite is run with different levels of authorization. + ## Integration tests The integration tests are defined in the Nix flake under the `hydraJobs.tests` attribute. These tests include everything that needs to interact with external services or run Nix in a non-trivial distributed setup. Because these tests are expensive and require more than what the standard github-actions setup provides, they only run on the master branch (on ). -You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}` +You can run them manually with `nix build .#hydraJobs.tests.{testName}` or `nix-build -A hydraJobs.tests.{testName}`. ## Installer tests diff --git a/doc/manual/src/glossary.md b/doc/manual/src/glossary.md index 0f3af42fd..6c0e32a43 100644 --- a/doc/manual/src/glossary.md +++ b/doc/manual/src/glossary.md @@ -12,7 +12,7 @@ For how Nix uses content addresses, see: - [Content-Addressing File System Objects](@docroot@/store/file-system-object/content-address.md) - - [content-addressed store object](#gloss-content-addressed-store-object) + - [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) - [content-addressed derivation](#gloss-content-addressed-derivation) Software Heritage's writing on [*Intrinsic and Extrinsic identifiers*](https://www.softwareheritage.org/2020/07/09/intrinsic-vs-extrinsic-identifiers) is also a good introduction to the value of content-addressing over other referencing schemes. @@ -71,10 +71,9 @@ [`__contentAddressed`](./language/advanced-attributes.md#adv-attr-__contentAddressed) attribute set to `true`. -- [fixed-output derivation]{#gloss-fixed-output-derivation} +- [fixed-output derivation]{#gloss-fixed-output-derivation} (FOD) - A derivation which includes the - [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute. + A [derivation] where a cryptographic hash of the [output] is determined in advance using the [`outputHash`](./language/advanced-attributes.md#adv-attr-outputHash) attribute, and where the [`builder`](@docroot@/language/derivations.md#attr-builder) executable has access to the network. - [store]{#gloss-store} @@ -120,7 +119,7 @@ A store object consists of a [file system object], [references][reference] to other store objects, and other metadata. It can be referred to by a [store path]. - See [Store Object](@docroot@/store/index.md#store-object) for details. + See [Store Object](@docroot@/store/store-object.md) for details. [store object]: #gloss-store-object @@ -137,9 +136,12 @@ - [content-addressed store object]{#gloss-content-addressed-store-object} - A [store object] whose [store path] is determined by its contents. + A [store object] which is [content-addressed](#gloss-content-address), + i.e. whose [store path] is determined by its contents. This includes derivations, the outputs of [content-addressed derivations](#gloss-content-addressed-derivation), and the outputs of [fixed-output derivations](#gloss-fixed-output-derivation). + See [Content-Addressing Store Objects](@docroot@/store/store-object/content-address.md) for details. + - [substitute]{#gloss-substitute} A substitute is a command invocation stored in the [Nix database] that @@ -166,7 +168,7 @@ - [impure derivation]{#gloss-impure-derivation} - [An experimental feature](#@docroot@/contributing/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, + [An experimental feature](#@docroot@/development/experimental-features.md#xp-feature-impure-derivations) that allows derivations to be explicitly marked as impure, so that they are always rebuilt, and their outputs not reused by subsequent calls to realise them. - [Nix database]{#gloss-nix-database} @@ -180,13 +182,18 @@ - [Nix expression]{#gloss-nix-expression} - 1. Commonly, a high-level description of software packages and compositions - thereof. Deploying software using Nix entails writing Nix - expressions for your packages. Nix expressions specify [derivations][derivation], - which are [instantiated][instantiate] into the Nix store as [store derivations][store derivation]. - These derivations can then be [realised][realise] to produce [outputs][output]. + A syntactically valid use of the [Nix language]. - 2. A syntactically valid use of the [Nix language]. For example, the contents of a `.nix` file form an expression. + > **Example** + > + > The contents of a `.nix` file form a Nix expression. + + Nix expressions specify [derivations][derivation], which are [instantiated][instantiate] into the Nix store as [store derivations][store derivation]. + These derivations can then be [realised][realise] to produce [outputs][output]. + + > **Example** + > + > Building and deploying software using Nix entails writing Nix expressions as a high-level description of packages and compositions thereof. - [reference]{#gloss-reference} @@ -310,7 +317,7 @@ - [package attribute set]{#package-attribute-set} - An [attribute set](@docroot@/language/values.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as + An [attribute set](@docroot@/language/types.md#attribute-set) containing the attribute `type = "derivation";` (derivation for historical reasons), as well as other attributes, such as - attributes that refer to the files of a [package], typically in the form of [derivation outputs](#output), - attributes that declare something about how the package is supposed to be installed or used, - other metadata or arbitrary attributes. @@ -323,9 +330,9 @@ See [String interpolation](./language/string-interpolation.md) for details. - [string]: ./language/values.md#type-string - [path]: ./language/values.md#type-path - [attribute name]: ./language/values.md#attribute-set + [string]: ./language/types.md#type-string + [path]: ./language/types.md#type-path + [attribute name]: ./language/types.md#attribute-set - [base directory]{#gloss-base-directory} @@ -351,7 +358,7 @@ Not yet stabilized functionality guarded by named experimental feature flags. These flags are enabled or disabled with the [`experimental-features`](./command-ref/conf-file.html#conf-experimental-features) setting. - See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/contributing/experimental-features.md). + See the contribution guide on the [purpose and lifecycle of experimental feaures](@docroot@/development/experimental-features.md). [Nix language]: ./language/index.md diff --git a/doc/manual/src/installation/building-source.md b/doc/manual/src/installation/building-source.md index 7dad9805a..d35cc18c2 100644 --- a/doc/manual/src/installation/building-source.md +++ b/doc/manual/src/installation/building-source.md @@ -1,31 +1,26 @@ # Building Nix from Source -After cloning Nix's Git repository, issue the following commands: +Nix is built with [Meson](https://mesonbuild.com/). +It is broken up into multiple Meson packages, which are optionally combined in a single project using Meson's [subprojects](https://mesonbuild.com/Subprojects.html) feature. -```console -$ autoreconf -vfi -$ ./configure options... -$ make -$ make install -``` +There are no mandatory extra steps to the building process: +generic Meson installation instructions like [this](https://mesonbuild.com/Quick-guide.html#using-meson-as-a-distro-packager) should work. -Nix requires GNU Make so you may need to invoke `gmake` instead. - -The installation path can be specified by passing the `--prefix=prefix` +The installation path can be specified by passing the `-Dprefix=prefix` to `configure`. The default installation directory is `/usr/local`. You can change this to any location you like. You must have write permission to the *prefix* path. Nix keeps its *store* (the place where packages are stored) in `/nix/store` by default. This can be changed using -`--with-store-dir=path`. +`-Dstore-dir=path`. > **Warning** -> +> > It is best *not* to change the Nix store from its default, since doing > so makes it impossible to use pre-built binaries from the standard > Nixpkgs channels — that is, all packages will need to be built from > source. Nix keeps state (such as its database and log files) in `/nix/var` by -default. This can be changed using `--localstatedir=path`. +default. This can be changed using `-Dlocalstatedir=path`. diff --git a/doc/manual/src/installation/index.md b/doc/manual/src/installation/index.md index dafdeb667..48725c1ba 100644 --- a/doc/manual/src/installation/index.md +++ b/doc/manual/src/installation/index.md @@ -14,8 +14,16 @@ This option requires either: * Linux running systemd, with SELinux disabled * MacOS +> **Updating to macOS 15 Sequoia** +> +> If you recently updated to macOS 15 Sequoia and are getting +> ```console +> error: the user '_nixbld1' in the group 'nixbld' does not exist +> ``` +> when running Nix commands, refer to GitHub issue [NixOS/nix#10892](https://github.com/NixOS/nix/issues/10892) for instructions to fix your installation without reinstalling. + ```console -$ bash <(curl -L https://nixos.org/nix/install) --daemon +$ curl -L https://nixos.org/nix/install | sh -s -- --daemon ``` ## Single-user @@ -28,7 +36,7 @@ cannot offer equivalent sharing, isolation, or security. This option is suitable for systems without systemd. ```console -$ bash <(curl -L https://nixos.org/nix/install) --no-daemon +$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon ``` ## Distributions diff --git a/doc/manual/src/installation/installing-binary.md b/doc/manual/src/installation/installing-binary.md index 385008d8c..6a1a5ddca 100644 --- a/doc/manual/src/installation/installing-binary.md +++ b/doc/manual/src/installation/installing-binary.md @@ -1,5 +1,13 @@ # Installing a Binary Distribution +> **Updating to macOS 15 Sequoia** +> +> If you recently updated to macOS 15 Sequoia and are getting +> ```console +> error: the user '_nixbld1' in the group 'nixbld' does not exist +> ``` +> when running Nix commands, refer to GitHub issue [NixOS/nix#10892](https://github.com/NixOS/nix/issues/10892) for instructions to fix your installation without reinstalling. + To install the latest version Nix, run the following command: ```console @@ -77,7 +85,7 @@ $ su root # Installing from a binary tarball You can also download a binary tarball that contains Nix and all its dependencies: -- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../contributing/hacking.md#platforms) +- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../development/building.md#platforms) - Download and unpack the tarball - Run the installer diff --git a/doc/manual/src/installation/prerequisites-source.md b/doc/manual/src/installation/prerequisites-source.md index 4aafa6d27..c346a0a4b 100644 --- a/doc/manual/src/installation/prerequisites-source.md +++ b/doc/manual/src/installation/prerequisites-source.md @@ -39,8 +39,6 @@ `pkgconfig` and the Boehm garbage collector, and pass the flag `--enable-gc` to `configure`. - For `bdw-gc` <= 8.2.4 Nix needs a [small patch](https://github.com/NixOS/nix/blob/ac4d2e7b857acdfeac35ac8a592bdecee2d29838/boehmgc-traceable_allocator-public.diff) to be applied. - - The `boost` library of version 1.66.0 or higher. It can be obtained from the official web site . diff --git a/doc/manual/src/installation/uninstall.md b/doc/manual/src/installation/uninstall.md index 590327fea..47689a16e 100644 --- a/doc/manual/src/installation/uninstall.md +++ b/doc/manual/src/installation/uninstall.md @@ -19,7 +19,7 @@ If you are on Linux with systemd: Remove files created by Nix: ```console -sudo rm -rf /etc/nix /etc/profile.d/nix.sh /etc/tmpfiles.d/nix-daemon.conf /nix ~root/.nix-channels ~root/.nix-defexpr ~root/.nix-profile +sudo rm -rf /etc/nix /etc/profile.d/nix.sh /etc/tmpfiles.d/nix-daemon.conf /nix ~root/.nix-channels ~root/.nix-defexpr ~root/.nix-profile ~root/.cache/nix ``` Remove build users and their group: @@ -43,6 +43,14 @@ which you may remove. ### macOS +> **Updating to macOS 15 Sequoia** +> +> If you recently updated to macOS 15 Sequoia and are getting +> ```console +> error: the user '_nixbld1' in the group 'nixbld' does not exist +> ``` +> when running Nix commands, refer to GitHub issue [NixOS/nix#10892](https://github.com/NixOS/nix/issues/10892) for instructions to fix your installation without reinstalling. + 1. If system-wide shell initialisation files haven't been altered since installing Nix, use the backups made by the installer: ```console @@ -92,7 +100,7 @@ which you may remove. LABEL=Nix\040Store /nix apfs rw,nobrowse ``` - by setting the cursor on the respective line using the error keys, and pressing `dd`, and then `:wq` to save the file. + by setting the cursor on the respective line using the arrow keys, and pressing `dd`, and then `:wq` to save the file. This will prevent automatic mounting of the Nix Store volume. @@ -133,7 +141,9 @@ which you may remove. diskutil list ``` - If you _do_ find a "Nix Store" volume, delete it by running `diskutil deleteVolume` with the store volume's `diskXsY` identifier. + If you _do_ find a "Nix Store" volume, delete it by running `diskutil apfs deleteVolume` with the store volume's `diskXsY` identifier. + + If you get an error that the volume is in use by the kernel, reboot and immediately delete the volume before starting any other process. > **Note** > diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index 3b8e48554..51b83fc8a 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -113,19 +113,18 @@ Derivations can declare some infrequently used optional attributes. > `nix-build`. If the [`configurable-impure-env` experimental - feature](@docroot@/contributing/experimental-features.md#xp-feature-configurable-impure-env) + feature](@docroot@/development/experimental-features.md#xp-feature-configurable-impure-env) is enabled, these environment variables can also be controlled through the [`impure-env`](@docroot@/command-ref/conf-file.md#conf-impure-env) configuration setting. - [`outputHash`]{#adv-attr-outputHash}; [`outputHashAlgo`]{#adv-attr-outputHashAlgo}; [`outputHashMode`]{#adv-attr-outputHashMode}\ - These attributes declare that the derivation is a so-called - *fixed-output derivation*, which means that a cryptographic hash of - the output is already known in advance. When the build of a - fixed-output derivation finishes, Nix computes the cryptographic - hash of the output and compares it to the hash declared with these - attributes. If there is a mismatch, the build fails. + These attributes declare that the derivation is a so-called *fixed-output derivation* (FOD), which means that a cryptographic hash of the output is already known in advance. + + As opposed to regular derivations, the [`builder`] executable of a fixed-output derivation has access to the network. + Nix computes a cryptographic hash of its output and compares that to the hash declared with these attributes. + If there is a mismatch, the derivation fails. The rationale for fixed-output derivations is derivations such as those produced by the `fetchurl` function. This function downloads a @@ -197,37 +196,40 @@ Derivations can declare some infrequently used optional attributes. `outputHashAlgo` can only be `null` when `outputHash` follows the SRI format. The `outputHashMode` attribute determines how the hash is computed. - It must be one of the following two values: + It must be one of the following values: - - - - `"flat"` - - The output must be a non-executable regular file; if it isn’t, the build fails. - The hash is - [simply computed over the contents of that file](@docroot@/store/file-system-object/content-address.md#serial-flat) - (so it’s equal to what Unix commands like `sha256sum` or `sha1sum` produce). + - [`"flat"`](@docroot@/store/store-object/content-address.md#method-flat) This is the default. - - `"recursive"` or `"nar"` + - [`"recursive"` or `"nar"`](@docroot@/store/store-object/content-address.md#method-nix-archive) - The hash is computed over the - [Nix Archive (NAR)](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) - dump of the output (i.e., the result of [`nix-store --dump`](@docroot@/command-ref/nix-store/dump.md)). - In this case, the output is allowed to be any [file system object], including directories and more. + > **Compatibility** + > + > `"recursive"` is the traditional way of indicating this, + > and is supported since 2005 (virtually the entire history of Nix). + > `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI), + > however support for it is only added in Nix version 2.21. - `"recursive"` is the traditional way of indicating this, - and is supported since 2005 (virtually the entire history of Nix). - `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI), - however support for it is only added in Nix version 2.21. + - [`"text"`](@docroot@/store/store-object/content-address.md#method-text) + + > **Warning** + > + > The use of this method for derivation outputs is part of the [`dynamic-derivations`][xp-feature-dynamic-derivations] experimental feature. + + - [`"git"`](@docroot@/store/store-object/content-address.md#method-git) + + > **Warning** + > + > This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. - [`__contentAddressed`]{#adv-attr-__contentAddressed} + > **Warning** - > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). + > This attribute is part of an [experimental feature](@docroot@/development/experimental-features.md). > > To use this attribute, you must enable the - > [`ca-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) experimental feature. + > [`ca-derivations`][xp-feature-ca-derivations] experimental feature. > For example, in [nix.conf](../command-ref/conf-file.md) you could add: > > ``` @@ -276,7 +278,9 @@ Derivations can declare some infrequently used optional attributes. > **Note** > - > If set to `false`, the [`builder`](./derivations.md#attr-builder) should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + > If set to `false`, the [`builder`] should be able to run on the system type specified in the [`system` attribute](./derivations.md#attr-system), since the derivation cannot be substituted. + + [`builder`]: ./derivations.md#attr-builder - [`__structuredAttrs`]{#adv-attr-structuredAttrs}\ If the special attribute `__structuredAttrs` is set to `true`, the other derivation @@ -298,6 +302,12 @@ Derivations can declare some infrequently used optional attributes. (associative) arrays. For example, the attribute `hardening.format = true` ends up as the Bash associative array element `${hardening[format]}`. + > **Warning** + > + > If set to `true`, other advanced attributes such as [`allowedReferences`](#adv-attr-allowedReferences), [`allowedReferences`](#adv-attr-allowedReferences), [`allowedRequisites`](#adv-attr-allowedRequisites), + [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), maxSize, and maxClosureSize. + will have no effect. + - [`outputChecks`]{#adv-attr-outputChecks}\ When using [structured attributes](#adv-attr-structuredAttrs), the `outputChecks` attribute allows defining checks per-output. @@ -359,3 +369,7 @@ Derivations can declare some infrequently used optional attributes. ``` ensures that the derivation can only be built on a machine with the `kvm` feature. + +[xp-feature-ca-derivations]: @docroot@/development/experimental-features.md#xp-feature-ca-derivations +[xp-feature-dynamic-derivations]: @docroot@/development/experimental-features.md#xp-feature-dynamic-derivations +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/language/builtin-constants-prefix.md b/doc/manual/src/language/builtin-constants-prefix.md deleted file mode 100644 index 50f43006d..000000000 --- a/doc/manual/src/language/builtin-constants-prefix.md +++ /dev/null @@ -1,5 +0,0 @@ -# Built-in Constants - -These constants are built into the Nix language evaluator: - -
diff --git a/doc/manual/src/language/builtin-constants-suffix.md b/doc/manual/src/language/builtin-constants-suffix.md deleted file mode 100644 index a74db2857..000000000 --- a/doc/manual/src/language/builtin-constants-suffix.md +++ /dev/null @@ -1 +0,0 @@ -
diff --git a/doc/manual/src/language/builtins-prefix.md b/doc/manual/src/language/builtins-prefix.md index 7b2321466..fb983bb7f 100644 --- a/doc/manual/src/language/builtins-prefix.md +++ b/doc/manual/src/language/builtins-prefix.md @@ -1,9 +1,11 @@ -# Built-in Functions +# Built-ins -This section lists the functions built into the Nix language evaluator. -All built-in functions are available through the global [`builtins`](./builtin-constants.md#builtins-builtins) constant. +This section lists the values and functions built into the Nix language evaluator. +All built-ins are available through the global [`builtins`](#builtins-builtins) constant. -For convenience, some built-ins can be accessed directly: +Some built-ins are also exposed directly in the global scope: + + - [`derivation`](#builtins-derivation) - [`import`](#builtins-import) diff --git a/doc/manual/src/language/constructs.md b/doc/manual/src/language/constructs.md index 4d75ea82c..41a180246 100644 --- a/doc/manual/src/language/constructs.md +++ b/doc/manual/src/language/constructs.md @@ -1,437 +1 @@ # Language Constructs - -## Recursive sets - -Recursive sets are like normal [attribute sets](./values.md#attribute-set), but the attributes can refer to each other. - -> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` - -Example: - -```nix -rec { - x = y; - y = 123; -}.x -``` - -This evaluates to `123`. - -Note that without `rec` the binding `x = y;` would -refer to the variable `y` in the surrounding scope, if one exists, and -would be invalid if no such variable exists. That is, in a normal -(non-recursive) set, attributes are not added to the lexical scope; in a -recursive set, they are. - -Recursive sets of course introduce the danger of infinite recursion. For -example, the expression - -```nix -rec { - x = y; - y = x; -}.x -``` - -will crash with an `infinite recursion encountered` error message. - -## Let-expressions - -A let-expression allows you to define local variables for an expression. - -> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* - -Example: - -```nix -let - x = "foo"; - y = "bar"; -in x + y -``` - -This evaluates to `"foobar"`. - -## Inheriting attributes - -When defining an [attribute set](./values.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). -This can be shortened using the `inherit` keyword. - -Example: - -```nix -let x = 123; in -{ - inherit x; - y = 456; -} -``` - -is equivalent to - -```nix -let x = 123; in -{ - x = x; - y = 456; -} -``` - -and both evaluate to `{ x = 123; y = 456; }`. - -> **Note** -> -> This works because `x` is added to the lexical scope by the `let` construct. - -It is also possible to inherit attributes from another attribute set. - -Example: - -In this fragment from `all-packages.nix`, - -```nix -graphviz = (import ../tools/graphics/graphviz) { - inherit fetchurl stdenv libpng libjpeg expat x11 yacc; - inherit (xorg) libXaw; -}; - -xorg = { - libX11 = ...; - libXaw = ...; - ... -} - -libpng = ...; -libjpg = ...; -... -``` - -the set used in the function call to the function defined in -`../tools/graphics/graphviz` inherits a number of variables from the -surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` -(the X Athena Widgets) from the `xorg` set. - -Summarizing the fragment - -```nix -... -inherit x y z; -inherit (src-set) a b c; -... -``` - -is equivalent to - -```nix -... -x = x; y = y; z = z; -a = src-set.a; b = src-set.b; c = src-set.c; -... -``` - -when used while defining local variables in a let-expression or while -defining a set. - -In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example - - -```nix -let - x = { a = 1; b = 2; }; - inherit (builtins) attrNames; -in -{ - names = attrNames x; -} -``` - -is equivalent to - -```nix -let - x = { a = 1; b = 2; }; -in -{ - names = builtins.attrNames x; -} -``` - -both evaluate to `{ names = [ "a" "b" ]; }`. - -## Functions - -Functions have the following form: - -```nix -pattern: body -``` - -The pattern specifies what the argument of the function must look like, -and binds variables in the body to (parts of) the argument. There are -three kinds of patterns: - - - If a pattern is a single identifier, then the function matches any - argument. Example: - - ```nix - let negate = x: !x; - concat = x: y: x + y; - in if negate true then concat "foo" "bar" else "" - ``` - - Note that `concat` is a function that takes one argument and returns - a function that takes another argument. This allows partial - parameterisation (i.e., only filling some of the arguments of a - function); e.g., - - ```nix - map (concat "foo") [ "bar" "bla" "abc" ] - ``` - - evaluates to `[ "foobar" "foobla" "fooabc" ]`. - - - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a - set containing the listed attributes, and binds the values of those - attributes to variables in the function body. For example, the - function - - ```nix - { x, y, z }: z + y + x - ``` - - can only be called with a set containing exactly the attributes `x`, - `y` and `z`. No other attributes are allowed. If you want to allow - additional arguments, you can use an ellipsis (`...`): - - ```nix - { x, y, z, ... }: z + y + x - ``` - - This works on any set that contains at least the three named - attributes. - - It is possible to provide *default values* for attributes, in - which case they are allowed to be missing. A default value is - specified by writing `name ? e`, where *e* is an arbitrary - expression. For example, - - ```nix - { x, y ? "foo", z ? "bar" }: z + y + x - ``` - - specifies a function that only requires an attribute named `x`, but - optionally accepts `y` and `z`. - - - An `@`-pattern provides a means of referring to the whole value - being matched: - - ```nix - args@{ x, y, z, ... }: z + y + x + args.a - ``` - - but can also be written as: - - ```nix - { x, y, z, ... } @ args: z + y + x + args.a - ``` - - Here `args` is bound to the argument *as passed*, which is further - matched against the pattern `{ x, y, z, ... }`. - The `@`-pattern makes mainly sense with an ellipsis(`...`) as - you can access attribute names as `a`, using `args.a`, which was - given as an additional attribute to the function. - - > **Warning** - > - > `args@` binds the name `args` to the attribute set that is passed to the function. - > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. - > - > For instance - > - > ```nix - > let - > f = args@{ a ? 23, ... }: [ a args ]; - > in - > f {} - > ``` - > - > is equivalent to - > - > ```nix - > let - > f = args @ { ... }: [ (args.a or 23) args ]; - > in - > f {} - > ``` - > - > and both expressions will evaluate to: - > - > ```nix - > [ 23 {} ] - > ``` - -Note that functions do not have names. If you want to give them a name, -you can bind them to an attribute, e.g., - -```nix -let concat = { x, y }: x + y; -in concat { x = "foo"; y = "bar"; } -``` - -## Conditionals - -Conditionals look like this: - -```nix -if e1 then e2 else e3 -``` - -where *e1* is an expression that should evaluate to a Boolean value -(`true` or `false`). - -## Assertions - -Assertions are generally used to check that certain requirements on or -between features and dependencies hold. They look like this: - -```nix -assert e1; e2 -``` - -where *e1* is an expression that should evaluate to a Boolean value. If -it evaluates to `true`, *e2* is returned; otherwise expression -evaluation is aborted and a backtrace is printed. - -Here is a Nix expression for the Subversion package that shows how -assertions can be used:. - -```nix -{ localServer ? false -, httpServer ? false -, sslSupport ? false -, pythonBindings ? false -, javaSwigBindings ? false -, javahlBindings ? false -, stdenv, fetchurl -, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null -}: - -assert localServer -> db4 != null; ① -assert httpServer -> httpd != null && httpd.expat == expat; ② -assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); ③ -assert pythonBindings -> swig != null && swig.pythonSupport; -assert javaSwigBindings -> swig != null && swig.javaSupport; -assert javahlBindings -> j2sdk != null; - -stdenv.mkDerivation { - name = "subversion-1.1.1"; - ... - openssl = if sslSupport then openssl else null; ④ - ... -} -``` - -The points of interest are: - -1. This assertion states that if Subversion is to have support for - local repositories, then Berkeley DB is needed. So if the Subversion - function is called with the `localServer` argument set to `true` but - the `db4` argument set to `null`, then the evaluation fails. - - Note that `->` is the [logical - implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) - Boolean operation. - -2. This is a more subtle condition: if Subversion is built with Apache - (`httpServer`) support, then the Expat library (an XML library) used - by Subversion should be same as the one used by Apache. This is - because in this configuration Subversion code ends up being linked - with Apache code, and if the Expat libraries do not match, a build- - or runtime link error or incompatibility might occur. - -3. This assertion says that in order for Subversion to have SSL support - (so that it can access `https` URLs), an OpenSSL library must be - passed. Additionally, it says that *if* Apache support is enabled, - then Apache's OpenSSL should match Subversion's. (Note that if - Apache support is not enabled, we don't care about Apache's - OpenSSL.) - -4. The conditional here is not really related to assertions, but is - worth pointing out: it ensures that if SSL support is disabled, then - the Subversion derivation is not dependent on OpenSSL, even if a - non-`null` value was passed. This prevents an unnecessary rebuild of - Subversion if OpenSSL changes. - -## With-expressions - -A *with-expression*, - -```nix -with e1; e2 -``` - -introduces the set *e1* into the lexical scope of the expression *e2*. -For instance, - -```nix -let as = { x = "foo"; y = "bar"; }; -in with as; x + y -``` - -evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes -of `as` to the lexical scope in the expression `x + y`. The most common -use of `with` is in conjunction with the `import` function. E.g., - -```nix -with (import ./definitions.nix); ... -``` - -makes all attributes defined in the file `definitions.nix` available as -if they were defined locally in a `let`-expression. - -The bindings introduced by `with` do not shadow bindings introduced by -other means, e.g. - -```nix -let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... -``` - -establishes the same scope as - -```nix -let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... -``` - -Variables coming from outer `with` expressions *are* shadowed: - -```nix -with { a = "outer"; }; -with { a = "inner"; }; -a -``` - -Does evaluate to `"inner"`. - -## Comments - -Comments can be single-line, started with a `#` character, or -inline/multi-line, enclosed within `/* ... */`. - -`#` comments last until the end of the line. - -`/*` comments run until the next occurrence of `*/`; this cannot be escaped. - -## Scoping rules - -Nix is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope), but with multiple scopes and shadowing rules. - -* primary scope --- explicitly-bound variables - * [`let`](#let-expressions) - * [`inherit`](#inheriting-attributes) - * function arguments - -* secondary scope --- implicitly-bound variables - * [`with`](#with-expressions) - -Primary scope takes precedence over secondary scope. -See [`with`](#with-expressions) for a detailed example. diff --git a/doc/manual/src/language/constructs/lookup-path.md b/doc/manual/src/language/constructs/lookup-path.md index e87d2922b..a2e80280b 100644 --- a/doc/manual/src/language/constructs/lookup-path.md +++ b/doc/manual/src/language/constructs/lookup-path.md @@ -4,11 +4,8 @@ > > *lookup-path* = `<` *identifier* [ `/` *identifier* ]... `>` -A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/values.md#type-path) if the identifier matches a search path entry. - -The value of a lookup path is determined by [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath). - -See [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile) for details on lookup path resolution. +A lookup path is an identifier with an optional path suffix that resolves to a [path value](@docroot@/language/types.md#type-path) if the identifier matches a search path entry in [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath). +The algorithm for lookup path resolution is described in the documentation on [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile). > **Example** > diff --git a/doc/manual/src/language/derivations.md b/doc/manual/src/language/derivations.md index b95900cdd..771b2bd91 100644 --- a/doc/manual/src/language/derivations.md +++ b/doc/manual/src/language/derivations.md @@ -12,7 +12,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Required -- [`name`]{#attr-name} ([String](@docroot@/language/values.md#type-string)) +- [`name`]{#attr-name} ([String](@docroot@/language/types.md#type-string)) A symbolic name for the derivation. It is added to the [store path] of the corresponding [store derivation] as well as to its [output paths](@docroot@/glossary.md#gloss-output-path). @@ -31,7 +31,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > The store derivation's path will be `/nix/store/-hello.drv`. > The [output](#attr-outputs) paths will be of the form `/nix/store/-hello[-]` -- [`system`]{#attr-system} ([String](@docroot@/language/values.md#type-string)) +- [`system`]{#attr-system} ([String](@docroot@/language/types.md#type-string)) The system type on which the [`builder`](#attr-builder) executable is meant to be run. @@ -64,9 +64,9 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > } > ``` > - > [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. + > [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) has the value of the [`system` configuration option], and defaults to the system type of the current Nix installation. -- [`builder`]{#attr-builder} ([Path](@docroot@/language/values.md#type-path) | [String](@docroot@/language/values.md#type-string)) +- [`builder`]{#attr-builder} ([Path](@docroot@/language/types.md#type-path) | [String](@docroot@/language/types.md#type-string)) Path to an executable that will perform the build. @@ -113,7 +113,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect ### Optional -- [`args`]{#attr-args} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`args`]{#attr-args} ([List](@docroot@/language/types.md#type-list) of [String](@docroot@/language/types.md#type-string)) Default: `[ ]` @@ -132,7 +132,7 @@ It outputs an attribute set, and produces a [store derivation] as a side effect > }; > ``` -- [`outputs`]{#attr-outputs} ([List](@docroot@/language/values.md#list) of [String](@docroot@/language/values.md#type-string)) +- [`outputs`]{#attr-outputs} ([List](@docroot@/language/types.md#type-list) of [String](@docroot@/language/types.md#type-string)) Default: `[ "out" ]` diff --git a/doc/manual/src/language/identifiers.md b/doc/manual/src/language/identifiers.md new file mode 100644 index 000000000..584a2f861 --- /dev/null +++ b/doc/manual/src/language/identifiers.md @@ -0,0 +1,51 @@ +# Identifiers + +An *identifier* is an [ASCII](https://en.wikipedia.org/wiki/ASCII) character sequence that: +- Starts with a letter (`a-z`, `A-Z`) or underscore (`_`) +- Can contain any number of: + - Letters (`a-z`, `A-Z`) + - Digits (`0-9`) + - Underscores (`_`) + - Apostrophes (`'`) + - Hyphens (`-`) +- Is not one of the [keywords](#keywords) + +> **Syntax** +> +> *identifier* ~ `[A-Za-z_][A-Za-z0-9_'-]*` + +# Names + +A *name* can be written as an [identifier](#identifier) or a [string literal](./string-literals.md). + +> **Syntax** +> +> *name* → *identifier* | *string* + +Names are used in [attribute sets](./syntax.md#attrs-literal), [`let` bindings](./syntax.md#let-expressions), and [`inherit`](./syntax.md#inheriting-attributes). +Two names are the same if they represent the same sequence of characters, regardless of whether they are written as identifiers or strings. + +# Keywords + +These keywords are reserved and cannot be used as [identifiers](#identifiers): + +- [`assert`](./syntax.md#assertions) +- [`else`][if] +- [`if`][if] +- [`in`][let] +- [`inherit`](./syntax.md#inheriting-attributes) +- [`let`][let] +- [`or`](./operators.md#attribute-selection) (see note) +- [`rec`](./syntax.md#recursive-sets) +- [`then`][if] +- [`with`](./syntax.md#with-expressions) + +[if]: ./syntax.md#conditionals +[let]: ./syntax.md#let-expressions + +> **Note** +> +> The Nix language evaluator currently allows `or` to be used as a name in some contexts, for backwards compatibility reasons. +> Users are advised not to rely on this. +> +> There are long-standing issues with how `or` is parsed as a name, which can't be resolved without making a breaking change to the language. diff --git a/doc/manual/src/language/index.md b/doc/manual/src/language/index.md index 3694480d7..2bfdbb8a0 100644 --- a/doc/manual/src/language/index.md +++ b/doc/manual/src/language/index.md @@ -53,7 +53,7 @@ This is an incomplete overview of language features, by example. - *Basic values ([primitives](@docroot@/language/values.md#primitives))* + *Basic values ([primitives](@docroot@/language/types.md#primitives))* @@ -71,7 +71,7 @@ This is an incomplete overview of language features, by example. - A [string](@docroot@/language/values.md#type-string) + A [string](@docroot@/language/types.md#type-string) @@ -102,7 +102,7 @@ This is an incomplete overview of language features, by example. - A [comment](@docroot@/language/constructs.md#comments). + A [comment](@docroot@/language/syntax.md#comments). @@ -130,7 +130,7 @@ This is an incomplete overview of language features, by example. - [Booleans](@docroot@/language/values.md#type-boolean) + [Booleans](@docroot@/language/types.md#type-boolean) @@ -142,7 +142,7 @@ This is an incomplete overview of language features, by example. - [Null](@docroot@/language/values.md#type-null) value + [Null](@docroot@/language/types.md#type-null) value @@ -154,7 +154,7 @@ This is an incomplete overview of language features, by example. - An [integer](@docroot@/language/values.md#type-number) + An [integer](@docroot@/language/types.md#type-int) @@ -166,7 +166,7 @@ This is an incomplete overview of language features, by example. - A [floating point number](@docroot@/language/values.md#type-number) + A [floating point number](@docroot@/language/types.md#type-float) @@ -178,7 +178,7 @@ This is an incomplete overview of language features, by example. - An absolute [path](@docroot@/language/values.md#type-path) + An absolute [path](@docroot@/language/types.md#type-path) @@ -190,7 +190,7 @@ This is an incomplete overview of language features, by example. - A [path](@docroot@/language/values.md#type-path) relative to the file containing this Nix expression + A [path](@docroot@/language/types.md#type-path) relative to the file containing this Nix expression @@ -202,7 +202,7 @@ This is an incomplete overview of language features, by example. - A home [path](@docroot@/language/values.md#type-path). Evaluates to the `"/.config"`. + A home [path](@docroot@/language/types.md#type-path). Evaluates to the `"/.config"`. @@ -238,7 +238,7 @@ This is an incomplete overview of language features, by example. - An [attribute set](@docroot@/language/values.md#attribute-set) with attributes named `x` and `y` + An [attribute set](@docroot@/language/types.md#attribute-set) with attributes named `x` and `y` @@ -262,7 +262,7 @@ This is an incomplete overview of language features, by example. - A [recursive set](@docroot@/language/constructs.md#recursive-sets), equivalent to `{ x = "foo"; y = "foobar"; }`. + A [recursive set](@docroot@/language/syntax.md#recursive-sets), equivalent to `{ x = "foo"; y = "foobar"; }`. @@ -278,7 +278,7 @@ This is an incomplete overview of language features, by example. - [Lists](@docroot@/language/values.md#list) with three elements. + [Lists](@docroot@/language/types.md#list) with three elements. @@ -362,7 +362,7 @@ This is an incomplete overview of language features, by example. - [Attribute selection](@docroot@/language/values.md#attribute-set) (evaluates to `1`) + [Attribute selection](@docroot@/language/types.md#attribute-set) (evaluates to `1`) @@ -374,7 +374,7 @@ This is an incomplete overview of language features, by example. - [Attribute selection](@docroot@/language/values.md#attribute-set) with default (evaluates to `3`) + [Attribute selection](@docroot@/language/types.md#attribute-set) with default (evaluates to `3`) @@ -410,7 +410,7 @@ This is an incomplete overview of language features, by example. - [Conditional expression](@docroot@/language/constructs.md#conditionals). + [Conditional expression](@docroot@/language/syntax.md#conditionals). @@ -422,7 +422,7 @@ This is an incomplete overview of language features, by example. - [Assertion](@docroot@/language/constructs.md#assertions) check (evaluates to `"yes!"`). + [Assertion](@docroot@/language/syntax.md#assertions) check (evaluates to `"yes!"`). @@ -434,7 +434,7 @@ This is an incomplete overview of language features, by example. - Variable definition. See [`let`-expressions](@docroot@/language/constructs.md#let-expressions). + Variable definition. See [`let`-expressions](@docroot@/language/syntax.md#let-expressions). @@ -448,7 +448,7 @@ This is an incomplete overview of language features, by example. Add all attributes from the given set to the scope (evaluates to `1`). - See [`with`-expressions](@docroot@/language/constructs.md#with-expressions) for details and shadowing caveats. + See [`with`-expressions](@docroot@/language/syntax.md#with-expressions) for details and shadowing caveats. @@ -462,7 +462,7 @@ This is an incomplete overview of language features, by example. Adds the variables to the current scope (attribute set or `let` binding). Desugars to `pkgs = pkgs; src = src;`. - See [Inheriting attributes](@docroot@/language/constructs.md#inheriting-attributes). + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). @@ -476,14 +476,14 @@ This is an incomplete overview of language features, by example. Adds the attributes, from the attribute set in parentheses, to the current scope (attribute set or `let` binding). Desugars to `lib = pkgs.lib; stdenv = pkgs.stdenv;`. - See [Inheriting attributes](@docroot@/language/constructs.md#inheriting-attributes). + See [Inheriting attributes](@docroot@/language/syntax.md#inheriting-attributes). - *[Functions](@docroot@/language/constructs.md#functions) (lambdas)* + *[Functions](@docroot@/language/syntax.md#functions) (lambdas)* @@ -500,7 +500,7 @@ This is an incomplete overview of language features, by example. - A [function](@docroot@/language/constructs.md#functions) that expects an integer and returns it increased by 1. + A [function](@docroot@/language/syntax.md#functions) that expects an integer and returns it increased by 1. @@ -512,7 +512,7 @@ This is an incomplete overview of language features, by example. - Curried [function](@docroot@/language/constructs.md#functions), equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. + Curried [function](@docroot@/language/syntax.md#functions), equivalent to `x: (y: x + y)`. Can be used like a function that takes two arguments and returns their sum. @@ -524,7 +524,7 @@ This is an incomplete overview of language features, by example. - A [function](@docroot@/language/constructs.md#functions) call (evaluates to 101) + A [function](@docroot@/language/syntax.md#functions) call (evaluates to 101) @@ -536,7 +536,7 @@ This is an incomplete overview of language features, by example. - A [function](@docroot@/language/constructs.md#functions) bound to a variable and subsequently called by name (evaluates to 103) + A [function](@docroot@/language/syntax.md#functions) bound to a variable and subsequently called by name (evaluates to 103) @@ -548,7 +548,7 @@ This is an incomplete overview of language features, by example. - A [function](@docroot@/language/constructs.md#functions) that expects a set with required attributes `x` and `y` and concatenates them + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and concatenates them @@ -560,7 +560,7 @@ This is an incomplete overview of language features, by example. - A [function](@docroot@/language/constructs.md#functions) that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attribute `x` and optional `y`, using `"bar"` as default value for `y` @@ -572,7 +572,7 @@ This is an incomplete overview of language features, by example. - A [function](@docroot@/language/constructs.md#functions) that expects a set with required attributes `x` and `y` and ignores any other attributes + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y` and ignores any other attributes @@ -586,7 +586,7 @@ This is an incomplete overview of language features, by example. - A [function](@docroot@/language/constructs.md#functions) that expects a set with required attributes `x` and `y`, and binds the whole set to `args` + A [function](@docroot@/language/syntax.md#functions) that expects a set with required attributes `x` and `y`, and binds the whole set to `args` diff --git a/doc/manual/src/language/meson.build b/doc/manual/src/language/meson.build new file mode 100644 index 000000000..97469e2f3 --- /dev/null +++ b/doc/manual/src/language/meson.build @@ -0,0 +1,20 @@ +builtins_md = custom_target( + command : [ + python.full_path(), + '@INPUT0@', + '@OUTPUT@', + '--' + ] + nix_eval_for_docs + [ + '--expr', + '(builtins.readFile @INPUT3@) + import @INPUT1@ (builtins.fromJSON (builtins.readFile ./@INPUT2@)) + (builtins.readFile @INPUT4@)', + ], + input : [ + '../../remove_before_wrapper.py', + '../../generate-builtins.nix', + language_json, + 'builtins-prefix.md', + 'builtins-suffix.md' + ], + output : 'builtins.md', + env : nix_env_for_docs, +) diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 311887e96..e2ed3fbed 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -3,7 +3,7 @@ | Name | Syntax | Associativity | Precedence | |----------------------------------------|--------------------------------------------|---------------|------------| | [Attribute selection] | *attrset* `.` *attrpath* \[ `or` *expr* \] | none | 1 | -| Function application | *func* *expr* | left | 2 | +| [Function application] | *func* *expr* | left | 2 | | [Arithmetic negation][arithmetic] | `-` *number* | none | 3 | | [Has attribute] | *attrset* `?` *attrpath* | none | 4 | | List concatenation | *list* `++` *list* | right | 5 | @@ -26,12 +26,16 @@ | Logical conjunction (`AND`) | *bool* `&&` *bool* | left | 12 | | Logical disjunction (`OR`) | *bool* \|\| *bool* | left | 13 | | [Logical implication] | *bool* `->` *bool* | right | 14 | +| [Pipe operator] (experimental) | *expr* `\|>` *func* | left | 15 | +| [Pipe operator] (experimental) | *func* `<\|` *expr* | right | 15 | -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[number]: ./values.md#type-number -[list]: ./values.md#list -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[number]: ./types.md#type-float +[list]: ./types.md#type-list +[attribute set]: ./types.md#type-attrs + + ## Attribute selection @@ -42,13 +46,23 @@ Select the attribute denoted by attribute path *attrpath* from [attribute set] *attrset*. If the attribute doesn’t exist, return the *expr* after `or` if provided, otherwise abort evaluation. -An attribute path is a dot-separated list of [attribute names](./values.md#attribute-set). +[Attribute selection]: #attribute-selection + +## Function application > **Syntax** > -> *attrpath* = *name* [ `.` *name* ]... +> *func* *expr* -[Attribute selection]: #attribute-selection +Apply the callable value *func* to the argument *expr*. Note the absence of any visible operator symbol. +A callable value is either: +- a [user-defined function][function] +- a [built-in][builtins] function +- an attribute set with a [`__functor` attribute](./syntax.md#attr-__functor) + +> **Warning** +> +> [List][list] items are also separated by whitespace, which means that function calls in list items must be enclosed by parentheses. ## Has attribute @@ -61,7 +75,7 @@ The result is a [Boolean] value. See also: [`builtins.hasAttr`](@docroot@/language/builtins.md#builtins-hasAttr) -[Boolean]: ./values.md#type-boolean +[Boolean]: ./types.md#type-bool [Has attribute]: #has-attribute @@ -69,8 +83,12 @@ After evaluating *attrset* and *attrpath*, the computational complexity is O(log ## Arithmetic -Numbers are type-compatible: -Pure integer operations will always return integers, whereas any operation involving at least one floating point number return a floating point number. +Numbers will retain their type unless mixed with other numeric types: +Pure integer operations will always return integers, whereas any operation involving at least one floating point number returns a floating point number. + +Evaluation of the following numeric operations throws an evaluation error: +- Division by zero +- Integer overflow, that is, any operation yielding a result outside of the representable range of [Nix language integers](./syntax.md#number-literal) See also [Comparison] and [Equality]. @@ -141,7 +159,7 @@ The result is a string. Update [attribute set] *attrset1* with names and values from *attrset2*. -The returned attribute set will have of all the attributes in *attrset1* and *attrset2*. +The returned attribute set will have all of the attributes in *attrset1* and *attrset2*. If an attribute name is present in both, the attribute value from the latter is taken. [Update]: #update @@ -172,7 +190,7 @@ All comparison operators are implemented in terms of `<`, and the following equi - Numbers are type-compatible, see [arithmetic] operators. - Floating point numbers only differ up to a limited precision. -[function]: ./constructs.md#functions +[function]: ./syntax.md#functions [Equality]: #equality @@ -182,3 +200,36 @@ Equivalent to `!`*b1* `||` *b2*. [Logical implication]: #logical-implication +## Pipe operators + +- *a* `|>` *b* is equivalent to *b* *a* +- *a* `<|` *b* is equivalent to *a* *b* + +> **Example** +> +> ``` +> nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 +> 9 +> +> nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 +> 7 +> ``` + +> **Warning** +> +> This syntax is part of an +> [experimental feature](@docroot@/development/experimental-features.md) +> and may change in future releases. +> +> To use this syntax, make sure the +> [`pipe-operators` experimental feature](@docroot@/development/experimental-features.md#xp-feature-pipe-operators) +> is enabled. +> For example, include the following in [`nix.conf`](@docroot@/command-ref/conf-file.md): +> +> ``` +> extra-experimental-features = pipe-operators +> ``` + +[Pipe operator]: #pipe-operators +[builtins]: ./builtins.md +[Function application]: #function-application diff --git a/doc/manual/src/language/scope.md b/doc/manual/src/language/scope.md new file mode 100644 index 000000000..9373324e2 --- /dev/null +++ b/doc/manual/src/language/scope.md @@ -0,0 +1,28 @@ +# Scoping rules + +A *scope* in the Nix language is a dictionary keyed by [name](./identifiers.md#names), mapping each name to an expression and a *definition type*. +The definition type is either *explicit* or *implicit*. +Each entry in this dictionary is a *definition*. + +Explicit definitions are created by the following expressions: +- [let-expressions](syntax.md#let-expressions) +- [recursive attribute set literals](syntax.md#recursive-sets) (`rec`) +- [function literals](syntax.md#functions) + +Implicit definitions are only created by [with-expressions](./syntax.md#with-expressions). + +Every expression is *enclosed* by a scope. +The outermost expression is enclosed by the [built-in, global scope](./builtins.md), which contains only explicit definitions. +The expressions listed above *extend* their enclosing scope by adding new definitions, or replacing existing ones with the same name. +An explicit definition can replace a definition of any type; an implicit definition can only replace another implicit definition. + +Each of the above expressions defines which of its subexpressions are enclosed by the extended scope. +In all other cases, the same scope that encloses an expression is the enclosing scope for its subexpressions. + +The Nix language is [statically scoped](https://en.wikipedia.org/wiki/Scope_(computer_science)#Lexical_scope); +the value of a variable is determined only by the variable's enclosing scope, and not by the dynamic context in which the variable is evaluated. + +> **Note** +> +> Expressions entered into the [Nix REPL](@docroot@/command-ref/new-cli/nix3-repl.md) are enclosed by a scope that can be extended by command line arguments or previous REPL commands. +> These ways of extending scope are not, strictly speaking, part of the Nix language. diff --git a/doc/manual/src/language/string-context.md b/doc/manual/src/language/string-context.md index 88ae0d8b0..6a3482cfd 100644 --- a/doc/manual/src/language/string-context.md +++ b/doc/manual/src/language/string-context.md @@ -111,7 +111,7 @@ It creates an [attribute set] representing the string context, which can be insp [`builtins.hasContext`]: ./builtins.md#builtins-hasContext [`builtins.getContext`]: ./builtins.md#builtins-getContext -[attribute set]: ./values.md#attribute-set +[attribute set]: ./types.md#attribute-set ## Clearing string contexts diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 1e2c4ad95..27780dcbb 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -4,9 +4,13 @@ String interpolation is a language feature where a [string], [path], or [attribu Such a construct is called *interpolated string*, and the expression inside is an [interpolated expression](#interpolated-expression). -[string]: ./values.md#type-string -[path]: ./values.md#type-path -[attribute set]: ./values.md#attribute-set +[string]: ./types.md#type-string +[path]: ./types.md#type-path +[attribute set]: ./types.md#attribute-set + +> **Syntax** +> +> *interpolation_element* → `${` *expression* `}` ## Examples @@ -43,6 +47,47 @@ configureFlags = " Note that Nix expressions and strings can be arbitrarily nested; in this case the outer string contains various interpolated expressions that themselves contain strings (e.g., `"-thread"`), some of which in turn contain interpolated expressions (e.g., `${mesa}`). +To write a literal `${` in an regular string, escape it with a backslash (`\`). + +> **Example** +> +> ```nix +> "echo \${PATH}" +> ``` +> +> "echo ${PATH}" + +To write a literal `${` in an indented string, escape it with two single quotes (`''`). + +> **Example** +> +> ```nix +> '' +> echo ''${PATH} +> '' +> ``` +> +> "echo ${PATH}\n" + +`$${` can be written literally in any string. + +> **Example** +> +> In Make, `$` in file names or recipes is represented as `$$`, see [GNU `make`: Basics of Variable Reference](https://www.gnu.org/software/make/manual/html_node/Reference.html#Basics-of-Variable-References). +> This can be expressed directly in the Nix language strings: +> +> ```nix +> '' +> MAKEVAR = Hello +> all: +> @export BASHVAR=world; echo $(MAKEVAR) $${BASHVAR} +> '' +> ``` +> +> "MAKEVAR = Hello\nall:\n\t@export BASHVAR=world; echo $(MAKEVAR) $\${BASHVAR}\n" + +See the [documentation on strings][string] for details. + ### Path Rather than writing diff --git a/doc/manual/src/language/string-literals.md b/doc/manual/src/language/string-literals.md new file mode 100644 index 000000000..8f4b75f3e --- /dev/null +++ b/doc/manual/src/language/string-literals.md @@ -0,0 +1,190 @@ +# String literals + +A *string literal* represents a [string](types.md#type-string) value. + +> **Syntax** +> +> *expression* → *string* +> +> *string* → `"` ( *string_char*\* [*interpolation_element*][string interpolation] )* *string_char*\* `"` +> +> *string* → `''` ( *indented_string_char*\* [*interpolation_element*][string interpolation] )* *indented_string_char*\* `''` +> +> *string* → *uri* +> +> *string_char* ~ `[^"$\\]|\$(?!\{)|\\.` +> +> *indented_string_char* ~ `[^$']|\$\$|\$(?!\{)|''[$']|''\\.|'(?!')` +> +> *uri* ~ `[A-Za-z][+\-.0-9A-Za-z]*:[!$%&'*+,\-./0-9:=?@A-Z_a-z~]+` + +Strings can be written in three ways. + +The most common way is to enclose the string between double quotes, e.g., `"foo bar"`. +Strings can span multiple lines. +The results of other expressions can be included into a string by enclosing them in `${ }`, a feature known as [string interpolation]. + +[string interpolation]: ./string-interpolation.md + +The following must be escaped to represent them within a string, by prefixing with a backslash (`\`): + +- Double quote (`"`) + +> **Example** +> +> ```nix +> "\"" +> ``` +> +> "\"" + +- Backslash (`\`) + +> **Example** +> +> ```nix +> "\\" +> ``` +> +> "\\" + +- Dollar sign followed by an opening curly bracket (`${`) – "dollar-curly" + +> **Example** +> +> ```nix +> "\${" +> ``` +> +> "\${" + +The newline, carriage return, and tab characters can be written as `\n`, `\r` and `\t`, respectively. + +A "double-dollar-curly" (`$${`) can be written literally. + +> **Example** +> +> ```nix +> "$${" +> ``` +> +> "$\${" + +String values are output on the terminal with Nix-specific escaping. +Strings written to files will contain the characters encoded by the escaping. + +The second way to write string literals is as an *indented string*, which is enclosed between pairs of *double single-quotes* (`''`), like so: + +```nix +'' +This is the first line. +This is the second line. + This is the third line. +'' +``` + +This kind of string literal intelligently strips indentation from +the start of each line. To be precise, it strips from each line a +number of spaces equal to the minimal indentation of the string as a +whole (disregarding the indentation of empty lines). For instance, +the first and second line are indented two spaces, while the third +line is indented four spaces. Thus, two spaces are stripped from +each line, so the resulting string is + +```nix +"This is the first line.\nThis is the second line.\n This is the third line.\n" +``` + +> **Note** +> +> Whitespace and newline following the opening `''` is ignored if there is no non-whitespace text on the initial line. + +> **Warning** +> +> Prefixed tab characters are not stripped. +> +> > **Example** +> > +> > The following indented string is prefixed with tabs: +> > +> >
''
+> > 	all:
+> > 		@echo hello
+> > ''
+> > 
+> > +> > "\tall:\n\t\t@echo hello\n" + +Indented strings support [string interpolation]. + +The following must be escaped to represent them in an indented string: + +- `$` is escaped by prefixing it with two single quotes (`''`) + +> **Example** +> +> ```nix +> '' +> ''$ +> '' +> ``` +> +> "$\n" + +- `''` is escaped by prefixing it with one single quote (`'`) + +> **Example** +> +> ```nix +> '' +> ''' +> '' +> ``` +> +> "''\n" + +These special characters are escaped as follows: +- Linefeed (`\n`): `''\n` +- Carriage return (`\r`): `''\r` +- Tab (`\t`): `''\t` + +`''\` escapes any other character. + +A "double-dollar-curly" (`$${`) can be written literally. + +> **Example** +> +> ```nix +> '' +> $${ +> '' +> ``` +> +> "$\${\n" + +Indented strings are primarily useful in that they allow multi-line +string literals to follow the indentation of the enclosing Nix +expression, and that less escaping is typically necessary for +strings representing languages such as shell scripts and +configuration files because `''` is much less common than `"`. +Example: + +```nix +stdenv.mkDerivation { +... +postInstall = + '' + mkdir $out/bin $out/etc + cp foo $out/bin + echo "Hello World" > $out/etc/foo.conf + ${if enableBar then "cp bar $out/bin" else ""} + ''; +... +} +``` + +Finally, as a convenience, *URIs* as defined in appendix B of +[RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as +is*, without quotes. For instance, the string +`"http://example.org/foo.tar.bz2"` can also be written as +`http://example.org/foo.tar.bz2`. diff --git a/doc/manual/src/language/syntax.md b/doc/manual/src/language/syntax.md new file mode 100644 index 000000000..506afbea1 --- /dev/null +++ b/doc/manual/src/language/syntax.md @@ -0,0 +1,705 @@ +# Language Constructs + +This section covers syntax and semantics of the Nix language. + +## Basic Literals + +### String {#string-literal} + +See [String literals](string-literals.md). + +### Number {#number-literal} + + + + Numbers, which can be *integers* (like `123`) or *floating point* + (like `123.43` or `.27e13`). + + Integers in the Nix language are 64-bit [two's complement] signed integers, with a range of -9223372036854775808 to 9223372036854775807, inclusive. + + [two's complement]: https://en.wikipedia.org/wiki/Two%27s_complement + + Note that negative numeric literals are actually parsed as unary negation of positive numeric literals. + This means that the minimum integer `-9223372036854775808` cannot be written as-is as a literal, since the positive number `9223372036854775808` is one past the maximum range. + + See [arithmetic] and [comparison] operators for semantics. + + [arithmetic]: ./operators.md#arithmetic + [comparison]: ./operators.md#comparison + +### Path {#path-literal} + + *Paths* can be expressed by path literals such as `./builder.sh`. + + A path literal must contain at least one slash to be recognised as such. + For instance, `builder.sh` is not a path: + it's parsed as an expression that selects the attribute `sh` from the variable `builder`. + + Path literals are resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory). + Path literals may also refer to absolute paths by starting with a slash. + + > **Note** + > + > Absolute paths make expressions less portable. + > In the case where a function translates a path literal into an absolute path string for a configuration file, it is recommended to write a string literal instead. + > This avoids some confusion about whether files at that location will be used during evaluation. + > It also avoids unintentional situations where some function might try to copy everything at the location into the store. + + If the first component of a path is a `~`, it is interpreted such that the rest of the path were relative to the user's home directory. + For example, `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. + Path literals that start with `~` are not allowed in [pure](@docroot@/command-ref/conf-file.md#conf-pure-eval) evaluation. + + Path literals can also include [string interpolation], besides being [interpolated into other expressions]. + + [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions + + At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. + + `a.${foo}/b.${bar}` is a syntactically valid number division operation. + `./a.${foo}/b.${bar}` is a path. + + [Lookup path](./constructs/lookup-path.md) literals such as `` also resolve to path values. + +## List {#list-literal} + +Lists are formed by enclosing a whitespace-separated list of values +between square brackets. For example, + +```nix +[ 123 ./foo.nix "abc" (f { x = y; }) ] +``` + +defines a list of four elements, the last being the result of a call to +the function `f`. Note that function calls have to be enclosed in +parentheses. If they had been omitted, e.g., + +```nix +[ 123 ./foo.nix "abc" f { x = y; } ] +``` + +the result would be a list of five elements, the fourth one being a +function and the fifth being a set. + +Note that lists are only lazy in values, and they are strict in length. + +Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). + +## Attribute Set {#attrs-literal} + +An attribute set is a collection of name-value-pairs called *attributes*. + +Attribute sets are written enclosed in curly brackets (`{ }`). +Attribute names and attribute values are separated by an equal sign (`=`). +Each value can be an arbitrary expression, terminated by a semicolon (`;`) + +An attribute name is a string without context, and is denoted by a [name] (an [identifier](./identifiers.md#identifiers) or [string literal](string-literals.md)). + +[name]: ./identifiers.md#names + +> **Syntax** +> +> *attrset* → `{` { *name* `=` *expr* `;` } `}` + +Attributes can appear in any order. +An attribute name may only occur once in each attribute set. + +> **Example** +> +> This defines an attribute set with attributes named: +> - `x` with the value `123`, an integer +> - `text` with the value `"Hello"`, a string +> - `y` where the value is the result of applying the function `f` to the attribute set `{ bla = 456; }` +> +> ```nix +> { +> x = 123; +> text = "Hello"; +> y = f { bla = 456; }; +> } +> ``` + +Attributes in nested attribute sets can be written using *attribute paths*. + +> **Syntax** +> +> *attrset* → `{` { *attrpath* `=` *expr* `;` } `}` + +An attribute path is a dot-separated list of [names][name]. + +> **Syntax** +> +> *attrpath* = *name* { `.` *name* } + + + +> **Example** +> +> ```nix +> { a.b.c = 1; a.b.d = 2; } +> ``` +> +> { +> a = { +> b = { +> c = 1; +> d = 2; +> }; +> }; +> } + +Attribute names can also be set implicitly by using the [`inherit` keyword](#inheriting-attributes). + +> **Example** +> +> ```nix +> { inherit (builtins) true; } +> ``` +> +> { true = true; } + +Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.a +``` + +This evaluates to `"Foo"`. + +It is possible to provide a default value in an attribute selection using the `or` keyword. + +Example: + +```nix +{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" +``` + +```nix +{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" +``` + +will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. + +You can use arbitrary double-quoted strings as attribute names: + +```nix +{ "$!@#?" = 123; }."$!@#?" +``` + +```nix +let bar = "bar"; in +{ "foo ${bar}" = 123; }."foo ${bar}" +``` + +Both will evaluate to `123`. + +Attribute names support [string interpolation]: + +```nix +let bar = "foo"; in +{ foo = 123; }.${bar} +``` + +```nix +let bar = "foo"; in +{ ${bar} = 123; }.foo +``` + +Both will evaluate to `123`. + +In the special case where an attribute name inside of a set declaration +evaluates to `null` (which is normally an error, as `null` cannot be coerced to +a string), that attribute is simply not added to the set: + +```nix +{ ${if foo then "bar" else null} = true; } +``` + +This will evaluate to `{}` if `foo` evaluates to `false`. + +A set that has a [`__functor`]{#attr-__functor} attribute whose value is callable (i.e. is +itself a function or a set with a `__functor` attribute whose value is +callable) can be applied as if it were a function, with the set itself +passed in first , e.g., + +```nix +let add = { __functor = self: x: x + self.x; }; + inc = add // { x = 1; }; +in inc 1 +``` + +evaluates to `2`. This can be used to attach metadata to a function +without the caller needing to treat it specially, or to implement a form +of object-oriented programming, for example. + +## Recursive sets + +Recursive sets are like normal [attribute sets](./types.md#attribute-set), but the attributes can refer to each other. + +> *rec-attrset* = `rec {` [ *name* `=` *expr* `;` `]`... `}` + +Example: + +```nix +rec { + x = y; + y = 123; +}.x +``` + +This evaluates to `123`. + +Note that without `rec` the binding `x = y;` would +refer to the variable `y` in the surrounding scope, if one exists, and +would be invalid if no such variable exists. That is, in a normal +(non-recursive) set, attributes are not added to the lexical scope; in a +recursive set, they are. + +Recursive sets of course introduce the danger of infinite recursion. For +example, the expression + +```nix +rec { + x = y; + y = x; +}.x +``` + +will crash with an `infinite recursion encountered` error message. + +## Let-expressions + +A let-expression allows you to define local variables for an expression. + +> *let-in* = `let` [ *identifier* = *expr* ]... `in` *expr* + +Example: + +```nix +let + x = "foo"; + y = "bar"; +in x + y +``` + +This evaluates to `"foobar"`. + +## Inheriting attributes + +When defining an [attribute set](./types.md#attribute-set) or in a [let-expression](#let-expressions) it is often convenient to copy variables from the surrounding lexical scope (e.g., when you want to propagate attributes). +This can be shortened using the `inherit` keyword. + +Example: + +```nix +let x = 123; in +{ + inherit x; + y = 456; +} +``` + +is equivalent to + +```nix +let x = 123; in +{ + x = x; + y = 456; +} +``` + +and both evaluate to `{ x = 123; y = 456; }`. + +> **Note** +> +> This works because `x` is added to the lexical scope by the `let` construct. + +It is also possible to inherit attributes from another attribute set. + +Example: + +In this fragment from `all-packages.nix`, + +```nix +graphviz = (import ../tools/graphics/graphviz) { + inherit fetchurl stdenv libpng libjpeg expat x11 yacc; + inherit (xorg) libXaw; +}; + +xorg = { + libX11 = ...; + libXaw = ...; + ... +} + +libpng = ...; +libjpg = ...; +... +``` + +the set used in the function call to the function defined in +`../tools/graphics/graphviz` inherits a number of variables from the +surrounding scope (`fetchurl` ... `yacc`), but also inherits `libXaw` +(the X Athena Widgets) from the `xorg` set. + +Summarizing the fragment + +```nix +... +inherit x y z; +inherit (src-set) a b c; +... +``` + +is equivalent to + +```nix +... +x = x; y = y; z = z; +a = src-set.a; b = src-set.b; c = src-set.c; +... +``` + +when used while defining local variables in a let-expression or while +defining a set. + +In a `let` expression, `inherit` can be used to selectively bring specific attributes of a set into scope. For example + + +```nix +let + x = { a = 1; b = 2; }; + inherit (builtins) attrNames; +in +{ + names = attrNames x; +} +``` + +is equivalent to + +```nix +let + x = { a = 1; b = 2; }; +in +{ + names = builtins.attrNames x; +} +``` + +both evaluate to `{ names = [ "a" "b" ]; }`. + +## Functions + +Functions have the following form: + +```nix +pattern: body +``` + +The pattern specifies what the argument of the function must look like, +and binds variables in the body to (parts of) the argument. There are +three kinds of patterns: + + - If a pattern is a single identifier, then the function matches any + argument. Example: + + ```nix + let negate = x: !x; + concat = x: y: x + y; + in if negate true then concat "foo" "bar" else "" + ``` + + Note that `concat` is a function that takes one argument and returns + a function that takes another argument. This allows partial + parameterisation (i.e., only filling some of the arguments of a + function); e.g., + + ```nix + map (concat "foo") [ "bar" "bla" "abc" ] + ``` + + evaluates to `[ "foobar" "foobla" "fooabc" ]`. + + - A *set pattern* of the form `{ name1, name2, …, nameN }` matches a + set containing the listed attributes, and binds the values of those + attributes to variables in the function body. For example, the + function + + ```nix + { x, y, z }: z + y + x + ``` + + can only be called with a set containing exactly the attributes `x`, + `y` and `z`. No other attributes are allowed. If you want to allow + additional arguments, you can use an ellipsis (`...`): + + ```nix + { x, y, z, ... }: z + y + x + ``` + + This works on any set that contains at least the three named + attributes. + + It is possible to provide *default values* for attributes, in + which case they are allowed to be missing. A default value is + specified by writing `name ? e`, where *e* is an arbitrary + expression. For example, + + ```nix + { x, y ? "foo", z ? "bar" }: z + y + x + ``` + + specifies a function that only requires an attribute named `x`, but + optionally accepts `y` and `z`. + + - An `@`-pattern provides a means of referring to the whole value + being matched: + + ```nix + args@{ x, y, z, ... }: z + y + x + args.a + ``` + + but can also be written as: + + ```nix + { x, y, z, ... } @ args: z + y + x + args.a + ``` + + Here `args` is bound to the argument *as passed*, which is further + matched against the pattern `{ x, y, z, ... }`. + The `@`-pattern makes mainly sense with an ellipsis(`...`) as + you can access attribute names as `a`, using `args.a`, which was + given as an additional attribute to the function. + + > **Warning** + > + > `args@` binds the name `args` to the attribute set that is passed to the function. + > In particular, `args` does *not* include any default values specified with `?` in the function's set pattern. + > + > For instance + > + > ```nix + > let + > f = args@{ a ? 23, ... }: [ a args ]; + > in + > f {} + > ``` + > + > is equivalent to + > + > ```nix + > let + > f = args @ { ... }: [ (args.a or 23) args ]; + > in + > f {} + > ``` + > + > and both expressions will evaluate to: + > + > ```nix + > [ 23 {} ] + > ``` + +Note that functions do not have names. If you want to give them a name, +you can bind them to an attribute, e.g., + +```nix +let concat = { x, y }: x + y; +in concat { x = "foo"; y = "bar"; } +``` + +## Conditionals + +Conditionals look like this: + +```nix +if e1 then e2 else e3 +``` + +where *e1* is an expression that should evaluate to a Boolean value +(`true` or `false`). + +## Assertions + +Assertions are generally used to check that certain requirements on or +between features and dependencies hold. They look like this: + +```nix +assert e1; e2 +``` + +where *e1* is an expression that should evaluate to a Boolean value. If +it evaluates to `true`, *e2* is returned; otherwise expression +evaluation is aborted and a backtrace is printed. + +Here is a Nix expression for the Subversion package that shows how +assertions can be used:. + +```nix +{ localServer ? false +, httpServer ? false +, sslSupport ? false +, pythonBindings ? false +, javaSwigBindings ? false +, javahlBindings ? false +, stdenv, fetchurl +, openssl ? null, httpd ? null, db4 ? null, expat, swig ? null, j2sdk ? null +}: + +assert localServer -> db4 != null; ① +assert httpServer -> httpd != null && httpd.expat == expat; ② +assert sslSupport -> openssl != null && (httpServer -> httpd.openssl == openssl); ③ +assert pythonBindings -> swig != null && swig.pythonSupport; +assert javaSwigBindings -> swig != null && swig.javaSupport; +assert javahlBindings -> j2sdk != null; + +stdenv.mkDerivation { + name = "subversion-1.1.1"; + ... + openssl = if sslSupport then openssl else null; ④ + ... +} +``` + +The points of interest are: + +1. This assertion states that if Subversion is to have support for + local repositories, then Berkeley DB is needed. So if the Subversion + function is called with the `localServer` argument set to `true` but + the `db4` argument set to `null`, then the evaluation fails. + + Note that `->` is the [logical + implication](https://en.wikipedia.org/wiki/Truth_table#Logical_implication) + Boolean operation. + +2. This is a more subtle condition: if Subversion is built with Apache + (`httpServer`) support, then the Expat library (an XML library) used + by Subversion should be same as the one used by Apache. This is + because in this configuration Subversion code ends up being linked + with Apache code, and if the Expat libraries do not match, a build- + or runtime link error or incompatibility might occur. + +3. This assertion says that in order for Subversion to have SSL support + (so that it can access `https` URLs), an OpenSSL library must be + passed. Additionally, it says that *if* Apache support is enabled, + then Apache's OpenSSL should match Subversion's. (Note that if + Apache support is not enabled, we don't care about Apache's + OpenSSL.) + +4. The conditional here is not really related to assertions, but is + worth pointing out: it ensures that if SSL support is disabled, then + the Subversion derivation is not dependent on OpenSSL, even if a + non-`null` value was passed. This prevents an unnecessary rebuild of + Subversion if OpenSSL changes. + +## With-expressions + +A *with-expression*, + +```nix +with e1; e2 +``` + +introduces the set *e1* into the lexical scope of the expression *e2*. +For instance, + +```nix +let as = { x = "foo"; y = "bar"; }; +in with as; x + y +``` + +evaluates to `"foobar"` since the `with` adds the `x` and `y` attributes +of `as` to the lexical scope in the expression `x + y`. The most common +use of `with` is in conjunction with the `import` function. E.g., + +```nix +with (import ./definitions.nix); ... +``` + +makes all attributes defined in the file `definitions.nix` available as +if they were defined locally in a `let`-expression. + +The bindings introduced by `with` do not shadow bindings introduced by +other means, e.g. + +```nix +let a = 3; in with { a = 1; }; let a = 4; in with { a = 2; }; ... +``` + +establishes the same scope as + +```nix +let a = 1; in let a = 2; in let a = 3; in let a = 4; in ... +``` + +Variables coming from outer `with` expressions *are* shadowed: + +```nix +with { a = "outer"; }; +with { a = "inner"; }; +a +``` + +Does evaluate to `"inner"`. + +## Comments + +- Inline comments start with `#` and run until the end of the line. + + > **Example** + > + > ```nix + > # A number + > 2 # Equals 1 + 1 + > ``` + > + > ```console + > 2 + > ``` + +- Block comments start with `/*` and run until the next occurrence of `*/`. + + > **Example** + > + > ```nix + > /* + > Block comments + > can span multiple lines. + > */ "hello" + > ``` + > + > ```console + > "hello" + > ``` + + This means that block comments cannot be nested. + + > **Example** + > + > ```nix + > /* /* nope */ */ 1 + > ``` + > + > ```console + > error: syntax error, unexpected '*' + > + > at «string»:1:15: + > + > 1| /* /* nope */ * + > | ^ + > ``` + + Consider escaping nested comments and unescaping them in post-processing. + + > **Example** + > + > ```nix + > /* /* nested *\/ */ 1 + > ``` + > + > ```console + > 1 + > ``` diff --git a/doc/manual/src/language/types.md b/doc/manual/src/language/types.md new file mode 100644 index 000000000..82184a8b0 --- /dev/null +++ b/doc/manual/src/language/types.md @@ -0,0 +1,120 @@ +# Data Types + +Every value in the Nix language has one of the following types: + +* [Integer](#type-int) +* [Float](#type-float) +* [Boolean](#type-bool) +* [String](#type-string) +* [Path](#type-path) +* [Null](#type-null) +* [Attribute set](#type-attrs) +* [List](#type-list) +* [Function](#type-function) +* [External](#type-external) + +## Primitives + +### Integer {#type-int} + +An _integer_ in the Nix language is a signed 64-bit integer. + +Non-negative integers can be expressed as [integer literals](syntax.md#number-literal). +Negative integers are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isInt`](builtins.md#builtins-isInt) can be used to determine if a value is an integer. + +### Float {#type-float} + +A _float_ in the Nix language is a 64-bit [IEEE 754](https://en.wikipedia.org/wiki/IEEE_754) floating-point number. + +Most non-negative floats can be expressed as [float literals](syntax.md#number-literal). +Negative floats are created with the [arithmetic negation operator](./operators.md#arithmetic). +The function [`builtins.isFloat`](builtins.md#builtins-isFloat) can be used to determine if a value is a float. + +### Boolean {#type-bool} + +A _boolean_ in the Nix language is one of _true_ or _false_. + + + +These values are available as attributes of [`builtins`](builtins.md#builtins-builtins) as [`builtins.true`](builtins.md#builtins-true) and [`builtins.false`](builtins.md#builtins-false). +The function [`builtins.isBool`](builtins.md#builtins-isBool) can be used to determine if a value is a boolean. + +### String {#type-string} + +A _string_ in the Nix language is an immutable, finite-length sequence of bytes, along with a [string context](string-context.md). +Nix does not assume or support working natively with character encodings. + +String values without string context can be expressed as [string literals](string-literals.md). +The function [`builtins.isString`](builtins.md#builtins-isString) can be used to determine if a value is a string. + +### Path {#type-path} + +A _path_ in the Nix language is an immutable, finite-length sequence of bytes starting with `/`, representing a POSIX-style, canonical file system path. +Path values are distinct from string values, even if they contain the same sequence of bytes. +Operations that produce paths will simplify the result as the standard C function [`realpath`] would, except that there is no symbolic link resolution. + +[`realpath`]: https://pubs.opengroup.org/onlinepubs/9699919799/functions/realpath.html + +Paths are suitable for referring to local files, and are often preferable over strings. +- Path values do not contain trailing or duplicate slashes, `.`, or `..`. +- Relative path literals are automatically resolved relative to their [base directory]. +- Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file. + +[base directory]: @docroot@/glossary.md#gloss-base-directory + +A file is not required to exist at a given path in order for that path value to be valid, but a path that is converted to a string with [string interpolation] or [string-and-path concatenation] must resolve to a readable file or directory which will be copied into the Nix store. +For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. +Operations such as [`import`] can also expect a path to resolve to a readable file or directory. + +[string interpolation]: string-interpolation.md#interpolated-expression +[string-and-path concatenation]: operators.md#string-and-path-concatenation +[`import`]: builtins.md#builtins-import + +> **Note** +> +> The Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. +> For example, assume you used a file path in an interpolated string during a `nix repl` session. +> Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. +> Use `:r` to reset the repl as needed. + +[store path]: @docroot@/store/store-path.md + +Path values can be expressed as [path literals](syntax.md#path-literal). +The function [`builtins.isPath`](builtins.md#builtins-isPath) can be used to determine if a value is a path. + +### Null {#type-null} + +There is a single value of type _null_ in the Nix language. + + + +This value is available as an attribute on the [`builtins`](builtins.md#builtins-builtins) attribute set as [`builtins.null`](builtins.md#builtins-null). + +## Compound values + +### Attribute set {#type-attrs} + + + +An attribute set can be constructed with an [attribute set literal](syntax.md#attrs-literal). +The function [`builtins.isAttrs`](builtins.md#builtins-isAttrs) can be used to determine if a value is an attribute set. + +### List {#type-list} + + + +A list can be constructed with a [list literal](syntax.md#list-literal). +The function [`builtins.isList`](builtins.md#builtins-isList) can be used to determine if a value is a list. + +## Function {#type-function} + + + +A function can be constructed with a [function expression](syntax.md#functions). +The function [`builtins.isFunction`](builtins.md#builtins-isFunction) can be used to determine if a value is a function. + +## External {#type-external} + +An _external_ value is an opaque value created by a Nix [plugin](../command-ref/conf-file.md#conf-plugin-files). +Such a value can be substituted in Nix expressions but only created and used by plugin code. diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index 4eb1887fa..e05f56025 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -1,280 +1 @@ # Data Types - -## Primitives - -- String - - *Strings* can be written in three ways. - - The most common way is to enclose the string between double quotes, - e.g., `"foo bar"`. Strings can span multiple lines. The special - characters `"` and `\` and the character sequence `${` must be - escaped by prefixing them with a backslash (`\`). Newlines, carriage - returns and tabs can be written as `\n`, `\r` and `\t`, - respectively. - - You can include the results of other expressions into a string by enclosing them in `${ }`, a feature known as [string interpolation]. - - [string interpolation]: ./string-interpolation.md - - The second way to write string literals is as an *indented string*, - which is enclosed between pairs of *double single-quotes*, like so: - - ```nix - '' - This is the first line. - This is the second line. - This is the third line. - '' - ``` - - This kind of string literal intelligently strips indentation from - the start of each line. To be precise, it strips from each line a - number of spaces equal to the minimal indentation of the string as a - whole (disregarding the indentation of empty lines). For instance, - the first and second line are indented two spaces, while the third - line is indented four spaces. Thus, two spaces are stripped from - each line, so the resulting string is - - ```nix - "This is the first line.\nThis is the second line.\n This is the third line.\n" - ``` - - Note that the whitespace and newline following the opening `''` is - ignored if there is no non-whitespace text on the initial line. - - Indented strings support [string interpolation]. - - Since `${` and `''` have special meaning in indented strings, you - need a way to quote them. `$` can be escaped by prefixing it with - `''` (that is, two single quotes), i.e., `''$`. `''` can be escaped - by prefixing it with `'`, i.e., `'''`. `$` removes any special - meaning from the following `$`. Linefeed, carriage-return and tab - characters can be written as `''\n`, `''\r`, `''\t`, and `''\` - escapes any other character. - - Indented strings are primarily useful in that they allow multi-line - string literals to follow the indentation of the enclosing Nix - expression, and that less escaping is typically necessary for - strings representing languages such as shell scripts and - configuration files because `''` is much less common than `"`. - Example: - - ```nix - stdenv.mkDerivation { - ... - postInstall = - '' - mkdir $out/bin $out/etc - cp foo $out/bin - echo "Hello World" > $out/etc/foo.conf - ${if enableBar then "cp bar $out/bin" else ""} - ''; - ... - } - ``` - - Finally, as a convenience, *URIs* as defined in appendix B of - [RFC 2396](http://www.ietf.org/rfc/rfc2396.txt) can be written *as - is*, without quotes. For instance, the string - `"http://example.org/foo.tar.bz2"` can also be written as - `http://example.org/foo.tar.bz2`. - -- Number - - Numbers, which can be *integers* (like `123`) or *floating point* - (like `123.43` or `.27e13`). - - See [arithmetic] and [comparison] operators for semantics. - - [arithmetic]: ./operators.md#arithmetic - [comparison]: ./operators.md#comparison - -- Path - - *Paths* are distinct from strings and can be expressed by path literals such as `./builder.sh`. - - Paths are suitable for referring to local files, and are often preferable over strings. - - Path values do not contain trailing slashes, `.` and `..`, as they are resolved when evaluating a path literal. - - Path literals are automatically resolved relative to their [base directory](@docroot@/glossary.md#gloss-base-directory). - - The files referred to by path values are automatically copied into the Nix store when used in a string interpolation or concatenation. - - Tooling can recognize path literals and provide additional features, such as autocompletion, refactoring automation and jump-to-file. - - A path literal must contain at least one slash to be recognised as such. - For instance, `builder.sh` is not a path: - it's parsed as an expression that selects the attribute `sh` from the variable `builder`. - - Path literals may also refer to absolute paths by starting with a slash. - - > **Note** - > - > Absolute paths make expressions less portable. - > In the case where a function translates a path literal into an absolute path string for a configuration file, it is recommended to write a string literal instead. - > This avoids some confusion about whether files at that location will be used during evaluation. - > It also avoids unintentional situations where some function might try to copy everything at the location into the store. - - If the first component of a path is a `~`, it is interpreted such that the rest of the path were relative to the user's home directory. - For example, `~/foo` would be equivalent to `/home/edolstra/foo` for a user whose home directory is `/home/edolstra`. - Path literals that start with `~` are not allowed in [pure](@docroot@/command-ref/conf-file.md#conf-pure-eval) evaluation. - - Paths can be used in [string interpolation] and string concatenation. - For instance, evaluating `"${./foo.txt}"` will cause `foo.txt` from the same directory to be copied into the Nix store and result in the string `"/nix/store/-foo.txt"`. - - Note that the Nix language assumes that all input files will remain _unchanged_ while evaluating a Nix expression. - For example, assume you used a file path in an interpolated string during a `nix repl` session. - Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. Use `:r` to reset the repl as needed. - - [store path]: @docroot@/store/store-path.md - - Path literals can also include [string interpolation], besides being [interpolated into other expressions]. - - [interpolated into other expressions]: ./string-interpolation.md#interpolated-expressions - - At least one slash (`/`) must appear *before* any interpolated expression for the result to be recognized as a path. - - `a.${foo}/b.${bar}` is a syntactically valid number division operation. - `./a.${foo}/b.${bar}` is a path. - - [Lookup path](./constructs/lookup-path.md) literals such as `` also resolve to path values. - -- Boolean - - *Booleans* with values `true` and `false`. - -- Null - - The null value, denoted as `null`. - -## List - -Lists are formed by enclosing a whitespace-separated list of values -between square brackets. For example, - -```nix -[ 123 ./foo.nix "abc" (f { x = y; }) ] -``` - -defines a list of four elements, the last being the result of a call to -the function `f`. Note that function calls have to be enclosed in -parentheses. If they had been omitted, e.g., - -```nix -[ 123 ./foo.nix "abc" f { x = y; } ] -``` - -the result would be a list of five elements, the fourth one being a -function and the fifth being a set. - -Note that lists are only lazy in values, and they are strict in length. - -Elements in a list can be accessed using [`builtins.elemAt`](./builtins.md#builtins-elemAt). - -## Attribute Set - -An attribute set is a collection of name-value-pairs (called *attributes*) enclosed in curly brackets (`{ }`). - -An attribute name can be an identifier or a [string](#string). -An identifier must start with a letter (`a-z`, `A-Z`) or underscore (`_`), and can otherwise contain letters (`a-z`, `A-Z`), numbers (`0-9`), underscores (`_`), apostrophes (`'`), or dashes (`-`). - -> **Syntax** -> -> *name* = *identifier* | *string* \ -> *identifier* ~ `[a-zA-Z_][a-zA-Z0-9_'-]*` - -Names and values are separated by an equal sign (`=`). -Each value is an arbitrary expression terminated by a semicolon (`;`). - -> **Syntax** -> -> *attrset* = `{` [ *name* `=` *expr* `;` ]... `}` - -Attributes can appear in any order. -An attribute name may only occur once. - -Example: - -```nix -{ - x = 123; - text = "Hello"; - y = f { bla = 456; }; -} -``` - -This defines a set with attributes named `x`, `text`, `y`. - -Attributes can be accessed with the [`.` operator](./operators.md#attribute-selection). - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.a -``` - -This evaluates to `"Foo"`. - -It is possible to provide a default value in an attribute selection using the `or` keyword. - -Example: - -```nix -{ a = "Foo"; b = "Bar"; }.c or "Xyzzy" -``` - -```nix -{ a = "Foo"; b = "Bar"; }.c.d.e.f.g or "Xyzzy" -``` - -will both evaluate to `"Xyzzy"` because there is no `c` attribute in the set. - -You can use arbitrary double-quoted strings as attribute names: - -```nix -{ "$!@#?" = 123; }."$!@#?" -``` - -```nix -let bar = "bar"; in -{ "foo ${bar}" = 123; }."foo ${bar}" -``` - -Both will evaluate to `123`. - -Attribute names support [string interpolation]: - -```nix -let bar = "foo"; in -{ foo = 123; }.${bar} -``` - -```nix -let bar = "foo"; in -{ ${bar} = 123; }.foo -``` - -Both will evaluate to `123`. - -In the special case where an attribute name inside of a set declaration -evaluates to `null` (which is normally an error, as `null` cannot be coerced to -a string), that attribute is simply not added to the set: - -```nix -{ ${if foo then "bar" else null} = true; } -``` - -This will evaluate to `{}` if `foo` evaluates to `false`. - -A set that has a `__functor` attribute whose value is callable (i.e. is -itself a function or a set with a `__functor` attribute whose value is -callable) can be applied as if it were a function, with the set itself -passed in first , e.g., - -```nix -let add = { __functor = self: x: x + self.x; }; - inc = add // { x = 1; }; -in inc 1 -``` - -evaluates to `2`. This can be used to attach metadata to a function -without the caller needing to treat it specially, or to implement a form -of object-oriented programming, for example. diff --git a/doc/manual/src/language/variables.md b/doc/manual/src/language/variables.md new file mode 100644 index 000000000..af6aff8a2 --- /dev/null +++ b/doc/manual/src/language/variables.md @@ -0,0 +1,10 @@ +# Variables + +A *variable* is an [identifier](identifiers.md) used as an expression. + +> **Syntax** +> +> *expression* → *identifier* + +A variable must have the same name as a definition in the [scope](./scope.md) that encloses it. +The value of a variable is the value of the corresponding expression in the enclosing scope. diff --git a/doc/manual/src/meson.build b/doc/manual/src/meson.build new file mode 100644 index 000000000..098a29897 --- /dev/null +++ b/doc/manual/src/meson.build @@ -0,0 +1,17 @@ +summary_rl_next = custom_target( + command : [ + bash, + '-euo', 'pipefail', + '-c', + ''' + if [ -e "@INPUT@" ]; then + echo ' - [Upcoming release](release-notes/rl-next.md)' + fi + ''', + ], + input : [ + rl_next_generated, + ], + capture: true, + output : 'SUMMARY-rl-next.md', +) diff --git a/doc/manual/src/package-management/copy-closure.md b/doc/manual/src/package-management/copy-closure.md deleted file mode 100644 index 14326298b..000000000 --- a/doc/manual/src/package-management/copy-closure.md +++ /dev/null @@ -1,34 +0,0 @@ -# Copying Closures via SSH - -The command `nix-copy-closure` copies a Nix store path along with all -its dependencies to or from another machine via the SSH protocol. It -doesn’t copy store paths that are already present on the target machine. -For example, the following command copies Firefox with all its -dependencies: - - $ nix-copy-closure --to alice@itchy.example.org $(type -p firefox) - -See the [manpage for `nix-copy-closure`](../command-ref/nix-copy-closure.md) for details. - -With `nix-store ---export` and `nix-store --import` you can write the closure of a store -path (that is, the path and all its dependencies) to a file, and then -unpack that file into another Nix store. For example, - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) > firefox.closure - -writes the closure of Firefox to a file. You can then copy this file to -another machine and install the closure: - - $ nix-store --import < firefox.closure - -Any store paths in the closure that are already present in the target -store are ignored. It is also possible to pipe the export into another -command, e.g. to copy and install a closure directly to/on another -machine: - - $ nix-store --export $(nix-store --query --requisites $(type -p firefox)) | bzip2 | \ - ssh alice@itchy.example.org "bunzip2 | nix-store --import" - -However, `nix-copy-closure` is generally more efficient because it only -copies paths that are not already present in the target Nix store. diff --git a/doc/manual/src/protocols/derivation-aterm.md b/doc/manual/src/protocols/derivation-aterm.md index e58b602a3..1ba757ae0 100644 --- a/doc/manual/src/protocols/derivation-aterm.md +++ b/doc/manual/src/protocols/derivation-aterm.md @@ -14,6 +14,6 @@ Derivations are serialised in one of the following formats: DrvWithVersion(, ...) ``` - The only `version-string`s that are in use today are for [experimental features](@docroot@/contributing/experimental-features.md): + The only `version-string`s that are in use today are for [experimental features](@docroot@/development/experimental-features.md): - - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + - `"xp-dyn-drv"` for the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. diff --git a/doc/manual/src/protocols/json/derivation.md b/doc/manual/src/protocols/json/derivation.md index 649d543cc..2f85340d6 100644 --- a/doc/manual/src/protocols/json/derivation.md +++ b/doc/manual/src/protocols/json/derivation.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. The JSON serialization of a @@ -18,10 +18,30 @@ is a JSON object with the following fields: Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields: - * `path`: The output path. + * `path`: + The output path, if it is known in advanced. + Otherwise, `null`. + + + * `method`: + For an output which will be [content addresed], a string representing the [method](@docroot@/store/store-object/content-address.md) of content addressing that is chosen. + Valid method strings are: + + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat) + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + - [`text`](@docroot@/store/store-object/content-address.md#method-text) + - [`git`](@docroot@/store/store-object/content-address.md#method-git) + + Otherwise, `null`. * `hashAlgo`: - For fixed-output derivations, the hashing algorithm (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a NAR hash rather than a flat file hash. + For an output which will be [content addresed], the name of the hash algorithm used. + Valid algorithm strings are: + + - `md5` + - `sha1` + - `sha256` + - `sha512` * `hash`: For fixed-output derivations, the expected content hash in base-16. @@ -32,7 +52,8 @@ is a JSON object with the following fields: > "outputs": { > "out": { > "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", - > "hashAlgo": "r:sha256", + > "method": "nar", + > "hashAlgo": "sha256", > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" > } > } diff --git a/doc/manual/src/protocols/json/index.md b/doc/manual/src/protocols/json/index.md new file mode 100644 index 000000000..1fcd1e62d --- /dev/null +++ b/doc/manual/src/protocols/json/index.md @@ -0,0 +1 @@ +# JSON Formats diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md index 22a14715f..6b4f48437 100644 --- a/doc/manual/src/protocols/json/store-object-info.md +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -3,7 +3,7 @@ > **Warning** > > This JSON format is currently -> [**experimental**](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) > and subject to change. Info about a [store object]. @@ -24,9 +24,11 @@ Info about a [store object]. An array of [store paths][store path], possibly including this one. -* `ca` (optional): +* `ca`: - Content address of this store object's file system object, used to compute its store path. + If the store object is [content-addressed], + this is the content address of this store object's file system object, used to compute its store path. + Otherwise (i.e. if it is [input-addressed]), this is `null`. [store path]: @docroot@/store/store-path.md [file system object]: @docroot@/store/file-system-object.md @@ -37,28 +39,30 @@ Info about a [store object]. These are not intrinsic properties of the store object. In other words, the same store object residing in different store could have different values for these properties. -* `deriver` (optional): +* `deriver`: - The path to the [derivation] from which this store object is produced. + If known, the path to the [derivation] from which this store object was produced. + Otherwise `null`. [derivation]: @docroot@/glossary.md#gloss-store-derivation * `registrationTime` (optional): - When this derivation was added to the store. + If known, when this derivation was added to the store. + Otherwise `null`. -* `ultimate` (optional): +* `ultimate`: Whether this store object is trusted because we built it ourselves, rather than substituted a build product from elsewhere. -* `signatures` (optional): +* `signatures`: Signatures claiming that this store object is what it claims to be. Not relevant for [content-addressed] store objects, but useful for [input-addressed] store objects. - [content-addressed]: @docroot@/glossary.md#gloss-content-addressed-store-object - [input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object +[content-addressed]: @docroot@/store/store-object/content-address.md +[input-addressed]: @docroot@/glossary.md#gloss-input-addressed-store-object ### `.narinfo` extra fields diff --git a/doc/manual/src/protocols/nix-archive.md b/doc/manual/src/protocols/nix-archive.md index bfc523b3d..640b527f1 100644 --- a/doc/manual/src/protocols/nix-archive.md +++ b/doc/manual/src/protocols/nix-archive.md @@ -29,7 +29,7 @@ regular = [ str("executable"), str("") ], str("contents"), str(contents); symlink = str("target"), str(target); (* side condition: directory entries must be ordered by their names *) -directory = str("type"), str("directory") { directory-entry }; +directory = { directory-entry }; directory-entry = str("entry"), str("("), str("name"), str(name), str("node"), nar-obj, str(")"); ``` diff --git a/doc/manual/src/protocols/store-path.md b/doc/manual/src/protocols/store-path.md index 657774238..8ec6f8201 100644 --- a/doc/manual/src/protocols/store-path.md +++ b/doc/manual/src/protocols/store-path.md @@ -36,18 +36,23 @@ where - `type` = one of: - ```ebnf - | "text" ( ":" store-path )* + | "text" { ":" store-path } ``` - for encoded derivations written to the store. + This is for the + ["Text"](@docroot@/store/store-object/content-address.md#method-text) + method of content addressing store objects. The optional trailing store paths are the references of the store object. - ```ebnf - | "source" ( ":" store-path )* + | "source" { ":" store-path } [ ":self" ] ``` - For paths copied to the store and hashed via a [Nix Archive (NAR)] and [SHA-256][sha-256]. - Just like in the text case, we can have the store objects referenced by their paths. + This is for the + ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) + method of content addressing store objects, + if the hash algorithm is [SHA-256]. + Just like in the "Text" case, we can have the store objects referenced by their paths. Additionally, we can have an optional `:self` label to denote self reference. - ```ebnf @@ -55,8 +60,12 @@ where ``` For either the outputs built from derivations, - paths copied to the store hashed that area single file hashed directly, or the via a hash algorithm other than [SHA-256][sha-256]. - (in that case "source" is used; this is only necessary for compatibility). + or content-addressed store objects that are not using one of the two above cases. + To be explicit about the latter, that is currently these methods: + + - ["Flat"](@docroot@/store/store-object/content-address.md#method-flat) + - ["Git"](@docroot@/store/store-object/content-address.md#method-git) + - ["Nix Archive"](@docroot@/store/store-object/content-address.md#method-nix-archive) if the hash algorithm is not [SHA-256]. `id` is the name of the output (usually, "out"). For content-addressed store objects, `id`, is always "out". @@ -73,7 +82,7 @@ where - if `type` = `"source:" ...`: - the the hash of the [Nix Archive (NAR)] serialization of the [file system object](@docroot@/store/file-system-object.md) of the store object. + the hash of the [Nix Archive (NAR)] serialization of the [file system object](@docroot@/store/file-system-object.md) of the store object. - if `type` = `"output:" id`: @@ -116,7 +125,7 @@ where Also note that NAR + SHA-256 must not use this case, and instead must use the `type` = `"source:" ...` case. [Nix Archive (NAR)]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive -[sha-256]: https://en.m.wikipedia.org/wiki/SHA-256 +[SHA-256]: https://en.m.wikipedia.org/wiki/SHA-256 ### Historical Note diff --git a/doc/manual/src/protocols/tarball-fetcher.md b/doc/manual/src/protocols/tarball-fetcher.md index 24ec7ae14..5cff05d66 100644 --- a/doc/manual/src/protocols/tarball-fetcher.md +++ b/doc/manual/src/protocols/tarball-fetcher.md @@ -41,4 +41,30 @@ Link: ///archive/.tar.gz +``` + +> **Example** +> +> +> ```nix +> # flake.nix +> { +> inputs = { +> foo.url = "https://gitea.example.org/some-person/some-flake/archive/main.tar.gz"; +> bar.url = "https://gitea.example.org/some-other-person/other-flake/archive/442793d9ec0584f6a6e82fa253850c8085bb150a.tar.gz"; +> qux = { +> url = "https://forgejo.example.org/another-person/some-non-flake-repo/archive/development.tar.gz"; +> flake = false; +> }; +> }; +> outputs = { foo, bar, qux }: { /* ... */ }; +> } +``` + [Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive diff --git a/doc/manual/src/release-notes/meson.build b/doc/manual/src/release-notes/meson.build new file mode 100644 index 000000000..d8bf154e1 --- /dev/null +++ b/doc/manual/src/release-notes/meson.build @@ -0,0 +1,24 @@ +rl_next_generated = custom_target( + command : [ + 'bash', + '-euo', + 'pipefail', + '-c', + ''' + if type -p build-release-notes > /dev/null; then + build-release-notes --change-authors @CURRENT_SOURCE_DIR@/../../change-authors.yml @CURRENT_SOURCE_DIR@/../../rl-next + elif type -p changelog-d > /dev/null; then + changelog-d @CURRENT_SOURCE_DIR@/../../rl-next + fi + @0@ @INPUT0@ @CURRENT_SOURCE_DIR@/../../rl-next > @DEPFILE@ + '''.format( + python.full_path(), + ), + ], + input : [ + generate_manual_deps, + ], + output : 'rl-next.md', + capture : true, + depfile : 'rl-next.d', +) diff --git a/doc/manual/src/release-notes/rl-2.18.md b/doc/manual/src/release-notes/rl-2.18.md index 4bbc52b50..eb26fc9e7 100644 --- a/doc/manual/src/release-notes/rl-2.18.md +++ b/doc/manual/src/release-notes/rl-2.18.md @@ -13,7 +13,7 @@ - The `discard-references` feature has been stabilized. This means that the - [unsafeDiscardReferences](@docroot@/contributing/experimental-features.md#xp-feature-discard-references) + [unsafeDiscardReferences](@docroot@/development/experimental-features.md#xp-feature-discard-references) attribute is no longer guarded by an experimental flag and can be used freely. @@ -21,7 +21,7 @@ This only affects `nix-build --json` when "building" non-derivation things like fetched sources, which is a no-op. - A new builtin [`outputOf`](@docroot@/language/builtins.md#builtins-outputOf) has been added. - It is part of the [`dynamic-derivations`](@docroot@/contributing/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. + It is part of the [`dynamic-derivations`](@docroot@/development/experimental-features.md#xp-feature-dynamic-derivations) experimental feature. - Flake follow paths at depths greater than 2 are now handled correctly, preventing "follows a non-existent input" errors. diff --git a/doc/manual/src/release-notes/rl-2.19.md b/doc/manual/src/release-notes/rl-2.19.md index ba6eb9c64..e2e2f85cc 100644 --- a/doc/manual/src/release-notes/rl-2.19.md +++ b/doc/manual/src/release-notes/rl-2.19.md @@ -17,8 +17,8 @@ - `nix-shell` shebang lines now support single-quoted arguments. -- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). - This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/contributing/experimental-features.md#xp-fetch-tree). +- `builtins.fetchTree` is now its own experimental feature, [`fetch-tree`](@docroot@/development/experimental-features.md#xp-fetch-tree). + This allows stabilising it independently of the rest of what is encompassed by [`flakes`](@docroot@/development/experimental-features.md#xp-fetch-tree). - The interface for creating and updating lock files has been overhauled: @@ -33,7 +33,7 @@ - The flake-specific flags `--recreate-lock-file` and `--update-input` have been removed from all commands operating on installables. They are superceded by `nix flake update`. -- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). +- Commit signature verification for the [`builtins.fetchGit`](@docroot@/language/builtins.md#builtins-fetchGit) is added as the new [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - [`nix path-info --json`](@docroot@/command-ref/new-cli/nix3-path-info.md) (experimental) now returns a JSON map rather than JSON list. diff --git a/doc/manual/src/release-notes/rl-2.23.md b/doc/manual/src/release-notes/rl-2.23.md new file mode 100644 index 000000000..249c183e6 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.23.md @@ -0,0 +1,102 @@ +# Release 2.23.0 (2024-06-03) + +- New builtin: `builtins.warn` [#306026](https://github.com/NixOS/nix/issues/306026) [#10592](https://github.com/NixOS/nix/pull/10592) + + `builtins.warn` behaves like `builtins.trace "warning: ${msg}"`, has an accurate log level, and is controlled by the options + [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace), + [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn) and + [`abort-on-warn`](@docroot@/command-ref/conf-file.md#conf-abort-on-warn). + +- Make `nix build --keep-going` consistent with `nix-build --keep-going` + + This means that if e.g. multiple fixed-output derivations fail to + build, all hash mismatches are displayed. + +- Modify `nix derivation {add,show}` JSON format [#9866](https://github.com/NixOS/nix/issues/9866) [#10722](https://github.com/NixOS/nix/pull/10722) + + The JSON format for derivations has been slightly revised to better conform to our [JSON guidelines](@docroot@/development/cli-guideline.md#returning-future-proof-json). + In particular, the hash algorithm and content addressing method of content-addresed derivation outputs are now separated into two fields `hashAlgo` and `method`, + rather than one field with an arcane `:`-separated format. + + This JSON format is only used by the experimental `nix derivation` family of commands, at this time. + Future revisions are expected as the JSON format is still not entirely in compliance even after these changes. + +- Warn on unknown settings anywhere in the command line [#10701](https://github.com/NixOS/nix/pull/10701) + + All `nix` commands will now properly warn when an unknown option is specified anywhere in the command line. + + Before: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + $ nix eval --expr '{}' --option foobar baz + { } + ``` + + After: + + ```console + $ nix-instantiate --option foobar baz --expr '{}' + warning: unknown setting 'foobar' + $ nix-instantiate '{}' --option foobar baz --expr + warning: unknown setting 'foobar' + $ nix eval --expr '{}' --option foobar baz + warning: unknown setting 'foobar' + { } + ``` + +- `nix env shell` is the new `nix shell`, and `nix shell` remains an accepted alias [#10504](https://github.com/NixOS/nix/issues/10504) [#10807](https://github.com/NixOS/nix/pull/10807) + + This is part of an effort to bring more structure to the CLI subcommands. + + `nix env` will be about the process environment. + Future commands may include `nix env run` and `nix env print-env`. + + It is also somewhat analogous to the [planned](https://github.com/NixOS/nix/issues/10504) `nix dev shell` (currently `nix develop`), which is less about environment variables, and more about running a development shell, which is a more powerful command, but also requires more setup. + +- Flake operations that expect derivations now print the failing value and its type [#10778](https://github.com/NixOS/nix/pull/10778) + + In errors like `flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path`, the message now includes the failing value and type. + + Before: + + ``` + error: flake output attribute 'nixosConfigurations.yuki.config' is not a derivation or path + ```` + + After: + + ``` + error: expected flake output attribute 'nixosConfigurations.yuki.config' to be a derivation or path but found a set: { appstream = «thunk»; assertions = «thunk»; boot = { bcache = «thunk»; binfmt = «thunk»; binfmtMiscRegistrations = «thunk»; blacklistedKernelModules = «thunk»; bootMount = «thunk»; bootspec = «thunk»; cleanTmpDir = «thunk»; consoleLogLevel = «thunk»; «43 attributes elided» }; «48 attributes elided» } + ``` + +- `fetchTree` now fetches Git repositories shallowly by default [#10028](https://github.com/NixOS/nix/pull/10028) + + `builtins.fetchTree` now clones Git repositories shallowly by default, which reduces network traffic and disk usage significantly in many cases. + + Previously, the default behavior was to clone the full history of a specific tag or branch (e.g. `ref`) and only afterwards extract the files of one specific revision. + + From now on, the `ref` and `allRefs` arguments will be ignored, except if shallow cloning is disabled by setting `shallow = false`. + + The defaults for `builtins.fetchGit` remain unchanged. Here, shallow cloning has to be enabled manually by passing `shallow = true`. + +- Store object info JSON format now uses `null` rather than omitting fields [#9995](https://github.com/NixOS/nix/pull/9995) + + The [store object info JSON format](@docroot@/protocols/json/store-object-info.md), used for e.g. `nix path-info`, no longer omits fields to indicate absent information, but instead includes the fields with a `null` value. + For example, `"ca": null` is used to indicate a store object that isn't content-addressed rather than omitting the `ca` field entirely. + This makes records of this sort more self-describing, and easier to consume programmatically. + + We will follow this design principle going forward; + the [JSON guidelines](@docroot@/development/json-guideline.md) in the contributing section have been updated accordingly. + +- Large path warnings [#10661](https://github.com/NixOS/nix/pull/10661) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using [the `warn-large-path-threshold` + setting](@docroot@/command-ref/conf-file.md#conf-warn-large-path-threshold), + e.g. `--warn-large-path-threshold 100M` will warn about paths larger + than 100 MiB. + diff --git a/doc/manual/src/release-notes/rl-2.24.md b/doc/manual/src/release-notes/rl-2.24.md new file mode 100644 index 000000000..08ec65be9 --- /dev/null +++ b/doc/manual/src/release-notes/rl-2.24.md @@ -0,0 +1,318 @@ +# Release 2.24.0 (2024-07-31) + +### Significant changes + +- Harden user sandboxing + + The build directory has been hardened against interference with the outside world by nesting it inside another directory owned by (and only readable by) the daemon user. + + This is a low severity security fix, [CVE-2024-38531](https://www.cve.org/CVERecord?id=CVE-2024-38531). + + Credit: [**@alois31**](https://github.com/alois31), [**Linus Heckemann (@lheckemann)**](https://github.com/lheckemann) + Co-authors: [**@edolstra**](https://github.com/edolstra) + +- `nix-shell ` looks for `shell.nix` [#496](https://github.com/NixOS/nix/issues/496) [#2279](https://github.com/NixOS/nix/issues/2279) [#4529](https://github.com/NixOS/nix/issues/4529) [#5431](https://github.com/NixOS/nix/issues/5431) [#11053](https://github.com/NixOS/nix/issues/11053) [#11057](https://github.com/NixOS/nix/pull/11057) + + `nix-shell $x` now looks for `$x/shell.nix` when `$x` resolves to a directory. + + Although this might be seen as a breaking change, its primarily interactive usage makes it a minor issue. + This adjustment addresses a commonly reported problem. + + This also applies to `nix-shell` shebang scripts. Consider the following example: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell -i bash + ``` + + This will now load `shell.nix` from the script's directory, if it exists; `default.nix` otherwise. + + The old behavior can be opted into by setting the option [`nix-shell-always-looks-for-shell-nix`](@docroot@/command-ref/conf-file.md#conf-nix-shell-always-looks-for-shell-nix) to `false`. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- `nix-repl`'s `:doc` shows documentation comments [#3904](https://github.com/NixOS/nix/issues/3904) [#10771](https://github.com/NixOS/nix/issues/10771) [#1652](https://github.com/NixOS/nix/pull/1652) [#9054](https://github.com/NixOS/nix/pull/9054) [#11072](https://github.com/NixOS/nix/pull/11072) + + `nix repl` has a `:doc` command that previously only rendered documentation for internally defined functions. + This feature has been extended to also render function documentation comments, in accordance with [RFC 145]. + + Example: + + ``` + nix-repl> :doc lib.toFunction + Function toFunction + … defined at /home/user/h/nixpkgs/lib/trivial.nix:1072:5 + + Turns any non-callable values into constant functions. Returns + callable values as is. + + Inputs + + v + + : Any value + + Examples + + :::{.example} + + ## lib.trivial.toFunction usage example + + | nix-repl> lib.toFunction 1 2 + | 1 + | + | nix-repl> lib.toFunction (x: x + 1) 2 + | 3 + + ::: + ``` + + Known limitations: + - It does not render documentation for "formals", such as `{ /** the value to return */ x, ... }: x`. + - Some extensions to markdown are not yet supported, as you can see in the example above. + + We'd like to acknowledge [Yingchi Long (@inclyc)](https://github.com/inclyc) for proposing a proof of concept for this functionality in [#9054](https://github.com/NixOS/nix/pull/9054), as well as [@sternenseemann](https://github.com/sternenseemann) and [Johannes Kirschbauer (@hsjobeki)](https://github.com/hsjobeki) for their contributions, proposals, and their work on [RFC 145]. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + + [RFC 145]: https://github.com/NixOS/rfcs/pull/145 + +### Other changes + +- Solve `cached failure of attribute X` [#9165](https://github.com/NixOS/nix/issues/9165) [#10513](https://github.com/NixOS/nix/issues/10513) [#10564](https://github.com/NixOS/nix/pull/10564) + + This eliminates all "cached failure of attribute X" messages by forcing evaluation of the original value when needed to show the exception to the user. This enhancement improves error reporting by providing the underlying message and stack trace. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Run the flake regressions test suite [#10603](https://github.com/NixOS/nix/pull/10603) + + This update introduces a GitHub action to run a subset of the [flake regressions test suite](https://github.com/NixOS/flake-regressions), which includes 259 flakes with their expected evaluation results. Currently, the action runs the first 25 flakes due to the full test suite's extensive runtime. A manually triggered action may be implemented later to run the entire test suite. + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Support unit prefixes in configuration settings [#10668](https://github.com/NixOS/nix/pull/10668) + + Configuration settings in Nix now support unit prefixes, allowing for more intuitive and readable configurations. For example, you can now specify [`--min-free 1G`](@docroot@/command-ref/opt-common.md#opt-min-free) to set the minimum free space to 1 gigabyte. + + This enhancement was extracted from [#7851](https://github.com/NixOS/nix/pull/7851) and is also useful for PR [#10661](https://github.com/NixOS/nix/pull/10661). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- `nix build`: show all FOD errors with `--keep-going` [#10734](https://github.com/NixOS/nix/pull/10734) + + The [`nix build`](@docroot@/command-ref/new-cli/nix3-build.md) command has been updated to improve the behavior of the [`--keep-going`] flag. Now, when `--keep-going` is used, all hash-mismatch errors of failing fixed-output derivations (FODs) are displayed, similar to the behavior for other build failures. This enhancement ensures that all relevant build errors are shown, making it easier for users to update multiple derivations at once or to diagnose and fix issues. + + Author: [**Jörg Thalheim (@Mic92)**](https://github.com/Mic92), [**Maximilian Bosch (@Ma27)**](https://github.com/Ma27) + + [`--keep-going`](@docroot@/command-ref/opt-common.md#opt-keep-going) + +- Build with Meson [#2503](https://github.com/NixOS/nix/issues/2503) [#10378](https://github.com/NixOS/nix/pull/10378) [#10855](https://github.com/NixOS/nix/pull/10855) [#10904](https://github.com/NixOS/nix/pull/10904) [#10908](https://github.com/NixOS/nix/pull/10908) [#10914](https://github.com/NixOS/nix/pull/10914) [#10933](https://github.com/NixOS/nix/pull/10933) [#10936](https://github.com/NixOS/nix/pull/10936) [#10954](https://github.com/NixOS/nix/pull/10954) [#10955](https://github.com/NixOS/nix/pull/10955) [#10963](https://github.com/NixOS/nix/pull/10963) [#10967](https://github.com/NixOS/nix/pull/10967) [#10973](https://github.com/NixOS/nix/pull/10973) [#11034](https://github.com/NixOS/nix/pull/11034) [#11054](https://github.com/NixOS/nix/pull/11054) [#11055](https://github.com/NixOS/nix/pull/11055) [#11060](https://github.com/NixOS/nix/pull/11060) [#11064](https://github.com/NixOS/nix/pull/11064) [#11155](https://github.com/NixOS/nix/pull/11155) + + These changes aim to replace the use of autotools and `make` with Meson for building various components of Nix. Additionally, each library is built in its own derivation, leveraging Meson's "subprojects" feature to allow a single development shell for building all libraries while also supporting separate builds. This approach aims to improve productivity and build modularity, compared to both make and a monolithic Meson-based derivation. + + Special thanks to everyone who has contributed to the Meson port, particularly [**@p01arst0rm**](https://github.com/p01arst0rm) and [**@Qyriad**](https://github.com/Qyriad). + + Authors: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314), [**Tom Bereknyei**](https://github.com/tomberek), [**Théophane Hufschmitt (@thufschmitt)**](https://github.com/thufschmitt), [**Valentin Gagarin (@fricklerhandwerk)**](https://github.com/fricklerhandwerk), [**Robert Hensing (@roberth)**](https://github.com/roberth) + Co-authors: [**@p01arst0rm**](https://github.com/p01arst0rm), [**@Qyriad**](https://github.com/Qyriad) + +- Evaluation cache: fix cache regressions [#10570](https://github.com/NixOS/nix/issues/10570) [#11086](https://github.com/NixOS/nix/pull/11086) + + This update addresses two bugs in the evaluation cache system: + + 1. Regression in #10570: The evaluation cache was not being persisted in `nix develop`. + 2. Nix could sometimes try to commit the evaluation cache SQLite transaction without there being an active transaction, resulting in non-error errors being printed. + + Author: [**Lexi Mattick (@kognise)**](https://github.com/kognise) + +- Introduce `libnixflake` [#9063](https://github.com/NixOS/nix/pull/9063) + + A new library, `libnixflake`, has been introduced to better separate the Flakes layer within Nix. This change refactors the codebase to encapsulate Flakes-specific functionality within its own library. + + See the commits in the pull request for detailed changes, with the only significant code modifications happening in the initial commit. + + This change was alluded to in [RFC 134](https://github.com/nixos/rfcs/blob/master/rfcs/0134-nix-store-layer.md) and is a step towards a more modular and maintainable codebase. + + Author: [**John Ericson (@Ericson2314)**](https://github.com/Ericson2314) + +- CLI options `--arg-from-file` and `--arg-from-stdin` [#9913](https://github.com/NixOS/nix/pull/9913) + +- The `--debugger` now prints source location information, instead of the + pointers of source location information. Before: + + ``` + nix-repl> :bt + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + 0x600001522598 + ``` + + After: + + ``` + 0: while evaluating the attribute 'python311.pythonForBuild.pkgs' + /nix/store/hg65h51xnp74ikahns9hyf3py5mlbbqq-source/overrides/default.nix:132:27 + + 131| + 132| bootstrappingBase = pkgs.${self.python.pythonAttr}.pythonForBuild.pkgs; + | ^ + 133| in + ``` + +- Stop vendoring `toml11` + + We don't apply any patches to it, and vendoring it locks users into + bugs (it hasn't been updated since its introduction in late 2021). + + Author: [**Winter (@winterqt)**](https://github.com/winterqt) + +- Rename hash format `base32` to `nix32` [#8678](https://github.com/NixOS/nix/pull/8678) + + Hash format `base32` was renamed to `nix32` since it used a special nix-specific character set for + [Base32](https://en.wikipedia.org/wiki/Base32). + + **Deprecation**: Use `nix32` instead of `base32` as `toHashFormat` + + For the builtin `convertHash`, the `toHashFormat` parameter now accepts the same hash formats as the `--to`/`--from` + parameters of the `nix hash conert` command: `"base16"`, `"nix32"`, `"base64"`, and `"sri"`. The former `"base32"` value + remains as a deprecated alias for `"nix32"`. Please convert your code from: + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "base32";} + ``` + + to + + ```nix + builtins.convertHash { inherit hash hashAlgo; toHashFormat = "nix32";} + ``` + +- Add `pipe-operators` experimental feature [#11131](https://github.com/NixOS/nix/pull/11131) + + This is a draft implementation of [RFC 0148](https://github.com/NixOS/rfcs/pull/148). + + The `pipe-operators` experimental feature adds [`<|` and `|>` operators][pipe operators] to the Nix language. + *a* `|>` *b* is equivalent to the function application *b* *a*, and + *a* `<|` *b* is equivalent to the function application *a* *b*. + + For example: + + ``` + nix-repl> 1 |> builtins.add 2 |> builtins.mul 3 + 9 + + nix-repl> builtins.add 1 <| builtins.mul 2 <| 3 + 7 + ``` + + `<|` and `|>` are right and left associative, respectively, and have lower precedence than any other operator. + These properties may change in future releases. + + See [the RFC](https://github.com/NixOS/rfcs/pull/148) for more examples and rationale. + + [pipe operators]: @docroot@/language/operators.md#pipe-operators + +- `nix-shell` shebang uses relative path [#4232](https://github.com/NixOS/nix/issues/4232) [#5088](https://github.com/NixOS/nix/pull/5088) [#11058](https://github.com/NixOS/nix/pull/11058) + + + Relative [path](@docroot@/language/types.md#type-path) literals in `nix-shell` shebang scripts' options are now resolved relative to the [script's location](@docroot@/glossary.md?highlight=base%20directory#gloss-base-directory). + Previously they were resolved relative to the current working directory. + + For example, consider the following script in `~/myproject/say-hi`: + + ```shell + #!/usr/bin/env nix-shell + #!nix-shell --expr 'import ./shell.nix' + #!nix-shell --arg toolset './greeting-tools.nix' + #!nix-shell -i bash + hello + ``` + + Older versions of `nix-shell` would resolve `shell.nix` relative to the current working directory, such as the user's home directory in this example: + + ```console + [hostname:~]$ ./myproject/say-hi + error: + … while calling the 'import' builtin + at «string»:1:2: + 1| (import ./shell.nix) + | ^ + + error: path '/home/user/shell.nix' does not exist + ``` + + Since this release, `nix-shell` resolves `shell.nix` relative to the script's location, and `~/myproject/shell.nix` is used. + + ```console + $ ./myproject/say-hi + Hello, world! + ``` + + **Opt-out** + + This is technically a breaking change, so we have added an option so you can adapt independently of your Nix update. + The old behavior can be opted into by setting the option [`nix-shell-shebang-arguments-relative-to-script`](@docroot@/command-ref/conf-file.md#conf-nix-shell-shebang-arguments-relative-to-script) to `false`. + This option will be removed in a future release. + + Author: [**Robert Hensing (@roberth)**](https://github.com/roberth) + +- Improve handling of tarballs that don't consist of a single top-level directory [#11195](https://github.com/NixOS/nix/pull/11195) + + In previous Nix releases, the tarball fetcher (used by `builtins.fetchTarball`) erroneously merged top-level directories into a single directory, and silently discarded top-level files that are not directories. This is no longer the case. The new behaviour is that *only* if the tarball consists of a single directory, the top-level path component of the files in the tarball is removed (similar to `tar`'s `--strip-components=1`). + + Author: [**Eelco Dolstra (@edolstra)**](https://github.com/edolstra) + +- Setting to warn about large paths [#10778](https://github.com/NixOS/nix/pull/10778) + + Nix can now warn when evaluation of a Nix expression causes a large + path to be copied to the Nix store. The threshold for this warning can + be configured using the `warn-large-path-threshold` setting, + e.g. `--warn-large-path-threshold 100M`. + + +# Contributors + +This release was made possible by the following 43 contributors: + +- Andreas Rammhold [**(@andir)**](https://github.com/andir) +- Andrew Marshall [**(@amarshall)**](https://github.com/amarshall) +- Brian McKenna [**(@puffnfresh)**](https://github.com/puffnfresh) +- Cameron [**(@SkamDart)**](https://github.com/SkamDart) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Corbin Simpson [**(@MostAwesomeDude)**](https://github.com/MostAwesomeDude) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- Emily [**(@emilazy)**](https://github.com/emilazy) +- Enno Richter [**(@elohmeier)**](https://github.com/elohmeier) +- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +- HaeNoe [**(@haenoe)**](https://github.com/haenoe) +- Hamir Mahal [**(@hamirmahal)**](https://github.com/hamirmahal) +- Harmen [**(@alicebob)**](https://github.com/alicebob) +- Ivan Trubach [**(@tie)**](https://github.com/tie) +- Jared Baur [**(@jmbaur)**](https://github.com/jmbaur) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Jonathan De Troye [**(@detroyejr)**](https://github.com/detroyejr) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Klemens Nanni [**(@klemensn)**](https://github.com/klemensn) +- Las Safin [**(@L-as)**](https://github.com/L-as) +- Lexi Mattick [**(@kognise)**](https://github.com/kognise) +- Matthew Bauer [**(@matthewbauer)**](https://github.com/matthewbauer) +- Max “Goldstein” Siling [**(@GoldsteinE)**](https://github.com/GoldsteinE) +- Mingye Wang [**(@Artoria2e5)**](https://github.com/Artoria2e5) +- Philip Taron [**(@philiptaron)**](https://github.com/philiptaron) +- Pierre Bourdon [**(@delroth)**](https://github.com/delroth) +- Pino Toscano [**(@pinotree)**](https://github.com/pinotree) +- RTUnreal [**(@RTUnreal)**](https://github.com/RTUnreal) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Romain Neil [**(@romain-neil)**](https://github.com/romain-neil) +- Ryan Hendrickson [**(@rhendric)**](https://github.com/rhendric) +- Sergei Trofimovich [**(@trofi)**](https://github.com/trofi) +- Shogo Takata [**(@pineapplehunter)**](https://github.com/pineapplehunter) +- Siddhant Kumar [**(@siddhantk232)**](https://github.com/siddhantk232) +- Silvan Mosberger [**(@infinisil)**](https://github.com/infinisil) +- Théophane Hufschmitt [**(@thufschmitt)**](https://github.com/thufschmitt) +- Valentin Gagarin [**(@fricklerhandwerk)**](https://github.com/fricklerhandwerk) +- Winter [**(@winterqt)**](https://github.com/winterqt) +- jade [**(@lf-)**](https://github.com/lf-) +- kirillrdy [**(@kirillrdy)**](https://github.com/kirillrdy) +- pennae [**(@pennae)**](https://github.com/pennae) +- poweredbypie [**(@poweredbypie)**](https://github.com/poweredbypie) +- tomberek [**(@tomberek)**](https://github.com/tomberek) diff --git a/doc/manual/src/release-notes/rl-2.4.md b/doc/manual/src/release-notes/rl-2.4.md index 8b566fc7b..dbec5a29d 100644 --- a/doc/manual/src/release-notes/rl-2.4.md +++ b/doc/manual/src/release-notes/rl-2.4.md @@ -23,7 +23,7 @@ more than 2800 commits from 195 contributors since release 2.3. * The **`nix` command** has seen a lot of work and is now almost at feature parity with the old command-line interface (the `nix-*` commands). It aims to be [more modern, consistent and pleasant to - use](../contributing/cli-guideline.md) than the old CLI. It is still + use](../development/cli-guideline.md) than the old CLI. It is still marked as experimental but its interface should not change much anymore in future releases. @@ -141,6 +141,8 @@ more than 2800 commits from 195 contributors since release 2.3. the evaluation cache. This is made possible by the hermetic evaluation model of flakes. + Intermediate results are not cached. + * The new `--offline` flag disables substituters and causes all locally cached tarballs and repositories to be considered up-to-date. diff --git a/doc/manual/src/store/concrete/file-system-object/content-address.md b/doc/manual/src/store/concrete/file-system-object/content-address.md index fc5be8c67..410d7fb7c 100644 --- a/doc/manual/src/store/concrete/file-system-object/content-address.md +++ b/doc/manual/src/store/concrete/file-system-object/content-address.md @@ -1,7 +1,9 @@ # Content-Addressing File System Objects For many operations, Nix needs to calculate [a content addresses](@docroot@/glossary.md#gloss-content-address) of [a file system object][file system object]. -Usually this is needed as part of content addressing [store objects], since store objects always have a root file system object. +Usually this is needed as part of +[content addressing store objects](../store-object/content-address.md), +since store objects always have a root file system object. But some command-line utilities also just work on "raw" file system objects, not part of any store object. Every content addressing scheme Nix uses ultimately involves feeding data into a [hash function](https://en.wikipedia.org/wiki/Hash_function), and getting back an opaque fixed-size digest which is deemed a content address. @@ -18,6 +20,9 @@ A single file object can just be hashed by its contents. This is not enough information to encode the fact that the file system object is a file, but if we *already* know that the FSO is a single non-executable file by other means, it is sufficient. +Because the hashed data is just the raw file, as is, this choice is good for compatibility with other systems. +For example, Unix commands like `sha256sum` or `sha1sum` will produce hashes for single files that match this. + ### Nix Archive (NAR) { #serial-nix-archive } For the other cases of [file system objects][file system object], especially directories with arbitrary descendents, we need a more complex serialisation format. @@ -69,7 +74,7 @@ every non-directory object is owned by a parent directory, and the entry that re However, if the root object is not a directory, then we have no way of knowing which one of an executable file, non-executable file, or symlink it is supposed to be. In response to this, we have decided to treat a bare file as non-executable file. -This is similar to do what we do with [flat serialisation](#flat), which also lacks this information. +This is similar to do what we do with [flat serialisation](#serial-flat), which also lacks this information. To avoid an address collision, attempts to hash a bare executable file or symlink will result in an error (just as would happen for flat serialisation also). Thus, Git can encode some, but not all of Nix's "File System Objects", and this sort of content-addressing is likewise partial. @@ -77,4 +82,4 @@ In the future, we may support a Git-like hash for such file system objects, or w [file system object]: ../file-system-object.md [store object]: ../store-object.md -[xp-feature-git-hashing]: @docroot@/contributing/experimental-features.md#xp-feature-git-hashing +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/concrete/store-object/content-address.md b/doc/manual/src/store/concrete/store-object/content-address.md new file mode 100644 index 000000000..02dce2836 --- /dev/null +++ b/doc/manual/src/store/concrete/store-object/content-address.md @@ -0,0 +1,95 @@ +# Content-Addressing Store Objects + +Just [like][fso-ca] [File System Objects][File System Object], +[Store Objects][Store Object] can also be [content-addressed](@docroot@/glossary.md#gloss-content-addressed), +unless they are [input-addressed](@docroot@/glossary.md#gloss-input-addressed-store-object). + +For store objects, the content address we produce will take the form of a [Store Path] rather than regular hash. +In particular, the content-addressing scheme will ensure that the digest of the store path is solely computed from the + +- file system object graph (the root one and its children, if it has any) +- references +- [store directory](../store-path.md#store-directory) +- name + +of the store object, and not any other information, which would not be an intrinsic property of that store object. + +For the full specification of the algorithms involved, see the [specification of store path digests][sp-spec]. + +[File System Object]: ../file-system-object.md +[Store Object]: ../store-object.md +[Store Path]: ../store-path.md + +## Content addressing each part of a store object + +### File System Objects + +With all currently supported store object content addressing methods, the file system object is always [content-addressed][fso-ca] first, and then that hash is incorporated into content address computation for the store object. + +### References + +With all currently supported store object content addressing methods, +other objects are referred to by their regular (string-encoded-) [store paths][Store Path]. + +Self-references however cannot be referred to by their path, because we are in the midst of describing how to compute that path! + +> The alternative would require finding as hash function fixed point, i.e. the solution to an equation in the form +> ``` +> digest = hash(..... || digest || ....) +> ``` +> which is computationally infeasible. +> As far as we know, this is equivalent to finding a hash collision. + +Instead we just have a "has self reference" boolean, which will end up affecting the digest. + +### Name and Store Directory + +These two items affect the digest in a way that is standard for store path digest computations and not specific to content-addressing. +Consult the [specification of store path digests][sp-spec] for further details. + +## Content addressing Methods + +For historical reasons, we don't support all features in all combinations. +Each currently supported method of content addressing chooses a single method of file system object hashing, and may offer some restrictions on references. +The names and store directories are unrestricted however. + +### Flat { #method-flat } + +This uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References are not supported: store objects with flat hashing *and* references can not be created. + +### Text { #method-text } + +This also uses the corresponding [Flat](../file-system-object/content-address.md#serial-flat) method of file system object content addressing. + +References to other store objects are supported, but self references are not. + +This is the only store-object content-addressing method that is not named identically with a corresponding file system object method. +It is somewhat obscure, mainly used for "drv files" +(derivations serialized as store objects in their ["ATerm" file format](@docroot@/protocols/derivation-aterm.md)). +Prefer another method if possible. + +### Nix Archive { #method-nix-archive } + +This uses the corresponding [Nix Archive](../file-system-object/content-address.md#serial-nix-archive) method of file system object content addressing. + +References (to other store objects and self references alike) are supported so long as the hash algorithm is SHA-256, but not (neither kind) otherwise. + +### Git { #method-git } + +> **Warning** +> +> This method is part of the [`git-hashing`][xp-feature-git-hashing] experimental feature. + +This uses the corresponding [Git](../file-system-object/content-address.md#serial-git) method of file system object content addressing. + +References are not supported. + +Only SHA-1 is supported at this time. +If [SHA-256-based Git](https://git-scm.com/docs/hash-function-transition) +becomes more widespread, this restriction will be revisited. + +[fso-ca]: ../file-system-object/content-address.md +[sp-spec]: @docroot@/protocols/store-path.md +[xp-feature-git-hashing]: @docroot@/development/experimental-features.md#xp-feature-git-hashing diff --git a/doc/manual/src/store/meson.build b/doc/manual/src/store/meson.build new file mode 100644 index 000000000..e3006020d --- /dev/null +++ b/doc/manual/src/store/meson.build @@ -0,0 +1,18 @@ +types_dir = custom_target( + command : [ + python.full_path(), + '@INPUT0@', + '@OUTPUT@', + '--' + ] + nix_eval_for_docs + [ + '--expr', + 'import @INPUT1@ (builtins.fromJSON (builtins.readFile ./@INPUT2@)).stores', + ], + input : [ + '../../remove_before_wrapper.py', + '../../generate-store-types.nix', + nix3_cli_json, + ], + output : 'types', + env : nix_env_for_docs, +) diff --git a/doc/manual/substitute.py b/doc/manual/substitute.py new file mode 100644 index 000000000..52cef4fa0 --- /dev/null +++ b/doc/manual/substitute.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 + +from pathlib import Path +import json +import os, os.path +import sys +import typing as t + +name = 'substitute.py' + +def log(*args: t.Any, **kwargs: t.Any) -> None: + kwargs['file'] = sys.stderr + print(f'{name}:', *args, **kwargs) + +def do_include(content: str, relative_md_path: Path, source_root: Path, search_path: Path) -> str: + assert not relative_md_path.is_absolute(), f'{relative_md_path=} from mdbook should be relative' + + md_path_abs = source_root / relative_md_path + var_abs = md_path_abs.parent + assert var_abs.is_dir(), f'supposed directory {var_abs} is not a directory (cwd={os.getcwd()})' + + lines = [] + for l in content.splitlines(keepends=True): + if l.strip().startswith("{{#include "): + requested = l.strip()[11:][:-2] + if requested.startswith("@generated@/"): + included = search_path / Path(requested[12:]) + requested = included.relative_to(search_path) + else: + included = source_root / relative_md_path.parent / requested + requested = included.resolve().relative_to(source_root) + assert included.exists(), f"{requested} not found at {included}" + lines.append(do_include(included.read_text(), requested, source_root, search_path) + "\n") + else: + lines.append(l) + return "".join(lines) + +def recursive_replace(data: dict[str, t.Any], book_root: Path, search_path: Path) -> dict[str, t.Any]: + match data: + case {'sections': sections}: + return data | dict( + sections = [recursive_replace(section, book_root, search_path) for section in sections], + ) + case {'Chapter': chapter}: + path_to_chapter = Path(chapter['path']) + chapter_content = chapter['content'] + + return data | dict( + Chapter = chapter | dict( + # first process includes. this must happen before docroot processing since + # mdbook does not see these included files, only the final agglomeration. + content = do_include( + chapter_content, + path_to_chapter, + book_root, + search_path + ).replace( + '@docroot@', + ("../" * len(path_to_chapter.parent.parts) or "./")[:-1] + ), + sub_items = [ + recursive_replace(sub_item, book_root, search_path) + for sub_item in chapter['sub_items'] + ], + ), + ) + + case rest: + assert False, f'should have been called on a dict, not {type(rest)=}\n\t{rest=}' + +def main() -> None: + + + if len(sys.argv) > 1 and sys.argv[1] == 'supports': + return 0 + + # includes pointing into @generated@ will look here + search_path = Path(os.environ['MDBOOK_SUBSTITUTE_SEARCH']) + + if len(sys.argv) > 1 and sys.argv[1] == 'summary': + print(do_include( + sys.stdin.read(), + Path('src/SUMMARY.md'), + Path(sys.argv[2]).resolve(), + search_path)) + return + + # mdbook communicates with us over stdin and stdout. + # It splorks us a JSON array, the first element describing the context, + # the second element describing the book itself, + # and then expects us to send it the modified book JSON over stdout. + + context, book = json.load(sys.stdin) + + # book_root is the directory where book contents leave (ie, src/) + book_root = Path(context['root']) / context['config']['book']['src'] + + # Find @var@ in all parts of our recursive book structure. + replaced_content = recursive_replace(book, book_root, search_path) + + replaced_content_str = json.dumps(replaced_content) + + # Give mdbook our changes. + print(replaced_content_str) + +try: + sys.exit(main()) +except AssertionError as e: + print(f'{name}: INTERNAL ERROR in mdbook preprocessor: {e}', file=sys.stderr) + print(f'this is a bug in {name}', file=sys.stderr) + raise diff --git a/flake.lock b/flake.lock index 409463ad8..b5d0b881c 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1673956053, - "narHash": "sha256-4gtG9iQuiKITOjNQQeQIpoIB6b16fm+504Ch3sNKLd8=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "35bb57c0c8d8b62bbfd284272c928ceb64ddbde9", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -23,11 +23,11 @@ ] }, "locked": { - "lastModified": 1712014858, - "narHash": "sha256-sB4SWl2lX95bExY2gMFG5HIzvva5AVMJd4Igm+GpZNw=", + "lastModified": 1719994518, + "narHash": "sha256-pQMhCCHyQGRzdfAkdJ4cIWiw+JNuWsTX7f0ZYSyz0VY=", "owner": "hercules-ci", "repo": "flake-parts", - "rev": "9126214d0a59633752a136528f5f3b9aa8565b7d", + "rev": "9227223f6d922fee3c7b190b2cc238a99527bbb7", "type": "github" }, "original": { @@ -36,53 +36,80 @@ "type": "github" } }, - "flake-utils": { + "git-hooks-nix": { + "inputs": { + "flake-compat": [], + "gitignore": [], + "nixpkgs": [ + "nixpkgs" + ], + "nixpkgs-stable": [ + "nixpkgs" + ] + }, "locked": { - "lastModified": 1667395993, - "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f", + "lastModified": 1721042469, + "narHash": "sha256-6FPUl7HVtvRHCCBQne7Ylp4p+dpP3P/OYuzjztZ4s70=", + "owner": "cachix", + "repo": "git-hooks.nix", + "rev": "f451c19376071a90d8c58ab1a953c6e9840527fd", "type": "github" }, "original": { - "owner": "numtide", - "repo": "flake-utils", + "owner": "cachix", + "repo": "git-hooks.nix", "type": "github" } }, "libgit2": { "flake": false, "locked": { - "lastModified": 1697646580, - "narHash": "sha256-oX4Z3S9WtJlwvj0uH9HlYcWv+x1hqp8mhXl7HsLu2f0=", + "lastModified": 1715853528, + "narHash": "sha256-J2rCxTecyLbbDdsyBWn9w7r3pbKRMkI9E7RvRgAqBdY=", "owner": "libgit2", "repo": "libgit2", - "rev": "45fd9ed7ae1a9b74b957ef4f337bc3c8b3df01b5", + "rev": "36f7e21ad757a3dacc58cf7944329da6bc1d6e96", "type": "github" }, "original": { "owner": "libgit2", + "ref": "v1.8.1", "repo": "libgit2", "type": "github" } }, "nixpkgs": { "locked": { - "lastModified": 1709083642, - "narHash": "sha256-7kkJQd4rZ+vFrzWu8sTRtta5D1kBG0LSRYAfhtmMlSo=", + "lastModified": 1723688146, + "narHash": "sha256-sqLwJcHYeWLOeP/XoLwAtYjr01TISlkOfz+NG82pbdg=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b550fe4b4776908ac2a861124307045f8e717c8e", + "rev": "c3d4ac725177c030b1e289015989da2ad9d56af0", "type": "github" }, "original": { "owner": "NixOS", - "ref": "release-23.11", + "ref": "nixos-24.05", "repo": "nixpkgs", "type": "github" } }, + "nixpkgs-23-11": { + "locked": { + "lastModified": 1717159533, + "narHash": "sha256-oamiKNfr2MS6yH64rUn99mIZjc45nGJlj9eGth/3Xuw=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "a62e6edd6d5e1fa0329b8653c801147986f8d446", + "type": "github" + } + }, "nixpkgs-regression": { "locked": { "lastModified": 1643052045, @@ -99,40 +126,15 @@ "type": "github" } }, - "pre-commit-hooks": { - "inputs": { - "flake-compat": [], - "flake-utils": "flake-utils", - "gitignore": [], - "nixpkgs": [ - "nixpkgs" - ], - "nixpkgs-stable": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1712897695, - "narHash": "sha256-nMirxrGteNAl9sWiOhoN5tIHyjBbVi5e2tgZUgZlK3Y=", - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "rev": "40e6053ecb65fcbf12863338a6dcefb3f55f1bf8", - "type": "github" - }, - "original": { - "owner": "cachix", - "repo": "pre-commit-hooks.nix", - "type": "github" - } - }, "root": { "inputs": { "flake-compat": "flake-compat", "flake-parts": "flake-parts", + "git-hooks-nix": "git-hooks-nix", "libgit2": "libgit2", "nixpkgs": "nixpkgs", - "nixpkgs-regression": "nixpkgs-regression", - "pre-commit-hooks": "pre-commit-hooks" + "nixpkgs-23-11": "nixpkgs-23-11", + "nixpkgs-regression": "nixpkgs-regression" } } }, diff --git a/flake.nix b/flake.nix index 987f25305..303779c2b 100644 --- a/flake.nix +++ b/flake.nix @@ -1,39 +1,31 @@ { description = "The purely functional package manager"; - # TODO switch to nixos-23.11-small - # https://nixpk.gs/pr-tracker.html?pr=291954 - inputs.nixpkgs.url = "github:NixOS/nixpkgs/release-23.11"; + inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; + inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; inputs.flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; - inputs.libgit2 = { url = "github:libgit2/libgit2"; flake = false; }; + inputs.libgit2 = { url = "github:libgit2/libgit2/v1.8.1"; flake = false; }; # dev tooling inputs.flake-parts.url = "github:hercules-ci/flake-parts"; - inputs.pre-commit-hooks.url = "github:cachix/pre-commit-hooks.nix"; + inputs.git-hooks-nix.url = "github:cachix/git-hooks.nix"; # work around https://github.com/NixOS/nix/issues/7730 inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; - inputs.pre-commit-hooks.inputs.nixpkgs.follows = "nixpkgs"; - inputs.pre-commit-hooks.inputs.nixpkgs-stable.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; + inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; # work around 7730 and https://github.com/NixOS/nix/issues/7807 - inputs.pre-commit-hooks.inputs.flake-compat.follows = ""; - inputs.pre-commit-hooks.inputs.gitignore.follows = ""; + inputs.git-hooks-nix.inputs.flake-compat.follows = ""; + inputs.git-hooks-nix.inputs.gitignore.follows = ""; outputs = inputs@{ self, nixpkgs, nixpkgs-regression, libgit2, ... }: let inherit (nixpkgs) lib; - inherit (lib) fileset; officialRelease = false; - version = lib.fileContents ./.version + versionSuffix; - versionSuffix = - if officialRelease - then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - linux32BitSystems = [ "i686-linux" ]; linux64BitSystems = [ "x86_64-linux" "aarch64-linux" ]; linuxSystems = linux32BitSystems ++ linux64BitSystems; @@ -45,6 +37,7 @@ "armv7l-unknown-linux-gnueabihf" "riscv64-unknown-linux-gnu" "x86_64-unknown-netbsd" + "x86_64-unknown-freebsd" "x86_64-w64-mingw32" ]; @@ -56,6 +49,18 @@ "stdenv" ]; + /** + `flatMapAttrs attrs f` applies `f` to each attribute in `attrs` and + merges the results into a single attribute set. + + This can be nested to form a build matrix where all the attributes + generated by the innermost `f` are returned as is. + (Provided that the names are unique.) + + See https://nixos.org/manual/nixpkgs/stable/index.html#function-library-lib.attrsets.concatMapAttrs + */ + flatMapAttrs = attrs: f: lib.concatMapAttrs f attrs; + forAllSystems = lib.genAttrs systems; forAllCrossSystems = lib.genAttrs crossSystems; @@ -104,28 +109,6 @@ cross = forAllCrossSystems (crossSystem: make-pkgs crossSystem "stdenv"); }); - installScriptFor = tarballs: - nixpkgsFor.x86_64-linux.native.callPackage ./scripts/installer.nix { - inherit tarballs; - }; - - testNixVersions = pkgs: client: daemon: - pkgs.callPackage ./package.nix { - pname = - "nix-tests" - + lib.optionalString - (lib.versionAtLeast daemon.version "2.4pre20211005" && - lib.versionAtLeast client.version "2.4pre20211005") - "-${client.version}-against-${daemon.version}"; - - inherit fileset; - - test-client = client; - test-daemon = daemon; - - doBuild = false; - }; - binaryTarball = nix: pkgs: pkgs.callPackage ./scripts/binary-tarball.nix { inherit nix; }; @@ -137,86 +120,24 @@ { nixStable = prev.nix; - default-busybox-sandbox-shell = final.busybox.override { - useMusl = true; - enableStatic = true; - enableMinimal = true; - extraConfig = '' - CONFIG_FEATURE_FANCY_ECHO y - CONFIG_FEATURE_SH_MATH y - CONFIG_FEATURE_SH_MATH_64 y - - CONFIG_ASH y - CONFIG_ASH_OPTIMIZE_FOR_SIZE y - - CONFIG_ASH_ALIAS y - CONFIG_ASH_BASH_COMPAT y - CONFIG_ASH_CMDCMD y - CONFIG_ASH_ECHO y - CONFIG_ASH_GETOPTS y - CONFIG_ASH_INTERNAL_GLOB y - CONFIG_ASH_JOB_CONTROL y - CONFIG_ASH_PRINTF y - CONFIG_ASH_TEST y - ''; - }; - - libgit2-nix = final.libgit2.overrideAttrs (attrs: { - src = libgit2; - version = libgit2.lastModifiedDate; - cmakeFlags = attrs.cmakeFlags or [] - ++ [ "-DUSE_SSH=exec" ]; + # A new scope, so that we can use `callPackage` to inject our own interdependencies + # without "polluting" the top level "`pkgs`" attrset. + # This also has the benefit of providing us with a distinct set of packages + # we can iterate over. + nixComponents = lib.makeScope final.nixDependencies.newScope (import ./packaging/components.nix { + inherit (final) lib; + inherit officialRelease; + src = self; }); - boehmgc-nix = (final.boehmgc.override { - enableLargeConfig = true; - }).overrideAttrs(o: { - patches = (o.patches or []) ++ [ - ./dep-patches/boehmgc-coroutine-sp-fallback.diff - - # https://github.com/ivmai/bdwgc/pull/586 - ./dep-patches/boehmgc-traceable_allocator-public.diff - ]; + # The dependencies are in their own scope, so that they don't have to be + # in Nixpkgs top level `pkgs` or `nixComponents`. + nixDependencies = lib.makeScope final.newScope (import ./packaging/dependencies.nix { + inherit inputs stdenv; + pkgs = final; }); - libseccomp-nix = final.libseccomp.overrideAttrs (_: rec { - version = "2.5.5"; - src = final.fetchurl { - url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; - hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; - }; - }); - - changelog-d-nix = final.buildPackages.callPackage ./misc/changelog-d.nix { }; - - nix = - let - officialRelease = false; - versionSuffix = - if officialRelease - then "" - else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified or "19700101")}_${self.shortRev or "dirty"}"; - - in final.callPackage ./package.nix { - inherit - fileset - stdenv - versionSuffix - ; - officialRelease = false; - boehmgc = final.boehmgc-nix; - libgit2 = final.libgit2-nix; - libseccomp = final.libseccomp-nix; - busybox-sandbox-shell = final.busybox-sandbox-shell or final.default-busybox-sandbox-shell; - } // { - # this is a proper separate downstream package, but put - # here also for back compat reasons. - perl-bindings = final.nix-perl-bindings; - }; - - nix-perl-bindings = final.callPackage ./perl { - inherit fileset stdenv; - }; + nix = final.nixComponents.nix; # See https://github.com/NixOS/nixpkgs/pull/214409 # Remove when fixed in this flake's nixpkgs @@ -229,159 +150,21 @@ in { # A Nixpkgs overlay that overrides the 'nix' and - # 'nix.perl-bindings' packages. + # 'nix-perl-bindings' packages. overlays.default = overlayFor (p: p.stdenv); - hydraJobs = { - - # Binary package for various platforms. - build = forAllSystems (system: self.packages.${system}.nix); - - shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); - - buildStatic = lib.genAttrs linux64BitSystems (system: self.packages.${system}.nix-static); - - buildCross = forAllCrossSystems (crossSystem: - lib.genAttrs ["x86_64-linux"] (system: self.packages.${system}."nix-${crossSystem}")); - - buildNoGc = forAllSystems (system: - self.packages.${system}.nix.override { enableGC = false; } - ); - - buildNoTests = forAllSystems (system: - self.packages.${system}.nix.override { - doCheck = false; - doInstallCheck = false; - installUnitTests = false; - } - ); - - # Toggles some settings for better coverage. Windows needs these - # library combinations, and Debian build Nix with GNU readline too. - buildReadlineNoMarkdown = forAllSystems (system: - self.packages.${system}.nix.override { - enableMarkdown = false; - readlineFlavor = "readline"; - } - ); - - # Perl bindings for various platforms. - perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nix.perl-bindings); - - # Binary tarball for various platforms, containing a Nix store - # with the closure of 'nix' package, and the second half of - # the installation script. - binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); - - binaryTarballCross = lib.genAttrs ["x86_64-linux"] (system: - forAllCrossSystems (crossSystem: - binaryTarball - self.packages.${system}."nix-${crossSystem}" - nixpkgsFor.${system}.cross.${crossSystem})); - - # The first half of the installation script. This is uploaded - # to https://nixos.org/nix/install. It downloads the binary - # tarball for the user's system and calls the second half of the - # installation script. - installerScript = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."i686-linux" - self.hydraJobs.binaryTarball."aarch64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - self.hydraJobs.binaryTarball."aarch64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" - ]; - installerScriptForGHA = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" - ]; - - # docker image with Nix inside - dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); - - # Line coverage analysis. - coverage = nixpkgsFor.x86_64-linux.native.nix.override { - pname = "nix-coverage"; - withCoverageChecks = true; - }; - - # API docs for Nix's unstable internal C++ interfaces. - internal-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { - inherit fileset; - doBuild = false; - enableInternalAPIDocs = true; - }; - - # API docs for Nix's C bindings. - external-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { - inherit fileset; - doBuild = false; - enableExternalAPIDocs = true; - }; - - # System tests. - tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { - - # Make sure that nix-env still produces the exact same result - # on a particular version of Nixpkgs. - evalNixpkgs = - let - inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; - in - runCommand "eval-nixos" { buildInputs = [ nix ]; } - '' - type -p nix-env - # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. - ( - set -x - time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages - [[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]] - ) - mkdir $out - ''; - - nixpkgsLibTests = - forAllSystems (system: - import (nixpkgs + "/lib/tests/release.nix") - { pkgs = nixpkgsFor.${system}.native; - nixVersions = [ self.packages.${system}.nix ]; - } - ); - }; - - metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { - pkgs = nixpkgsFor.x86_64-linux.native; - nixpkgs = nixpkgs-regression; - }; - - installTests = forAllSystems (system: - let pkgs = nixpkgsFor.${system}.native; in - pkgs.runCommand "install-tests" { - againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; - againstCurrentUnstable = - # FIXME: temporarily disable this on macOS because of #3605. - if system == "x86_64-linux" - then testNixVersions pkgs pkgs.nix pkgs.nixUnstable - else null; - # Disabled because the latest stable version doesn't handle - # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work - # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; - } "touch $out"); - - installerTests = import ./tests/installer { - binaryTarballs = self.hydraJobs.binaryTarball; - inherit nixpkgsFor; - }; - + hydraJobs = import ./packaging/hydra.nix { + inherit + inputs + binaryTarball + forAllCrossSystems + forAllSystems + lib + linux64BitSystems + nixpkgsFor + self + officialRelease + ; }; checks = forAllSystems (system: { @@ -391,30 +174,116 @@ rl-next = let pkgs = nixpkgsFor.${system}.native; in pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' - LANG=C.UTF-8 ${pkgs.changelog-d-nix}/bin/changelog-d ${./doc/manual/rl-next} >$out + LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${./doc/manual/rl-next} >$out ''; + repl-completion = nixpkgsFor.${system}.native.callPackage ./tests/repl-completion.nix { }; } // (lib.optionalAttrs (builtins.elem system linux64BitSystems)) { dockerImage = self.hydraJobs.dockerImage.${system}; } // (lib.optionalAttrs (!(builtins.elem system linux32BitSystems))) { # Some perl dependencies are broken on i686-linux. # Since the support is only best-effort there, disable the perl # bindings - perlBindings = self.hydraJobs.perlBindings.${system}; - } // devFlake.checks.${system} or {} + + # Temporarily disabled because GitHub Actions OOM issues. Once + # the old build system is gone and we are back to one build + # system, we should reenable this. + #perlBindings = self.hydraJobs.perlBindings.${system}; + } + /* + # Add "passthru" tests + // flatMapAttrs ({ + "" = nixpkgsFor.${system}.native; + } // lib.optionalAttrs (! nixpkgsFor.${system}.native.stdenv.hostPlatform.isDarwin) { + # TODO: enable static builds for darwin, blocked on: + # https://github.com/NixOS/nixpkgs/issues/320448 + # TODO: disabled to speed up GHA CI. + #"static-" = nixpkgsFor.${system}.static; + }) + (nixpkgsPrefix: nixpkgs: + flatMapAttrs nixpkgs.nixComponents + (pkgName: pkg: + flatMapAttrs pkg.tests or {} + (testName: test: { + "${nixpkgsPrefix}${pkgName}-${testName}" = test; + }) + ) + // lib.optionalAttrs (nixpkgs.stdenv.hostPlatform == nixpkgs.stdenv.buildPlatform) { + "${nixpkgsPrefix}nix-functional-tests" = nixpkgs.nixComponents.nix-functional-tests; + } + ) + */ + // devFlake.checks.${system} or {} ); - packages = forAllSystems (system: rec { - inherit (nixpkgsFor.${system}.native) nix changelog-d-nix; - default = nix; - } // (lib.optionalAttrs (builtins.elem system linux64BitSystems) { - nix-static = nixpkgsFor.${system}.static.nix; + packages = forAllSystems (system: + { # Here we put attributes that map 1:1 into packages., ie + # for which we don't apply the full build matrix such as cross or static. + inherit (nixpkgsFor.${system}.native) + changelog-d; + default = self.packages.${system}.nix-ng; + nix-manual = nixpkgsFor.${system}.native.nixComponents.nix-manual; + nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-internal-api-docs; + nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents.nix-external-api-docs; + } + # We need to flatten recursive attribute sets of derivations to pass `flake check`. + // flatMapAttrs + { # Components we'll iterate over in the upcoming lambda + "nix" = { }; + "nix-util" = { }; + "nix-util-c" = { }; + "nix-util-test-support" = { }; + "nix-util-tests" = { }; + + "nix-store" = { }; + "nix-store-c" = { }; + "nix-store-test-support" = { }; + "nix-store-tests" = { }; + + "nix-fetchers" = { }; + "nix-fetchers-tests" = { }; + + "nix-expr" = { }; + "nix-expr-c" = { }; + "nix-expr-test-support" = { }; + "nix-expr-tests" = { }; + + "nix-flake" = { }; + "nix-flake-tests" = { }; + + "nix-main" = { }; + "nix-main-c" = { }; + + "nix-cmd" = { }; + + "nix-cli" = { }; + + "nix-functional-tests" = { supportsCross = false; }; + + "nix-perl-bindings" = { supportsCross = false; }; + "nix-ng" = { }; + } + (pkgName: { supportsCross ? true }: { + # These attributes go right into `packages.`. + "${pkgName}" = nixpkgsFor.${system}.native.nixComponents.${pkgName}; + "${pkgName}-static" = nixpkgsFor.${system}.static.nixComponents.${pkgName}; + } + // lib.optionalAttrs supportsCross (flatMapAttrs (lib.genAttrs crossSystems (_: { })) (crossSystem: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${crossSystem}" = nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName}; + })) + // flatMapAttrs (lib.genAttrs stdenvs (_: { })) (stdenvName: {}: { + # These attributes go right into `packages.`. + "${pkgName}-${stdenvName}" = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nixComponents.${pkgName}; + }) + ) + // lib.optionalAttrs (builtins.elem system linux64BitSystems) { dockerImage = let pkgs = nixpkgsFor.${system}.native; - image = import ./docker.nix { inherit pkgs; tag = version; }; + image = import ./docker.nix { inherit pkgs; tag = pkgs.nix.version; }; in pkgs.runCommand - "docker-image-tarball-${version}" + "docker-image-tarball-${pkgs.nix.version}" { meta.description = "Docker image with Nix for ${system}"; } '' mkdir -p $out/nix-support @@ -422,25 +291,28 @@ ln -s ${image} $image echo "file binary-dist $image" >> $out/nix-support/hydra-build-products ''; - } // builtins.listToAttrs (map - (crossSystem: { - name = "nix-${crossSystem}"; - value = nixpkgsFor.${system}.cross.${crossSystem}.nix; - }) - crossSystems) - // builtins.listToAttrs (map - (stdenvName: { - name = "nix-${stdenvName}"; - value = nixpkgsFor.${system}.stdenvs."${stdenvName}Packages".nix; - }) - stdenvs))); + }); devShells = let makeShell = pkgs: stdenv: (pkgs.nix.override { inherit stdenv; forDevShell = true; }).overrideAttrs (attrs: let + buildCanExecuteHost = stdenv.buildPlatform.canExecute stdenv.hostPlatform; modular = devFlake.getSystem stdenv.buildPlatform.system; + transformFlag = prefix: flag: + assert builtins.isString flag; + let + rest = builtins.substring 2 (builtins.stringLength flag) flag; + in + "-D${prefix}:${rest}"; + havePerl = stdenv.buildPlatform == stdenv.hostPlatform && stdenv.hostPlatform.isUnix; + ignoreCrossFile = flags: builtins.filter (flag: !(lib.strings.hasInfix "cross-file" flag)) flags; in { pname = "shell-for-" + attrs.pname; + + # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop + version = lib.fileContents ./.version; + name = attrs.pname; + installFlags = "sysconfdir=$(out)/etc"; shellHook = '' PATH=$prefix/bin:$PATH @@ -455,13 +327,44 @@ src = null; env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev pkgs.nixDependencies.boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib pkgs.nixDependencies.boost}/lib"; # For `make format`, to work without installing pre-commit _NIX_PRE_COMMIT_HOOKS_CONFIG = "${(pkgs.formats.yaml { }).generate "pre-commit-config.yaml" modular.pre-commit.settings.rawConfig}"; }; + mesonFlags = + map (transformFlag "libutil") (ignoreCrossFile pkgs.nixComponents.nix-util.mesonFlags) + ++ map (transformFlag "libstore") (ignoreCrossFile pkgs.nixComponents.nix-store.mesonFlags) + ++ map (transformFlag "libfetchers") (ignoreCrossFile pkgs.nixComponents.nix-fetchers.mesonFlags) + ++ lib.optionals havePerl (map (transformFlag "perl") (ignoreCrossFile pkgs.nixComponents.nix-perl-bindings.mesonFlags)) + ++ map (transformFlag "libexpr") (ignoreCrossFile pkgs.nixComponents.nix-expr.mesonFlags) + ++ map (transformFlag "libcmd") (ignoreCrossFile pkgs.nixComponents.nix-cmd.mesonFlags) + ; + nativeBuildInputs = attrs.nativeBuildInputs or [] + ++ pkgs.nixComponents.nix-util.nativeBuildInputs + ++ pkgs.nixComponents.nix-store.nativeBuildInputs + ++ pkgs.nixComponents.nix-fetchers.nativeBuildInputs + ++ lib.optionals havePerl pkgs.nixComponents.nix-perl-bindings.nativeBuildInputs + ++ lib.optionals buildCanExecuteHost pkgs.nixComponents.nix-manual.externalNativeBuildInputs + ++ pkgs.nixComponents.nix-internal-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-external-api-docs.nativeBuildInputs + ++ pkgs.nixComponents.nix-functional-tests.externalNativeBuildInputs + ++ lib.optional + (!buildCanExecuteHost + # Hack around https://github.com/nixos/nixpkgs/commit/bf7ad8cfbfa102a90463433e2c5027573b462479 + && !(stdenv.hostPlatform.isWindows && stdenv.buildPlatform.isDarwin) + && stdenv.hostPlatform.emulatorAvailable pkgs.buildPackages + && lib.meta.availableOn stdenv.buildPlatform (stdenv.hostPlatform.emulator pkgs.buildPackages)) + pkgs.buildPackages.mesonEmulatorHook ++ [ + pkgs.buildPackages.cmake + pkgs.buildPackages.shellcheck + pkgs.buildPackages.changelog-d modular.pre-commit.settings.package (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) @@ -469,7 +372,15 @@ # TODO: Remove the darwin check once # https://github.com/NixOS/nixpkgs/pull/291814 is available ++ lib.optional (stdenv.cc.isClang && !stdenv.buildPlatform.isDarwin) pkgs.buildPackages.bear - ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) pkgs.buildPackages.clang-tools; + ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) (lib.hiPrio pkgs.buildPackages.clang-tools); + + buildInputs = attrs.buildInputs or [] + ++ [ + pkgs.gtest + pkgs.rapidcheck + ] + ++ lib.optional havePerl pkgs.perl + ; }); in forAllSystems (system: diff --git a/maintainers/README.md b/maintainers/README.md index bfa0cb5a1..b92833497 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -30,7 +30,6 @@ We aim to achieve this by improving the contributor experience and attracting mo ## Members - Eelco Dolstra (@edolstra) – Team lead -- Théophane Hufschmitt (@thufschmitt) - Valentin Gagarin (@fricklerhandwerk) - Thomas Bereknyei (@tomberek) - Robert Hensing (@roberth) @@ -51,7 +50,7 @@ The team meets twice a week (times are denoted in the [Europe/Amsterdam](https:/ - mark it as draft if it is blocked on the contributor - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again. -- Work meeting: [Mondays 13:00-15:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) +- Work meeting: [Mondays 14:00-16:00 Europe/Amsterdam](https://www.google.com/calendar/event?eid=Ym52NDdzYnRic2NzcDcybjZiNDhpNzhpa3NfMjAyNDA1MTNUMTIwMDAwWiBiOW81MmZvYnFqYWs4b3E4bGZraGczdDBxZ0Bn) 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json new file mode 100644 index 000000000..cddc1a6e7 --- /dev/null +++ b/maintainers/data/release-credits-email-to-handle.json @@ -0,0 +1,52 @@ +{ + "bogus": "bogus", + "edolstra@gmail.com": "edolstra", + "roberth@users.noreply.github.com": "roberth", + "toscano.pino@tiscali.it": "pinotree", + "valentin@gagarin.work": "fricklerhandwerk", + "mr.trubach@icloud.com": "tie", + "robert@roberthensing.nl": "roberth", + "lix@jade.fyi": "lf-", + "cole.e.helbling@outlook.com": "cole-h", + "joerg@thalheim.io": "Mic92", + "John.Ericson@Obsidian.Systems": "Ericson2314", + "ryan.hendrickson@alum.mit.edu": "rhendric", + "67135060+poweredbypie@users.noreply.github.com": "poweredbypie", + "detroyejr@outlook.com": "detroyejr", + "silvan.mosberger@tweag.io": "infinisil", + "vcs@emily.moe": "emilazy", + "farid.m.zakaria@gmail.com": "fzakaria", + "22859658+RTUnreal@users.noreply.github.com": "RTUnreal", + "me@las.rs": "L-as", + "philip.taron@gmail.com": "philiptaron", + "root@goldstein.rs": "GoldsteinE", + "tomberek@users.noreply.github.com": "tomberek", + "lexi.mattick@neuralink.com": "kognise", + "andrew@johnandrewmarshall.com": "amarshall", + "contact@romain-neil.fr": "romain-neil", + "Mic92@users.noreply.github.com": "Mic92", + "valentin.gagarin@tweag.io": "fricklerhandwerk", + "siddhantk232@gmail.com": "siddhantk232", + "kn@openbsd.org": "klemensn", + "slyich@gmail.com": "trofi", + "theophane.hufschmitt@tweag.io": "thufschmitt", + "alicebob@lijzij.de": "alicebob", + "winter@winter.cafe": "winterqt", + "brian@brianmckenna.org": "puffnfresh", + "git@haenoe.party": "haenoe", + "peshogo@gmail.com": "pineapplehunter", + "poweredbypie@users.noreply.github.com": "poweredbypie", + "arthur200126@gmail.com": "Artoria2e5", + "tomberek@gmail.com": "tomberek", + "jaredbaur@fastmail.com": "jmbaur", + "andreas@rammhold.de": "andir", + "hamirmahal@gmail.com": "hamirmahal", + "git@JohnEricson.me": "Ericson2314", + "8763518+SkamDart@users.noreply.github.com": "SkamDart", + "kirillrdy@gmail.com": "kirillrdy", + "pennae@lix.systems": "pennae", + "delroth@gmail.com": "delroth", + "enno@nerdworks.de": "elohmeier", + "mjbauer95@gmail.com": "matthewbauer", + "MostAwesomeDude@gmail.com": "MostAwesomeDude" +} \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json new file mode 100644 index 000000000..abf9ed05b --- /dev/null +++ b/maintainers/data/release-credits-handle-to-name.json @@ -0,0 +1,45 @@ +{ + "fzakaria": "Farid Zakaria", + "kognise": "Lexi Mattick", + "L-as": "Las Safin", + "haenoe": "HaeNoe", + "andir": "Andreas Rammhold", + "matthewbauer": "Matthew Bauer", + "emilazy": "Emily", + "pineapplehunter": "Shogo Takata", + "RTUnreal": null, + "jmbaur": "Jared Baur", + "Ericson2314": "John Ericson", + "pinotree": "Pino Toscano", + "tie": "Ivan Trubach", + "poweredbypie": null, + "fricklerhandwerk": "Valentin Gagarin", + "Mic92": "J\u00f6rg Thalheim", + "alicebob": "Harmen", + "elohmeier": "Enno Richter", + "delroth": "Pierre Bourdon", + "kirillrdy": null, + "thufschmitt": "Th\u00e9ophane Hufschmitt", + "detroyejr": "Jonathan De Troye", + "klemensn": "Klemens Nanni", + "tomberek": null, + "rhendric": "Ryan Hendrickson", + "philiptaron": "Philip Taron", + "puffnfresh": "Brian McKenna", + "lf-": "jade", + "romain-neil": "Romain Neil", + "hamirmahal": "Hamir Mahal", + "edolstra": "Eelco Dolstra", + "Artoria2e5": "Mingye Wang", + "SkamDart": "Cameron", + "roberth": "Robert Hensing", + "amarshall": "Andrew Marshall", + "trofi": "Sergei Trofimovich", + "cole-h": "Cole Helbling", + "infinisil": "Silvan Mosberger", + "siddhantk232": "Siddhant Kumar", + "winterqt": "Winter", + "GoldsteinE": "Max \u201cGoldstein\u201d Siling", + "pennae": null, + "MostAwesomeDude": "Corbin Simpson" +} \ No newline at end of file diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 351a01fcb..fb286208d 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -2,7 +2,7 @@ { imports = [ - inputs.pre-commit-hooks.flakeModule + inputs.git-hooks-nix.flakeModule ]; perSystem = { config, pkgs, ... }: { @@ -10,425 +10,652 @@ # https://flake.parts/options/pre-commit-hooks-nix.html#options pre-commit.settings = { hooks = { - clang-format.enable = true; + clang-format = { + enable = true; + excludes = [ + # We don't want to format test data + # ''tests/(?!nixos/).*\.nix'' + ''^tests/unit/[^/]*/data/.*$'' + + # Don't format vendored code + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + + # We haven't applied formatting to these files yet + ''^doc/manual/redirects\.js$'' + ''^doc/manual/theme/highlight\.js$'' + ''^precompiled-headers\.h$'' + ''^src/build-remote/build-remote\.cc$'' + ''^src/libcmd/built-path\.cc$'' + ''^src/libcmd/built-path\.hh$'' + ''^src/libcmd/command\.cc$'' + ''^src/libcmd/command\.hh$'' + ''^src/libcmd/common-eval-args\.cc$'' + ''^src/libcmd/common-eval-args\.hh$'' + ''^src/libcmd/editor-for\.cc$'' + ''^src/libcmd/installable-attr-path\.cc$'' + ''^src/libcmd/installable-attr-path\.hh$'' + ''^src/libcmd/installable-derived-path\.cc$'' + ''^src/libcmd/installable-derived-path\.hh$'' + ''^src/libcmd/installable-flake\.cc$'' + ''^src/libcmd/installable-flake\.hh$'' + ''^src/libcmd/installable-value\.cc$'' + ''^src/libcmd/installable-value\.hh$'' + ''^src/libcmd/installables\.cc$'' + ''^src/libcmd/installables\.hh$'' + ''^src/libcmd/legacy\.hh$'' + ''^src/libcmd/markdown\.cc$'' + ''^src/libcmd/misc-store-flags\.cc$'' + ''^src/libcmd/repl-interacter\.cc$'' + ''^src/libcmd/repl-interacter\.hh$'' + ''^src/libcmd/repl\.cc$'' + ''^src/libcmd/repl\.hh$'' + ''^src/libexpr-c/nix_api_expr\.cc$'' + ''^src/libexpr-c/nix_api_external\.cc$'' + ''^src/libexpr/attr-path\.cc$'' + ''^src/libexpr/attr-path\.hh$'' + ''^src/libexpr/attr-set\.cc$'' + ''^src/libexpr/attr-set\.hh$'' + ''^src/libexpr/eval-cache\.cc$'' + ''^src/libexpr/eval-cache\.hh$'' + ''^src/libexpr/eval-error\.cc$'' + ''^src/libexpr/eval-inline\.hh$'' + ''^src/libexpr/eval-settings\.cc$'' + ''^src/libexpr/eval-settings\.hh$'' + ''^src/libexpr/eval\.cc$'' + ''^src/libexpr/eval\.hh$'' + ''^src/libexpr/function-trace\.cc$'' + ''^src/libexpr/gc-small-vector\.hh$'' + ''^src/libexpr/get-drvs\.cc$'' + ''^src/libexpr/get-drvs\.hh$'' + ''^src/libexpr/json-to-value\.cc$'' + ''^src/libexpr/nixexpr\.cc$'' + ''^src/libexpr/nixexpr\.hh$'' + ''^src/libexpr/parser-state\.hh$'' + ''^src/libexpr/pos-table\.hh$'' + ''^src/libexpr/primops\.cc$'' + ''^src/libexpr/primops\.hh$'' + ''^src/libexpr/primops/context\.cc$'' + ''^src/libexpr/primops/fetchClosure\.cc$'' + ''^src/libexpr/primops/fetchMercurial\.cc$'' + ''^src/libexpr/primops/fetchTree\.cc$'' + ''^src/libexpr/primops/fromTOML\.cc$'' + ''^src/libexpr/print-ambiguous\.cc$'' + ''^src/libexpr/print-ambiguous\.hh$'' + ''^src/libexpr/print-options\.hh$'' + ''^src/libexpr/print\.cc$'' + ''^src/libexpr/print\.hh$'' + ''^src/libexpr/search-path\.cc$'' + ''^src/libexpr/symbol-table\.hh$'' + ''^src/libexpr/value-to-json\.cc$'' + ''^src/libexpr/value-to-json\.hh$'' + ''^src/libexpr/value-to-xml\.cc$'' + ''^src/libexpr/value-to-xml\.hh$'' + ''^src/libexpr/value\.hh$'' + ''^src/libexpr/value/context\.cc$'' + ''^src/libexpr/value/context\.hh$'' + ''^src/libfetchers/attrs\.cc$'' + ''^src/libfetchers/cache\.cc$'' + ''^src/libfetchers/cache\.hh$'' + ''^src/libfetchers/fetch-settings\.cc$'' + ''^src/libfetchers/fetch-settings\.hh$'' + ''^src/libfetchers/fetch-to-store\.cc$'' + ''^src/libfetchers/fetchers\.cc$'' + ''^src/libfetchers/fetchers\.hh$'' + ''^src/libfetchers/filtering-source-accessor\.cc$'' + ''^src/libfetchers/filtering-source-accessor\.hh$'' + ''^src/libfetchers/fs-source-accessor\.cc$'' + ''^src/libfetchers/fs-source-accessor\.hh$'' + ''^src/libfetchers/git-utils\.cc$'' + ''^src/libfetchers/git-utils\.hh$'' + ''^src/libfetchers/github\.cc$'' + ''^src/libfetchers/indirect\.cc$'' + ''^src/libfetchers/memory-source-accessor\.cc$'' + ''^src/libfetchers/path\.cc$'' + ''^src/libfetchers/registry\.cc$'' + ''^src/libfetchers/registry\.hh$'' + ''^src/libfetchers/tarball\.cc$'' + ''^src/libfetchers/tarball\.hh$'' + ''^src/libfetchers/git\.cc$'' + ''^src/libfetchers/mercurial\.cc$'' + ''^src/libflake/flake/config\.cc$'' + ''^src/libflake/flake/flake\.cc$'' + ''^src/libflake/flake/flake\.hh$'' + ''^src/libflake/flake/flakeref\.cc$'' + ''^src/libflake/flake/flakeref\.hh$'' + ''^src/libflake/flake/lockfile\.cc$'' + ''^src/libflake/flake/lockfile\.hh$'' + ''^src/libflake/flake/url-name\.cc$'' + ''^src/libmain/common-args\.cc$'' + ''^src/libmain/common-args\.hh$'' + ''^src/libmain/loggers\.cc$'' + ''^src/libmain/loggers\.hh$'' + ''^src/libmain/progress-bar\.cc$'' + ''^src/libmain/shared\.cc$'' + ''^src/libmain/shared\.hh$'' + ''^src/libmain/unix/stack\.cc$'' + ''^src/libstore/binary-cache-store\.cc$'' + ''^src/libstore/binary-cache-store\.hh$'' + ''^src/libstore/build-result\.hh$'' + ''^src/libstore/builtins\.hh$'' + ''^src/libstore/builtins/buildenv\.cc$'' + ''^src/libstore/builtins/buildenv\.hh$'' + ''^src/libstore/common-protocol-impl\.hh$'' + ''^src/libstore/common-protocol\.cc$'' + ''^src/libstore/common-protocol\.hh$'' + ''^src/libstore/common-ssh-store-config\.hh$'' + ''^src/libstore/content-address\.cc$'' + ''^src/libstore/content-address\.hh$'' + ''^src/libstore/daemon\.cc$'' + ''^src/libstore/daemon\.hh$'' + ''^src/libstore/derivations\.cc$'' + ''^src/libstore/derivations\.hh$'' + ''^src/libstore/derived-path-map\.cc$'' + ''^src/libstore/derived-path-map\.hh$'' + ''^src/libstore/derived-path\.cc$'' + ''^src/libstore/derived-path\.hh$'' + ''^src/libstore/downstream-placeholder\.cc$'' + ''^src/libstore/downstream-placeholder\.hh$'' + ''^src/libstore/dummy-store\.cc$'' + ''^src/libstore/export-import\.cc$'' + ''^src/libstore/filetransfer\.cc$'' + ''^src/libstore/filetransfer\.hh$'' + ''^src/libstore/gc-store\.hh$'' + ''^src/libstore/globals\.cc$'' + ''^src/libstore/globals\.hh$'' + ''^src/libstore/http-binary-cache-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.cc$'' + ''^src/libstore/legacy-ssh-store\.hh$'' + ''^src/libstore/length-prefixed-protocol-helper\.hh$'' + ''^src/libstore/linux/personality\.cc$'' + ''^src/libstore/linux/personality\.hh$'' + ''^src/libstore/local-binary-cache-store\.cc$'' + ''^src/libstore/local-fs-store\.cc$'' + ''^src/libstore/local-fs-store\.hh$'' + ''^src/libstore/log-store\.cc$'' + ''^src/libstore/log-store\.hh$'' + ''^src/libstore/machines\.cc$'' + ''^src/libstore/machines\.hh$'' + ''^src/libstore/make-content-addressed\.cc$'' + ''^src/libstore/make-content-addressed\.hh$'' + ''^src/libstore/misc\.cc$'' + ''^src/libstore/names\.cc$'' + ''^src/libstore/names\.hh$'' + ''^src/libstore/nar-accessor\.cc$'' + ''^src/libstore/nar-accessor\.hh$'' + ''^src/libstore/nar-info-disk-cache\.cc$'' + ''^src/libstore/nar-info-disk-cache\.hh$'' + ''^src/libstore/nar-info\.cc$'' + ''^src/libstore/nar-info\.hh$'' + ''^src/libstore/outputs-spec\.cc$'' + ''^src/libstore/outputs-spec\.hh$'' + ''^src/libstore/parsed-derivations\.cc$'' + ''^src/libstore/path-info\.cc$'' + ''^src/libstore/path-info\.hh$'' + ''^src/libstore/path-references\.cc$'' + ''^src/libstore/path-regex\.hh$'' + ''^src/libstore/path-with-outputs\.cc$'' + ''^src/libstore/path\.cc$'' + ''^src/libstore/path\.hh$'' + ''^src/libstore/pathlocks\.cc$'' + ''^src/libstore/pathlocks\.hh$'' + ''^src/libstore/profiles\.cc$'' + ''^src/libstore/profiles\.hh$'' + ''^src/libstore/realisation\.cc$'' + ''^src/libstore/realisation\.hh$'' + ''^src/libstore/remote-fs-accessor\.cc$'' + ''^src/libstore/remote-fs-accessor\.hh$'' + ''^src/libstore/remote-store-connection\.hh$'' + ''^src/libstore/remote-store\.cc$'' + ''^src/libstore/remote-store\.hh$'' + ''^src/libstore/s3-binary-cache-store\.cc$'' + ''^src/libstore/s3\.hh$'' + ''^src/libstore/serve-protocol-impl\.cc$'' + ''^src/libstore/serve-protocol-impl\.hh$'' + ''^src/libstore/serve-protocol\.cc$'' + ''^src/libstore/serve-protocol\.hh$'' + ''^src/libstore/sqlite\.cc$'' + ''^src/libstore/sqlite\.hh$'' + ''^src/libstore/ssh-store\.cc$'' + ''^src/libstore/ssh\.cc$'' + ''^src/libstore/ssh\.hh$'' + ''^src/libstore/store-api\.cc$'' + ''^src/libstore/store-api\.hh$'' + ''^src/libstore/store-dir-config\.hh$'' + ''^src/libstore/build/derivation-goal\.cc$'' + ''^src/libstore/build/derivation-goal\.hh$'' + ''^src/libstore/build/drv-output-substitution-goal\.cc$'' + ''^src/libstore/build/drv-output-substitution-goal\.hh$'' + ''^src/libstore/build/entry-points\.cc$'' + ''^src/libstore/build/goal\.cc$'' + ''^src/libstore/build/goal\.hh$'' + ''^src/libstore/unix/build/hook-instance\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.cc$'' + ''^src/libstore/unix/build/local-derivation-goal\.hh$'' + ''^src/libstore/build/substitution-goal\.cc$'' + ''^src/libstore/build/substitution-goal\.hh$'' + ''^src/libstore/build/worker\.cc$'' + ''^src/libstore/build/worker\.hh$'' + ''^src/libstore/builtins/fetchurl\.cc$'' + ''^src/libstore/builtins/unpack-channel\.cc$'' + ''^src/libstore/gc\.cc$'' + ''^src/libstore/local-overlay-store\.cc$'' + ''^src/libstore/local-overlay-store\.hh$'' + ''^src/libstore/local-store\.cc$'' + ''^src/libstore/local-store\.hh$'' + ''^src/libstore/unix/user-lock\.cc$'' + ''^src/libstore/unix/user-lock\.hh$'' + ''^src/libstore/optimise-store\.cc$'' + ''^src/libstore/unix/pathlocks\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.cc$'' + ''^src/libstore/posix-fs-canonicalise\.hh$'' + ''^src/libstore/uds-remote-store\.cc$'' + ''^src/libstore/uds-remote-store\.hh$'' + ''^src/libstore/windows/build\.cc$'' + ''^src/libstore/worker-protocol-impl\.hh$'' + ''^src/libstore/worker-protocol\.cc$'' + ''^src/libstore/worker-protocol\.hh$'' + ''^src/libutil-c/nix_api_util_internal\.h$'' + ''^src/libutil/archive\.cc$'' + ''^src/libutil/archive\.hh$'' + ''^src/libutil/args\.cc$'' + ''^src/libutil/args\.hh$'' + ''^src/libutil/args/root\.hh$'' + ''^src/libutil/callback\.hh$'' + ''^src/libutil/canon-path\.cc$'' + ''^src/libutil/canon-path\.hh$'' + ''^src/libutil/chunked-vector\.hh$'' + ''^src/libutil/closure\.hh$'' + ''^src/libutil/comparator\.hh$'' + ''^src/libutil/compute-levels\.cc$'' + ''^src/libutil/config-impl\.hh$'' + ''^src/libutil/config\.cc$'' + ''^src/libutil/config\.hh$'' + ''^src/libutil/current-process\.cc$'' + ''^src/libutil/current-process\.hh$'' + ''^src/libutil/english\.cc$'' + ''^src/libutil/english\.hh$'' + ''^src/libutil/error\.cc$'' + ''^src/libutil/error\.hh$'' + ''^src/libutil/exit\.hh$'' + ''^src/libutil/experimental-features\.cc$'' + ''^src/libutil/experimental-features\.hh$'' + ''^src/libutil/file-content-address\.cc$'' + ''^src/libutil/file-content-address\.hh$'' + ''^src/libutil/file-descriptor\.cc$'' + ''^src/libutil/file-descriptor\.hh$'' + ''^src/libutil/file-path-impl\.hh$'' + ''^src/libutil/file-path\.hh$'' + ''^src/libutil/file-system\.cc$'' + ''^src/libutil/file-system\.hh$'' + ''^src/libutil/finally\.hh$'' + ''^src/libutil/fmt\.hh$'' + ''^src/libutil/fs-sink\.cc$'' + ''^src/libutil/fs-sink\.hh$'' + ''^src/libutil/git\.cc$'' + ''^src/libutil/git\.hh$'' + ''^src/libutil/hash\.cc$'' + ''^src/libutil/hash\.hh$'' + ''^src/libutil/hilite\.cc$'' + ''^src/libutil/hilite\.hh$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/json-impls\.hh$'' + ''^src/libutil/json-utils\.cc$'' + ''^src/libutil/json-utils\.hh$'' + ''^src/libutil/linux/cgroup\.cc$'' + ''^src/libutil/linux/namespaces\.cc$'' + ''^src/libutil/logging\.cc$'' + ''^src/libutil/logging\.hh$'' + ''^src/libutil/lru-cache\.hh$'' + ''^src/libutil/memory-source-accessor\.cc$'' + ''^src/libutil/memory-source-accessor\.hh$'' + ''^src/libutil/pool\.hh$'' + ''^src/libutil/position\.cc$'' + ''^src/libutil/position\.hh$'' + ''^src/libutil/posix-source-accessor\.cc$'' + ''^src/libutil/posix-source-accessor\.hh$'' + ''^src/libutil/processes\.hh$'' + ''^src/libutil/ref\.hh$'' + ''^src/libutil/references\.cc$'' + ''^src/libutil/references\.hh$'' + ''^src/libutil/regex-combinators\.hh$'' + ''^src/libutil/serialise\.cc$'' + ''^src/libutil/serialise\.hh$'' + ''^src/libutil/signals\.hh$'' + ''^src/libutil/signature/local-keys\.cc$'' + ''^src/libutil/signature/local-keys\.hh$'' + ''^src/libutil/signature/signer\.cc$'' + ''^src/libutil/signature/signer\.hh$'' + ''^src/libutil/source-accessor\.cc$'' + ''^src/libutil/source-accessor\.hh$'' + ''^src/libutil/source-path\.cc$'' + ''^src/libutil/source-path\.hh$'' + ''^src/libutil/split\.hh$'' + ''^src/libutil/suggestions\.cc$'' + ''^src/libutil/suggestions\.hh$'' + ''^src/libutil/sync\.hh$'' + ''^src/libutil/terminal\.cc$'' + ''^src/libutil/terminal\.hh$'' + ''^src/libutil/thread-pool\.cc$'' + ''^src/libutil/thread-pool\.hh$'' + ''^src/libutil/topo-sort\.hh$'' + ''^src/libutil/types\.hh$'' + ''^src/libutil/unix/file-descriptor\.cc$'' + ''^src/libutil/unix/file-path\.cc$'' + ''^src/libutil/unix/monitor-fd\.hh$'' + ''^src/libutil/unix/processes\.cc$'' + ''^src/libutil/unix/signals-impl\.hh$'' + ''^src/libutil/unix/signals\.cc$'' + ''^src/libutil/unix-domain-socket\.cc$'' + ''^src/libutil/unix/users\.cc$'' + ''^src/libutil/url-parts\.hh$'' + ''^src/libutil/url\.cc$'' + ''^src/libutil/url\.hh$'' + ''^src/libutil/users\.cc$'' + ''^src/libutil/users\.hh$'' + ''^src/libutil/util\.cc$'' + ''^src/libutil/util\.hh$'' + ''^src/libutil/variant-wrapper\.hh$'' + ''^src/libutil/windows/file-descriptor\.cc$'' + ''^src/libutil/windows/file-path\.cc$'' + ''^src/libutil/windows/processes\.cc$'' + ''^src/libutil/windows/users\.cc$'' + ''^src/libutil/windows/windows-error\.cc$'' + ''^src/libutil/windows/windows-error\.hh$'' + ''^src/libutil/xml-writer\.cc$'' + ''^src/libutil/xml-writer\.hh$'' + ''^src/nix-build/nix-build\.cc$'' + ''^src/nix-channel/nix-channel\.cc$'' + ''^src/nix-collect-garbage/nix-collect-garbage\.cc$'' + ''^src/nix-env/buildenv.nix$'' + ''^src/nix-env/nix-env\.cc$'' + ''^src/nix-env/user-env\.cc$'' + ''^src/nix-env/user-env\.hh$'' + ''^src/nix-instantiate/nix-instantiate\.cc$'' + ''^src/nix-store/dotgraph\.cc$'' + ''^src/nix-store/graphml\.cc$'' + ''^src/nix-store/nix-store\.cc$'' + ''^src/nix/add-to-store\.cc$'' + ''^src/nix/app\.cc$'' + ''^src/nix/build\.cc$'' + ''^src/nix/bundle\.cc$'' + ''^src/nix/cat\.cc$'' + ''^src/nix/config-check\.cc$'' + ''^src/nix/config\.cc$'' + ''^src/nix/copy\.cc$'' + ''^src/nix/derivation-add\.cc$'' + ''^src/nix/derivation-show\.cc$'' + ''^src/nix/derivation\.cc$'' + ''^src/nix/develop\.cc$'' + ''^src/nix/diff-closures\.cc$'' + ''^src/nix/dump-path\.cc$'' + ''^src/nix/edit\.cc$'' + ''^src/nix/eval\.cc$'' + ''^src/nix/flake\.cc$'' + ''^src/nix/fmt\.cc$'' + ''^src/nix/hash\.cc$'' + ''^src/nix/log\.cc$'' + ''^src/nix/ls\.cc$'' + ''^src/nix/main\.cc$'' + ''^src/nix/make-content-addressed\.cc$'' + ''^src/nix/nar\.cc$'' + ''^src/nix/optimise-store\.cc$'' + ''^src/nix/path-from-hash-part\.cc$'' + ''^src/nix/path-info\.cc$'' + ''^src/nix/prefetch\.cc$'' + ''^src/nix/profile\.cc$'' + ''^src/nix/realisation\.cc$'' + ''^src/nix/registry\.cc$'' + ''^src/nix/repl\.cc$'' + ''^src/nix/run\.cc$'' + ''^src/nix/run\.hh$'' + ''^src/nix/search\.cc$'' + ''^src/nix/sigs\.cc$'' + ''^src/nix/store-copy-log\.cc$'' + ''^src/nix/store-delete\.cc$'' + ''^src/nix/store-gc\.cc$'' + ''^src/nix/store-info\.cc$'' + ''^src/nix/store-repair\.cc$'' + ''^src/nix/store\.cc$'' + ''^src/nix/unix/daemon\.cc$'' + ''^src/nix/upgrade-nix\.cc$'' + ''^src/nix/verify\.cc$'' + ''^src/nix/why-depends\.cc$'' + + ''^tests/functional/plugins/plugintest\.cc'' + ''^tests/functional/test-libstoreconsumer/main\.cc'' + ''^tests/nixos/ca-fd-leak/sender\.c'' + ''^tests/nixos/ca-fd-leak/smuggler\.c'' + ''^tests/nixos/user-sandboxing/attacker\.c'' + ''^tests/unit/libexpr-support/tests/libexpr\.hh'' + ''^tests/unit/libexpr-support/tests/value/context\.cc'' + ''^tests/unit/libexpr-support/tests/value/context\.hh'' + ''^tests/unit/libexpr/derived-path\.cc'' + ''^tests/unit/libexpr/error_traces\.cc'' + ''^tests/unit/libexpr/eval\.cc'' + ''^tests/unit/libexpr/json\.cc'' + ''^tests/unit/libexpr/main\.cc'' + ''^tests/unit/libexpr/primops\.cc'' + ''^tests/unit/libexpr/search-path\.cc'' + ''^tests/unit/libexpr/trivial\.cc'' + ''^tests/unit/libexpr/value/context\.cc'' + ''^tests/unit/libexpr/value/print\.cc'' + ''^tests/unit/libfetchers/public-key\.cc'' + ''^tests/unit/libflake/flakeref\.cc'' + ''^tests/unit/libflake/url-name\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.cc'' + ''^tests/unit/libstore-support/tests/derived-path\.hh'' + ''^tests/unit/libstore-support/tests/nix_api_store\.hh'' + ''^tests/unit/libstore-support/tests/outputs-spec\.cc'' + ''^tests/unit/libstore-support/tests/outputs-spec\.hh'' + ''^tests/unit/libstore-support/tests/path\.cc'' + ''^tests/unit/libstore-support/tests/path\.hh'' + ''^tests/unit/libstore-support/tests/protocol\.hh'' + ''^tests/unit/libstore/common-protocol\.cc'' + ''^tests/unit/libstore/content-address\.cc'' + ''^tests/unit/libstore/derivation\.cc'' + ''^tests/unit/libstore/derived-path\.cc'' + ''^tests/unit/libstore/downstream-placeholder\.cc'' + ''^tests/unit/libstore/machines\.cc'' + ''^tests/unit/libstore/nar-info-disk-cache\.cc'' + ''^tests/unit/libstore/nar-info\.cc'' + ''^tests/unit/libstore/outputs-spec\.cc'' + ''^tests/unit/libstore/path-info\.cc'' + ''^tests/unit/libstore/path\.cc'' + ''^tests/unit/libstore/serve-protocol\.cc'' + ''^tests/unit/libstore/worker-protocol\.cc'' + ''^tests/unit/libutil-support/tests/characterization\.hh'' + ''^tests/unit/libutil-support/tests/hash\.cc'' + ''^tests/unit/libutil-support/tests/hash\.hh'' + ''^tests/unit/libutil/args\.cc'' + ''^tests/unit/libutil/canon-path\.cc'' + ''^tests/unit/libutil/chunked-vector\.cc'' + ''^tests/unit/libutil/closure\.cc'' + ''^tests/unit/libutil/compression\.cc'' + ''^tests/unit/libutil/config\.cc'' + ''^tests/unit/libutil/file-content-address\.cc'' + ''^tests/unit/libutil/git\.cc'' + ''^tests/unit/libutil/hash\.cc'' + ''^tests/unit/libutil/hilite\.cc'' + ''^tests/unit/libutil/json-utils\.cc'' + ''^tests/unit/libutil/logging\.cc'' + ''^tests/unit/libutil/lru-cache\.cc'' + ''^tests/unit/libutil/pool\.cc'' + ''^tests/unit/libutil/references\.cc'' + ''^tests/unit/libutil/suggestions\.cc'' + ''^tests/unit/libutil/url\.cc'' + ''^tests/unit/libutil/xml-writer\.cc'' + ]; + }; + shellcheck = { + enable = true; + excludes = [ + # We haven't linted these files yet + ''^config/install-sh$'' + ''^misc/bash/completion\.sh$'' + ''^misc/fish/completion\.fish$'' + ''^misc/zsh/completion\.zsh$'' + ''^scripts/create-darwin-volume\.sh$'' + ''^scripts/install-darwin-multi-user\.sh$'' + ''^scripts/install-multi-user\.sh$'' + ''^scripts/install-nix-from-closure\.sh$'' + ''^scripts/install-systemd-multi-user\.sh$'' + ''^src/nix/get-env\.sh$'' + ''^tests/functional/build\.sh$'' + ''^tests/functional/ca/build-dry\.sh$'' + ''^tests/functional/ca/build-with-garbage-path\.sh$'' + ''^tests/functional/ca/common\.sh$'' + ''^tests/functional/ca/concurrent-builds\.sh$'' + ''^tests/functional/ca/eval-store\.sh$'' + ''^tests/functional/ca/gc\.sh$'' + ''^tests/functional/ca/import-from-derivation\.sh$'' + ''^tests/functional/ca/new-build-cmd\.sh$'' + ''^tests/functional/ca/nix-shell\.sh$'' + ''^tests/functional/ca/post-hook\.sh$'' + ''^tests/functional/ca/recursive\.sh$'' + ''^tests/functional/ca/repl\.sh$'' + ''^tests/functional/ca/selfref-gc\.sh$'' + ''^tests/functional/ca/why-depends\.sh$'' + ''^tests/functional/characterisation-test-infra\.sh$'' + ''^tests/functional/check\.sh$'' + ''^tests/functional/common/vars-and-functions\.sh$'' + ''^tests/functional/completions\.sh$'' + ''^tests/functional/compute-levels\.sh$'' + ''^tests/functional/config\.sh$'' + ''^tests/functional/db-migration\.sh$'' + ''^tests/functional/debugger\.sh$'' + ''^tests/functional/dependencies\.builder0\.sh$'' + ''^tests/functional/dependencies\.sh$'' + ''^tests/functional/dump-db\.sh$'' + ''^tests/functional/dyn-drv/build-built-drv\.sh$'' + ''^tests/functional/dyn-drv/common\.sh$'' + ''^tests/functional/dyn-drv/dep-built-drv\.sh$'' + ''^tests/functional/dyn-drv/eval-outputOf\.sh$'' + ''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$'' + ''^tests/functional/dyn-drv/recursive-mod-json\.sh$'' + ''^tests/functional/eval-store\.sh$'' + ''^tests/functional/eval\.sh$'' + ''^tests/functional/export-graph\.sh$'' + ''^tests/functional/export\.sh$'' + ''^tests/functional/extra-sandbox-profile\.sh$'' + ''^tests/functional/fetchClosure\.sh$'' + ''^tests/functional/fetchGit\.sh$'' + ''^tests/functional/fetchGitRefs\.sh$'' + ''^tests/functional/fetchGitSubmodules\.sh$'' + ''^tests/functional/fetchGitVerification\.sh$'' + ''^tests/functional/fetchMercurial\.sh$'' + ''^tests/functional/fetchurl\.sh$'' + ''^tests/functional/fixed\.builder1\.sh$'' + ''^tests/functional/fixed\.builder2\.sh$'' + ''^tests/functional/fixed\.sh$'' + ''^tests/functional/flakes/absolute-paths\.sh$'' + ''^tests/functional/flakes/check\.sh$'' + ''^tests/functional/flakes/common\.sh$'' + ''^tests/functional/flakes/config\.sh$'' + ''^tests/functional/flakes/develop\.sh$'' + ''^tests/functional/flakes/flakes\.sh$'' + ''^tests/functional/flakes/follow-paths\.sh$'' + ''^tests/functional/flakes/prefetch\.sh$'' + ''^tests/functional/flakes/run\.sh$'' + ''^tests/functional/flakes/show\.sh$'' + ''^tests/functional/fmt\.sh$'' + ''^tests/functional/fmt\.simple\.sh$'' + ''^tests/functional/gc-auto\.sh$'' + ''^tests/functional/gc-concurrent\.builder\.sh$'' + ''^tests/functional/gc-concurrent\.sh$'' + ''^tests/functional/gc-concurrent2\.builder\.sh$'' + ''^tests/functional/gc-non-blocking\.sh$'' + ''^tests/functional/gc\.sh$'' + ''^tests/functional/git-hashing/common\.sh$'' + ''^tests/functional/git-hashing/simple\.sh$'' + ''^tests/functional/hash-convert\.sh$'' + ''^tests/functional/help\.sh$'' + ''^tests/functional/impure-derivations\.sh$'' + ''^tests/functional/impure-env\.sh$'' + ''^tests/functional/impure-eval\.sh$'' + ''^tests/functional/install-darwin\.sh$'' + ''^tests/functional/lang\.sh$'' + ''^tests/functional/legacy-ssh-store\.sh$'' + ''^tests/functional/linux-sandbox\.sh$'' + ''^tests/functional/local-overlay-store/add-lower-inner\.sh$'' + ''^tests/functional/local-overlay-store/add-lower\.sh$'' + ''^tests/functional/local-overlay-store/bad-uris\.sh$'' + ''^tests/functional/local-overlay-store/build-inner\.sh$'' + ''^tests/functional/local-overlay-store/build\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init-inner\.sh$'' + ''^tests/functional/local-overlay-store/check-post-init\.sh$'' + ''^tests/functional/local-overlay-store/common\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-duplicate\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs-inner\.sh$'' + ''^tests/functional/local-overlay-store/delete-refs\.sh$'' + ''^tests/functional/local-overlay-store/gc-inner\.sh$'' + ''^tests/functional/local-overlay-store/gc\.sh$'' + ''^tests/functional/local-overlay-store/optimise-inner\.sh$'' + ''^tests/functional/local-overlay-store/optimise\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add-inner\.sh$'' + ''^tests/functional/local-overlay-store/redundant-add\.sh$'' + ''^tests/functional/local-overlay-store/remount\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle-inner\.sh$'' + ''^tests/functional/local-overlay-store/stale-file-handle\.sh$'' + ''^tests/functional/local-overlay-store/verify-inner\.sh$'' + ''^tests/functional/local-overlay-store/verify\.sh$'' + ''^tests/functional/logging\.sh$'' + ''^tests/functional/misc\.sh$'' + ''^tests/functional/multiple-outputs\.sh$'' + ''^tests/functional/nar-access\.sh$'' + ''^tests/functional/nested-sandboxing\.sh$'' + ''^tests/functional/nested-sandboxing/command\.sh$'' + ''^tests/functional/nix-build\.sh$'' + ''^tests/functional/nix-channel\.sh$'' + ''^tests/functional/nix-collect-garbage-d\.sh$'' + ''^tests/functional/nix-copy-ssh-common\.sh$'' + ''^tests/functional/nix-copy-ssh-ng\.sh$'' + ''^tests/functional/nix-copy-ssh\.sh$'' + ''^tests/functional/nix-daemon-untrusting\.sh$'' + ''^tests/functional/nix-profile\.sh$'' + ''^tests/functional/nix-shell\.sh$'' + ''^tests/functional/nix_path\.sh$'' + ''^tests/functional/optimise-store\.sh$'' + ''^tests/functional/output-normalization\.sh$'' + ''^tests/functional/parallel\.builder\.sh$'' + ''^tests/functional/parallel\.sh$'' + ''^tests/functional/pass-as-file\.sh$'' + ''^tests/functional/path-from-hash-part\.sh$'' + ''^tests/functional/path-info\.sh$'' + ''^tests/functional/placeholders\.sh$'' + ''^tests/functional/plugins\.sh$'' + ''^tests/functional/post-hook\.sh$'' + ''^tests/functional/pure-eval\.sh$'' + ''^tests/functional/push-to-store-old\.sh$'' + ''^tests/functional/push-to-store\.sh$'' + ''^tests/functional/read-only-store\.sh$'' + ''^tests/functional/readfile-context\.sh$'' + ''^tests/functional/recursive\.sh$'' + ''^tests/functional/referrers\.sh$'' + ''^tests/functional/remote-store\.sh$'' + ''^tests/functional/repair\.sh$'' + ''^tests/functional/restricted\.sh$'' + ''^tests/functional/search\.sh$'' + ''^tests/functional/secure-drv-outputs\.sh$'' + ''^tests/functional/selfref-gc\.sh$'' + ''^tests/functional/shell\.sh$'' + ''^tests/functional/shell\.shebang\.sh$'' + ''^tests/functional/simple\.builder\.sh$'' + ''^tests/functional/supplementary-groups\.sh$'' + ''^tests/functional/toString-path\.sh$'' + ''^tests/functional/user-envs-migration\.sh$'' + ''^tests/functional/user-envs-test-case\.sh$'' + ''^tests/functional/user-envs\.builder\.sh$'' + ''^tests/functional/user-envs\.sh$'' + ''^tests/functional/why-depends\.sh$'' + ''^tests/functional/zstd\.sh$'' + ''^tests/unit/libutil/data/git/check-data\.sh$'' + ]; + }; # TODO: nixfmt, https://github.com/NixOS/nixfmt/issues/153 }; - - excludes = [ - # We don't want to format test data - # ''tests/(?!nixos/).*\.nix'' - ''^tests/.*'' - - # Don't format vendored code - ''^src/toml11/.*'' - ''^doc/manual/redirects\.js$'' - ''^doc/manual/theme/highlight\.js$'' - - # We haven't applied formatting to these files yet - ''^doc/manual/redirects\.js$'' - ''^doc/manual/theme/highlight\.js$'' - ''^precompiled-headers\.h$'' - ''^src/build-remote/build-remote\.cc$'' - ''^src/libcmd/built-path\.cc$'' - ''^src/libcmd/built-path\.hh$'' - ''^src/libcmd/command\.cc$'' - ''^src/libcmd/command\.hh$'' - ''^src/libcmd/common-eval-args\.cc$'' - ''^src/libcmd/common-eval-args\.hh$'' - ''^src/libcmd/editor-for\.cc$'' - ''^src/libcmd/installable-attr-path\.cc$'' - ''^src/libcmd/installable-attr-path\.hh$'' - ''^src/libcmd/installable-derived-path\.cc$'' - ''^src/libcmd/installable-derived-path\.hh$'' - ''^src/libcmd/installable-flake\.cc$'' - ''^src/libcmd/installable-flake\.hh$'' - ''^src/libcmd/installable-value\.cc$'' - ''^src/libcmd/installable-value\.hh$'' - ''^src/libcmd/installables\.cc$'' - ''^src/libcmd/installables\.hh$'' - ''^src/libcmd/legacy\.hh$'' - ''^src/libcmd/markdown\.cc$'' - ''^src/libcmd/misc-store-flags\.cc$'' - ''^src/libcmd/repl-interacter\.cc$'' - ''^src/libcmd/repl-interacter\.hh$'' - ''^src/libcmd/repl\.cc$'' - ''^src/libcmd/repl\.hh$'' - ''^src/libexpr-c/nix_api_expr\.cc$'' - ''^src/libexpr-c/nix_api_external\.cc$'' - ''^src/libexpr/attr-path\.cc$'' - ''^src/libexpr/attr-path\.hh$'' - ''^src/libexpr/attr-set\.cc$'' - ''^src/libexpr/attr-set\.hh$'' - ''^src/libexpr/eval-cache\.cc$'' - ''^src/libexpr/eval-cache\.hh$'' - ''^src/libexpr/eval-error\.cc$'' - ''^src/libexpr/eval-inline\.hh$'' - ''^src/libexpr/eval-settings\.cc$'' - ''^src/libexpr/eval-settings\.hh$'' - ''^src/libexpr/eval\.cc$'' - ''^src/libexpr/eval\.hh$'' - ''^src/libexpr/flake/config\.cc$'' - ''^src/libexpr/flake/flake\.cc$'' - ''^src/libexpr/flake/flake\.hh$'' - ''^src/libexpr/flake/flakeref\.cc$'' - ''^src/libexpr/flake/flakeref\.hh$'' - ''^src/libexpr/flake/lockfile\.cc$'' - ''^src/libexpr/flake/lockfile\.hh$'' - ''^src/libexpr/flake/url-name\.cc$'' - ''^src/libexpr/function-trace\.cc$'' - ''^src/libexpr/gc-small-vector\.hh$'' - ''^src/libexpr/get-drvs\.cc$'' - ''^src/libexpr/get-drvs\.hh$'' - ''^src/libexpr/json-to-value\.cc$'' - ''^src/libexpr/nixexpr\.cc$'' - ''^src/libexpr/nixexpr\.hh$'' - ''^src/libexpr/parser-state\.hh$'' - ''^src/libexpr/pos-table\.hh$'' - ''^src/libexpr/primops\.cc$'' - ''^src/libexpr/primops\.hh$'' - ''^src/libexpr/primops/context\.cc$'' - ''^src/libexpr/primops/fetchClosure\.cc$'' - ''^src/libexpr/primops/fetchMercurial\.cc$'' - ''^src/libexpr/primops/fetchTree\.cc$'' - ''^src/libexpr/primops/fromTOML\.cc$'' - ''^src/libexpr/print-ambiguous\.cc$'' - ''^src/libexpr/print-ambiguous\.hh$'' - ''^src/libexpr/print-options\.hh$'' - ''^src/libexpr/print\.cc$'' - ''^src/libexpr/print\.hh$'' - ''^src/libexpr/search-path\.cc$'' - ''^src/libexpr/symbol-table\.hh$'' - ''^src/libexpr/value-to-json\.cc$'' - ''^src/libexpr/value-to-json\.hh$'' - ''^src/libexpr/value-to-xml\.cc$'' - ''^src/libexpr/value-to-xml\.hh$'' - ''^src/libexpr/value\.hh$'' - ''^src/libexpr/value/context\.cc$'' - ''^src/libexpr/value/context\.hh$'' - ''^src/libfetchers/attrs\.cc$'' - ''^src/libfetchers/cache\.cc$'' - ''^src/libfetchers/cache\.hh$'' - ''^src/libfetchers/fetch-settings\.cc$'' - ''^src/libfetchers/fetch-settings\.hh$'' - ''^src/libfetchers/fetch-to-store\.cc$'' - ''^src/libfetchers/fetchers\.cc$'' - ''^src/libfetchers/fetchers\.hh$'' - ''^src/libfetchers/filtering-source-accessor\.cc$'' - ''^src/libfetchers/filtering-source-accessor\.hh$'' - ''^src/libfetchers/fs-source-accessor\.cc$'' - ''^src/libfetchers/fs-source-accessor\.hh$'' - ''^src/libfetchers/git-utils\.cc$'' - ''^src/libfetchers/git-utils\.hh$'' - ''^src/libfetchers/github\.cc$'' - ''^src/libfetchers/indirect\.cc$'' - ''^src/libfetchers/memory-source-accessor\.cc$'' - ''^src/libfetchers/path\.cc$'' - ''^src/libfetchers/registry\.cc$'' - ''^src/libfetchers/registry\.hh$'' - ''^src/libfetchers/tarball\.cc$'' - ''^src/libfetchers/tarball\.hh$'' - ''^src/libfetchers/unix/git\.cc$'' - ''^src/libfetchers/unix/mercurial\.cc$'' - ''^src/libmain/common-args\.cc$'' - ''^src/libmain/common-args\.hh$'' - ''^src/libmain/loggers\.cc$'' - ''^src/libmain/loggers\.hh$'' - ''^src/libmain/progress-bar\.cc$'' - ''^src/libmain/shared\.cc$'' - ''^src/libmain/shared\.hh$'' - ''^src/libmain/unix/stack\.cc$'' - ''^src/libstore/binary-cache-store\.cc$'' - ''^src/libstore/binary-cache-store\.hh$'' - ''^src/libstore/build-result\.hh$'' - ''^src/libstore/builtins\.hh$'' - ''^src/libstore/builtins/buildenv\.cc$'' - ''^src/libstore/builtins/buildenv\.hh$'' - ''^src/libstore/common-protocol-impl\.hh$'' - ''^src/libstore/common-protocol\.cc$'' - ''^src/libstore/common-protocol\.hh$'' - ''^src/libstore/content-address\.cc$'' - ''^src/libstore/content-address\.hh$'' - ''^src/libstore/daemon\.cc$'' - ''^src/libstore/daemon\.hh$'' - ''^src/libstore/derivations\.cc$'' - ''^src/libstore/derivations\.hh$'' - ''^src/libstore/derived-path-map\.cc$'' - ''^src/libstore/derived-path-map\.hh$'' - ''^src/libstore/derived-path\.cc$'' - ''^src/libstore/derived-path\.hh$'' - ''^src/libstore/downstream-placeholder\.cc$'' - ''^src/libstore/downstream-placeholder\.hh$'' - ''^src/libstore/dummy-store\.cc$'' - ''^src/libstore/export-import\.cc$'' - ''^src/libstore/filetransfer\.cc$'' - ''^src/libstore/filetransfer\.hh$'' - ''^src/libstore/gc-store\.hh$'' - ''^src/libstore/globals\.cc$'' - ''^src/libstore/globals\.hh$'' - ''^src/libstore/http-binary-cache-store\.cc$'' - ''^src/libstore/legacy-ssh-store\.cc$'' - ''^src/libstore/legacy-ssh-store\.hh$'' - ''^src/libstore/length-prefixed-protocol-helper\.hh$'' - ''^src/libstore/linux/personality\.cc$'' - ''^src/libstore/linux/personality\.hh$'' - ''^src/libstore/local-binary-cache-store\.cc$'' - ''^src/libstore/local-fs-store\.cc$'' - ''^src/libstore/local-fs-store\.hh$'' - ''^src/libstore/log-store\.cc$'' - ''^src/libstore/log-store\.hh$'' - ''^src/libstore/machines\.cc$'' - ''^src/libstore/machines\.hh$'' - ''^src/libstore/make-content-addressed\.cc$'' - ''^src/libstore/make-content-addressed\.hh$'' - ''^src/libstore/misc\.cc$'' - ''^src/libstore/names\.cc$'' - ''^src/libstore/names\.hh$'' - ''^src/libstore/nar-accessor\.cc$'' - ''^src/libstore/nar-accessor\.hh$'' - ''^src/libstore/nar-info-disk-cache\.cc$'' - ''^src/libstore/nar-info-disk-cache\.hh$'' - ''^src/libstore/nar-info\.cc$'' - ''^src/libstore/nar-info\.hh$'' - ''^src/libstore/outputs-spec\.cc$'' - ''^src/libstore/outputs-spec\.hh$'' - ''^src/libstore/parsed-derivations\.cc$'' - ''^src/libstore/path-info\.cc$'' - ''^src/libstore/path-info\.hh$'' - ''^src/libstore/path-references\.cc$'' - ''^src/libstore/path-regex\.hh$'' - ''^src/libstore/path-with-outputs\.cc$'' - ''^src/libstore/path\.cc$'' - ''^src/libstore/path\.hh$'' - ''^src/libstore/pathlocks\.cc$'' - ''^src/libstore/pathlocks\.hh$'' - ''^src/libstore/profiles\.cc$'' - ''^src/libstore/profiles\.hh$'' - ''^src/libstore/realisation\.cc$'' - ''^src/libstore/realisation\.hh$'' - ''^src/libstore/remote-fs-accessor\.cc$'' - ''^src/libstore/remote-fs-accessor\.hh$'' - ''^src/libstore/remote-store-connection\.hh$'' - ''^src/libstore/remote-store\.cc$'' - ''^src/libstore/remote-store\.hh$'' - ''^src/libstore/s3-binary-cache-store\.cc$'' - ''^src/libstore/s3\.hh$'' - ''^src/libstore/serve-protocol-impl\.cc$'' - ''^src/libstore/serve-protocol-impl\.hh$'' - ''^src/libstore/serve-protocol\.cc$'' - ''^src/libstore/serve-protocol\.hh$'' - ''^src/libstore/sqlite\.cc$'' - ''^src/libstore/sqlite\.hh$'' - ''^src/libstore/ssh-store-config\.hh$'' - ''^src/libstore/ssh-store\.cc$'' - ''^src/libstore/ssh\.cc$'' - ''^src/libstore/ssh\.hh$'' - ''^src/libstore/store-api\.cc$'' - ''^src/libstore/store-api\.hh$'' - ''^src/libstore/store-dir-config\.hh$'' - ''^src/libstore/unix/build/derivation-goal\.cc$'' - ''^src/libstore/unix/build/derivation-goal\.hh$'' - ''^src/libstore/unix/build/drv-output-substitution-goal\.cc$'' - ''^src/libstore/unix/build/drv-output-substitution-goal\.hh$'' - ''^src/libstore/unix/build/entry-points\.cc$'' - ''^src/libstore/unix/build/goal\.cc$'' - ''^src/libstore/unix/build/goal\.hh$'' - ''^src/libstore/unix/build/hook-instance\.cc$'' - ''^src/libstore/unix/build/local-derivation-goal\.cc$'' - ''^src/libstore/unix/build/local-derivation-goal\.hh$'' - ''^src/libstore/unix/build/substitution-goal\.cc$'' - ''^src/libstore/unix/build/substitution-goal\.hh$'' - ''^src/libstore/unix/build/worker\.cc$'' - ''^src/libstore/unix/build/worker\.hh$'' - ''^src/libstore/unix/builtins/fetchurl\.cc$'' - ''^src/libstore/unix/builtins/unpack-channel\.cc$'' - ''^src/libstore/gc\.cc$'' - ''^src/libstore/unix/local-overlay-store\.cc$'' - ''^src/libstore/unix/local-overlay-store\.hh$'' - ''^src/libstore/local-store\.cc$'' - ''^src/libstore/local-store\.hh$'' - ''^src/libstore/unix/lock\.cc$'' - ''^src/libstore/unix/lock\.hh$'' - ''^src/libstore/optimise-store\.cc$'' - ''^src/libstore/unix/pathlocks\.cc$'' - ''^src/libstore/posix-fs-canonicalise\.cc$'' - ''^src/libstore/posix-fs-canonicalise\.hh$'' - ''^src/libstore/uds-remote-store\.cc$'' - ''^src/libstore/uds-remote-store\.hh$'' - ''^src/libstore/windows/build\.cc$'' - ''^src/libstore/worker-protocol-impl\.hh$'' - ''^src/libstore/worker-protocol\.cc$'' - ''^src/libstore/worker-protocol\.hh$'' - ''^src/libutil-c/nix_api_util_internal\.h$'' - ''^src/libutil/archive\.cc$'' - ''^src/libutil/archive\.hh$'' - ''^src/libutil/args\.cc$'' - ''^src/libutil/args\.hh$'' - ''^src/libutil/args/root\.hh$'' - ''^src/libutil/callback\.hh$'' - ''^src/libutil/canon-path\.cc$'' - ''^src/libutil/canon-path\.hh$'' - ''^src/libutil/chunked-vector\.hh$'' - ''^src/libutil/closure\.hh$'' - ''^src/libutil/comparator\.hh$'' - ''^src/libutil/compute-levels\.cc$'' - ''^src/libutil/config-impl\.hh$'' - ''^src/libutil/config\.cc$'' - ''^src/libutil/config\.hh$'' - ''^src/libutil/current-process\.cc$'' - ''^src/libutil/current-process\.hh$'' - ''^src/libutil/english\.cc$'' - ''^src/libutil/english\.hh$'' - ''^src/libutil/environment-variables\.cc$'' - ''^src/libutil/error\.cc$'' - ''^src/libutil/error\.hh$'' - ''^src/libutil/exit\.hh$'' - ''^src/libutil/experimental-features\.cc$'' - ''^src/libutil/experimental-features\.hh$'' - ''^src/libutil/file-content-address\.cc$'' - ''^src/libutil/file-content-address\.hh$'' - ''^src/libutil/file-descriptor\.cc$'' - ''^src/libutil/file-descriptor\.hh$'' - ''^src/libutil/file-path-impl\.hh$'' - ''^src/libutil/file-path\.hh$'' - ''^src/libutil/file-system\.cc$'' - ''^src/libutil/file-system\.hh$'' - ''^src/libutil/finally\.hh$'' - ''^src/libutil/fmt\.hh$'' - ''^src/libutil/fs-sink\.cc$'' - ''^src/libutil/fs-sink\.hh$'' - ''^src/libutil/git\.cc$'' - ''^src/libutil/git\.hh$'' - ''^src/libutil/hash\.cc$'' - ''^src/libutil/hash\.hh$'' - ''^src/libutil/hilite\.cc$'' - ''^src/libutil/hilite\.hh$'' - ''^src/libutil/source-accessor\.hh$'' - ''^src/libutil/json-impls\.hh$'' - ''^src/libutil/json-utils\.cc$'' - ''^src/libutil/json-utils\.hh$'' - ''^src/libutil/linux/cgroup\.cc$'' - ''^src/libutil/linux/namespaces\.cc$'' - ''^src/libutil/logging\.cc$'' - ''^src/libutil/logging\.hh$'' - ''^src/libutil/lru-cache\.hh$'' - ''^src/libutil/memory-source-accessor\.cc$'' - ''^src/libutil/memory-source-accessor\.hh$'' - ''^src/libutil/pool\.hh$'' - ''^src/libutil/position\.cc$'' - ''^src/libutil/position\.hh$'' - ''^src/libutil/posix-source-accessor\.cc$'' - ''^src/libutil/posix-source-accessor\.hh$'' - ''^src/libutil/processes\.hh$'' - ''^src/libutil/ref\.hh$'' - ''^src/libutil/references\.cc$'' - ''^src/libutil/references\.hh$'' - ''^src/libutil/regex-combinators\.hh$'' - ''^src/libutil/serialise\.cc$'' - ''^src/libutil/serialise\.hh$'' - ''^src/libutil/signals\.hh$'' - ''^src/libutil/signature/local-keys\.cc$'' - ''^src/libutil/signature/local-keys\.hh$'' - ''^src/libutil/signature/signer\.cc$'' - ''^src/libutil/signature/signer\.hh$'' - ''^src/libutil/source-accessor\.cc$'' - ''^src/libutil/source-accessor\.hh$'' - ''^src/libutil/source-path\.cc$'' - ''^src/libutil/source-path\.hh$'' - ''^src/libutil/split\.hh$'' - ''^src/libutil/suggestions\.cc$'' - ''^src/libutil/suggestions\.hh$'' - ''^src/libutil/sync\.hh$'' - ''^src/libutil/terminal\.cc$'' - ''^src/libutil/terminal\.hh$'' - ''^src/libutil/thread-pool\.cc$'' - ''^src/libutil/thread-pool\.hh$'' - ''^src/libutil/topo-sort\.hh$'' - ''^src/libutil/types\.hh$'' - ''^src/libutil/unix/file-descriptor\.cc$'' - ''^src/libutil/unix/file-path\.cc$'' - ''^src/libutil/unix/monitor-fd\.hh$'' - ''^src/libutil/unix/processes\.cc$'' - ''^src/libutil/unix/signals-impl\.hh$'' - ''^src/libutil/unix/signals\.cc$'' - ''^src/libutil/unix-domain-socket\.cc$'' - ''^src/libutil/unix/users\.cc$'' - ''^src/libutil/url-parts\.hh$'' - ''^src/libutil/url\.cc$'' - ''^src/libutil/url\.hh$'' - ''^src/libutil/users\.cc$'' - ''^src/libutil/users\.hh$'' - ''^src/libutil/util\.cc$'' - ''^src/libutil/util\.hh$'' - ''^src/libutil/variant-wrapper\.hh$'' - ''^src/libutil/windows/environment-variables\.cc$'' - ''^src/libutil/windows/file-descriptor\.cc$'' - ''^src/libutil/windows/file-path\.cc$'' - ''^src/libutil/windows/processes\.cc$'' - ''^src/libutil/windows/users\.cc$'' - ''^src/libutil/windows/windows-error\.cc$'' - ''^src/libutil/windows/windows-error\.hh$'' - ''^src/libutil/xml-writer\.cc$'' - ''^src/libutil/xml-writer\.hh$'' - ''^src/nix-build/nix-build\.cc$'' - ''^src/nix-channel/nix-channel\.cc$'' - ''^src/nix-collect-garbage/nix-collect-garbage\.cc$'' - ''^src/nix-env/buildenv.nix$'' - ''^src/nix-env/nix-env\.cc$'' - ''^src/nix-env/user-env\.cc$'' - ''^src/nix-env/user-env\.hh$'' - ''^src/nix-instantiate/nix-instantiate\.cc$'' - ''^src/nix-store/dotgraph\.cc$'' - ''^src/nix-store/graphml\.cc$'' - ''^src/nix-store/nix-store\.cc$'' - ''^src/nix/add-to-store\.cc$'' - ''^src/nix/app\.cc$'' - ''^src/nix/build\.cc$'' - ''^src/nix/bundle\.cc$'' - ''^src/nix/cat\.cc$'' - ''^src/nix/config-check\.cc$'' - ''^src/nix/config\.cc$'' - ''^src/nix/copy\.cc$'' - ''^src/nix/derivation-add\.cc$'' - ''^src/nix/derivation-show\.cc$'' - ''^src/nix/derivation\.cc$'' - ''^src/nix/develop\.cc$'' - ''^src/nix/diff-closures\.cc$'' - ''^src/nix/dump-path\.cc$'' - ''^src/nix/edit\.cc$'' - ''^src/nix/eval\.cc$'' - ''^src/nix/flake\.cc$'' - ''^src/nix/fmt\.cc$'' - ''^src/nix/hash\.cc$'' - ''^src/nix/log\.cc$'' - ''^src/nix/ls\.cc$'' - ''^src/nix/main\.cc$'' - ''^src/nix/make-content-addressed\.cc$'' - ''^src/nix/nar\.cc$'' - ''^src/nix/optimise-store\.cc$'' - ''^src/nix/path-from-hash-part\.cc$'' - ''^src/nix/path-info\.cc$'' - ''^src/nix/prefetch\.cc$'' - ''^src/nix/profile\.cc$'' - ''^src/nix/realisation\.cc$'' - ''^src/nix/registry\.cc$'' - ''^src/nix/repl\.cc$'' - ''^src/nix/run\.cc$'' - ''^src/nix/run\.hh$'' - ''^src/nix/search\.cc$'' - ''^src/nix/sigs\.cc$'' - ''^src/nix/store-copy-log\.cc$'' - ''^src/nix/store-delete\.cc$'' - ''^src/nix/store-gc\.cc$'' - ''^src/nix/store-info\.cc$'' - ''^src/nix/store-repair\.cc$'' - ''^src/nix/store\.cc$'' - ''^src/nix/unix/daemon\.cc$'' - ''^src/nix/upgrade-nix\.cc$'' - ''^src/nix/verify\.cc$'' - ''^src/nix/why-depends\.cc$'' - ]; }; - }; # We'll be pulling from this in the main flake diff --git a/maintainers/format.sh b/maintainers/format.sh new file mode 100755 index 000000000..a2a6d8b41 --- /dev/null +++ b/maintainers/format.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +if ! type -p pre-commit &>/dev/null; then + echo "format.sh: pre-commit not found. Please use \`nix develop\`."; + exit 1; +fi; +if test -z "$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then + echo "format.sh: _NIX_PRE_COMMIT_HOOKS_CONFIG not set. Please use \`nix develop\`."; + exit 1; +fi; +pre-commit run --config "$_NIX_PRE_COMMIT_HOOKS_CONFIG" --all-files diff --git a/maintainers/local.mk b/maintainers/local.mk index 88d594d67..e81517eda 100644 --- a/maintainers/local.mk +++ b/maintainers/local.mk @@ -3,13 +3,6 @@ print-top-help += echo ' format: Format source code' # This uses the cached .pre-commit-hooks.yaml file +fmt_script := $(d)/format.sh format: - @if ! type -p pre-commit &>/dev/null; then \ - echo "make format: pre-commit not found. Please use \`nix develop\`."; \ - exit 1; \ - fi; \ - if test -z "$$_NIX_PRE_COMMIT_HOOKS_CONFIG"; then \ - echo "make format: _NIX_PRE_COMMIT_HOOKS_CONFIG not set. Please use \`nix develop\`."; \ - exit 1; \ - fi; \ - pre-commit run --config $$_NIX_PRE_COMMIT_HOOKS_CONFIG --all-files + @$(fmt_script) diff --git a/maintainers/release-credits b/maintainers/release-credits new file mode 100755 index 000000000..7a5c87d7d --- /dev/null +++ b/maintainers/release-credits @@ -0,0 +1,184 @@ +#!/usr/bin/env nix +# vim: set filetype=python: +#!nix develop --impure --expr +#!nix `` +#!nix let flake = builtins.getFlake ("git+file://" + toString ../.); +#!nix pkgs = flake.inputs.nixpkgs.legacyPackages.${builtins.currentSystem}; +#!nix in pkgs.mkShell { nativeBuildInputs = [ +#!nix (pkgs.python3.withPackages (ps: with ps; [ requests ])) +#!nix ]; } +#!nix `` --command python3 + +# This script lists out the contributors for a given release. +# It must be run from the root of the Nix repository. + +import os +import sys +import json +import requests + +github_token = os.environ.get("GITHUB_TOKEN") +if not github_token: + print("GITHUB_TOKEN is not set. If you hit the rate limit, set it", file=sys.stderr) + # Might be ok, as we have a cache. + # raise ValueError("GITHUB_TOKEN must be set") + +# 1. Read the current version in .version +version = os.environ.get("VERSION") +if not version: + version = open(".version").read().strip() + +print(f"Generating release credits for Nix {version}", file=sys.stderr) + +# 2. Compute previous version +vcomponents = version.split(".") +if len(vcomponents) >= 2: + prev_version = f"{vcomponents[0]}.{int(vcomponents[1])-1}.0" +else: + raise ValueError(".version must have at least two components") + +# For unreleased versions +endref = "HEAD" +# For older releases +# endref = version + +# 2. Find the merge base between the current version and the previous version +mergeBase = os.popen(f"git merge-base {prev_version} {endref}").read().strip() +print(f"Merge base between {prev_version} and {endref} is {mergeBase}", file=sys.stderr) + +# 3. Find the date of the merge base +mergeBaseDate = os.popen(f"git show -s --format=%ci {mergeBase}").read().strip()[0:10] +print(f"Merge base date is {mergeBaseDate}", file=sys.stderr) + +# 4. Get the commits between the merge base and the current version + +def get_commits(): + raw = os.popen(f"git log --pretty=format:'%H\t%an\t%ae' {mergeBase}..{endref}").read().strip() + lines = raw.split("\n") + return [ { "hash": items[0], "author": items[1], "email": items[2] } + for line in lines + for items in (line.split("\t"),) + ] + +def commits_to_first_commit_by_email(commits): + by_email = dict() + for commit in commits: + email = commit["email"] + if email not in by_email: + by_email[email] = commit + return by_email + + +samples = commits_to_first_commit_by_email(get_commits()) + +# For quick testing, only pick two samples from the dict +# samples = dict(list(samples.items())[:2]) + +# Query the GitHub API to get handle +def get_github_commit(commit): + url = f"https://api.github.com/repos/NixOS/nix/commits/{commit['hash']}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + return response.json() + +class Cache: + def __init__(self, filename, require = True): + self.filename = filename + try: + with open(filename, "r") as f: + self.values = json.load(f) + except FileNotFoundError: + if require: + raise + self.values = dict() + def save(self): + with open(self.filename, "w") as f: + json.dump(self.values, f, indent=4) + print(f"Saved cache to {self.filename}", file=sys.stderr) + +# The email to handle cache maps email addresses to either +# - a handle (string) +# - None (if no handle was found) +email_to_handle_cache = Cache("maintainers/data/release-credits-email-to-handle.json") + +handles = set() +emails = dict() + +for sample in samples: + s = samples[sample] + email = s["email"] + if not email in email_to_handle_cache.values: + print(f"Querying GitHub API for {s['hash']}, to get handle for {s['email']}") + ghc = get_github_commit(samples[sample]) + gha = ghc["author"] + if gha and gha["login"]: + handle = gha["login"] + print(f"Handle: {handle}") + email_to_handle_cache.values[email] = handle + else: + print(f"Found no handle for {s['email']}") + email_to_handle_cache.values[email] = None + handle = email_to_handle_cache.values[email] + if handle is not None: + handles.add(handle) + else: + emails[email] = s["author"] + +# print(email_to_handle_cache.values) + +email_to_handle_cache.save() + +handle_to_name_cache = Cache("maintainers/data/release-credits-handle-to-name.json") + +print(f"Found {len(handles)} handles", file=sys.stderr) + +for handle in handles: + if not handle in handle_to_name_cache.values: + print(f"Querying GitHub API for {handle}, to get name", file=sys.stderr) + url = f"https://api.github.com/users/{handle}" + headers = {'Authorization': f'token {github_token}'} + response = requests.get(url, headers=headers) + response.raise_for_status() + user = response.json() + name = user["name"] + print(f"Name: {name}", file=sys.stderr) + handle_to_name_cache.values[handle] = name + +handle_to_name_cache.save() + +entries = list() + +def is_bot(handle): + return handle.endswith("[bot]") + +for handle in handles: + name = handle_to_name_cache.values[handle] + if name is None: + # This way it looks more regular + name = handle + + # Note to bots: I'm only doing this because it would distract humans, and + # therefore delay your creation. I'm sorry; it's for your benefit. + if not is_bot(handle): + entries += [ f"- {name} [**(@{handle})**](https://github.com/{handle})" ] + +def shuffle(entries): + salt = os.urandom(16) + return sorted(entries, key=lambda x: hash((x, salt))) + +# Fair ordering is undecidable +entries = shuffle(entries) + +# For a sanity check, we could sort the entries by handle instead. +# entries = sorted(entries) + +print("") +print(f"This release was made possible by the following {len(entries)} contributors:") +print("") + +for entry in entries: + print(entry) + +for email in emails: + print(f"- {emails[email]}") diff --git a/maintainers/release-notes b/maintainers/release-notes index 2d84485c1..c0c4ee734 100755 --- a/maintainers/release-notes +++ b/maintainers/release-notes @@ -1,5 +1,6 @@ #!/usr/bin/env nix -#!nix shell .#changelog-d-nix --command bash +# vim: set filetype=bash: +#!nix shell .#changelog-d --command bash # --- CONFIGURATION --- @@ -151,6 +152,13 @@ section_title="Release $version_full ($DATE)" echo "# $section_title" echo changelog-d doc/manual/rl-next | sed -e 's/ *$//' + + if ! $IS_PATCH; then + echo + echo "# Contributors" + echo + VERSION=$version_full ./maintainers/release-credits + fi ) | tee -a $file log "Wrote $file" diff --git a/maintainers/release-process.md b/maintainers/release-process.md index da6886ea9..7a2b3c0a7 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -39,6 +39,10 @@ release: * Proof-read / edit / rearrange the release notes if needed. Breaking changes and highlights should go to the top. +* Run `maintainers/release-credits` to make sure the credits script works + and produces a sensible output. Some emails might not automatically map to + a GitHub handle. + * Push. ```console diff --git a/maintainers/upload-release.pl b/maintainers/upload-release.pl index 9e73524a6..8a470c7cc 100755 --- a/maintainers/upload-release.pl +++ b/maintainers/upload-release.pl @@ -42,7 +42,7 @@ my $flakeUrl = $evalInfo->{flake}; my $flakeInfo = decode_json(`nix flake metadata --json "$flakeUrl"` or die) if $flakeUrl; my $nixRev = ($flakeInfo ? $flakeInfo->{revision} : $evalInfo->{jobsetevalinputs}->{nix}->{revision}) or die; -my $buildInfo = decode_json(fetch("$evalUrl/job/build.x86_64-linux", 'application/json')); +my $buildInfo = decode_json(fetch("$evalUrl/job/build.nix.x86_64-linux", 'application/json')); #print Dumper($buildInfo); my $releaseName = $buildInfo->{nixname}; @@ -91,7 +91,7 @@ sub getStorePath { sub copyManual { my $manual; eval { - $manual = getStorePath("build.x86_64-linux", "doc"); + $manual = getStorePath("build.nix.x86_64-linux", "doc"); }; if ($@) { warn "$@"; @@ -112,7 +112,7 @@ sub copyManual { system("xz -d < '$manualNar' | nix-store --restore $tmpDir/manual.tmp") == 0 or die "unable to unpack $manualNar\n"; rename("$tmpDir/manual.tmp/share/doc/nix/manual", "$tmpDir/manual") or die; - system("rm -rf '$tmpDir/manual.tmp'") == 0 or die; + File::Path::remove_tree("$tmpDir/manual.tmp", {safe => 1}); } system("aws s3 sync '$tmpDir/manual' s3://$releasesBucketName/$releaseDir/manual") == 0 @@ -240,11 +240,12 @@ if ($haveDocker) { # Upload nix-fallback-paths.nix. write_file("$tmpDir/fallback-paths.nix", "{\n" . - " x86_64-linux = \"" . getStorePath("build.x86_64-linux") . "\";\n" . - " i686-linux = \"" . getStorePath("build.i686-linux") . "\";\n" . - " aarch64-linux = \"" . getStorePath("build.aarch64-linux") . "\";\n" . - " x86_64-darwin = \"" . getStorePath("build.x86_64-darwin") . "\";\n" . - " aarch64-darwin = \"" . getStorePath("build.aarch64-darwin") . "\";\n" . + " x86_64-linux = \"" . getStorePath("build.nix.x86_64-linux") . "\";\n" . + " i686-linux = \"" . getStorePath("build.nix.i686-linux") . "\";\n" . + " aarch64-linux = \"" . getStorePath("build.nix.aarch64-linux") . "\";\n" . + " riscv64-linux = \"" . getStorePath("buildCross.nix.riscv64-unknown-linux-gnu.x86_64-linux") . "\";\n" . + " x86_64-darwin = \"" . getStorePath("build.nix.x86_64-darwin") . "\";\n" . + " aarch64-darwin = \"" . getStorePath("build.nix.aarch64-darwin") . "\";\n" . "}\n"); # Upload release files to S3. @@ -280,3 +281,6 @@ system("git remote update origin") == 0 or die; system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die; system("git push --tags") == 0 or die; system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die if $isLatest; + +File::Path::remove_tree($narCache, {safe => 1}); +File::Path::remove_tree($tmpDir, {safe => 1}); diff --git a/meson.build b/meson.build new file mode 100644 index 000000000..636d38b08 --- /dev/null +++ b/meson.build @@ -0,0 +1,50 @@ +# This is just a stub project to include all the others as subprojects +# for development shell purposes + +project('nix-dev-shell', 'cpp', + version : files('.version'), + default_options : [ + 'localstatedir=/nix/var', + ] +) + +# Internal Libraries +subproject('libutil') +subproject('libstore') +subproject('libfetchers') +subproject('libexpr') +subproject('libflake') +subproject('libmain') +subproject('libcmd') + +# Executables +subproject('nix') + +# Docs +subproject('internal-api-docs') +subproject('external-api-docs') +if not meson.is_cross_build() + subproject('nix-manual') +endif + +# External C wrapper libraries +subproject('libutil-c') +subproject('libstore-c') +subproject('libexpr-c') +subproject('libmain-c') + +# Language Bindings +if not meson.is_cross_build() + subproject('perl') +endif + +# Testing +subproject('nix-util-test-support') +subproject('nix-util-tests') +subproject('nix-store-test-support') +subproject('nix-store-tests') +subproject('nix-fetchers-tests') +subproject('nix-expr-test-support') +subproject('nix-expr-tests') +subproject('nix-flake-tests') +subproject('nix-functional-tests') diff --git a/misc/bash/completion.sh b/misc/bash/completion.sh index 9af695f5a..c4ba96cd3 100644 --- a/misc/bash/completion.sh +++ b/misc/bash/completion.sh @@ -12,9 +12,16 @@ function _complete_nix { elif [[ $completion == attrs ]]; then compopt -o nospace fi - else - COMPREPLY+=("$completion") + continue fi + + if [[ "${cur}" =~ "=" ]]; then + # drop everything up to the first =. if a = is included, bash assumes this to be + # an arg=value argument and the completion gets mangled (see #11208) + completion="${completion#*=}" + fi + + COMPREPLY+=("${completion}") done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}" 2>/dev/null) __ltrim_colon_completions "$cur" } diff --git a/misc/bash/meson.build b/misc/bash/meson.build new file mode 100644 index 000000000..8a97a02cb --- /dev/null +++ b/misc/bash/meson.build @@ -0,0 +1,8 @@ +configure_file( + input : 'completion.sh', + output : 'nix', + install : true, + install_dir : get_option('datadir') / 'bash-completion' / 'completions', + install_mode : 'rw-r--r--', + copy : true, +) diff --git a/misc/changelog-d.cabal.nix b/misc/changelog-d.cabal.nix deleted file mode 100644 index 76f9353cd..000000000 --- a/misc/changelog-d.cabal.nix +++ /dev/null @@ -1,31 +0,0 @@ -{ mkDerivation, aeson, base, bytestring, cabal-install-parsers -, Cabal-syntax, containers, directory, filepath, frontmatter -, generic-lens-lite, lib, mtl, optparse-applicative, parsec, pretty -, regex-applicative, text, pkgs -}: -let rev = "f30f6969e9cd8b56242309639d58acea21c99d06"; -in -mkDerivation { - pname = "changelog-d"; - version = "0.1"; - src = pkgs.fetchurl { - name = "changelog-d-${rev}.tar.gz"; - url = "https://codeberg.org/roberth/changelog-d/archive/${rev}.tar.gz"; - hash = "sha256-8a2+i5u7YoszAgd5OIEW0eYUcP8yfhtoOIhLJkylYJ4="; - } // { inherit rev; }; - isLibrary = false; - isExecutable = true; - libraryHaskellDepends = [ - aeson base bytestring cabal-install-parsers Cabal-syntax containers - directory filepath frontmatter generic-lens-lite mtl parsec pretty - regex-applicative text - ]; - executableHaskellDepends = [ - base bytestring Cabal-syntax directory filepath - optparse-applicative - ]; - doHaddock = false; - description = "Concatenate changelog entries into a single one"; - license = lib.licenses.gpl3Plus; - mainProgram = "changelog-d"; -} diff --git a/misc/changelog-d.nix b/misc/changelog-d.nix deleted file mode 100644 index 1b20f4596..000000000 --- a/misc/changelog-d.nix +++ /dev/null @@ -1,31 +0,0 @@ -# Taken temporarily from -{ - callPackage, - lib, - haskell, - haskellPackages, -}: - -let - hsPkg = haskellPackages.callPackage ./changelog-d.cabal.nix { }; - - addCompletions = haskellPackages.generateOptparseApplicativeCompletions ["changelog-d"]; - - haskellModifications = - lib.flip lib.pipe [ - addCompletions - haskell.lib.justStaticExecutables - ]; - - mkDerivationOverrides = finalAttrs: oldAttrs: { - - version = oldAttrs.version + "-git-${lib.strings.substring 0 7 oldAttrs.src.rev}"; - - meta = oldAttrs.meta // { - homepage = "https://codeberg.org/roberth/changelog-d"; - maintainers = [ lib.maintainers.roberth ]; - }; - - }; -in - (haskellModifications hsPkg).overrideAttrs mkDerivationOverrides diff --git a/misc/fish/meson.build b/misc/fish/meson.build new file mode 100644 index 000000000..e7e89b438 --- /dev/null +++ b/misc/fish/meson.build @@ -0,0 +1,8 @@ +configure_file( + input : 'completion.fish', + output : 'nix.fish', + install : true, + install_dir : get_option('datadir') / 'fish' / 'vendor_completions.d', + install_mode : 'rw-r--r--', + copy : true, +) diff --git a/misc/launchd/org.nixos.nix-daemon.plist.in b/misc/launchd/org.nixos.nix-daemon.plist.in index e1470cf99..664608305 100644 --- a/misc/launchd/org.nixos.nix-daemon.plist.in +++ b/misc/launchd/org.nixos.nix-daemon.plist.in @@ -2,11 +2,6 @@ - EnvironmentVariables - - OBJC_DISABLE_INITIALIZE_FORK_SAFETY - YES - Label org.nixos.nix-daemon KeepAlive diff --git a/misc/meson.build b/misc/meson.build new file mode 100644 index 000000000..a6d1f944b --- /dev/null +++ b/misc/meson.build @@ -0,0 +1,5 @@ +subdir('bash') +subdir('fish') +subdir('zsh') + +subdir('systemd') diff --git a/misc/systemd/meson.build b/misc/systemd/meson.build new file mode 100644 index 000000000..6ccb6a873 --- /dev/null +++ b/misc/systemd/meson.build @@ -0,0 +1,25 @@ +foreach config : [ 'nix-daemon.socket', 'nix-daemon.service' ] + configure_file( + input : config + '.in', + output : config, + install : true, + install_dir : get_option('prefix') / 'lib/systemd/system', + install_mode : 'rw-r--r--', + configuration : { + 'storedir' : store_dir, + 'localstatedir' : localstatedir, + 'bindir' : bindir, + }, + ) +endforeach + +configure_file( + input : 'nix-daemon.conf.in', + output : 'nix-daemon.conf', + install : true, + install_dir : get_option('prefix') / 'lib/tmpfiles.d', + install_mode : 'rw-r--r--', + configuration : { + 'localstatedir' : localstatedir, + }, +) diff --git a/misc/systemd/nix-daemon.service.in b/misc/systemd/nix-daemon.service.in index 45fbea02c..b3055cfe2 100644 --- a/misc/systemd/nix-daemon.service.in +++ b/misc/systemd/nix-daemon.service.in @@ -11,6 +11,7 @@ ExecStart=@@bindir@/nix-daemon nix-daemon --daemon KillMode=process LimitNOFILE=1048576 TasksMax=1048576 +Delegate=yes [Install] WantedBy=multi-user.target diff --git a/misc/systemv/nix-daemon b/misc/systemv/nix-daemon index fea537167..e8326f947 100755 --- a/misc/systemv/nix-daemon +++ b/misc/systemv/nix-daemon @@ -34,6 +34,7 @@ else fi # Source function library. +# shellcheck source=/dev/null . /etc/init.d/functions LOCKFILE=/var/lock/subsys/nix-daemon @@ -41,14 +42,20 @@ RUNDIR=/var/run/nix PIDFILE=${RUNDIR}/nix-daemon.pid RETVAL=0 -base=${0##*/} +# https://www.shellcheck.net/wiki/SC3004 +# Check if gettext exists +if ! type gettext > /dev/null 2>&1 +then + # If not, create a dummy function that returns the input verbatim + gettext() { printf '%s' "$1"; } +fi start() { mkdir -p ${RUNDIR} chown ${NIX_DAEMON_USER}:${NIX_DAEMON_USER} ${RUNDIR} - echo -n $"Starting nix daemon... " + printf '%s' "$(gettext 'Starting nix daemon... ')" daemonize -u $NIX_DAEMON_USER -p ${PIDFILE} $NIX_DAEMON_BIN $NIX_DAEMON_OPTS RETVAL=$? @@ -58,7 +65,7 @@ start() { } stop() { - echo -n $"Shutting down nix daemon: " + printf '%s' "$(gettext 'Shutting down nix daemon: ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN RETVAL=$? [ $RETVAL -eq 0 ] && rm -f ${LOCKFILE} ${PIDFILE} @@ -67,7 +74,7 @@ stop() { } reload() { - echo -n $"Reloading nix daemon... " + printf '%s' "$(gettext 'Reloading nix daemon... ')" killproc -p ${PIDFILE} $NIX_DAEMON_BIN -HUP RETVAL=$? echo @@ -105,7 +112,7 @@ case "$1" in fi ;; *) - echo $"Usage: $0 {start|stop|status|restart|condrestart}" + printf '%s' "$(gettext "Usage: $0 {start|stop|status|restart|condrestart}")" exit 2 ;; esac diff --git a/misc/zsh/meson.build b/misc/zsh/meson.build new file mode 100644 index 000000000..f3d0426e7 --- /dev/null +++ b/misc/zsh/meson.build @@ -0,0 +1,10 @@ +foreach script : [ [ 'completion.zsh', '_nix' ], [ 'run-help-nix' ] ] + configure_file( + input : script[0], + output : script.get(1, script[0]), + install : true, + install_dir : get_option('datadir') / 'zsh/site-functions', + install_mode : 'rw-r--r--', + copy : true, + ) +endforeach diff --git a/mk/common-test.sh b/mk/common-test.sh index 2abea7887..817422c40 100644 --- a/mk/common-test.sh +++ b/mk/common-test.sh @@ -1,19 +1,23 @@ +# shellcheck shell=bash + # Remove overall test dir (at most one of the two should match) and # remove file extension. -test_name=$(echo -n "$test" | sed \ - -e "s|^tests/unit/[^/]*/data/||" \ + +test_name=$(echo -n "${test?must be defined by caller (test runner)}" | sed \ + -e "s|^src/[^/]*-test/data/||" \ -e "s|^tests/functional/||" \ -e "s|\.sh$||" \ ) +# shellcheck disable=SC2016 TESTS_ENVIRONMENT=( "TEST_NAME=$test_name" 'NIX_REMOTE=' 'PS4=+(${BASH_SOURCE[0]-$0}:$LINENO) ' ) -: ${BASH:=/usr/bin/env bash} +read -r -a bash <<< "${BASH:-/usr/bin/env bash}" run () { - cd "$(dirname $1)" && env "${TESTS_ENVIRONMENT[@]}" $BASH -x -e -u -o pipefail $(basename $1) + cd "$(dirname "$1")" && env "${TESTS_ENVIRONMENT[@]}" "${bash[@]}" -x -e -u -o pipefail "$(basename "$1")" } diff --git a/mk/platform.mk b/mk/platform.mk index fe960dedf..22c114a20 100644 --- a/mk/platform.mk +++ b/mk/platform.mk @@ -29,4 +29,8 @@ ifdef HOST_OS HOST_SOLARIS = 1 HOST_UNIX = 1 endif + ifeq ($(HOST_KERNEL), gnu) + HOST_HURD = 1 + HOST_UNIX = 1 + endif endif diff --git a/mk/precompiled-headers.mk b/mk/precompiled-headers.mk index cdd3daecd..f2803eb79 100644 --- a/mk/precompiled-headers.mk +++ b/mk/precompiled-headers.mk @@ -8,7 +8,7 @@ GCH = $(buildprefix)precompiled-headers.h.gch $(GCH): precompiled-headers.h @rm -f $@ @mkdir -p "$(dir $@)" - $(trace-gen) $(CXX) -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) + $(trace-gen) $(CXX) -c -x c++-header -o $@ $< $(GLOBAL_CXXFLAGS) $(GCH_CXXFLAGS) clean-files += $(GCH) diff --git a/mk/run-test.sh b/mk/run-test.sh index 1256bfcf7..7f9f1d5f8 100755 --- a/mk/run-test.sh +++ b/mk/run-test.sh @@ -26,12 +26,13 @@ run_test () { run_test -if [ $status -eq 0 ]; then +if [[ "$status" = 0 ]]; then echo "$post_run_msg [${green}PASS$normal]" -elif [ $status -eq 99 ]; then +elif [[ "$status" = 77 ]]; then echo "$post_run_msg [${yellow}SKIP$normal]" else echo "$post_run_msg [${red}FAIL$normal]" + # shellcheck disable=SC2001 echo "$log" | sed 's/^/ /' exit "$status" fi diff --git a/package.nix b/package.nix index cf1654c6a..8ab184667 100644 --- a/package.nix +++ b/package.nix @@ -1,12 +1,10 @@ { lib -, fetchurl , stdenv , releaseTools , autoconf-archive , autoreconfHook , aws-sdk-cpp , boehmgc -, buildPackages , nlohmann_json , bison , boost @@ -15,12 +13,10 @@ , curl , editline , readline -, fileset , flex , git , gtest , jq -, doxygen , libarchive , libcpuid , libgit2 @@ -36,7 +32,8 @@ , pkg-config , rapidcheck , sqlite -, util-linux +, toml11 +, unixtools , xz , busybox-sandbox-shell ? null @@ -50,11 +47,10 @@ , pname ? "nix" -, versionSuffix ? "" -, officialRelease ? false +, version +, versionSuffix -# Whether to build Nix. Useful to skip for tasks like (a) just -# generating API docs or (b) testing existing pre-built versions of Nix +# Whether to build Nix. Useful to skip for tasks like testing existing pre-built versions of Nix , doBuild ? true # Run the unit tests as part of the build. See `installUnitTests` for an @@ -64,7 +60,7 @@ # Run the functional tests as part of the build. , doInstallCheck ? test-client != null || __forDefaults.canRunInstalled -# Check test coverage of Nix. Probably want to use with with at least +# Check test coverage of Nix. Probably want to use with at least # one of `doCHeck` or `doInstallCheck` enabled. , withCoverageChecks ? false @@ -93,11 +89,6 @@ # - readline , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" -# Whether to build the internal/external API docs, can be done separately from -# everything else. -, enableInternalAPIDocs ? forDevShell -, enableExternalAPIDocs ? forDevShell - # Whether to install unit tests. This is useful when cross compiling # since we cannot run them natively during the build, but can do so # later. @@ -120,7 +111,7 @@ }: let - version = lib.fileContents ./.version + versionSuffix; + inherit (lib) fileset; # selected attributes with defaults, will be used to define some # things which should instead be gotten via `finalAttrs` in order to @@ -180,20 +171,11 @@ in { ./doc ./misc ./precompiled-headers.h - ./src + (fileset.difference ./src ./src/perl) ./COPYING ./scripts/local.mk - ] ++ lib.optionals buildUnitTests [ + ] ++ lib.optionals enableManual [ ./doc/manual - ] ++ lib.optionals enableInternalAPIDocs [ - ./doc/internal-api - ] ++ lib.optionals enableExternalAPIDocs [ - ./doc/external-api - ] ++ lib.optionals (enableInternalAPIDocs || enableExternalAPIDocs) [ - # Source might not be compiled, but still must be available - # for Doxygen to gather comments. - ./src - ./tests/unit ] ++ lib.optionals buildUnitTests [ ./tests/unit ] ++ lib.optionals doInstallCheck [ @@ -207,8 +189,10 @@ in { ++ lib.optional doBuild "dev" # If we are doing just build or just docs, the one thing will use # "out". We only need additional outputs if we are doing both. - ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs || enableExternalAPIDocs)) "doc" - ++ lib.optional installUnitTests "check"; + ++ lib.optional (doBuild && enableManual) "doc" + ++ lib.optional installUnitTests "check" + ++ lib.optional doCheck "testresults" + ; nativeBuildInputs = [ autoconf-archive @@ -228,12 +212,11 @@ in { man # for testing `nix-* --help` ] ++ lib.optionals (doInstallCheck || enableManual) [ jq # Also for custom mdBook preprocessor. - ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux - ++ lib.optional (enableInternalAPIDocs || enableExternalAPIDocs) doxygen + ] ++ lib.optional stdenv.hostPlatform.isStatic unixtools.hexdump ; - buildInputs = lib.optionals doBuild [ - boost + buildInputs = lib.optionals doBuild ( + [ brotli bzip2 curl @@ -242,6 +225,7 @@ in { libsodium openssl sqlite + toml11 xz ({ inherit readline editline; }.${readlineFlavor}) ] ++ lib.optionals enableMarkdown [ @@ -253,47 +237,22 @@ in { ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid # There have been issues building these dependencies ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) - (aws-sdk-cpp.override { - apis = ["s3" "transfer"]; - customMemoryManagement = false; - }) - ; + aws-sdk-cpp + ); - propagatedBuildInputs = [ + propagatedBuildInputs = lib.optionals doBuild ([ + boost nlohmann_json - ] ++ lib.optional enableGC boehmgc; + ] ++ lib.optional enableGC boehmgc + ); dontBuild = !attrs.doBuild; doCheck = attrs.doCheck; - disallowedReferences = [ boost ]; - - preConfigure = lib.optionalString (doBuild && ! stdenv.hostPlatform.isStatic) ( - '' - # Copy libboost_context so we don't get all of Boost in our closure. - # https://github.com/NixOS/nixpkgs/issues/45462 - mkdir -p $out/lib - cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib - rm -f $out/lib/*.a - '' + lib.optionalString stdenv.hostPlatform.isLinux '' - chmod u+w $out/lib/*.so.* - patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.* - '' + lib.optionalString stdenv.hostPlatform.isDarwin '' - for LIB in $out/lib/*.dylib; do - chmod u+w $LIB - install_name_tool -id $LIB $LIB - install_name_tool -delete_rpath ${boost}/lib/ $LIB || true - done - install_name_tool -change ${boost}/lib/libboost_system.dylib $out/lib/libboost_system.dylib $out/lib/libboost_thread.dylib - '' - ); - configureFlags = [ (lib.enableFeature doBuild "build") (lib.enableFeature buildUnitTests "unit-tests") (lib.enableFeature doInstallCheck "functional-tests") - (lib.enableFeature enableInternalAPIDocs "internal-api-docs") - (lib.enableFeature enableExternalAPIDocs "external-api-docs") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature enableGC "gc") (lib.enableFeature enableMarkdown "markdown") @@ -317,9 +276,11 @@ in { makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; - installTargets = lib.optional doBuild "install" - ++ lib.optional enableInternalAPIDocs "internal-api-html" - ++ lib.optional enableExternalAPIDocs "external-api-html"; + preCheck = '' + mkdir $testresults + ''; + + installTargets = lib.optional doBuild "install"; installFlags = "sysconfdir=$(out)/etc"; @@ -334,22 +295,10 @@ in { lib.optionalString stdenv.hostPlatform.isStatic '' mkdir -p $out/nix-support echo "file binary-dist $out/bin/nix" >> $out/nix-support/hydra-build-products - '' + lib.optionalString stdenv.isDarwin '' - install_name_tool \ - -change ${boost}/lib/libboost_context.dylib \ - $out/lib/libboost_context.dylib \ - $out/lib/libnixutil.dylib '' ) + lib.optionalString enableManual '' mkdir -p ''${!outputDoc}/nix-support echo "doc manual ''${!outputDoc}/share/doc/nix/manual" >> ''${!outputDoc}/nix-support/hydra-build-products - '' + lib.optionalString enableInternalAPIDocs '' - mkdir -p ''${!outputDoc}/nix-support - echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products - '' - + lib.optionalString enableExternalAPIDocs '' - mkdir -p ''${!outputDoc}/nix-support - echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products ''; # So the check output gets links for DLLs in the out output. @@ -376,17 +325,11 @@ in { preInstallCheck = lib.optionalString (! doBuild) '' mkdir -p src/nix-channel - '' - # See https://github.com/NixOS/nix/issues/2523 - # Occurs often in tests since https://github.com/NixOS/nix/pull/9900 - + lib.optionalString stdenv.hostPlatform.isDarwin '' - export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES ''; separateDebugInfo = !stdenv.hostPlatform.isStatic; - # TODO `releaseTools.coverageAnalysis` in Nixpkgs needs to be updated - # to work with `strictDeps`. + # TODO Always true after https://github.com/NixOS/nixpkgs/issues/318564 strictDeps = !withCoverageChecks; hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; diff --git a/packaging/components.nix b/packaging/components.nix new file mode 100644 index 000000000..4c18dc6a3 --- /dev/null +++ b/packaging/components.nix @@ -0,0 +1,71 @@ +{ + lib, + src, + officialRelease, +}: + +scope: + +let + inherit (scope) callPackage; + + baseVersion = lib.fileContents ../.version; + + versionSuffix = lib.optionalString (!officialRelease) "pre"; + + fineVersionSuffix = lib.optionalString + (!officialRelease) + "pre${builtins.substring 0 8 (src.lastModifiedDate or src.lastModified or "19700101")}_${src.shortRev or "dirty"}"; + + fineVersion = baseVersion + fineVersionSuffix; +in + +# This becomes the pkgs.nixComponents attribute set +{ + version = baseVersion + versionSuffix; + inherit versionSuffix; + + nix = callPackage ../package.nix { + version = fineVersion; + versionSuffix = fineVersionSuffix; + }; + + nix-util = callPackage ../src/libutil/package.nix { }; + nix-util-c = callPackage ../src/libutil-c/package.nix { }; + nix-util-test-support = callPackage ../tests/unit/libutil-support/package.nix { }; + nix-util-tests = callPackage ../tests/unit/libutil/package.nix { }; + + nix-store = callPackage ../src/libstore/package.nix { }; + nix-store-c = callPackage ../src/libstore-c/package.nix { }; + nix-store-test-support = callPackage ../tests/unit/libstore-support/package.nix { }; + nix-store-tests = callPackage ../tests/unit/libstore/package.nix { }; + + nix-fetchers = callPackage ../src/libfetchers/package.nix { }; + nix-fetchers-tests = callPackage ../tests/unit/libfetchers/package.nix { }; + + nix-expr = callPackage ../src/libexpr/package.nix { }; + nix-expr-c = callPackage ../src/libexpr-c/package.nix { }; + nix-expr-test-support = callPackage ../tests/unit/libexpr-support/package.nix { }; + nix-expr-tests = callPackage ../tests/unit/libexpr/package.nix { }; + + nix-flake = callPackage ../src/libflake/package.nix { }; + nix-flake-tests = callPackage ../tests/unit/libflake/package.nix { }; + + nix-main = callPackage ../src/libmain/package.nix { }; + nix-main-c = callPackage ../src/libmain-c/package.nix { }; + + nix-cmd = callPackage ../src/libcmd/package.nix { }; + + nix-cli = callPackage ../src/nix/package.nix { version = fineVersion; }; + + nix-functional-tests = callPackage ../src/nix-functional-tests/package.nix { version = fineVersion; }; + + nix-manual = callPackage ../doc/manual/package.nix { version = fineVersion; }; + nix-internal-api-docs = callPackage ../src/internal-api-docs/package.nix { version = fineVersion; }; + nix-external-api-docs = callPackage ../src/external-api-docs/package.nix { version = fineVersion; }; + + nix-perl-bindings = callPackage ../src/perl/package.nix { }; + + # Will replace `nix` once the old build system is gone. + nix-ng = callPackage ../packaging/everything.nix { }; +} diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix new file mode 100644 index 000000000..e5f4c0f91 --- /dev/null +++ b/packaging/dependencies.nix @@ -0,0 +1,185 @@ +# These overrides are applied to the dependencies of the Nix components. + +{ + # Flake inputs; used for sources + inputs, + + # The raw Nixpkgs, not affected by this scope + pkgs, + + stdenv, +}: + +let + prevStdenv = stdenv; +in + +let + inherit (pkgs) lib; + + root = ../.; + + stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 + then darwinStdenv + else prevStdenv; + + # Fix the following error with the default x86_64-darwin SDK: + # + # error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer + # + # Despite the use of the 10.13 deployment target here, the aligned + # allocation function Clang uses with this setting actually works + # all the way back to 10.6. + darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; }; + + # Nixpkgs implements this by returning a subpath into the fetched Nix sources. + resolvePath = p: p; + + # Indirection for Nixpkgs to override when package.nix files are vendored + filesetToSource = lib.fileset.toSource; + + localSourceLayer = finalAttrs: prevAttrs: + let + workDirPath = + # Ideally we'd pick finalAttrs.workDir, but for now `mkDerivation` has + # the requirement that everything except passthru and meta must be + # serialized by mkDerivation, which doesn't work for this. + prevAttrs.workDir; + + workDirSubpath = lib.path.removePrefix root workDirPath; + sources = assert prevAttrs.fileset._type == "fileset"; prevAttrs.fileset; + src = lib.fileset.toSource { fileset = sources; inherit root; }; + + in + { + sourceRoot = "${src.name}/" + workDirSubpath; + inherit src; + + # Clear what `derivation` can't/shouldn't serialize; see prevAttrs.workDir. + fileset = null; + workDir = null; + }; + + # Work around weird `--as-needed` linker behavior with BSD, see + # https://github.com/mesonbuild/meson/issues/3593 + bsdNoLinkAsNeeded = finalAttrs: prevAttrs: + lib.optionalAttrs stdenv.hostPlatform.isBSD { + mesonFlags = [ (lib.mesonBool "b_asneeded" false) ] ++ prevAttrs.mesonFlags or []; + }; + + miscGoodPractice = finalAttrs: prevAttrs: + { + strictDeps = prevAttrs.strictDeps or true; + enableParallelBuilding = true; + }; +in +scope: { + inherit stdenv; + + aws-sdk-cpp = (pkgs.aws-sdk-cpp.override { + apis = [ "s3" "transfer" ]; + customMemoryManagement = false; + }).overrideAttrs { + # only a stripped down version is built, which takes a lot less resources + # to build, so we don't need a "big-parallel" machine. + requiredSystemFeatures = [ ]; + }; + + libseccomp = pkgs.libseccomp.overrideAttrs (_: rec { + version = "2.5.5"; + src = pkgs.fetchurl { + url = "https://github.com/seccomp/libseccomp/releases/download/v${version}/libseccomp-${version}.tar.gz"; + hash = "sha256-JIosik2bmFiqa69ScSw0r+/PnJ6Ut23OAsHJqiX7M3U="; + }; + }); + + boehmgc = pkgs.boehmgc.override { + enableLargeConfig = true; + }; + + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. + boost = (pkgs.boost.override { + extraB2Args = [ + "--with-container" + "--with-context" + "--with-coroutine" + ]; + }).overrideAttrs (old: { + # Need to remove `--with-*` to use `--with-libraries=...` + buildPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.buildPhase; + installPhase = lib.replaceStrings [ "--without-python" ] [ "" ] old.installPhase; + }); + + libgit2 = pkgs.libgit2.overrideAttrs (attrs: { + src = inputs.libgit2; + version = inputs.libgit2.lastModifiedDate; + cmakeFlags = attrs.cmakeFlags or [] + ++ [ "-DUSE_SSH=exec" ]; + nativeBuildInputs = attrs.nativeBuildInputs or [] + # gitMinimal does not build on Windows. See packbuilder patch. + ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ + # Needed for `git apply`; see `prePatch` + pkgs.buildPackages.gitMinimal + ]; + # Only `git apply` can handle git binary patches + prePatch = attrs.prePatch or "" + + lib.optionalString (!stdenv.hostPlatform.isWindows) '' + patch() { + git apply + } + ''; + patches = attrs.patches or [] + ++ [ + ./patches/libgit2-mempack-thin-packfile.patch + ] + # gitMinimal does not build on Windows, but fortunately this patch only + # impacts interruptibility + ++ lib.optionals (!stdenv.hostPlatform.isWindows) [ + # binary patch; see `prePatch` + ./patches/libgit2-packbuilder-callback-interruptible.patch + ]; + }); + + busybox-sandbox-shell = pkgs.busybox-sandbox-shell or (pkgs.busybox.override { + useMusl = true; + enableStatic = true; + enableMinimal = true; + extraConfig = '' + CONFIG_FEATURE_FANCY_ECHO y + CONFIG_FEATURE_SH_MATH y + CONFIG_FEATURE_SH_MATH_64 y + + CONFIG_ASH y + CONFIG_ASH_OPTIMIZE_FOR_SIZE y + + CONFIG_ASH_ALIAS y + CONFIG_ASH_BASH_COMPAT y + CONFIG_ASH_CMDCMD y + CONFIG_ASH_ECHO y + CONFIG_ASH_GETOPTS y + CONFIG_ASH_INTERNAL_GLOB y + CONFIG_ASH_JOB_CONTROL y + CONFIG_ASH_PRINTF y + CONFIG_ASH_TEST y + ''; + }); + + # TODO change in Nixpkgs, Windows works fine. First commit of + # https://github.com/NixOS/nixpkgs/pull/322977 backported will fix. + toml11 = pkgs.toml11.overrideAttrs (old: { + meta.platforms = lib.platforms.all; + }); + + inherit resolvePath filesetToSource; + + mkMesonDerivation = f: let + exts = [ + miscGoodPractice + bsdNoLinkAsNeeded + localSourceLayer + ]; + in stdenv.mkDerivation + (lib.extends + (lib.foldr lib.composeExtensions (_: _: {}) exts) + f); +} diff --git a/packaging/everything.nix b/packaging/everything.nix new file mode 100644 index 000000000..ae2f93da0 --- /dev/null +++ b/packaging/everything.nix @@ -0,0 +1,130 @@ +{ + lib, + stdenv, + buildEnv, + + nix-util, + nix-util-c, + nix-util-test-support, + nix-util-tests, + + nix-store, + nix-store-c, + nix-store-test-support, + nix-store-tests, + + nix-fetchers, + nix-fetchers-tests, + + nix-expr, + nix-expr-c, + nix-expr-test-support, + nix-expr-tests, + + nix-flake, + nix-flake-tests, + + nix-main, + nix-main-c, + + nix-cmd, + + nix-cli, + + nix-functional-tests, + + nix-manual, + nix-internal-api-docs, + nix-external-api-docs, + + nix-perl-bindings, +}: + +(buildEnv { + name = "nix-${nix-cli.version}"; + paths = [ + nix-util + nix-util-c + nix-util-test-support + nix-util-tests + + nix-store + nix-store-c + nix-store-test-support + nix-store-tests + + nix-fetchers + nix-fetchers-tests + + nix-expr + nix-expr-c + nix-expr-test-support + nix-expr-tests + + nix-flake + nix-flake-tests + + nix-main + nix-main-c + + nix-cmd + + nix-cli + + nix-manual + nix-internal-api-docs + nix-external-api-docs + + ] ++ lib.optionals (stdenv.buildPlatform.canExecute stdenv.hostPlatform) [ + nix-perl-bindings + ]; + + meta.mainProgram = "nix"; +}).overrideAttrs (finalAttrs: prevAttrs: { + doCheck = true; + doInstallCheck = true; + + checkInputs = [ + # Actually run the unit tests too + nix-util-tests.tests.run + nix-store-tests.tests.run + nix-expr-tests.tests.run + nix-flake-tests.tests.run + ]; + installCheckInputs = [ + nix-functional-tests + ]; + passthru = prevAttrs.passthru // { + /** + These are the libraries that are part of the Nix project. They are used + by the Nix CLI and other tools. + + If you need to use these libraries in your project, we recommend to use + the `-c` C API libraries exclusively, if possible. + + We also recommend that you build the complete package to ensure that the unit tests pass. + You could do this in CI, or by passing it in an unused environment variable. e.g in a `mkDerivation` call: + + ```nix + buildInputs = [ nix.libs.nix-util-c nix.libs.nix-store-c ]; + # Make sure the nix libs we use are ok + unusedInputsForTests = [ nix ]; + disallowedReferences = nix.all; + ``` + */ + libs = { + inherit + nix-util + nix-util-c + nix-store + nix-store-c + nix-fetchers + nix-expr + nix-expr-c + nix-flake + nix-main + nix-main-c + ; + }; + }; +}) diff --git a/packaging/hydra.nix b/packaging/hydra.nix new file mode 100644 index 000000000..cba1b2583 --- /dev/null +++ b/packaging/hydra.nix @@ -0,0 +1,214 @@ +{ inputs +, binaryTarball +, forAllCrossSystems +, forAllSystems +, lib +, linux64BitSystems +, nixpkgsFor +, self +, officialRelease +}: +let + inherit (inputs) nixpkgs nixpkgs-regression; + + installScriptFor = tarballs: + nixpkgsFor.x86_64-linux.native.callPackage ../scripts/installer.nix { + inherit tarballs; + }; + + testNixVersions = pkgs: client: daemon: + pkgs.nixComponents.callPackage ../package.nix { + pname = + "nix-tests" + + lib.optionalString + (lib.versionAtLeast daemon.version "2.4pre20211005" && + lib.versionAtLeast client.version "2.4pre20211005") + "-${client.version}-against-${daemon.version}"; + + test-client = client; + test-daemon = daemon; + + doBuild = false; + + # This could be more accurate, but a shorter version will match the + # fine version with rev. This functionality is already covered in + # the normal test, so it's fine. + version = pkgs.nixComponents.version; + versionSuffix = pkgs.nixComponents.versionSuffix; + }; + + # Technically we could just return `pkgs.nixComponents`, but for Hydra it's + # convention to transpose it, and to transpose it efficiently, we need to + # enumerate them manually, so that we don't evaluate unnecessary package sets. + forAllPackages = lib.genAttrs [ + "nix" + "nix-util" + "nix-util-c" + "nix-util-test-support" + "nix-util-tests" + "nix-store" + "nix-store-c" + "nix-store-test-support" + "nix-store-tests" + "nix-fetchers" + "nix-fetchers-tests" + "nix-expr" + "nix-expr-c" + "nix-expr-test-support" + "nix-expr-tests" + "nix-flake" + "nix-flake-tests" + "nix-main" + "nix-main-c" + "nix-cmd" + "nix-cli" + "nix-functional-tests" + "nix-ng" + ]; +in +{ + # Binary package for various platforms. + build = forAllPackages (pkgName: + forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.${pkgName})); + + shellInputs = forAllSystems (system: self.devShells.${system}.default.inputDerivation); + + buildStatic = forAllPackages (pkgName: + lib.genAttrs linux64BitSystems (system: nixpkgsFor.${system}.static.nixComponents.${pkgName})); + + buildCross = forAllPackages (pkgName: + # Hack to avoid non-evaling package + (if pkgName == "nix-functional-tests" then lib.flip builtins.removeAttrs ["x86_64-w64-mingw32"] else lib.id) + (forAllCrossSystems (crossSystem: + lib.genAttrs [ "x86_64-linux" ] (system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents.${pkgName})))); + + buildNoGc = forAllSystems (system: + self.packages.${system}.nix.override { enableGC = false; } + ); + + buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-cli); + + # Toggles some settings for better coverage. Windows needs these + # library combinations, and Debian build Nix with GNU readline too. + buildReadlineNoMarkdown = forAllSystems (system: + self.packages.${system}.nix.override { + enableMarkdown = false; + readlineFlavor = "readline"; + } + ); + + # Perl bindings for various platforms. + perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents.nix-perl-bindings); + + # Binary tarball for various platforms, containing a Nix store + # with the closure of 'nix' package, and the second half of + # the installation script. + binaryTarball = forAllSystems (system: binaryTarball nixpkgsFor.${system}.native.nix nixpkgsFor.${system}.native); + + binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] (system: + forAllCrossSystems (crossSystem: + binaryTarball + nixpkgsFor.${system}.cross.${crossSystem}.nix + nixpkgsFor.${system}.cross.${crossSystem})); + + # The first half of the installation script. This is uploaded + # to https://nixos.org/nix/install. It downloads the binary + # tarball for the user's system and calls the second half of the + # installation script. + installerScript = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."i686-linux" + self.hydraJobs.binaryTarball."aarch64-linux" + self.hydraJobs.binaryTarball."x86_64-darwin" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + installerScriptForGHA = installScriptFor [ + # Native + self.hydraJobs.binaryTarball."x86_64-linux" + self.hydraJobs.binaryTarball."aarch64-darwin" + # Cross + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" + self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" + ]; + + # docker image with Nix inside + dockerImage = lib.genAttrs linux64BitSystems (system: self.packages.${system}.dockerImage); + + # Line coverage analysis. + coverage = nixpkgsFor.x86_64-linux.native.nix.override { + pname = "nix-coverage"; + withCoverageChecks = true; + }; + + # Nix's manual + manual = nixpkgsFor.x86_64-linux.native.nixComponents.nix-manual; + + # API docs for Nix's unstable internal C++ interfaces. + internal-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-internal-api-docs; + + # API docs for Nix's C bindings. + external-api-docs = nixpkgsFor.x86_64-linux.native.nixComponents.nix-external-api-docs; + + # System tests. + tests = import ../tests/nixos { inherit lib nixpkgs nixpkgsFor self; } // { + + # Make sure that nix-env still produces the exact same result + # on a particular version of Nixpkgs. + evalNixpkgs = + let + inherit (nixpkgsFor.x86_64-linux.native) runCommand nix; + in + runCommand "eval-nixos" { buildInputs = [ nix ]; } + '' + type -p nix-env + # Note: we're filtering out nixos-install-tools because https://github.com/NixOS/nixpkgs/pull/153594#issuecomment-1020530593. + ( + set -x + time nix-env --store dummy:// -f ${nixpkgs-regression} -qaP --drv-path | sort | grep -v nixos-install-tools > packages + [[ $(sha1sum < packages | cut -c1-40) = e01b031fc9785a572a38be6bc473957e3b6faad7 ]] + ) + mkdir $out + ''; + + nixpkgsLibTests = + forAllSystems (system: + import (nixpkgs + "/lib/tests/test-with-nix.nix") + { + lib = nixpkgsFor.${system}.native.lib; + nix = self.packages.${system}.nix; + pkgs = nixpkgsFor.${system}.native; + } + ); + }; + + metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { + pkgs = nixpkgsFor.x86_64-linux.native; + nixpkgs = nixpkgs-regression; + }; + + installTests = forAllSystems (system: + let pkgs = nixpkgsFor.${system}.native; in + pkgs.runCommand "install-tests" + { + againstSelf = testNixVersions pkgs pkgs.nix pkgs.pkgs.nix; + againstCurrentLatest = + # FIXME: temporarily disable this on macOS because of #3605. + if system == "x86_64-linux" + then testNixVersions pkgs pkgs.nix pkgs.nixVersions.latest + else null; + # Disabled because the latest stable version doesn't handle + # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work + # againstLatestStable = testNixVersions pkgs pkgs.nix pkgs.nixStable; + } "touch $out"); + + installerTests = import ../tests/installer { + binaryTarballs = self.hydraJobs.binaryTarball; + inherit nixpkgsFor; + }; +} diff --git a/packaging/patches/libgit2-mempack-thin-packfile.patch b/packaging/patches/libgit2-mempack-thin-packfile.patch new file mode 100644 index 000000000..fb74b1683 --- /dev/null +++ b/packaging/patches/libgit2-mempack-thin-packfile.patch @@ -0,0 +1,282 @@ +commit 9bacade4a3ef4b6b26e2c02f549eef0e9eb9eaa2 +Author: Robert Hensing +Date: Sun Aug 18 20:20:36 2024 +0200 + + Add unoptimized git_mempack_write_thin_pack + +diff --git a/include/git2/sys/mempack.h b/include/git2/sys/mempack.h +index 17da590a3..3688bdd50 100644 +--- a/include/git2/sys/mempack.h ++++ b/include/git2/sys/mempack.h +@@ -44,6 +44,29 @@ GIT_BEGIN_DECL + */ + GIT_EXTERN(int) git_mempack_new(git_odb_backend **out); + ++/** ++ * Write a thin packfile with the objects in the memory store. ++ * ++ * A thin packfile is a packfile that does not contain its transitive closure of ++ * references. This is useful for efficiently distributing additions to a ++ * repository over the network, but also finds use in the efficient bulk ++ * addition of objects to a repository, locally. ++ * ++ * This operation performs the (shallow) insert operations into the ++ * `git_packbuilder`, but does not write the packfile to disk; ++ * see `git_packbuilder_write_buf`. ++ * ++ * It also does not reset the memory store; see `git_mempack_reset`. ++ * ++ * @note This function may or may not write trees and blobs that are not ++ * referenced by commits. Currently everything is written, but this ++ * behavior may change in the future as the packer is optimized. ++ * ++ * @param backend The mempack backend ++ * @param pb The packbuilder to use to write the packfile ++ */ ++GIT_EXTERN(int) git_mempack_write_thin_pack(git_odb_backend *backend, git_packbuilder *pb); ++ + /** + * Dump all the queued in-memory writes to a packfile. + * +diff --git a/src/libgit2/odb_mempack.c b/src/libgit2/odb_mempack.c +index 6f27f45f8..0b61e2b66 100644 +--- a/src/libgit2/odb_mempack.c ++++ b/src/libgit2/odb_mempack.c +@@ -132,6 +132,35 @@ cleanup: + return err; + } + ++int git_mempack_write_thin_pack(git_odb_backend *backend, git_packbuilder *pb) ++{ ++ struct memory_packer_db *db = (struct memory_packer_db *)backend; ++ const git_oid *oid; ++ size_t iter = 0; ++ int err = -1; ++ ++ /* TODO: Implement the recency heuristics. ++ For this it probably makes sense to only write what's referenced ++ through commits, an option I've carved out for you in the docs. ++ wrt heuristics: ask your favorite LLM to translate https://git-scm.com/docs/pack-heuristics/en ++ to actual normal reference documentation. */ ++ while (true) { ++ err = git_oidmap_iterate(NULL, db->objects, &iter, &oid); ++ if (err == GIT_ITEROVER) { ++ err = 0; ++ break; ++ } ++ if (err != 0) ++ return err; ++ ++ err = git_packbuilder_insert(pb, oid, NULL); ++ if (err != 0) ++ return err; ++ } ++ ++ return 0; ++} ++ + int git_mempack_dump( + git_buf *pack, + git_repository *repo, +diff --git a/tests/libgit2/mempack/thinpack.c b/tests/libgit2/mempack/thinpack.c +new file mode 100644 +index 000000000..604a4dda2 +--- /dev/null ++++ b/tests/libgit2/mempack/thinpack.c +@@ -0,0 +1,196 @@ ++#include "clar_libgit2.h" ++#include "git2/indexer.h" ++#include "git2/odb_backend.h" ++#include "git2/tree.h" ++#include "git2/types.h" ++#include "git2/sys/mempack.h" ++#include "git2/sys/odb_backend.h" ++#include "util.h" ++ ++static git_repository *_repo; ++static git_odb_backend * _mempack_backend; ++ ++void test_mempack_thinpack__initialize(void) ++{ ++ git_odb *odb; ++ ++ _repo = cl_git_sandbox_init_new("mempack_thinpack_repo"); ++ ++ cl_git_pass(git_mempack_new(&_mempack_backend)); ++ cl_git_pass(git_repository_odb(&odb, _repo)); ++ cl_git_pass(git_odb_add_backend(odb, _mempack_backend, 999)); ++ git_odb_free(odb); ++} ++ ++void _mempack_thinpack__cleanup(void) ++{ ++ cl_git_sandbox_cleanup(); ++} ++ ++/* ++ Generating a packfile for an unchanged repo works and produces an empty packfile. ++ Even if we allow this scenario to be detected, it shouldn't misbehave if the ++ application is unaware of it. ++*/ ++void test_mempack_thinpack__empty(void) ++{ ++ git_packbuilder *pb; ++ int version; ++ int n; ++ git_buf buf = GIT_BUF_INIT; ++ ++ git_packbuilder_new(&pb, _repo); ++ ++ cl_git_pass(git_mempack_write_thin_pack(_mempack_backend, pb)); ++ cl_git_pass(git_packbuilder_write_buf(&buf, pb)); ++ cl_assert_in_range(12, buf.size, 1024 /* empty packfile is >0 bytes, but certainly not that big */); ++ cl_assert(buf.ptr[0] == 'P'); ++ cl_assert(buf.ptr[1] == 'A'); ++ cl_assert(buf.ptr[2] == 'C'); ++ cl_assert(buf.ptr[3] == 'K'); ++ version = (buf.ptr[4] << 24) | (buf.ptr[5] << 16) | (buf.ptr[6] << 8) | buf.ptr[7]; ++ /* Subject to change. https://git-scm.com/docs/pack-format: Git currently accepts version number 2 or 3 but generates version 2 only.*/ ++ cl_assert_equal_i(2, version); ++ n = (buf.ptr[8] << 24) | (buf.ptr[9] << 16) | (buf.ptr[10] << 8) | buf.ptr[11]; ++ cl_assert_equal_i(0, n); ++ git_buf_dispose(&buf); ++ ++ git_packbuilder_free(pb); ++} ++ ++#define LIT_LEN(x) x, sizeof(x) - 1 ++ ++/* ++ Check that git_mempack_write_thin_pack produces a thin packfile. ++*/ ++void test_mempack_thinpack__thin(void) ++{ ++ /* Outline: ++ - Create tree 1 ++ - Flush to packfile A ++ - Create tree 2 ++ - Flush to packfile B ++ ++ Tree 2 has a new blob and a reference to a blob from tree 1. ++ ++ Expectation: ++ - Packfile B is thin and does not contain the objects from packfile A ++ */ ++ ++ ++ git_oid oid_blob_1; ++ git_oid oid_blob_2; ++ git_oid oid_blob_3; ++ git_oid oid_tree_1; ++ git_oid oid_tree_2; ++ git_treebuilder *tb; ++ ++ git_packbuilder *pb; ++ git_buf buf = GIT_BUF_INIT; ++ git_indexer *indexer; ++ git_indexer_progress stats; ++ char pack_dir_path[1024]; ++ ++ char sbuf[1024]; ++ const char * repo_path; ++ const char * pack_name_1; ++ const char * pack_name_2; ++ git_str pack_path_1 = GIT_STR_INIT; ++ git_str pack_path_2 = GIT_STR_INIT; ++ git_odb_backend * pack_odb_backend_1; ++ git_odb_backend * pack_odb_backend_2; ++ ++ ++ cl_assert_in_range(0, snprintf(pack_dir_path, sizeof(pack_dir_path), "%s/objects/pack", git_repository_path(_repo)), sizeof(pack_dir_path)); ++ ++ /* Create tree 1 */ ++ ++ cl_git_pass(git_blob_create_from_buffer(&oid_blob_1, _repo, LIT_LEN("thinpack blob 1"))); ++ cl_git_pass(git_blob_create_from_buffer(&oid_blob_2, _repo, LIT_LEN("thinpack blob 2"))); ++ ++ ++ cl_git_pass(git_treebuilder_new(&tb, _repo, NULL)); ++ cl_git_pass(git_treebuilder_insert(NULL, tb, "blob1", &oid_blob_1, GIT_FILEMODE_BLOB)); ++ cl_git_pass(git_treebuilder_insert(NULL, tb, "blob2", &oid_blob_2, GIT_FILEMODE_BLOB)); ++ cl_git_pass(git_treebuilder_write(&oid_tree_1, tb)); ++ ++ /* Flush */ ++ ++ cl_git_pass(git_packbuilder_new(&pb, _repo)); ++ cl_git_pass(git_mempack_write_thin_pack(_mempack_backend, pb)); ++ cl_git_pass(git_packbuilder_write_buf(&buf, pb)); ++ cl_git_pass(git_indexer_new(&indexer, pack_dir_path, 0, NULL, NULL)); ++ cl_git_pass(git_indexer_append(indexer, buf.ptr, buf.size, &stats)); ++ cl_git_pass(git_indexer_commit(indexer, &stats)); ++ pack_name_1 = strdup(git_indexer_name(indexer)); ++ cl_assert(pack_name_1); ++ git_buf_dispose(&buf); ++ git_mempack_reset(_mempack_backend); ++ git_indexer_free(indexer); ++ git_packbuilder_free(pb); ++ ++ /* Create tree 2 */ ++ ++ cl_git_pass(git_treebuilder_clear(tb)); ++ /* blob 1 won't be used, but we add it anyway to test that just "declaring" an object doesn't ++ necessarily cause its inclusion in the next thin packfile. It must only be included if new. */ ++ cl_git_pass(git_blob_create_from_buffer(&oid_blob_1, _repo, LIT_LEN("thinpack blob 1"))); ++ cl_git_pass(git_blob_create_from_buffer(&oid_blob_3, _repo, LIT_LEN("thinpack blob 3"))); ++ cl_git_pass(git_treebuilder_insert(NULL, tb, "blob1", &oid_blob_1, GIT_FILEMODE_BLOB)); ++ cl_git_pass(git_treebuilder_insert(NULL, tb, "blob3", &oid_blob_3, GIT_FILEMODE_BLOB)); ++ cl_git_pass(git_treebuilder_write(&oid_tree_2, tb)); ++ ++ /* Flush */ ++ ++ cl_git_pass(git_packbuilder_new(&pb, _repo)); ++ cl_git_pass(git_mempack_write_thin_pack(_mempack_backend, pb)); ++ cl_git_pass(git_packbuilder_write_buf(&buf, pb)); ++ cl_git_pass(git_indexer_new(&indexer, pack_dir_path, 0, NULL, NULL)); ++ cl_git_pass(git_indexer_append(indexer, buf.ptr, buf.size, &stats)); ++ cl_git_pass(git_indexer_commit(indexer, &stats)); ++ pack_name_2 = strdup(git_indexer_name(indexer)); ++ cl_assert(pack_name_2); ++ git_buf_dispose(&buf); ++ git_mempack_reset(_mempack_backend); ++ git_indexer_free(indexer); ++ git_packbuilder_free(pb); ++ git_treebuilder_free(tb); ++ ++ /* Assertions */ ++ ++ assert(pack_name_1); ++ assert(pack_name_2); ++ ++ repo_path = git_repository_path(_repo); ++ ++ snprintf(sbuf, sizeof(sbuf), "objects/pack/pack-%s.pack", pack_name_1); ++ git_str_joinpath(&pack_path_1, repo_path, sbuf); ++ snprintf(sbuf, sizeof(sbuf), "objects/pack/pack-%s.pack", pack_name_2); ++ git_str_joinpath(&pack_path_2, repo_path, sbuf); ++ ++ /* If they're the same, something definitely went wrong. */ ++ cl_assert(strcmp(pack_name_1, pack_name_2) != 0); ++ ++ cl_git_pass(git_odb_backend_one_pack(&pack_odb_backend_1, pack_path_1.ptr)); ++ cl_assert(pack_odb_backend_1->exists(pack_odb_backend_1, &oid_blob_1)); ++ cl_assert(pack_odb_backend_1->exists(pack_odb_backend_1, &oid_blob_2)); ++ cl_assert(!pack_odb_backend_1->exists(pack_odb_backend_1, &oid_blob_3)); ++ cl_assert(pack_odb_backend_1->exists(pack_odb_backend_1, &oid_tree_1)); ++ cl_assert(!pack_odb_backend_1->exists(pack_odb_backend_1, &oid_tree_2)); ++ ++ cl_git_pass(git_odb_backend_one_pack(&pack_odb_backend_2, pack_path_2.ptr)); ++ /* blob 1 is already in the packfile 1, so packfile 2 must not include it, in order to be _thin_. */ ++ cl_assert(!pack_odb_backend_2->exists(pack_odb_backend_2, &oid_blob_1)); ++ cl_assert(!pack_odb_backend_2->exists(pack_odb_backend_2, &oid_blob_2)); ++ cl_assert(pack_odb_backend_2->exists(pack_odb_backend_2, &oid_blob_3)); ++ cl_assert(!pack_odb_backend_2->exists(pack_odb_backend_2, &oid_tree_1)); ++ cl_assert(pack_odb_backend_2->exists(pack_odb_backend_2, &oid_tree_2)); ++ ++ pack_odb_backend_1->free(pack_odb_backend_1); ++ pack_odb_backend_2->free(pack_odb_backend_2); ++ free((void *)pack_name_1); ++ free((void *)pack_name_2); ++ git_str_dispose(&pack_path_1); ++ git_str_dispose(&pack_path_2); ++ ++} diff --git a/packaging/patches/libgit2-packbuilder-callback-interruptible.patch b/packaging/patches/libgit2-packbuilder-callback-interruptible.patch new file mode 100644 index 000000000..c67822ff7 --- /dev/null +++ b/packaging/patches/libgit2-packbuilder-callback-interruptible.patch @@ -0,0 +1,930 @@ +commit e9823c5da4fa977c46bcb97167fbdd0d70adb5ff +Author: Robert Hensing +Date: Mon Aug 26 20:07:04 2024 +0200 + + Make packbuilder interruptible using progress callback + + Forward errors from packbuilder->progress_cb + + This allows the callback to terminate long-running operations when + the application is interrupted. + +diff --git a/include/git2/pack.h b/include/git2/pack.h +index 0f6bd2ab9..bee72a6c0 100644 +--- a/include/git2/pack.h ++++ b/include/git2/pack.h +@@ -247,6 +247,9 @@ typedef int GIT_CALLBACK(git_packbuilder_progress)( + * @param progress_cb Function to call with progress information during + * pack building. Be aware that this is called inline with pack building + * operations, so performance may be affected. ++ * When progress_cb returns an error, the pack building process will be ++ * aborted and the error will be returned from the invoked function. ++ * `pb` must then be freed. + * @param progress_cb_payload Payload for progress callback. + * @return 0 or an error code + */ +diff --git a/src/libgit2/pack-objects.c b/src/libgit2/pack-objects.c +index b2d80cba9..7c331c2d5 100644 +--- a/src/libgit2/pack-objects.c ++++ b/src/libgit2/pack-objects.c +@@ -932,6 +932,9 @@ static int report_delta_progress( + { + int ret; + ++ if (pb->failure) ++ return pb->failure; ++ + if (pb->progress_cb) { + uint64_t current_time = git_time_monotonic(); + uint64_t elapsed = current_time - pb->last_progress_report_time; +@@ -943,8 +946,10 @@ static int report_delta_progress( + GIT_PACKBUILDER_DELTAFICATION, + count, pb->nr_objects, pb->progress_cb_payload); + +- if (ret) ++ if (ret) { ++ pb->failure = ret; + return git_error_set_after_callback(ret); ++ } + } + } + +@@ -976,7 +981,10 @@ static int find_deltas(git_packbuilder *pb, git_pobject **list, + } + + pb->nr_deltified += 1; +- report_delta_progress(pb, pb->nr_deltified, false); ++ if ((error = report_delta_progress(pb, pb->nr_deltified, false)) < 0) { ++ GIT_ASSERT(git_packbuilder__progress_unlock(pb) == 0); ++ goto on_error; ++ } + + po = *list++; + (*list_size)--; +@@ -1124,6 +1132,10 @@ struct thread_params { + size_t depth; + size_t working; + size_t data_ready; ++ ++ /* A pb->progress_cb can stop the packing process by returning an error. ++ When that happens, all threads observe the error and stop voluntarily. */ ++ bool stopped; + }; + + static void *threaded_find_deltas(void *arg) +@@ -1133,7 +1145,12 @@ static void *threaded_find_deltas(void *arg) + while (me->remaining) { + if (find_deltas(me->pb, me->list, &me->remaining, + me->window, me->depth) < 0) { +- ; /* TODO */ ++ me->stopped = true; ++ GIT_ASSERT_WITH_RETVAL(git_packbuilder__progress_lock(me->pb) == 0, NULL); ++ me->working = false; ++ git_cond_signal(&me->pb->progress_cond); ++ GIT_ASSERT_WITH_RETVAL(git_packbuilder__progress_unlock(me->pb) == 0, NULL); ++ return NULL; + } + + GIT_ASSERT_WITH_RETVAL(git_packbuilder__progress_lock(me->pb) == 0, NULL); +@@ -1175,8 +1192,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list, + pb->nr_threads = git__online_cpus(); + + if (pb->nr_threads <= 1) { +- find_deltas(pb, list, &list_size, window, depth); +- return 0; ++ return find_deltas(pb, list, &list_size, window, depth); + } + + p = git__mallocarray(pb->nr_threads, sizeof(*p)); +@@ -1195,6 +1211,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list, + p[i].depth = depth; + p[i].working = 1; + p[i].data_ready = 0; ++ p[i].stopped = 0; + + /* try to split chunks on "path" boundaries */ + while (sub_size && sub_size < list_size && +@@ -1262,7 +1279,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list, + (!victim || victim->remaining < p[i].remaining)) + victim = &p[i]; + +- if (victim) { ++ if (victim && !target->stopped) { + sub_size = victim->remaining / 2; + list = victim->list + victim->list_size - sub_size; + while (sub_size && list[0]->hash && +@@ -1286,7 +1303,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list, + } + target->list_size = sub_size; + target->remaining = sub_size; +- target->working = 1; ++ target->working = 1; /* even when target->stopped, so that we don't process this thread again */ + GIT_ASSERT(git_packbuilder__progress_unlock(pb) == 0); + + if (git_mutex_lock(&target->mutex)) { +@@ -1299,7 +1316,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list, + git_cond_signal(&target->cond); + git_mutex_unlock(&target->mutex); + +- if (!sub_size) { ++ if (target->stopped || !sub_size) { + git_thread_join(&target->thread, NULL); + git_cond_free(&target->cond); + git_mutex_free(&target->mutex); +@@ -1308,7 +1325,7 @@ static int ll_find_deltas(git_packbuilder *pb, git_pobject **list, + } + + git__free(p); +- return 0; ++ return pb->failure; + } + + #else +@@ -1319,6 +1336,7 @@ int git_packbuilder__prepare(git_packbuilder *pb) + { + git_pobject **delta_list; + size_t i, n = 0; ++ int error; + + if (pb->nr_objects == 0 || pb->done) + return 0; /* nothing to do */ +@@ -1327,8 +1345,10 @@ int git_packbuilder__prepare(git_packbuilder *pb) + * Although we do not report progress during deltafication, we + * at least report that we are in the deltafication stage + */ +- if (pb->progress_cb) +- pb->progress_cb(GIT_PACKBUILDER_DELTAFICATION, 0, pb->nr_objects, pb->progress_cb_payload); ++ if (pb->progress_cb) { ++ if ((error = pb->progress_cb(GIT_PACKBUILDER_DELTAFICATION, 0, pb->nr_objects, pb->progress_cb_payload)) < 0) ++ return git_error_set_after_callback(error); ++ } + + delta_list = git__mallocarray(pb->nr_objects, sizeof(*delta_list)); + GIT_ERROR_CHECK_ALLOC(delta_list); +@@ -1345,31 +1365,33 @@ int git_packbuilder__prepare(git_packbuilder *pb) + + if (n > 1) { + git__tsort((void **)delta_list, n, type_size_sort); +- if (ll_find_deltas(pb, delta_list, n, ++ if ((error = ll_find_deltas(pb, delta_list, n, + GIT_PACK_WINDOW + 1, +- GIT_PACK_DEPTH) < 0) { ++ GIT_PACK_DEPTH)) < 0) { + git__free(delta_list); +- return -1; ++ return error; + } + } + +- report_delta_progress(pb, pb->nr_objects, true); ++ error = report_delta_progress(pb, pb->nr_objects, true); + + pb->done = true; + git__free(delta_list); +- return 0; ++ return error; + } + +-#define PREPARE_PACK if (git_packbuilder__prepare(pb) < 0) { return -1; } ++#define PREPARE_PACK error = git_packbuilder__prepare(pb); if (error < 0) { return error; } + + int git_packbuilder_foreach(git_packbuilder *pb, int (*cb)(void *buf, size_t size, void *payload), void *payload) + { ++ int error; + PREPARE_PACK; + return write_pack(pb, cb, payload); + } + + int git_packbuilder__write_buf(git_str *buf, git_packbuilder *pb) + { ++ int error; + PREPARE_PACK; + + return write_pack(pb, &write_pack_buf, buf); +diff --git a/src/libgit2/pack-objects.h b/src/libgit2/pack-objects.h +index bbc8b9430..380a28ebe 100644 +--- a/src/libgit2/pack-objects.h ++++ b/src/libgit2/pack-objects.h +@@ -100,6 +100,10 @@ struct git_packbuilder { + uint64_t last_progress_report_time; + + bool done; ++ ++ /* A non-zero error code in failure causes all threads to shut themselves ++ down. Some functions will return this error code. */ ++ volatile int failure; + }; + + int git_packbuilder__write_buf(git_str *buf, git_packbuilder *pb); +diff --git a/tests/libgit2/pack/cancel.c b/tests/libgit2/pack/cancel.c +new file mode 100644 +index 000000000..a0aa9716a +--- /dev/null ++++ b/tests/libgit2/pack/cancel.c +@@ -0,0 +1,240 @@ ++#include "clar_libgit2.h" ++#include "futils.h" ++#include "pack.h" ++#include "hash.h" ++#include "iterator.h" ++#include "vector.h" ++#include "posix.h" ++#include "hash.h" ++#include "pack-objects.h" ++ ++static git_repository *_repo; ++static git_revwalk *_revwalker; ++static git_packbuilder *_packbuilder; ++static git_indexer *_indexer; ++static git_vector _commits; ++static int _commits_is_initialized; ++static git_indexer_progress _stats; ++ ++extern bool git_disable_pack_keep_file_checks; ++ ++static void pack_packbuilder_init(const char *sandbox) { ++ _repo = cl_git_sandbox_init(sandbox); ++ /* cl_git_pass(p_chdir(sandbox)); */ ++ cl_git_pass(git_revwalk_new(&_revwalker, _repo)); ++ cl_git_pass(git_packbuilder_new(&_packbuilder, _repo)); ++ cl_git_pass(git_vector_init(&_commits, 0, NULL)); ++ _commits_is_initialized = 1; ++ memset(&_stats, 0, sizeof(_stats)); ++ p_fsync__cnt = 0; ++} ++ ++void test_pack_cancel__initialize(void) ++{ ++ pack_packbuilder_init("small.git"); ++} ++ ++void test_pack_cancel__cleanup(void) ++{ ++ git_oid *o; ++ unsigned int i; ++ ++ cl_git_pass(git_libgit2_opts(GIT_OPT_ENABLE_FSYNC_GITDIR, 0)); ++ cl_git_pass(git_libgit2_opts(GIT_OPT_DISABLE_PACK_KEEP_FILE_CHECKS, false)); ++ ++ if (_commits_is_initialized) { ++ _commits_is_initialized = 0; ++ git_vector_foreach(&_commits, i, o) { ++ git__free(o); ++ } ++ git_vector_free(&_commits); ++ } ++ ++ git_packbuilder_free(_packbuilder); ++ _packbuilder = NULL; ++ ++ git_revwalk_free(_revwalker); ++ _revwalker = NULL; ++ ++ git_indexer_free(_indexer); ++ _indexer = NULL; ++ ++ /* cl_git_pass(p_chdir("..")); */ ++ cl_git_sandbox_cleanup(); ++ _repo = NULL; ++} ++ ++static int seed_packbuilder(void) ++{ ++ int error; ++ git_oid oid, *o; ++ unsigned int i; ++ ++ git_revwalk_sorting(_revwalker, GIT_SORT_TIME); ++ cl_git_pass(git_revwalk_push_ref(_revwalker, "HEAD")); ++ ++ while (git_revwalk_next(&oid, _revwalker) == 0) { ++ o = git__malloc(sizeof(git_oid)); ++ cl_assert(o != NULL); ++ git_oid_cpy(o, &oid); ++ cl_git_pass(git_vector_insert(&_commits, o)); ++ } ++ ++ git_vector_foreach(&_commits, i, o) { ++ if((error = git_packbuilder_insert(_packbuilder, o, NULL)) < 0) ++ return error; ++ } ++ ++ git_vector_foreach(&_commits, i, o) { ++ git_object *obj; ++ cl_git_pass(git_object_lookup(&obj, _repo, o, GIT_OBJECT_COMMIT)); ++ error = git_packbuilder_insert_tree(_packbuilder, ++ git_commit_tree_id((git_commit *)obj)); ++ git_object_free(obj); ++ if (error < 0) ++ return error; ++ } ++ ++ return 0; ++} ++ ++static int fail_stage; ++ ++static int packbuilder_cancel_after_n_calls_cb(int stage, uint32_t current, uint32_t total, void *payload) ++{ ++ ++ /* Force the callback to run again on the next opportunity regardless ++ of how fast we're running. */ ++ _packbuilder->last_progress_report_time = 0; ++ ++ if (stage == fail_stage) { ++ int *calls = (int *)payload; ++ int n = *calls; ++ /* Always decrement, including past zero. This way the error is only ++ triggered once, making sure it is picked up immediately. */ ++ --*calls; ++ if (n == 0) ++ return GIT_EUSER; ++ } ++ ++ return 0; ++} ++ ++static void test_cancel(int n) ++{ ++ ++ int calls_remaining = n; ++ int err; ++ git_buf buf = GIT_BUF_INIT; ++ ++ /* Switch to a small repository, so that `packbuilder_cancel_after_n_calls_cb` ++ can hack the time to call the callback on every opportunity. */ ++ ++ cl_git_pass(git_packbuilder_set_callbacks(_packbuilder, &packbuilder_cancel_after_n_calls_cb, &calls_remaining)); ++ err = seed_packbuilder(); ++ if (!err) ++ err = git_packbuilder_write_buf(&buf, _packbuilder); ++ ++ cl_assert_equal_i(GIT_EUSER, err); ++} ++void test_pack_cancel__cancel_after_add_0(void) ++{ ++ fail_stage = GIT_PACKBUILDER_ADDING_OBJECTS; ++ test_cancel(0); ++} ++ ++void test_pack_cancel__cancel_after_add_1(void) ++{ ++ cl_skip(); ++ fail_stage = GIT_PACKBUILDER_ADDING_OBJECTS; ++ test_cancel(1); ++} ++ ++void test_pack_cancel__cancel_after_delta_0(void) ++{ ++ fail_stage = GIT_PACKBUILDER_DELTAFICATION; ++ test_cancel(0); ++} ++ ++void test_pack_cancel__cancel_after_delta_1(void) ++{ ++ fail_stage = GIT_PACKBUILDER_DELTAFICATION; ++ test_cancel(1); ++} ++ ++void test_pack_cancel__cancel_after_delta_0_threaded(void) ++{ ++#ifdef GIT_THREADS ++ git_packbuilder_set_threads(_packbuilder, 8); ++ fail_stage = GIT_PACKBUILDER_DELTAFICATION; ++ test_cancel(0); ++#else ++ cl_skip(); ++#endif ++} ++ ++void test_pack_cancel__cancel_after_delta_1_threaded(void) ++{ ++#ifdef GIT_THREADS ++ git_packbuilder_set_threads(_packbuilder, 8); ++ fail_stage = GIT_PACKBUILDER_DELTAFICATION; ++ test_cancel(1); ++#else ++ cl_skip(); ++#endif ++} ++ ++static int foreach_cb(void *buf, size_t len, void *payload) ++{ ++ git_indexer *idx = (git_indexer *) payload; ++ cl_git_pass(git_indexer_append(idx, buf, len, &_stats)); ++ return 0; ++} ++ ++void test_pack_cancel__foreach(void) ++{ ++ git_indexer *idx; ++ ++ seed_packbuilder(); ++ ++#ifdef GIT_EXPERIMENTAL_SHA256 ++ cl_git_pass(git_indexer_new(&idx, ".", GIT_OID_SHA1, NULL)); ++#else ++ cl_git_pass(git_indexer_new(&idx, ".", 0, NULL, NULL)); ++#endif ++ ++ cl_git_pass(git_packbuilder_foreach(_packbuilder, foreach_cb, idx)); ++ cl_git_pass(git_indexer_commit(idx, &_stats)); ++ git_indexer_free(idx); ++} ++ ++static int foreach_cancel_cb(void *buf, size_t len, void *payload) ++{ ++ git_indexer *idx = (git_indexer *)payload; ++ cl_git_pass(git_indexer_append(idx, buf, len, &_stats)); ++ return (_stats.total_objects > 2) ? -1111 : 0; ++} ++ ++void test_pack_cancel__foreach_with_cancel(void) ++{ ++ git_indexer *idx; ++ ++ seed_packbuilder(); ++ ++#ifdef GIT_EXPERIMENTAL_SHA256 ++ cl_git_pass(git_indexer_new(&idx, ".", GIT_OID_SHA1, NULL)); ++#else ++ cl_git_pass(git_indexer_new(&idx, ".", 0, NULL, NULL)); ++#endif ++ ++ cl_git_fail_with( ++ git_packbuilder_foreach(_packbuilder, foreach_cancel_cb, idx), -1111); ++ git_indexer_free(idx); ++} ++ ++void test_pack_cancel__keep_file_check(void) ++{ ++ assert(!git_disable_pack_keep_file_checks); ++ cl_git_pass(git_libgit2_opts(GIT_OPT_DISABLE_PACK_KEEP_FILE_CHECKS, true)); ++ assert(git_disable_pack_keep_file_checks); ++} +diff --git a/tests/resources/small.git/HEAD b/tests/resources/small.git/HEAD +new file mode 100644 +index 0000000000000000000000000000000000000000..cb089cd89a7d7686d284d8761201649346b5aa1c +GIT binary patch +literal 23 +ecmXR)O|w!cN=+-)&qz&7Db~+TEG|hc;sO9;xClW2 + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/config b/tests/resources/small.git/config +new file mode 100644 +index 0000000000000000000000000000000000000000..07d359d07cf1ed0c0074fdad71ffff5942f0adfa +GIT binary patch +literal 66 +zcmaz}&M!)h<>D+#Eyypk5{uv*03B5png9R* + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/description b/tests/resources/small.git/description +new file mode 100644 +index 0000000000000000000000000000000000000000..498b267a8c7812490d6479839c5577eaaec79d62 +GIT binary patch +literal 73 +zcmWH|%S+5nO;IRHEyyp$t+PQ$;d2LNXyJgRZve!Elw`VEGWs$&r??@ +Q$yWgB0LrH#Y0~2Y0PnOK(EtDd + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/hooks/applypatch-msg.sample b/tests/resources/small.git/hooks/applypatch-msg.sample +new file mode 100755 +index 0000000000000000000000000000000000000000..dcbf8167fa503f96ff6a39c68409007eadc9b1f3 +GIT binary patch +literal 535 +zcmY+AX;Q;542A#a6e8^~FyI8r&I~hf2QJ{GO6(?HuvEG*+#R{4EI%zhfA8r{j%sh$ +zHE~E-UtQd8{bq4@*S%jq3@bmxwQDXGv#o!N`o3AHMw3xD)hy0#>&E&zzl%vRffomqo=v6>_2NRa#TwDdYvTVQyueO*15Nlo%=#DXgC0bhF3vTa`LQGaO9;jeD$OP?~ +za$G4Q{z+Q_{5V?5h;a-noM$P{<>Q~j4o7u%#P6^o^16{y*jU=-K8GYD_dUtdj4FSx +zSC0C!DvAnv%S!4dgk +XB^)11aoGMJPCqWs%IS0YSv(eBT&%T6 + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/hooks/commit-msg.sample b/tests/resources/small.git/hooks/commit-msg.sample +new file mode 100755 +index 0000000000000000000000000000000000000000..f3780f92349638ebe32f6baf24c7c3027675d7c9 +GIT binary patch +literal 953 +zcmaJy@-{3h^^Cx;#d0zEA@DDc$nY4ez&|=%jTg@_HU*ub=!!y$xW09TSjlj +z(`I@QCsM`!9&80$I98wsQ8yK#)Orb<8re8FjkKh630D$QUDwi~(gkX=RunYm$rDjk +zlp%RUSnzA#6yjdG5?T?2DcYKp+v_lts0ljn&bh3J0bD5@N@1UKZ190O6ZeWr-BuZ^ +zWRebCX%(%=Xoj#(xYk1Cjtr!=tyBesf@m6}8zY6Ijbz9i9ziI_jG9MvR +zDH*e>^ga9IR?2wrSrAVm;eButj4Y>7(E2?b~jsu>& +zRKCJ7bp#19sqYh627wD%D9R$8=Ml$TNlumDypl~$jBu*G>5fIR^FB0h0Ex&TGZNr> +zL5hs1_K>taRb!|ThN9ns7^@4MXKP+6aGI_UK)T-M#rcP$;kN(Vcf#P)+5GzWa{l@J +z>-E{`$1iiNVYxq27}j;uo%;)r3kJI2xCFF~Ux;$Q%) +wjbk6JlDCM`jU&P+UVOvg`|iYl<7~9k>HHB4I;pdlQ=I-^$DrHaN$@lH1?P!0U;qFB + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/hooks/fsmonitor-watchman.sample b/tests/resources/small.git/hooks/fsmonitor-watchman.sample +new file mode 100755 +index 0000000000000000000000000000000000000000..41184ebc318c159f51cd1ebe2290559805df89d8 +GIT binary patch +literal 4777 +zcmbtYYi}F368$Xwipg4lq(BeHMvzvH-4;n7DGJBPqq#tw3aed8+IU5-m)yvL>;Cqh +z8FFRGj$`9CA8aoJ?j^$%==FV``-=rhLcPW`McSytRm~mEO7_&_cAVZrf1fFy*ha@8oe%*-aBYE +zcjzZg>LOkgxuUr-XJnHyD;zmPnRaSc#!k_P*d_BttRdc+J6G7za5#+^Y1nkc2Oowk`ya47uUR3Feu?B(w;S{(VYzxh}q-=#zP@uxSx{wbyPUMFU;K(06)$o{07&3yI?q{GqMcQ1c_^M<0< +zF4acAV)Il-V(rCTC1(;bsZ*}bl8dmejAk~yb`B}!^0;g^(o9kGUfZfDOvyp@x4OQt +zSgWh6T|3eq;9MFs8-#z+FDM1h(IjRUP|``PxupgJ7CUHOH90gbgl^2~97`?_X{P)) +zB*$r1cDlF-%azKND}?Gv`2K8-9v5e`gQoft=j?T<&a13c^!wY_$D`5z-X1g?ty&6- +zQN50{8?bUk9AI->^W@~~nkOghHIC2YN+AXkLQG_2-{Pq3%{`3KUMeG$iIn%%^6*NYb +zn|_BdV#C)n4565VccX;uT8&z3vSi!HXGbUj2B!R +zdz~&#fk#L-&k$fLwo$4?>12g@AXOKFekuo#6EHB%gmpD?1eyh%N8s{2wGoTu +z*@6cEZ^ZW!FAF_|JL`NkV7k}0ow|-2jHwbgH0;c@Dq*o?@&c*HnGdyx6^su8Qk%2{ +z*ye(dxO*6-&>qn1+zw}tc6;=sOX{4WB=VqjTS^))y1jlX2Q;=e!qMmFA5lC$#;BxC +z=Y%tRpWxb+_uQAvAw7Q{HGV#R$xb&udLCzZ+HN?kTyB};1EJ8UlQ5!>5eGW@)RX0n +zkjj>EF!3=0Gl^8dzv$B^NMGRxJoqN4A`xq-@wCbrx*u2NmIJ1xZ%H +zh;{|4T3(!E9sY#Ni(wUJYs1MmIc9bl)(4Nl3_wD_BWB>i<1S(LX7m*{Q7PU$muMS* +zM!%0EZx-Vw=Zey;erC?SNxF;pY@^A%-krqzfLV2meBp1vWdyArFYn`DD19T)Hw(?n +z)}{NP(Lk(o*?gl#B@pP7^*r|=;PIDT4|F#{2Hzh-AL0Rv$6uT;n|WzE4=slK?on@(fZeGhRgQCu56qB +z{+n81Az96qnQjMY*-*r-KV*7;Z#4QuJRJJV$M^KdldiMhj?ImK6~FvwJ*L5a){QoM=L5TYHkGO1$UrO3`a>{?Opw|b +zG(#59NQ#jFL9v~vgOVkM@^^(^A}onOE))yWEwhIlk&{ZyseZ^O0b=w8&O=BK{k<5B +k^Q-B@eG}LeHrquz%(SVEp_N)VhYZikCW__82JXfD17`J9Qvd(} + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/hooks/pre-applypatch.sample b/tests/resources/small.git/hooks/pre-applypatch.sample +new file mode 100755 +index 0000000000000000000000000000000000000000..625837e25f91421b8809a097f4a3103dd387ef31 +GIT binary patch +literal 481 +zcmY+ATTa6;5Jms9iouO45IBJXEg&Jm9@v1LPHMM_ZR|;#6tQh$71hSXq*MxP;V& +zj0cY7SCL=x4`a46sF)C>94Gk%=3q$W2s;j6iHtB2$R0%gix4oK@&T~=ALd_o*CKxt +I-`Pv{1Bpzc>;M1& + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/hooks/pre-commit.sample b/tests/resources/small.git/hooks/pre-commit.sample +new file mode 100755 +index 0000000000000000000000000000000000000000..10b39b2e26981b8f87ea424e735ef87359066dbb +GIT binary patch +literal 1706 +zcmZuxU2ohr5PY_N#pZ0-F<{-v&v-X^RA+u>k}E$4d&uD7=g_fA8+pNNV=4s0|iD3p<=DTXClTS +zXV23tJ;ECmN@M0j@zUAKEYW@3bv!SeYZ8ZH`YQNTApFVNc;F|9r5p4TqGs=>8E?6y +zi|gY{iM#PG1nL?UE9YCnWTk72kgZPG*Usqw!~Qd3c?~@w2?%eg@~)+VlSs6N5Yf2^ +zz;owF#K#r^&KMq1A`oqVGFpD&-!Pv|Rc +zO3KSqA@h9nSc%bm`0)Amk6*J}@14J*1-219l%%7D!Pl}UK>|lVi0Dfgu2jN3WC!uL +z0ej??b2iSehVgdnWHmZV4kUo*QL#aiIp}U=9x)IXk}JJ7VQ;CI9Rtn5e0VcjbYcVt+`x5D+svCGD;Z5hm*E$jSEQZ%SQ(}oLgslTvrKK@9Qf#b!hajVFnp9@oIix;NcI9Wk +xjnh0ya!AWet{I7YpD;y6HXyzI*lfSvH=o6*7mJZPkuaYpm>vzZ`wyGEBtOQPo|pgt + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/hooks/pre-push.sample b/tests/resources/small.git/hooks/pre-push.sample +new file mode 100755 +index 0000000000000000000000000000000000000000..02cbd80c287f959fe33975bb66c56293e3f5b396 +GIT binary patch +literal 1431 +zcmaJ>U60!~5PUX&#a1@z9B{IIZkjLT0t5kq9#8~D(I5{+8&J~9;#ndUk~-ZT`r|uG +z$#K$$J{TsKs*LP1}9!GoZ@4I4myMMG_di|of +z%?llx{O8TS-#^;(OioEmPy%kwWQBA1OMzV{hsQ8XFzS1k!~YQoLa5 +zhtP1fA$q6VmMbbAC_9)4I628k*O5J$NR19uHe4QYDK<==I~SQk)Nu%xQ~KH +z53w=!ke(FGb_PpnZfd*+hnXDTn;2*`u^~;?+5C~cn?bRka7NR%06%e6O91{MAgN6J +zmlO8{Biw4&wr&&(z4p3eln`E}XR9m9bNYZ7Ibrg(4yZIXrfgD7N*AFD7L3YSM#j}% +zo__rOS5fr;@8UM<6cl+cv_$YB$PQ&9dv($eM*))g!_cu!QcSh-mqE9i#QDZT)=o#` +z?8!RtE?w6p?GkGZ-6yt_p~5~4ecu|Sf^)6096%h*q-eNiEA1;Xwg)p~Q&iGSG7-IQ +z9aII&`ps$WOojFA`*bjGkFk|E@sHHuD}W^d`7YJ3YE^zrQnqR +zGoq?;YGKe)93o|_=^f%3U1KYZGPOXRRxK7w`UUbMMa3<86OmVH!EKP$8RCrn9mWX+ +zC?9yF!fRVLmud3hF<}x;;sR}f(*r}6Gap3fR6zLHR~kbMgD{98N`L+r&?3p~*0+FX +zcAL%j=(SO}xTJUTvA`&Lf`2mv4koPG9&|;2+68$XxiXKL@ma;l5d2^5Ba_rPh_DHI-u1#&_upttZXp;no03$20|NFiM +zK#D#xQ>!Z3JkX8T-LDVm!B5j7y_{;JDmmTTef+K1oIiPzeEr+Ai*<2PUgnG4^ZB>p +z_fkAvoR1emuf~ri^K$-px=4#D-vY9w& +z`bCv#2zVn=YnJyeNey(Y +zRh`9vtLw~A+5zsjp|W0Nsa|29Rm!B>OoG5a+vi;ari8O>KkU!KAWg_fa3btK2x*_@ +z0bEc7J;Ubghm}n9bOi(Sv_B66nQ7U)J7f0fO}8Wuf*uorcIgEG +zOHc|-V6+HlRhOP}?Cn?@5iwSl43abmBA^2lyL$+cpabCGVES+v^j^FO_}?FIp%En%Ll?Z*7*}TwrZyg5OSZ9rY-`aU~Mc-jjv{Ll)FLMgtB4ujktfQ`Xhqrka +zT=P!A;9w^;Z?PqpLwOLu=cj3L>TdUKw2;DMu)`oVkj}#bcDx4tYg=j%D`+i{W~fVM +zVmZ>W9VMyin9c-0KzI_;iZ-g|OyzuG`Yq%(%dvl;ifnVr0;jWE&S`z|rQu=!yHBBO +zx`OJ;oOQ(KKM<$(bC38o>pD0%|HA(E0TRw7qj$fJ_pRN+7Nm>dSC(gLg{(`t+5Z=?o+}wXU4tHy+&%F&aRhFebeEhR2R5|$#Ycbp^w@t +zTl%=f1t=w+WpJzF<|CE@?SCNAz)%9?w33lQ8vrHJqPfH9@}qs*QXOG71W=ylx;wOB +zcx!Bj^)Yy6WX$a^vBkBJ5CobqlaDx_B0c<3b+8)f84LCrt;e;qxc+7>VbwVK{skNv!wvBiTa^9Iu +zkwP;VK)jH$WJ{`MRwAA9fal!y0dtV;FWg8PTkWU>CwnqD>1ZX2B@;$DlX%C5MI+}{ +z9xQVnffR*~v2KAUj*hCdgul~`bk#mk`o>zk9)<2Uc8?hUZAEvd!`9em)~$Z)zev>w^8 +zyAgCP_$&Y)7HSQ84`xG}OeTavaEswwF|8Xpi5iZzZa@hCiv(J-%bfFC&)HLlO+Rhw +zG6g?9eL5&A!SuJnQ6}LxG%tU+@vZ`i+!+Rz6iYvsTdhnPo7lW{m-}{hya@viX4)XZ +zngaw+j;gloB#|UwI@8sOmQpc`h+bicQJnQIB5eifIMQNgD2+oai33m!34~xU|0Azj +zhu$8z+T5^;Pxx@d{N)pzOJLSa^e;aDf$W%N5XcOf!mGC9l9j$Ev2h6N+6ZQC+CJzl +zaM7?S!SrFLS2DASjj(h6y1WN3N?|bmqmyzm!&nLoE|`rKBOc_yDF$a#FsUn!IQf(t +zdC&Us(kQz*7mvH^j*^MC@>wTDb}g%~sx*ng#>{@lR=XG-Z5_ +z#<9*Oh0joMzt;nS)ObAp)347`D=}r-;nV!TbIq&xrGRGsF6fZg+!VkfUei@_&l-M& +zPqQ+Dw)RV}+)I8RuqAxa`Pv8e&!_gXS=e2-un>=Ktn}-;%lLZxaVn?Q>yZCb2R3Wk +z77zr%;Rq&h|2ncqyKYmFI0148JVY7Q$V5p=dWj+Qqpu%i|xp2C=WaOb2Wudn^h0EcD%$p9YVU1fnoRV9`(cy(vv6K>FXS!2jY>1GnU--7)4usH&K +zao*&P^@9~YmUe|ZdLW@C>H;!*Vt3>Nw4M*;=?j(TBD#O@XCv0|MEhA;z}kTFRv@`tPHhp=&Yh +zg%Zhg4i7o_k{a5i&f5;tZ==%}^Sn4aD_6%qs_XAuJt&EumdH4Yu`UjT<-+XHTuHss+b +YOmM2;hq8Egm*4=7_P9T{21QBYH*F=mfB*mh + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/hooks/prepare-commit-msg.sample b/tests/resources/small.git/hooks/prepare-commit-msg.sample +new file mode 100755 +index 0000000000000000000000000000000000000000..b1970da1b6d3f42f00069fd17c325de72cda812e +GIT binary patch +literal 1702 +zcmb_cTW{Mo6n>t6#i?x6xmZ$SFLf{QfG*3r0L?Pg?px55l8$UTGO3bO;spKi{V3XX +z))weX0X>M9bNMcZ-6yG%>(n}JI2|25dr}WZBP@ih?JX^+@ +zu#5O48P>yRX(mfDIhYP)doc1&TADZa@ZGpusJ$6G+e$ZMcmC +zoOosDQPS}l{H?YPsq(4;0SGkATa9eeqAaDcjq8n2wALbFwU@2i@FAaRV!=uw-nwx1gKn2SvY +z>Ff>;2sg!+Hxfkwv1lsiii=p6WenF=5)6LZcQaZ=aS_}+-4Y&?!@HWh|<^gJ21!|T@+%On#w6azxPHV}XsRbe*w +zR_TZ2XEsQa1lPK~biYqg@0-RW@5J1@=<87cFzEUABdCoFH2CZo?}l(Z*!OFqUxo>K +z_d`l#4d9|H6;VPT{X?^{VJ>oL|D7K{BJwwqB>`YcPoGk+9hbvHnoQ{EM|kPgD_`wk +zKm4#2xu;-y`RAm!=L_BnLvJ8$AZm8@?)v<%vwvsw8AF2x6!mTT;c72A_~U9nIq0ST +zv)N0!I!^1p=g8-RQfx5)E_Mb_4I2vtQpI30XZ&t-9h5!Hn + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/hooks/push-to-checkout.sample b/tests/resources/small.git/hooks/push-to-checkout.sample +new file mode 100755 +index 0000000000000000000000000000000000000000..a80611e18896f212c390d845e49a3f6d5693b41d +GIT binary patch +literal 2840 +zcmai0U31$u5PXh)#YOS7cE^-rw@uolNhe9&aUS|HtvhX>G$45tVUYj>fRdF?|9kfU +zNR~aG=E)WbEbeyq7JTw}ZuHIE2kUtL<AoeCNptd-NM1aZLhESzC;I`+Ns +zfmNNjdAp^W8#Q*}l>CT7RB9F5(BbI8ly2l~+E};JW|>&d1)=epZ-8vm8ppkbEVn#R +zt30a5A-c(YQR8eM5%;|UAnO>rt!&@x@G@yp+92%w-}%(5P_+P&Wf_zb$f-Qrl5(7z +z2ah(bkE;!DK(&aAMuQ%1TS>ai?wSXCOCSj=_}8x4IbCx^$}9q)whwv)SBt| +zg#MX4;;Oau`m=MI9(^&zPbueY@~>3*ixX%mvR5m_1&nAg@ZKvY1E$O}&EtLiG;mhV +z1xhMIm~fGjmf_#{62f`y;09?I7M1W2tWQvz<}i9lR>OpQyUJi45_&*pQus&EkwY<> +zI|ZAx=*3i9a-)g)hXkvO7>UJ5MNgL(Z+-wpXVcgbSgpmFmbf1~DPA(OVGI&FNLeIE +zNH!_aiH$vsif$_j7=T2{cS(!DOI`~bn@)vSd-0d7xL=DF;UNP|tW}4ih>DvHtu9tY_pbJ6x(6E*hxgC +zzNDao%qlr-IE%YGbS4hF!n!on7#W3$bX-_hbZAaws^nHu#)Dx=WzdbJ>AKzAy@T$x +zSWE^x9+|TEHVEPyaPYa0DOChp?AeHSBBDbZNokQpAY{lE!7geZI=jV)G^2@l)&91Zb1+`T+oq9wWF +zRV~kGTGce0O~p^6mj{kT5kL(pv>r;Lvd7VDX*P>A^Th`$3cWO0L81p4Ysdo3ZP1(SrR-peEdTo;-@bkB((G +zPHYQXUL!@Q$e(OQ;R9r%@Afz+50I7>*^^c&&|E*r-jN)LH=pM4AqMwWxSv|nqjddE +Z4{_hwv8!W(T +zYw`X3V>TCdnSD1ru8&`j=2DIPbCT@SnIgUw>$+lEYP}+x8(BMYnr=iT3*ndq)xzaV +z>I+qjv}vC#8_9M+b1p#uNS0M0)q

8!3p_LRQ0MA3M`!2foxzRUjbFY@}O~(ki=S +zqscnq8cU*dY)D$$cqE}n)V0yIk>CNKHCrndOtSP*HbOb;nbwAHSb;R+gs^?^Dve%) +zoW}t(*D}$>O3ab0TS^-;J|u&sb-PkZzo#kn*#xYt(;FGuwzSb^g&RDiGcOz9TB;Hu`nJh)$W=C=XCSm2AY=$w3G3P-V#Oo+N*;#2 +z4ijJ-pBZ=;T(RTgp_HYrD!uW-dTMfkuqY5jwOy)~gM;#=P^i{!l7`pXTS^s(&^{RU +zydaw}OpS#^D1cXM8?FW+fh`t7D(g;yr6|}fdaNtZBx3hlK~IpkTu3!Qq%R+zAo#t}Bs8^3$vHD+-TGT@`F>H1Cc#WAVW;&$S6%fE2d6@kLS0g&ihIM{}0z +z8#XhD>b>3{(BH|Px7}&lJ4%y1v(CihZJx@8MPoGdl*BJGD;usf*iS7%;{Joe; +zNFuBa>*~o&qETDPo~u&~$FxE1xb^x&(CbE`Y3GfsibL2rl+L;>P6j&Y3U>K$mkp*6 +zd`Q{<^+^&;GskGjwD-%!boR&i-TCA9UOR|@=GYb5x#+dhd7fkaVIR^pol`Mv+rUbmZ43dVL6^S7g3{NsPiG$iy$5EDB% +z6KIgnb$H(n&t3e4E6d4V7w^B?JS}JkG)PM6+X3Co`SQs($O*AA+MG~{S7RJ=cy-l& +z>~%3y`tjfx2>uOutB_^s +ziwG=e=ch|FQ0IkN91US7rhdQkXhwwt$gU0WEVDjo=IPb+?6PC=s8}J*ua(Ms))`UL +fi$|vMHn?H_tSE3ettp-hLlsZCxaLX8(nU;bVRB;Ce6@s#eu2|WvLz>- +zvy(&>Gyfp@+BtKnpqWkKi^+v{4jn_pNw_zeuxETifiGO|)w}OANj2n2D^K=o3j6P6uOL70#cbA{uzWXDlk1wr9GV1X(2W{RuTvjXV +zCmd8u +zH%V`94=q3)Dk)PHNrnFC(T1)Om6f{Usj;u1R->&XoCYVK2V3ZlgZuF?N}1+33OER*x +z*9Z=L=zI8CN>A_^jYjt0F$psO$sL=38q5q|SG)qCN6{^>RFh5E&l5GZ$pEahnF&d+ +z5c>64t}uJPkf~_!VUj#&N%nC-gUMj%=@B=!V>&}xtj2%@-mOm#rQUSJ3(ccmc+fza +znZ#uxF>N?QN5UrIEd!5RgHEfW#;(nKYF+D<*rdshJ$X-z2OZ2X;)nn@KSVdVhaA?}@3;6gZxb4v +zozoWSr{{+!h}zGpumG3H`=AvWpm^9kW;J$Jp^Xl*?8ckr`fqN%c|Z;VC0|cM4vSrk +zH_O8Yvh85nvJp^;``wo8=z0f`FWg?`>gO#y1hjX1{}rTlg9rwIKia8eyGexA3GnuR +z`Rg~XZoW;0pA)vI8=p5!+6sIn#C^FCvR>ffv39h6SCNi9v);%WD;WZ`of_MgwyRWy +z-yY%n*Y>X89W-v4`Ff%bx$Vkn}$!Ay}rnY6F$m-Kg*KD_+;Lx#g4|^&N +I02NaX#p`nv=Kufz + +literal 0 +HcmV?d00001 + +diff --git a/tests/resources/small.git/objects/af/5626b4a114abcb82d63db7c8082c3c4756e51b b/tests/resources/small.git/objects/af/5626b4a114abcb82d63db7c8082c3c4756e51b +new file mode 100644 +index 0000000000000000000000000000000000000000..822bc151862ec3763cf2d3fa2372b93bbd3a4b65 +GIT binary patch +literal 30 +mcmb>0i}&W3IZ_@1U=^!a~EV1casc=c+{&un1qQN*i9hD|0|m(2n|iwp*q%W +z%N;b$hu%cM`$TMo*~EnC1BFP&Pfj~;jZVKXQ96s_PhV<-XAROi+@-v8dBLUa`!;GB +k^iXlEv8$>R)1G>9th&t3j;s7J{?^9n|7U^`%mXoWC24Q^m!3%@{ + +literal 0 +HcmV?d00001 + diff --git a/scripts/bigsur-nixbld-user-migration.sh b/scripts/bigsur-nixbld-user-migration.sh index f1619fd56..bd9eeb920 100755 --- a/scripts/bigsur-nixbld-user-migration.sh +++ b/scripts/bigsur-nixbld-user-migration.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash -((NEW_NIX_FIRST_BUILD_UID=301)) +((NEW_NIX_FIRST_BUILD_UID=351)) -id_available(){ - dscl . list /Users UniqueID | grep -E '\b'$1'\b' >/dev/null +id_unavailable(){ + dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null } change_nixbld_names_and_ids(){ @@ -15,7 +15,7 @@ change_nixbld_names_and_ids(){ while read -r name uid; do echo " Checking $name (uid: $uid)" # iterate for a clean ID - while id_available "$next_id"; do + while id_unavailable "$next_id"; do ((next_id++)) if ((next_id >= 400)); then echo "We've hit UID 400 without placing all of your users :(" @@ -26,18 +26,18 @@ change_nixbld_names_and_ids(){ fi done - if [[ $name == _* ]]; then + if [[ "$name" == _* ]]; then echo " It looks like $name has already been renamed--skipping." else # first 3 are cleanup, it's OK if they aren't here - sudo dscl . delete /Users/$name dsAttrTypeNative:_writers_passwd &>/dev/null || true - sudo dscl . change /Users/$name NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true + sudo dscl . delete "/Users/$name" dsAttrTypeNative:_writers_passwd &>/dev/null || true + sudo dscl . change "/Users/$name" NFSHomeDirectory "/private/var/empty 1" "/var/empty" &>/dev/null || true # remove existing user from group - sudo dseditgroup -o edit -t user -d $name nixbld || true - sudo dscl . change /Users/$name UniqueID $uid $next_id - sudo dscl . change /Users/$name RecordName $name _$name + sudo dseditgroup -o edit -t user -d "$name" nixbld || true + sudo dscl . change "/Users/$name" UniqueID "$uid" "$next_id" + sudo dscl . change "/Users/$name" RecordName "$name" "_$name" # add renamed user to group - sudo dseditgroup -o edit -t user -a _$name nixbld + sudo dseditgroup -o edit -t user -a "_$name" nixbld echo " $name migrated to _$name (uid: $next_id)" fi done < <(dscl . list /Users UniqueID | grep nixbld | sort -n -k2) diff --git a/scripts/check-hydra-status.sh b/scripts/check-hydra-status.sh deleted file mode 100644 index e62705e94..000000000 --- a/scripts/check-hydra-status.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -# set -x - - -# mapfile BUILDS_FOR_LATEST_EVAL < <( -# curl -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ -# jq -r '.evals[0].builds[] | @sh') -BUILDS_FOR_LATEST_EVAL=$( -curl -sS -H 'Accept: application/json' https://hydra.nixos.org/jobset/nix/master/evals | \ - jq -r '.evals[0].builds[]') - -someBuildFailed=0 - -for buildId in $BUILDS_FOR_LATEST_EVAL; do - buildInfo=$(curl --fail -sS -H 'Accept: application/json' "https://hydra.nixos.org/build/$buildId") - - finished=$(echo "$buildInfo" | jq -r '.finished') - - if [[ $finished = 0 ]]; then - continue - fi - - buildStatus=$(echo "$buildInfo" | jq -r '.buildstatus') - - if [[ $buildStatus != 0 ]]; then - someBuildFailed=1 - echo "Job “$(echo "$buildInfo" | jq -r '.job')” failed on hydra: $buildInfo" - fi -done - -exit "$someBuildFailed" diff --git a/scripts/flake-regressions.sh b/scripts/flake-regressions.sh new file mode 100755 index 000000000..d76531134 --- /dev/null +++ b/scripts/flake-regressions.sh @@ -0,0 +1,27 @@ +#! /usr/bin/env bash + +set -e + +echo "Nix version:" +nix --version + +cd flake-regressions + +status=0 + +flakes=$(find tests -mindepth 3 -maxdepth 3 -type d -not -path '*/.*' | sort | head -n25) + +echo "Running flake tests..." + +for flake in $flakes; do + + if ! REGENERATE=0 ./eval-flake.sh "$flake"; then + status=1 + echo "❌ $flake" + else + echo "✅ $flake" + fi + +done + +exit "$status" diff --git a/scripts/install-darwin-multi-user.sh b/scripts/install-darwin-multi-user.sh index 24c9052f9..89c66b8f4 100644 --- a/scripts/install-darwin-multi-user.sh +++ b/scripts/install-darwin-multi-user.sh @@ -4,7 +4,17 @@ set -eu set -o pipefail # System specific settings -export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-301}" +# Notes: +# - up to macOS Big Sur we used the same GID/UIDs as Linux (30000:30001-32) +# - we changed UID to 301 because Big Sur updates failed into recovery mode +# we're targeting the 200-400 UID range for role users mentioned in the +# usage note for sysadminctl +# - we changed UID to 351 because Sequoia now uses UIDs 300-304 for its own +# daemon users +# - we changed GID to 350 alongside above just because it hides the nixbld +# group from the Users & Groups settings panel :) +export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-351}" +export NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-350}" export NIX_BUILD_USER_NAME_TEMPLATE="_nixbld%d" readonly NIX_DAEMON_DEST=/Library/LaunchDaemons/org.nixos.nix-daemon.plist diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index ad3ee8881..a487d459f 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -23,10 +23,10 @@ readonly RED='\033[31m' # installer allows overriding build user count to speed up installation # as creating each user takes non-trivial amount of time on macos readonly NIX_USER_COUNT=${NIX_USER_COUNT:-32} -readonly NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-30000}" readonly NIX_BUILD_GROUP_NAME="nixbld" # each system specific installer must set these: # NIX_FIRST_BUILD_UID +# NIX_BUILD_GROUP_ID # NIX_BUILD_USER_NAME_TEMPLATE # Please don't change this. We don't support it, because the # default shell profile that comes with Nix doesn't support it. @@ -530,9 +530,7 @@ It seems the build group $NIX_BUILD_GROUP_NAME already exists, but with the UID $primary_group_id. This script can't really handle that right now, so I'm going to give up. -You can fix this by editing this script and changing the -NIX_BUILD_GROUP_ID variable near the top to from $NIX_BUILD_GROUP_ID -to $primary_group_id and re-run. +You can export NIX_BUILD_GROUP_ID=$primary_group_id and re-run. EOF else row " Exists" "Yes" @@ -754,7 +752,7 @@ I will: (if it does, I will tell you how to clean them up.) - create local users (see the list above for the users I'll make) - create a local group ($NIX_BUILD_GROUP_NAME) - - install Nix in to $NIX_ROOT + - install Nix in $NIX_ROOT - create a configuration file in /etc/nix - set up the "default profile" by creating some Nix-related files in $ROOT_HOME diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index a62ed7e3a..a79a69990 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -5,6 +5,7 @@ set -o pipefail # System specific settings export NIX_FIRST_BUILD_UID="${NIX_FIRST_BUILD_UID:-30001}" +export NIX_BUILD_GROUP_ID="${NIX_BUILD_GROUP_ID:-30000}" export NIX_BUILD_USER_NAME_TEMPLATE="nixbld%d" readonly SERVICE_SRC=/lib/systemd/system/nix-daemon.service diff --git a/scripts/install.in b/scripts/install.in index 7d2e52b26..b4e808d8e 100755 --- a/scripts/install.in +++ b/scripts/install.in @@ -50,6 +50,11 @@ case "$(uname -s).$(uname -m)" in path=@tarballPath_armv7l-linux@ system=armv7l-linux ;; + Linux.riscv64) + hash=@tarballHash_riscv64-linux@ + path=@tarballPath_riscv64-linux@ + system=riscv64-linux + ;; Darwin.x86_64) hash=@tarballHash_x86_64-darwin@ path=@tarballPath_x86_64-darwin@ diff --git a/scripts/meson.build b/scripts/meson.build new file mode 100644 index 000000000..777da42b1 --- /dev/null +++ b/scripts/meson.build @@ -0,0 +1,20 @@ +configure_file( + input : 'nix-profile.sh.in', + output : 'nix-profile.sh', + configuration : { + 'localstatedir': localstatedir, + } +) + +foreach rc : [ '.sh', '.fish', '-daemon.sh', '-daemon.fish' ] + configure_file( + input : 'nix-profile' + rc + '.in', + output : 'nix' + rc, + install : true, + install_dir : get_option('profile-dir'), + install_mode : 'rw-r--r--', + configuration : { + 'localstatedir': localstatedir, + }, + ) +endforeach diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index 0ec72e797..59c00d491 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -1,4 +1,5 @@ # Only execute this file once per shell. +# This file is tested by tests/installer/default.nix. if [ -n "${__ETC_PROFILE_NIX_SOURCED:-}" ]; then return; fi __ETC_PROFILE_NIX_SOURCED=1 @@ -9,11 +10,9 @@ else NIX_LINK_NEW=$HOME/.local/state/nix/profile fi if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" -else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 else @@ -26,6 +25,7 @@ else printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 fi fi + NIX_LINK="$NIX_LINK_NEW" fi export NIX_PROFILES="@localstatedir@/nix/profiles/default $NIX_LINK" @@ -52,7 +52,7 @@ elif [ -e /etc/pki/tls/certs/ca-bundle.crt ]; then # Fedora, CentOS else # Fall back to what is in the nix profiles, favouring whatever is defined last. check_nix_profiles() { - if [ -n "$ZSH_VERSION" ]; then + if [ -n "${ZSH_VERSION:-}" ]; then # Zsh by default doesn't split words in unquoted parameter expansion. # Set local_options for these options to be reverted at the end of the function # and shwordsplit to force splitting words in $NIX_PROFILES below. diff --git a/scripts/nix-profile.sh.in b/scripts/nix-profile.sh.in index 44bc96e89..2d6bf6e95 100644 --- a/scripts/nix-profile.sh.in +++ b/scripts/nix-profile.sh.in @@ -1,30 +1,34 @@ -if [ -n "$HOME" ] && [ -n "$USER" ]; then +# This file is tested by tests/installer/default.nix. +if [ -n "${HOME-}" ] && [ -n "${USER-}" ]; then # Set up the per-user profile. - NIX_LINK="$HOME/.nix-profile" - if [ -n "${XDG_STATE_HOME-}" ]; then - NIX_LINK_NEW="$XDG_STATE_HOME/nix/profile" + if [ -n "${NIX_STATE_HOME-}" ]; then + NIX_LINK="$NIX_STATE_HOME/profile" else - NIX_LINK_NEW="$HOME/.local/state/nix/profile" - fi - if [ -e "$NIX_LINK_NEW" ]; then - NIX_LINK="$NIX_LINK_NEW" - else - if [ -t 2 ] && [ -e "$NIX_LINK_NEW" ]; then - warning="\033[1;35mwarning:\033[0m" - printf "$warning Both %s and legacy %s exist; using the latter.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 - if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then - printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 - else - # This should be an exceptionally rare occasion: the only way to get it would be to - # 1. Update to newer Nix; - # 2. Remove .nix-profile; - # 3. Set the $NIX_LINK_NEW to something other than the default user profile; - # 4. Roll back to older Nix. - # If someone did all that, they can probably figure out how to migrate the profile. - printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 + NIX_LINK="$HOME/.nix-profile" + if [ -n "${XDG_STATE_HOME-}" ]; then + NIX_LINK_NEW="$XDG_STATE_HOME/nix/profile" + else + NIX_LINK_NEW="$HOME/.local/state/nix/profile" + fi + if [ -e "$NIX_LINK_NEW" ]; then + if [ -t 2 ] && [ -e "$NIX_LINK" ]; then + warning="\033[1;35mwarning:\033[0m" + printf "$warning Both %s and legacy %s exist; using the former.\n" "$NIX_LINK_NEW" "$NIX_LINK" 1>&2 + if [ "$(realpath "$NIX_LINK")" = "$(realpath "$NIX_LINK_NEW")" ]; then + printf " Since the profiles match, you can safely delete either of them.\n" 1>&2 + else + # This should be an exceptionally rare occasion: the only way to get it would be to + # 1. Update to newer Nix; + # 2. Remove .nix-profile; + # 3. Set the $NIX_LINK_NEW to something other than the default user profile; + # 4. Roll back to older Nix. + # If someone did all that, they can probably figure out how to migrate the profile. + printf "$warning Profiles do not match. You should manually migrate from %s to %s.\n" "$NIX_LINK" "$NIX_LINK_NEW" 1>&2 + fi fi + NIX_LINK="$NIX_LINK_NEW" fi fi diff --git a/scripts/sequoia-nixbld-user-migration.sh b/scripts/sequoia-nixbld-user-migration.sh new file mode 100755 index 000000000..88e801706 --- /dev/null +++ b/scripts/sequoia-nixbld-user-migration.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash + +set -eo pipefail + +((NEW_NIX_FIRST_BUILD_UID=351)) +((TEMP_NIX_FIRST_BUILD_UID=31000)) + +nix_user_n() { + printf "_nixbld%d" "$1" +} + +id_unavailable(){ + dscl . list /Users UniqueID | grep -E '\b'"$1"'\b' >/dev/null +} + +any_nixbld(){ + dscl . list /Users UniqueID | grep -E '\b_nixbld' >/dev/null +} + +dsclattr() { + dscl . -read "$1" | awk "/$2/ { print \$2 }" +} + +re_create_nixbld_user(){ + local name uid + + name="$1" + uid="$2" + gid="$3" + + sudo /usr/bin/dscl . -create "/Users/$name" "UniqueID" "$uid" + sudo /usr/bin/dscl . -create "/Users/$name" "IsHidden" "1" + sudo /usr/bin/dscl . -create "/Users/$name" "NFSHomeDirectory" "/var/empty" + sudo /usr/bin/dscl . -create "/Users/$name" "RealName" "Nix build user $name" + sudo /usr/bin/dscl . -create "/Users/$name" "UserShell" "/sbin/nologin" + sudo /usr/bin/dscl . -create "/Users/$name" "PrimaryGroupID" "$gid" +} + +hit_id_cap(){ + echo "We've hit UID 400 without placing all of your users :(" + echo "You should use the commands in this script as a starting" + echo "point to review your UID-space and manually move the" + echo "remaining users (or delete them, if you don't need them)." +} + +# evacuate the role-uid space to simplify final placement logic +temporarily_move_existing_nixbld_uids(){ + local name uid next_id user_n + + ((next_id=TEMP_NIX_FIRST_BUILD_UID)) + + echo "" + echo "Step 1: move existing _nixbld users out of the destination UID range." + + while read -r name uid; do + # iterate for a clean ID + while id_unavailable "$next_id"; do + ((next_id++)) + # We really want to get these all placed, but I guess there's + # some risk we iterate forever--so we'll give up after 9k uids. + if ((next_id >= 40000)); then + echo "We've hit UID 40000 without temporarily placing all of your users :(" + echo "You should use the commands in this script as a starting" + echo "point to review your UID-space and manually move the" + echo "remaining users to any open UID over 1000." + exit 1 + fi + done + sudo dscl . -create "/Users/$name" UniqueID "$next_id" + echo " Temporarily moved $name from uid $uid -> $next_id" + + done < <(dscl . list /Users UniqueID | grep _nixbld | sort -n -k2) +} + +change_nixbld_uids(){ + local existing_gid name next_id user_n + + ((next_id=NEW_NIX_FIRST_BUILD_UID)) + ((user_n=1)) + name="$(nix_user_n "$user_n")" + existing_gid="$(dsclattr "/Groups/nixbld" "PrimaryGroupID")" + + # we know that we have *some* nixbld users, but macOS may have + # already clobbered the first few users if this system has been + # upgraded + + echo "" + echo "Step 2: re-create missing early _nixbld# users." + + until dscl . read "/Users/$name" &>/dev/null; do + # iterate for a clean ID + while id_unavailable "$next_id"; do + ((next_id++)) + if ((next_id >= 400)); then + hit_id_cap + exit 1 + fi + done + + re_create_nixbld_user "$name" "$next_id" "$existing_gid" + echo " $name was missing; created with uid: $next_id" + + ((user_n++)) + name="$(nix_user_n "$user_n")" + done + + echo "" + echo "Step 3: relocate remaining _nixbld# UIDs to $next_id+" + + # start at first _nixbld# not re-created above and increment + # until _nixbld doesn't exist + while dscl . read "/Users/$name" &>/dev/null; do + # iterate for a clean ID + while id_unavailable "$next_id"; do + ((next_id++)) + if ((next_id >= 400)); then + hit_id_cap + exit 1 + fi + done + + sudo dscl . -create "/Users/$name" UniqueID "$next_id" + echo " $name migrated to uid: $next_id" + + ((user_n++)) + name="$(nix_user_n "$user_n")" + done + + if ((user_n == 1)); then + echo "Didn't find _nixbld1. Perhaps you have single-user Nix?" + exit 1 + else + echo "Migrated $((user_n - 1)) users. If you want to double-check, try:" + echo "dscl . list /Users UniqueID | grep _nixbld | sort -n -k2" + fi +} +needs_migration(){ + local name uid next_id user_n + + ((next_id=NEW_NIX_FIRST_BUILD_UID)) + ((user_n=1)) + + while read -r name uid; do + expected_name="$(nix_user_n "$user_n")" + if [[ "$expected_name" != "$name" ]]; then + return 0 + fi + if [[ "$next_id" != "$uid" ]]; then + return 0 + fi + ((next_id++)) + ((user_n++)) + done < <(dscl . list /Users UniqueID | grep _nixbld | sort -n -k2) + return 1 +} + + +if any_nixbld; then + if needs_migration; then + echo "Attempting to migrate _nixbld users." + temporarily_move_existing_nixbld_uids + change_nixbld_uids + else + echo "_nixbld users already appear to be migrated." + fi +else + echo "Didn't find any _nixbld users. Perhaps you have single-user Nix?" + exit 1 +fi diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 18eee830b..82ad7d862 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -11,11 +11,13 @@ #include "machines.hh" #include "shared.hh" +#include "plugin.hh" #include "pathlocks.hh" #include "globals.hh" #include "serialise.hh" #include "build-result.hh" #include "store-api.hh" +#include "strings.hh" #include "derivations.hh" #include "local-store.hh" #include "legacy.hh" @@ -37,7 +39,7 @@ static std::string currentLoad; static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot) { - return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true); + return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri.render()), slot), true); } static bool allSupportedLocally(Store & store, const std::set& requiredFeatures) { @@ -135,7 +137,7 @@ static int main_build_remote(int argc, char * * argv) Machine * bestMachine = nullptr; uint64_t bestLoad = 0; for (auto & m : machines) { - debug("considering building on remote machine '%s'", m.storeUri); + debug("considering building on remote machine '%s'", m.storeUri.render()); if (m.enabled && m.systemSupported(neededSystem) && @@ -232,17 +234,16 @@ static int main_build_remote(int argc, char * * argv) lock = -1; try { + storeUri = bestMachine->storeUri.render(); - Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", bestMachine->storeUri)); + Activity act(*logger, lvlTalkative, actUnknown, fmt("connecting to '%s'", storeUri)); sshStore = bestMachine->openStore(); sshStore->connect(); - storeUri = bestMachine->storeUri; - } catch (std::exception & e) { auto msg = chomp(drainFD(5, false)); printError("cannot build on '%s': %s%s", - bestMachine->storeUri, e.what(), + storeUri, e.what(), msg.empty() ? "" : ": " + msg); bestMachine->enabled = false; continue; @@ -262,7 +263,20 @@ connected: auto inputs = readStrings(source); auto wantedOutputs = readStrings(source); - AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true); + AutoCloseFD uploadLock; + { + auto setUpdateLock = [&](auto && fileName){ + uploadLock = openLockFile(currentLoad + "/" + escapeUri(fileName) + ".upload-lock", true); + }; + try { + setUpdateLock(storeUri); + } catch (SysError & e) { + if (e.errNo != ENAMETOOLONG) throw; + // Try again hashing the store URL so we have a shorter path + auto h = hashString(HashAlgorithm::MD5, storeUri); + setUpdateLock(h.to_string(HashFormat::Base64, false)); + } + } { Activity act(*logger, lvlTalkative, actUnknown, fmt("waiting for the upload lock to '%s'", storeUri)); diff --git a/doc/external-api/.gitignore b/src/external-api-docs/.gitignore similarity index 100% rename from doc/external-api/.gitignore rename to src/external-api-docs/.gitignore diff --git a/src/external-api-docs/.version b/src/external-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/external-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/doc/external-api/README.md b/src/external-api-docs/README.md similarity index 93% rename from doc/external-api/README.md rename to src/external-api-docs/README.md index 167c02199..8760ac88b 100644 --- a/doc/external-api/README.md +++ b/src/external-api-docs/README.md @@ -27,7 +27,7 @@ appreciated. The following examples, for simplicity, don't include error handling. See the [Handling errors](@ref errors) section for more information. -# Embedding the Nix Evaluator +# Embedding the Nix Evaluator{#nix_evaluator_example} In this example we programmatically start the Nix language evaluator with a dummy store (that has no store paths and cannot be written to), and evaluate the @@ -46,9 +46,9 @@ Nix expression `builtins.nixVersion`. // NOTE: This example lacks all error handling. Production code must check for // errors, as some return values will be undefined. -void my_get_string_cb(const char * start, unsigned int n, char ** user_data) +void my_get_string_cb(const char * start, unsigned int n, void * user_data) { - *user_data = strdup(start); + *((char **) user_data) = strdup(start); } int main() @@ -63,7 +63,7 @@ int main() nix_value_force(NULL, state, value); char * version; - nix_get_string(NULL, value, my_get_string_cb, version); + nix_get_string(NULL, value, my_get_string_cb, &version); printf("Nix version: %s\n", version); free(version); diff --git a/doc/external-api/doxygen.cfg.in b/src/external-api-docs/doxygen.cfg.in similarity index 91% rename from doc/external-api/doxygen.cfg.in rename to src/external-api-docs/doxygen.cfg.in index cd8b4989b..1be71d895 100644 --- a/doc/external-api/doxygen.cfg.in +++ b/src/external-api-docs/doxygen.cfg.in @@ -12,7 +12,9 @@ PROJECT_NAME = "Nix" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = @PACKAGE_VERSION@ +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -36,10 +38,10 @@ GENERATE_LATEX = NO # so they can expand variables despite configure variables. INPUT = \ - src/libutil-c \ - src/libexpr-c \ - src/libstore-c \ - doc/external-api/README.md + @src@/src/libutil-c \ + @src@/src/libexpr-c \ + @src@/src/libstore-c \ + @src@/doc/external-api/README.md FILE_PATTERNS = nix_api_*.h *.md @@ -49,7 +51,6 @@ FILE_PATTERNS = nix_api_*.h *.md # RECURSIVE has no effect here. # This tag requires that the tag SEARCH_INCLUDES is set to YES. -INCLUDE_PATH = @RAPIDCHECK_HEADERS@ EXCLUDE_PATTERNS = *_internal.h GENERATE_TREEVIEW = YES OPTIMIZE_OUTPUT_FOR_C = YES diff --git a/src/external-api-docs/meson.build b/src/external-api-docs/meson.build new file mode 100644 index 000000000..62474ffe4 --- /dev/null +++ b/src/external-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-external-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())), + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'external-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/external-api', + build_always_stale : true, +) diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix new file mode 100644 index 000000000..743b3e9b7 --- /dev/null +++ b/src/external-api-docs/package.nix @@ -0,0 +1,59 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-external-api-docs"; + inherit version; + + workDir = ./.; + fileset = + let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "h"); + in + fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + ./README.md + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../libexpr-c) + (cpp ../libstore-c) + (cpp ../libutil-c) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/doc/internal-api/.gitignore b/src/internal-api-docs/.gitignore similarity index 100% rename from doc/internal-api/.gitignore rename to src/internal-api-docs/.gitignore diff --git a/src/internal-api-docs/.version b/src/internal-api-docs/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/internal-api-docs/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/doc/internal-api/doxygen.cfg.in b/src/internal-api-docs/doxygen.cfg.in similarity index 84% rename from doc/internal-api/doxygen.cfg.in rename to src/internal-api-docs/doxygen.cfg.in index 6c6c325bd..f1ef75b38 100644 --- a/doc/internal-api/doxygen.cfg.in +++ b/src/internal-api-docs/doxygen.cfg.in @@ -12,7 +12,9 @@ PROJECT_NAME = "Nix" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = @PACKAGE_VERSION@ +PROJECT_NUMBER = @PROJECT_NUMBER@ + +OUTPUT_DIRECTORY = @OUTPUT_DIRECTORY@ # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -36,27 +38,27 @@ GENERATE_LATEX = NO # so they can expand variables despite configure variables. INPUT = \ - src/libcmd \ - src/libexpr \ - src/libexpr/flake \ - tests/unit/libexpr \ - tests/unit/libexpr/value \ - tests/unit/libexpr/test \ - tests/unit/libexpr/test/value \ - src/libexpr/value \ - src/libfetchers \ - src/libmain \ - src/libstore \ - src/libstore/build \ - src/libstore/builtins \ - tests/unit/libstore \ - tests/unit/libstore/test \ - src/libutil \ - tests/unit/libutil \ - tests/unit/libutil/test \ - src/nix \ - src/nix-env \ - src/nix-store + @src@/libcmd \ + @src@/libexpr \ + @src@/libexpr/flake \ + @src@/nix-expr-tests \ + @src@/nix-expr-tests/value \ + @src@/nix-expr-test-support/test \ + @src@/nix-expr-test-support/test/value \ + @src@/libexpr/value \ + @src@/libfetchers \ + @src@/libmain \ + @src@/libstore \ + @src@/libstore/build \ + @src@/libstore/builtins \ + @src@/nix-store-tests \ + @src@/nix-store-test-support/test \ + @src@/libutil \ + @src@/nix-util-tests \ + @src@/nix-util-test-support/test \ + @src@/nix \ + @src@/nix-env \ + @src@/nix-store # If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names # in the source code. If set to NO, only conditional compilation will be diff --git a/src/internal-api-docs/meson.build b/src/internal-api-docs/meson.build new file mode 100644 index 000000000..54eb7e5dd --- /dev/null +++ b/src/internal-api-docs/meson.build @@ -0,0 +1,31 @@ +project('nix-internal-api-docs', + version : files('.version'), + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +doxygen_cfg = configure_file( + input : 'doxygen.cfg.in', + output : 'doxygen.cfg', + configuration : { + 'PROJECT_NUMBER': meson.project_version(), + 'OUTPUT_DIRECTORY' : meson.current_build_dir(), + 'src' : fs.parent(fs.parent(meson.project_source_root())) / 'src', + }, +) + +doxygen = find_program('doxygen', native : true, required : true) + +custom_target( + 'internal-api-docs', + command : [ doxygen , doxygen_cfg ], + input : [ + doxygen_cfg, + ], + output : 'html', + install : true, + install_dir : get_option('datadir') / 'doc/nix/internal-api', + build_always_stale : true, +) diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix new file mode 100644 index 000000000..07ca6d4d9 --- /dev/null +++ b/src/internal-api-docs/package.nix @@ -0,0 +1,54 @@ +{ lib +, mkMesonDerivation + +, meson +, ninja +, doxygen + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-internal-api-docs"; + inherit version; + + workDir = ./.; + fileset = let + cpp = fileset.fileFilter (file: file.hasExt "cc" || file.hasExt "hh"); + in fileset.unions [ + ./.version + ../../.version + ./meson.build + ./doxygen.cfg.in + # Source is not compiled, but still must be available for Doxygen + # to gather comments. + (cpp ../.) + ]; + + nativeBuildInputs = [ + meson + ninja + doxygen + ]; + + preConfigure = + '' + chmod u+w ./.version + echo ${finalAttrs.version} > ./.version + ''; + + postInstall = '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + meta = { + platforms = lib.platforms.all; + }; +}) diff --git a/src/libcmd/.version b/src/libcmd/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libcmd/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libcmd/build-utils-meson b/src/libcmd/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libcmd/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index c5eb93c5d..905e70f32 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -1,6 +1,7 @@ #include "built-path.hh" #include "derivations.hh" #include "store-api.hh" +#include "comparator.hh" #include @@ -8,30 +9,24 @@ namespace nix { -#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ - bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ - { \ - const MY_TYPE* me = this; \ - auto fields1 = std::tie(*me->drvPath, me->FIELD); \ - me = &other; \ - auto fields2 = std::tie(*me->drvPath, me->FIELD); \ - return fields1 COMPARATOR fields2; \ - } -#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleBuiltPathBuilt, + *me->drvPath, + me->output); -#define FIELD_TYPE std::pair -CMP(SingleBuiltPath, SingleBuiltPathBuilt, output) -#undef FIELD_TYPE +// Custom implementation to avoid `ref` ptr equality -#define FIELD_TYPE std::map -CMP(SingleBuiltPath, BuiltPathBuilt, outputs) -#undef FIELD_TYPE - -#undef CMP -#undef CMP_ONE +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + BuiltPathBuilt ::, + BuiltPathBuilt, + *me->drvPath, + me->outputs); StorePath SingleBuiltPath::outPath() const { diff --git a/src/libcmd/built-path.hh b/src/libcmd/built-path.hh index 99917e0ee..dc78d3e59 100644 --- a/src/libcmd/built-path.hh +++ b/src/libcmd/built-path.hh @@ -18,7 +18,8 @@ struct SingleBuiltPathBuilt { static SingleBuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - DECLARE_CMP(SingleBuiltPathBuilt); + bool operator ==(const SingleBuiltPathBuilt &) const noexcept; + std::strong_ordering operator <=>(const SingleBuiltPathBuilt &) const noexcept; }; using _SingleBuiltPathRaw = std::variant< @@ -33,6 +34,9 @@ struct SingleBuiltPath : _SingleBuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = SingleBuiltPathBuilt; + bool operator == (const SingleBuiltPath &) const = default; + auto operator <=> (const SingleBuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } @@ -59,11 +63,13 @@ struct BuiltPathBuilt { ref drvPath; std::map outputs; + bool operator == (const BuiltPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //std::strong_ordering operator <=> (const BuiltPathBuilt &) const noexcept; + std::string to_string(const StoreDirConfig & store) const; static BuiltPathBuilt parse(const StoreDirConfig & store, std::string_view, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - - DECLARE_CMP(BuiltPathBuilt); }; using _BuiltPathRaw = std::variant< @@ -82,6 +88,10 @@ struct BuiltPath : _BuiltPathRaw { using Opaque = DerivedPathOpaque; using Built = BuiltPathBuilt; + bool operator == (const BuiltPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BuiltPath &) const = default; + inline const Raw & raw() const { return static_cast(*this); } diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 543250da3..6d8bfc19b 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -1,3 +1,5 @@ +#include + #include "command.hh" #include "markdown.hh" #include "store-api.hh" @@ -6,8 +8,7 @@ #include "nixexpr.hh" #include "profiles.hh" #include "repl.hh" - -#include +#include "strings.hh" extern char * * environ __attribute__((weak)); @@ -126,14 +127,9 @@ ref EvalCommand::getEvalState() { if (!evalState) { evalState = - #if HAVE_BOEHMGC - std::allocate_shared(traceable_allocator(), - lookupPath, getEvalStore(), getStore()) - #else - std::make_shared( - lookupPath, getEvalStore(), getStore()) - #endif - ; + std::allocate_shared( + traceable_allocator(), + lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()); evalState->repair = repair; diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index 155b43b70..ccbf957d9 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -1,18 +1,59 @@ +#include "fetch-settings.hh" #include "eval-settings.hh" #include "common-eval-args.hh" #include "shared.hh" +#include "config-global.hh" #include "filetransfer.hh" #include "eval.hh" #include "fetchers.hh" #include "registry.hh" #include "flake/flakeref.hh" +#include "flake/settings.hh" #include "store-api.hh" #include "command.hh" #include "tarball.hh" #include "fetch-to-store.hh" +#include "compatibility-settings.hh" +#include "eval-settings.hh" namespace nix { +namespace fs { using namespace std::filesystem; } + +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + +EvalSettings evalSettings { + settings.readOnlyMode, + { + { + "flake", + [](ref store, std::string_view rest) { + experimentalFeatureSettings.require(Xp::Flakes); + // FIXME `parseFlakeRef` should take a `std::string_view`. + auto flakeRef = parseFlakeRef(fetchSettings, std::string { rest }, {}, true, false); + debug("fetching flake search path element '%s''", rest); + auto storePath = flakeRef.resolve(store).fetchTree(store).first; + return store->toRealPath(storePath); + }, + }, + }, +}; + +static GlobalConfig::Register rEvalSettings(&evalSettings); + + +flake::Settings flakeSettings; + +static GlobalConfig::Register rFlakeSettings(&flakeSettings); + + +CompatibilitySettings compatibilitySettings {}; + +static GlobalConfig::Register rCompatibilitySettings(&compatibilitySettings); + + MixEvalArgs::MixEvalArgs() { addFlag({ @@ -20,7 +61,7 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the value *expr* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "expr"}, - .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr(expr)}); }} + .handler = {[&](std::string name, std::string expr) { autoArgs.insert_or_assign(name, AutoArg{AutoArgExpr{expr}}); }} }); addFlag({ @@ -28,7 +69,7 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the string *string* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "string"}, - .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString(s)}); }}, + .handler = {[&](std::string name, std::string s) { autoArgs.insert_or_assign(name, AutoArg{AutoArgString{s}}); }}, }); addFlag({ @@ -36,7 +77,7 @@ MixEvalArgs::MixEvalArgs() .description = "Pass the contents of file *path* as the argument *name* to Nix functions.", .category = category, .labels = {"name", "path"}, - .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile(path)}); }}, + .handler = {[&](std::string name, std::string path) { autoArgs.insert_or_assign(name, AutoArg{AutoArgFile{path}}); }}, .completer = completePath }); @@ -52,75 +93,11 @@ MixEvalArgs::MixEvalArgs() .longName = "include", .shortName = 'I', .description = R"( - Add *path* to the Nix search path. The Nix search path is - initialized from the colon-separated [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment - variable, and is used to look up the location of Nix expressions using [paths](@docroot@/language/values.md#type-path) enclosed in angle - brackets (i.e., ``). + Add *path* to search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md) - For instance, passing + This option may be given multiple times. - ``` - -I /home/eelco/Dev - -I /etc/nixos - ``` - - will cause Nix to look for paths relative to `/home/eelco/Dev` and - `/etc/nixos`, in that order. This is equivalent to setting the - `NIX_PATH` environment variable to - - ``` - /home/eelco/Dev:/etc/nixos - ``` - - It is also possible to match paths against a prefix. For example, - passing - - ``` - -I nixpkgs=/home/eelco/Dev/nixpkgs-branch - -I /etc/nixos - ``` - - will cause Nix to search for `` in - `/home/eelco/Dev/nixpkgs-branch/path` and `/etc/nixos/nixpkgs/path`. - - If a path in the Nix search path starts with `http://` or `https://`, - it is interpreted as the URL of a tarball that will be downloaded and - unpacked to a temporary location. The tarball must consist of a single - top-level directory. For example, passing - - ``` - -I nixpkgs=https://github.com/NixOS/nixpkgs/archive/master.tar.gz - ``` - - tells Nix to download and use the current contents of the `master` - branch in the `nixpkgs` repository. - - The URLs of the tarballs from the official `nixos.org` channels - (see [the manual page for `nix-channel`](../nix-channel.md)) can be - abbreviated as `channel:`. For instance, the - following two flags are equivalent: - - ``` - -I nixpkgs=channel:nixos-21.05 - -I nixpkgs=https://nixos.org/channels/nixos-21.05/nixexprs.tar.xz - ``` - - You can also fetch source trees using [flake URLs](./nix3-flake.md#url-like-syntax) and add them to the - search path. For instance, - - ``` - -I nixpkgs=flake:nixpkgs - ``` - - specifies that the prefix `nixpkgs` shall refer to the source tree - downloaded from the `nixpkgs` entry in the flake registry. Similarly, - - ``` - -I nixpkgs=flake:github:NixOS/nixpkgs/nixos-22.05 - ``` - - makes `` refer to a particular branch of the - `NixOS/nixpkgs` repository on GitHub. + Paths added through `-I` take precedence over the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path) and the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH). )", .category = category, .labels = {"path"}, @@ -144,8 +121,8 @@ MixEvalArgs::MixEvalArgs() .category = category, .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { - auto from = parseFlakeRef(_from, absPath(".")); - auto to = parseFlakeRef(_to, absPath(".")); + auto from = parseFlakeRef(fetchSettings, _from, fs::current_path().string()); + auto to = parseFlakeRef(fetchSettings, _to, fs::current_path().string()); fetchers::Attrs extraAttrs; if (to.subdir != "") extraAttrs["dir"] = to.subdir; fetchers::overrideRegistry(from.input, to.input, extraAttrs); @@ -175,7 +152,7 @@ Bindings * MixEvalArgs::getAutoArgs(EvalState & state) auto v = state.allocValue(); std::visit(overloaded { [&](const AutoArgExpr & arg) { - state.mkThunk_(*v, state.parseExprFromString(arg.expr, state.rootPath("."))); + state.mkThunk_(*v, state.parseExprFromString(arg.expr, compatibilitySettings.nixShellShebangArgumentsRelativeToScript ? state.rootPath(absPath(getCommandBaseDir())) : state.rootPath("."))); }, [&](const AutoArgString & arg) { v->mkString(arg.s); @@ -196,14 +173,16 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas { if (EvalSettings::isPseudoUrl(s)) { auto accessor = fetchers::downloadTarball( - EvalSettings::resolvePseudoUrl(s)).accessor; + state.store, + state.fetchSettings, + EvalSettings::resolvePseudoUrl(s)); auto storePath = fetchToStore(*state.store, SourcePath(accessor), FetchMode::Copy); return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } else if (hasPrefix(s, "flake:")) { experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(std::string(s.substr(6)), {}, true, false); + auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto storePath = flakeRef.resolve(state.store).fetchTree(state.store).first; return state.rootPath(CanonPath(state.store->toRealPath(storePath))); } diff --git a/src/libcmd/common-eval-args.hh b/src/libcmd/common-eval-args.hh index 75cb19334..c62365b32 100644 --- a/src/libcmd/common-eval-args.hh +++ b/src/libcmd/common-eval-args.hh @@ -11,10 +11,37 @@ namespace nix { class Store; + +namespace fetchers { struct Settings; } + class EvalState; +struct EvalSettings; +struct CompatibilitySettings; class Bindings; struct SourcePath; +namespace flake { struct Settings; } + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern EvalSettings evalSettings; + +/** + * @todo Get rid of global setttings variables + */ +extern flake::Settings flakeSettings; + +/** + * Settings that control behaviors that have changed since Nix 2.3. + */ +extern CompatibilitySettings compatibilitySettings; + struct MixEvalArgs : virtual Args, virtual MixRepair { static constexpr auto category = "Common evaluation options"; diff --git a/src/libcmd/compatibility-settings.hh b/src/libcmd/compatibility-settings.hh new file mode 100644 index 000000000..a129a957a --- /dev/null +++ b/src/libcmd/compatibility-settings.hh @@ -0,0 +1,36 @@ +#pragma once +#include "config.hh" + +namespace nix { +struct CompatibilitySettings : public Config +{ + + CompatibilitySettings() = default; + + // Added in Nix 2.24, July 2024. + Setting nixShellAlwaysLooksForShellNix{this, true, "nix-shell-always-looks-for-shell-nix", R"( + Before Nix 2.24, [`nix-shell`](@docroot@/command-ref/nix-shell.md) would only look at `shell.nix` if it was in the working directory - when no file was specified. + + Since Nix 2.24, `nix-shell` always looks for a `shell.nix`, whether that's in the working directory, or in a directory that was passed as an argument. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; + + // Added in Nix 2.24, July 2024. + Setting nixShellShebangArgumentsRelativeToScript{ + this, true, "nix-shell-shebang-arguments-relative-to-script", R"( + Before Nix 2.24, relative file path expressions in arguments in a `nix-shell` shebang were resolved relative to the working directory. + + Since Nix 2.24, `nix-shell` resolves these paths in a manner that is relative to the [base directory](@docroot@/glossary.md#gloss-base-directory), defined as the script's directory. + + You may set this to `false` to temporarily revert to the behavior of Nix 2.23 and older. + + Using this setting is not recommended. + It will be deprecated and removed. + )"}; +}; + +}; diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 3ec1c1614..8917e7a01 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -75,6 +75,8 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() std::set outputsToInstall; for (auto & output : packageInfo.queryOutputs(false, true)) outputsToInstall.insert(output.first); + if (outputsToInstall.empty()) + outputsToInstall.insert("out"); return OutputsSpec::Names { std::move(outputsToInstall) }; }, [&](const ExtendedOutputsSpec::Explicit & e) -> OutputsSpec { diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 6ff837ddc..6c9ee6748 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -43,20 +43,6 @@ std::vector InstallableFlake::getActualAttrPaths() return res; } -Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) -{ - auto vFlake = state.allocValue(); - - callFlake(state, lockedFlake, *vFlake); - - auto aOutputs = vFlake->attrs()->get(state.symbols.create("outputs")); - assert(aOutputs); - - state.forceValue(*aOutputs->value, aOutputs->value->determinePos(noPos)); - - return aOutputs->value; -} - static std::string showAttrPaths(const std::vector & paths) { std::string s; @@ -106,19 +92,24 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() fmt("while evaluating the flake output attribute '%s'", attrPath))) { return { *derivedPathWithInfo }; + } else { + throw Error( + "expected flake output attribute '%s' to be a derivation or path but found %s: %s", + attrPath, + showType(v), + ValuePrinter(*this->state, v, errorPrintOptions) + ); } - else - throw Error("flake output attribute '%s' is not a derivation or path", attrPath); } auto drvPath = attr->forceDerivation(); - std::optional priority; + std::optional priority; if (attr->maybeGetAttr(state->sOutputSpecified)) { } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { if (auto aPriority = aMeta->maybeGetAttr("priority")) - priority = aPriority->getInt(); + priority = aPriority->getInt().value; } return {{ @@ -205,7 +196,8 @@ std::shared_ptr InstallableFlake::getLockedFlake() const flake::LockFlags lockFlagsApplyConfig = lockFlags; // FIXME why this side effect? lockFlagsApplyConfig.applyNixConfig = true; - _lockedFlake = std::make_shared(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); + _lockedFlake = std::make_shared(lockFlake( + flakeSettings, *state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; } diff --git a/src/libcmd/installable-flake.hh b/src/libcmd/installable-flake.hh index 314918c14..8e0a232ef 100644 --- a/src/libcmd/installable-flake.hh +++ b/src/libcmd/installable-flake.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "common-eval-args.hh" #include "installable-value.hh" namespace nix { @@ -52,8 +53,6 @@ struct InstallableFlake : InstallableValue std::vector getActualAttrPaths(); - Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); - DerivedPathsWithInfo toDerivedPaths() override; std::pair toValue(EvalState & state) override; @@ -80,7 +79,7 @@ struct InstallableFlake : InstallableValue */ static inline FlakeRef defaultNixpkgsFlakeRef() { - return FlakeRef::fromAttrs({{"type","indirect"}, {"id", "nixpkgs"}}); + return FlakeRef::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", "nixpkgs"}}); } ref openEvalCache( diff --git a/src/libcmd/installable-value.hh b/src/libcmd/installable-value.hh index f300d392b..60207cd23 100644 --- a/src/libcmd/installable-value.hh +++ b/src/libcmd/installable-value.hh @@ -40,7 +40,7 @@ struct ExtraPathInfoValue : ExtraPathInfo /** * An optional priority for use with "build envs". See Package */ - std::optional priority; + std::optional priority; /** * The attribute path associated with this value. The idea is @@ -66,7 +66,7 @@ struct ExtraPathInfoValue : ExtraPathInfo }; /** - * An Installable which corresponds a Nix langauge value, in addition to + * An Installable which corresponds a Nix language value, in addition to * a collection of \ref DerivedPath "derived paths". */ struct InstallableValue : Installable diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 43e312540..f9d6d8ce8 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -27,8 +27,12 @@ #include +#include "strings-inline.hh" + namespace nix { +namespace fs { using namespace std::filesystem; } + void completeFlakeInputPath( AddCompletions & completions, ref evalState, @@ -84,7 +88,7 @@ MixFlakeOptions::MixFlakeOptions() > **DEPRECATED** > - > Use [`--no-use-registries`](#opt-no-use-registries) instead. + > Use [`--no-use-registries`](@docroot@/command-ref/conf-file.md#conf-use-registries) instead. )", .category = category, .handler = {[&]() { @@ -129,7 +133,7 @@ MixFlakeOptions::MixFlakeOptions() lockFlags.writeLockFile = false; lockFlags.inputOverrides.insert_or_assign( flake::parseInputPath(inputPath), - parseFlakeRef(flakeRef, absPath(getCommandBaseDir()), true)); + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir()), true)); }}, .completer = {[&](AddCompletions & completions, size_t n, std::string_view prefix) { if (n == 0) { @@ -170,14 +174,15 @@ MixFlakeOptions::MixFlakeOptions() .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); auto flake = flake::lockFlake( + flakeSettings, *evalState, - parseFlakeRef(flakeRef, absPath(getCommandBaseDir())), + parseFlakeRef(fetchSettings, flakeRef, absPath(getCommandBaseDir())), { .writeLockFile = false }); for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes if (auto input3 = std::dynamic_pointer_cast(input2)) { overrideRegistry( - fetchers::Input::fromAttrs({{"type","indirect"}, {"id", inputName}}), + fetchers::Input::fromAttrs(fetchSettings, {{"type","indirect"}, {"id", inputName}}), input3->lockedRef.input, {}); } @@ -289,10 +294,10 @@ void SourceExprCommand::completeInstallable(AddCompletions & completions, std::s if (v2.type() == nAttrs) { for (auto & i : *v2.attrs()) { - std::string name = state->symbols[i.name]; + std::string_view name = state->symbols[i.name]; if (name.find(searchWord) == 0) { if (prefix_ == "") - completions.add(name); + completions.add(std::string(name)); else completions.add(prefix_ + "." + name); } @@ -338,10 +343,11 @@ void completeFlakeRefWithFragment( auto flakeRefS = std::string(prefix.substr(0, hash)); // TODO: ideally this would use the command base directory instead of assuming ".". - auto flakeRef = parseFlakeRef(expandTilde(flakeRefS), absPath(".")); + auto flakeRef = parseFlakeRef(fetchSettings, expandTilde(flakeRefS), fs::current_path().string()); auto evalCache = openEvalCache(*evalState, - std::make_shared(lockFlake(*evalState, flakeRef, lockFlags))); + std::make_shared(lockFlake( + flakeSettings, *evalState, flakeRef, lockFlags))); auto root = evalCache->getRoot(); @@ -372,6 +378,7 @@ void completeFlakeRefWithFragment( auto attrPath2 = (*attr)->getAttrPath(attr2); /* Strip the attrpath prefix. */ attrPath2.erase(attrPath2.begin(), attrPath2.begin() + attrPathPrefix.size()); + // FIXME: handle names with dots completions.add(flakeRefS + "#" + prefixRoot + concatStringsSep(".", evalState->symbols.resolve(attrPath2))); } } @@ -403,7 +410,7 @@ void completeFlakeRef(AddCompletions & completions, ref store, std::strin Args::completeDir(completions, 0, prefix); /* Look for registry entries that match the prefix. */ - for (auto & registry : fetchers::getRegistries(store)) { + for (auto & registry : fetchers::getRegistries(fetchSettings, store)) { for (auto & entry : registry->entries) { auto from = entry.from.to_string(); if (!hasPrefix(prefix, "flake:") && hasPrefix(from, "flake:")) { @@ -534,7 +541,8 @@ Installables SourceExprCommand::parseInstallables( } try { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, absPath(getCommandBaseDir())); + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, std::string { prefix }, absPath(getCommandBaseDir())); result.push_back(make_ref( this, getEvalState(), @@ -601,6 +609,37 @@ std::vector Installable::build( return res; } +static void throwBuildErrors( + std::vector & buildResults, + const Store & store) +{ + std::vector failed; + for (auto & buildResult : buildResults) { + if (!buildResult.success()) { + failed.push_back(buildResult); + } + } + + auto failedResult = failed.begin(); + if (failedResult != failed.end()) { + if (failed.size() == 1) { + failedResult->rethrow(); + } else { + StringSet failedPaths; + for (; failedResult != failed.end(); failedResult++) { + if (!failedResult->errorMsg.empty()) { + logError(ErrorInfo{ + .level = lvlError, + .msg = failedResult->errorMsg, + }); + } + failedPaths.insert(failedResult->path.to_string(store)); + } + throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths))); + } + } +} + std::vector, BuiltPathWithResult>> Installable::build2( ref evalStore, ref store, @@ -662,10 +701,9 @@ std::vector, BuiltPathWithResult>> Installable::build if (settings.printMissing) printMissing(store, pathsToBuild, lvlInfo); - for (auto & buildResult : store->buildPathsWithResults(pathsToBuild, bMode, evalStore)) { - if (!buildResult.success()) - buildResult.rethrow(); - + auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore); + throwBuildErrors(buildResults, *store); + for (auto & buildResult : buildResults) { for (auto & aux : backmap[buildResult.path]) { std::visit(overloaded { [&](const DerivedPath::Built & bfd) { @@ -821,6 +859,7 @@ std::vector RawInstallablesCommand::getFlakeRefsForCompletion() std::vector res; for (auto i : rawInstallables) res.push_back(parseFlakeRefWithFragment( + fetchSettings, expandTilde(i), absPath(getCommandBaseDir())).first); return res; @@ -843,6 +882,7 @@ std::vector InstallableCommand::getFlakeRefsForCompletion() { return { parseFlakeRefWithFragment( + fetchSettings, expandTilde(_installable), absPath(getCommandBaseDir())).first }; diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index 9aa33a9d3..a270333f4 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -6,10 +6,10 @@ libcmd_DIR := $(d) libcmd_SOURCES := $(wildcard $(d)/*.cc) -libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libmain) +libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS) -libcmd_LIBS = libstore libutil libexpr libmain libfetchers +libcmd_LIBS = libutil libstore libfetchers libflake libexpr libmain $(eval $(call install-file-in, $(buildprefix)$(d)/nix-cmd.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index 88c3f640b..6a0d05d9f 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -1,21 +1,23 @@ #include "markdown.hh" -#include "util.hh" +#include "environment-variables.hh" +#include "error.hh" #include "finally.hh" #include "terminal.hh" #if HAVE_LOWDOWN -# include -# include +# include +# include #endif namespace nix { -std::string renderMarkdownToTerminal(std::string_view markdown) -{ #if HAVE_LOWDOWN +static std::string doRenderMarkdownToTerminal(std::string_view markdown) +{ int windowWidth = getWindowSize().second; - struct lowdown_opts opts { + struct lowdown_opts opts + { .type = LOWDOWN_TERM, .maxdepth = 20, .cols = (size_t) std::max(windowWidth - 5, 60), @@ -51,9 +53,21 @@ std::string renderMarkdownToTerminal(std::string_view markdown) throw Error("allocation error while rendering Markdown"); return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY()); -#else - return std::string(markdown); -#endif } +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + if (auto e = getEnv("_NIX_TEST_RAW_MARKDOWN"); e && *e == "1") + return std::string(markdown); + else + return doRenderMarkdownToTerminal(markdown); } + +#else +std::string renderMarkdownToTerminal(std::string_view markdown) +{ + return std::string(markdown); +} +#endif + +} // namespace nix diff --git a/src/libcmd/markdown.hh b/src/libcmd/markdown.hh index a04d32a4f..66db1736c 100644 --- a/src/libcmd/markdown.hh +++ b/src/libcmd/markdown.hh @@ -1,10 +1,17 @@ #pragma once ///@file -#include "types.hh" +#include namespace nix { +/** + * Render the given Markdown text to the terminal. + * + * If Nix is compiled without Markdown support, this function will return the input text as-is. + * + * The renderer takes into account the terminal width, and wraps text accordingly. + */ std::string renderMarkdownToTerminal(std::string_view markdown); } diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build new file mode 100644 index 000000000..c484cf998 --- /dev/null +++ b/src/libcmd/meson.build @@ -0,0 +1,130 @@ +project('nix-cmd', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), + dependency('nix-expr'), + dependency('nix-flake'), + dependency('nix-main'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +lowdown = dependency('lowdown', version : '>= 0.9.0', required : get_option('markdown')) +deps_private += lowdown +configdata.set('HAVE_LOWDOWN', lowdown.found().to_int()) + +readline_flavor = get_option('readline-flavor') +if readline_flavor == 'editline' + editline = dependency('libeditline', 'editline', version : '>=1.14') + deps_private += editline +elif readline_flavor == 'readline' + readline = dependency('readline') + deps_private += readline + configdata.set( + 'USE_READLINE', + 1, + description: 'Use readline instead of editline', + ) +else + error('illegal editline flavor', readline_flavor) +endif + +config_h = configure_file( + configuration : configdata, + output : 'config-cmd.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', + '-include', 'config-main.hh', + '-include', 'config-cmd.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'built-path.cc', + 'command-installable-value.cc', + 'command.cc', + 'common-eval-args.cc', + 'editor-for.cc', + 'installable-attr-path.cc', + 'installable-derived-path.cc', + 'installable-flake.cc', + 'installable-value.cc', + 'installables.cc', + 'legacy.cc', + 'markdown.cc', + 'misc-store-flags.cc', + 'network-proxy.cc', + 'repl-interacter.cc', + 'repl.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'built-path.hh', + 'command-installable-value.hh', + 'command.hh', + 'common-eval-args.hh', + 'compatibility-settings.hh', + 'editor-for.hh', + 'installable-attr-path.hh', + 'installable-derived-path.hh', + 'installable-flake.hh', + 'installable-value.hh', + 'installables.hh', + 'legacy.hh', + 'markdown.hh', + 'misc-store-flags.hh', + 'network-proxy.hh', + 'repl-interacter.hh', + 'repl.hh', +) + +this_library = library( + 'nixcmd', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libcmd/meson.options b/src/libcmd/meson.options new file mode 100644 index 000000000..79ae4fa55 --- /dev/null +++ b/src/libcmd/meson.options @@ -0,0 +1,15 @@ +# vim: filetype=meson + +option( + 'markdown', + type: 'feature', + description: 'Enable Markdown rendering in the Nix binary (requires lowdown)', +) + +option( + 'readline-flavor', + type : 'combo', + choices : ['editline', 'readline'], + value : 'editline', + description : 'Which library to use for nice line editing with the Nix language REPL', +) diff --git a/src/libcmd/misc-store-flags.cc b/src/libcmd/misc-store-flags.cc index 063a9dd9e..06552c032 100644 --- a/src/libcmd/misc-store-flags.cc +++ b/src/libcmd/misc-store-flags.cc @@ -103,27 +103,27 @@ Args::Flag contentAddressMethod(ContentAddressMethod * method) return Args::Flag { .longName = "mode", // FIXME indentation carefully made for context, this is messed up. - /* FIXME link to store object content-addressing not file system - object content addressing once we have that page. */ .description = R"( How to compute the content-address of the store object. One of: - - `nar` (the default): + - [`nar`](@docroot@/store/store-object/content-address.md#method-nix-archive) + (the default): Serialises the input as a [Nix Archive](@docroot@/store/file-system-object/content-address.md#serial-nix-archive) and passes that to the hash function. - - `flat`: + - [`flat`](@docroot@/store/store-object/content-address.md#method-flat): Assumes that the input is a single file and [directly passes](@docroot@/store/file-system-object/content-address.md#serial-flat) it to the hash function. - - `text`: Like `flat`, but used for + - [`text`](@docroot@/store/store-object/content-address.md#method-text): + Like `flat`, but used for [derivations](@docroot@/glossary.md#store-derivation) serialized in store object and [`builtins.toFile`](@docroot@/language/builtins.html#builtins-toFile). For advanced use-cases only; - for regular usage prefer `nar` and `flat. + for regular usage prefer `nar` and `flat`. )", .labels = {"content-address-method"}, .handler = {[method](std::string s) { diff --git a/src/libcmd/network-proxy.cc b/src/libcmd/network-proxy.cc index 633b2c005..738bf6147 100644 --- a/src/libcmd/network-proxy.cc +++ b/src/libcmd/network-proxy.cc @@ -1,7 +1,6 @@ #include "network-proxy.hh" #include -#include #include "environment-variables.hh" @@ -13,7 +12,10 @@ static StringSet getAllVariables() { StringSet variables = lowercaseVariables; for (const auto & variable : lowercaseVariables) { - variables.insert(boost::to_upper_copy(variable)); + std::string upperVariable; + std::transform( + variable.begin(), variable.end(), upperVariable.begin(), [](unsigned char c) { return std::toupper(c); }); + variables.insert(std::move(upperVariable)); } return variables; } @@ -25,7 +27,10 @@ static StringSet getExcludingNoProxyVariables() static const StringSet excludeVariables{"no_proxy", "NO_PROXY"}; StringSet variables; std::set_difference( - networkProxyVariables.begin(), networkProxyVariables.end(), excludeVariables.begin(), excludeVariables.end(), + networkProxyVariables.begin(), + networkProxyVariables.end(), + excludeVariables.begin(), + excludeVariables.end(), std::inserter(variables, variables.begin())); return variables; } diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix new file mode 100644 index 000000000..cde494901 --- /dev/null +++ b/src/libcmd/package.nix @@ -0,0 +1,104 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-store +, nix-fetchers +, nix-expr +, nix-flake +, nix-main +, editline +, readline +, lowdown +, nlohmann_json + +# Configuration Options + +, version + +# Whether to enable Markdown rendering in the Nix binary. +, enableMarkdown ? !stdenv.hostPlatform.isWindows + +# Which interactive line editor library to use for Nix's repl. +# +# Currently supported choices are: +# +# - editline (default) +# - readline +, readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-cmd"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + ({ inherit editline readline; }.${readlineFlavor}) + ] ++ lib.optional enableMarkdown lowdown; + + propagatedBuildInputs = [ + nix-util + nix-store + nix-fetchers + nix-expr + nix-flake + nix-main + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "markdown" enableMarkdown) + (lib.mesonOption "readline-flavor" readlineFlavor) + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libcmd/repl-interacter.cc b/src/libcmd/repl-interacter.cc index eb4361e25..187af46ea 100644 --- a/src/libcmd/repl-interacter.cc +++ b/src/libcmd/repl-interacter.cc @@ -18,7 +18,8 @@ extern "C" { #include "finally.hh" #include "repl-interacter.hh" #include "file-system.hh" -#include "libcmd/repl.hh" +#include "repl.hh" +#include "environment-variables.hh" namespace nix { @@ -34,6 +35,7 @@ void sigintHandler(int signo) static detail::ReplCompleterMixin * curRepl; // ugly +#ifndef USE_READLINE static char * completionCallback(char * s, int * match) { auto possible = curRepl->completePrefix(s); @@ -73,7 +75,7 @@ static int listPossibleCallback(char * s, char *** avp) { auto possible = curRepl->completePrefix(s); - if (possible.size() > (INT_MAX / sizeof(char *))) + if (possible.size() > (std::numeric_limits::max() / sizeof(char *))) throw Error("too many completions"); int ac = 0; @@ -100,6 +102,7 @@ static int listPossibleCallback(char * s, char *** avp) return ac; } +#endif ReadlineLikeInteracter::Guard ReadlineLikeInteracter::init(detail::ReplCompleterMixin * repl) { @@ -175,10 +178,23 @@ bool ReadlineLikeInteracter::getLine(std::string & input, ReplPromptType promptT return true; } + // editline doesn't echo the input to the output when non-interactive, unlike readline + // this results in a different behavior when running tests. The echoing is + // quite useful for reading the test output, so we add it here. + if (auto e = getEnv("_NIX_TEST_REPL_ECHO"); s && e && *e == "1") + { +#ifndef USE_READLINE + // This is probably not right for multi-line input, but we don't use that + // in the characterisation tests, so it's fine. + std::cout << promptForType(promptType) << s << std::endl; +#endif + } + if (!s) return false; input += s; input += '\n'; + return true; } diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index a069dd52c..f292f06bb 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -1,16 +1,14 @@ #include #include #include -#include -#include "libcmd/repl-interacter.hh" +#include "error.hh" +#include "repl-interacter.hh" #include "repl.hh" #include "ansicolor.hh" #include "shared.hh" #include "eval.hh" -#include "eval-cache.hh" -#include "eval-inline.hh" #include "eval-settings.hh" #include "attr-path.hh" #include "signals.hh" @@ -28,11 +26,10 @@ #include "markdown.hh" #include "local-fs-store.hh" #include "print.hh" +#include "ref.hh" +#include "value.hh" -#if HAVE_BOEHMGC -#define GC_INCLUDE_NEW -#include -#endif +#include "strings.hh" namespace nix { @@ -60,9 +57,7 @@ enum class ProcessLineResult { struct NixRepl : AbstractNixRepl , detail::ReplCompleterMixin - #if HAVE_BOEHMGC , gc - #endif { size_t debugTraceIndex; @@ -75,10 +70,14 @@ struct NixRepl int displ; StringSet varNames; + RunNix * runNixPtr; + + void runNix(Path program, const Strings & args, const std::optional & input = {}); + std::unique_ptr interacter; NixRepl(const LookupPath & lookupPath, nix::ref store,ref state, - std::function getValues); + std::function getValues, RunNix * runNix); virtual ~NixRepl() = default; ReplExitStatus mainLoop() override; @@ -123,31 +122,16 @@ std::string removeWhitespace(std::string s) NixRepl::NixRepl(const LookupPath & lookupPath, nix::ref store, ref state, - std::function getValues) + std::function getValues, RunNix * runNix = nullptr) : AbstractNixRepl(state) , debugTraceIndex(0) , getValues(getValues) , staticEnv(new StaticEnv(nullptr, state->staticBaseEnv.get())) - , interacter(make_unique(getDataDir() + "/nix/repl-history")) + , runNixPtr{runNix} + , interacter(make_unique(getDataDir() + "/repl-history")) { } -void runNix(Path program, const Strings & args, - const std::optional & input = {}) -{ - auto subprocessEnv = getEnv(); - subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); - - runProgram2(RunOptions { - .program = settings.nixBinDir+ "/" + program, - .args = args, - .environment = subprocessEnv, - .input = input, - }); - - return; -} - static std::ostream & showDebugTrace(std::ostream & out, const PosTable & positions, const DebugTrace & dt) { if (dt.isError) @@ -214,7 +198,7 @@ ReplExitStatus NixRepl::mainLoop() case ProcessLineResult::PromptAgain: break; default: - abort(); + unreachable(); } } catch (ParseError & e) { if (e.msg().find("unexpected end of file") != std::string::npos) { @@ -260,6 +244,7 @@ StringSet NixRepl::completePrefix(const std::string & prefix) auto dir = std::string(cur, 0, slash); auto prefix2 = std::string(cur, slash + 1); for (auto & entry : std::filesystem::directory_iterator{dir == "" ? "/" : dir}) { + checkInterrupt(); auto name = entry.path().filename().string(); if (name[0] != '.' && hasPrefix(name, prefix2)) completions.insert(prev + entry.path().string()); @@ -304,6 +289,8 @@ StringSet NixRepl::completePrefix(const std::string & prefix) // Quietly ignore evaluation errors. } catch (BadURL & e) { // Quietly ignore BadURL flake-related errors. + } catch (FileNotFound & e) { + // Quietly ignore non-existent file beeing `import`-ed. } } @@ -508,13 +495,9 @@ ProcessLineResult NixRepl::processLine(std::string line) auto editor = args.front(); args.pop_front(); - // avoid garbling the editor with the progress bar - logger->pause(); - Finally resume([&]() { logger->resume(); }); - // runProgram redirects stdout to a StringSink, // using runProgram2 to allow editors to display their UI - runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args }); + runProgram2(RunOptions { .program = editor, .lookupPath = true, .args = args , .isInteractive = true }); // Reload right after exiting the editor state->resetFileCache(); @@ -615,6 +598,35 @@ ProcessLineResult NixRepl::processLine(std::string line) else if (command == ":doc") { Value v; + + auto expr = parseString(arg); + std::string fallbackName; + PosIdx fallbackPos; + DocComment fallbackDoc; + if (auto select = dynamic_cast(expr)) { + Value vAttrs; + auto name = select->evalExceptFinalSelect(*state, *env, vAttrs); + fallbackName = state->symbols[name]; + + state->forceAttrs(vAttrs, noPos, "while evaluating an attribute set to look for documentation"); + auto attrs = vAttrs.attrs(); + assert(attrs); + auto attr = attrs->get(name); + if (!attr) { + // When missing, trigger the normal exception + // e.g. :doc builtins.foo + // behaves like + // nix-repl> builtins.foo + // error: attribute 'foo' missing + evalString(arg, v); + assert(false); + } + if (attr->pos) { + fallbackPos = attr->pos; + fallbackDoc = state->getDocCommentForPos(fallbackPos); + } + } + evalString(arg, v); if (auto doc = state->getDoc(v)) { std::string markdown; @@ -632,6 +644,19 @@ ProcessLineResult NixRepl::processLine(std::string line) markdown += stripIndentation(doc->doc); logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else if (fallbackPos) { + std::ostringstream ss; + ss << "Attribute `" << fallbackName << "`\n\n"; + ss << " … defined at " << state->positions[fallbackPos] << "\n\n"; + if (fallbackDoc) { + ss << fallbackDoc.getInnerText(state->positions); + } else { + ss << "No documentation found.\n\n"; + } + + auto markdown = toView(ss); + logger->cout(trim(renderMarkdownToTerminal(markdown))); + } else throw Error("value does not have documentation"); } @@ -689,14 +714,21 @@ void NixRepl::loadFlake(const std::string & flakeRefS) if (flakeRefS.empty()) throw Error("cannot use ':load-flake' without a path specified. (Use '.' for the current working directory.)"); - auto flakeRef = parseFlakeRef(flakeRefS, absPath("."), true); + std::filesystem::path cwd; + try { + cwd = std::filesystem::current_path(); + } catch (std::filesystem::filesystem_error & e) { + throw SysError("cannot determine current working directory"); + } + + auto flakeRef = parseFlakeRef(fetchSettings, flakeRefS, cwd.string(), true); if (evalSettings.pureEval && !flakeRef.input.isLocked()) throw Error("cannot use ':load-flake' on locked flake reference '%s' (use --impure to override)", flakeRefS); Value v; flake::callFlake(*state, - flake::lockFlake(*state, flakeRef, + flake::lockFlake(flakeSettings, *state, flakeRef, flake::LockFlags { .updateLockFile = false, .useRegistries = !evalSettings.pureEval, @@ -789,9 +821,18 @@ void NixRepl::evalString(std::string s, Value & v) } +void NixRepl::runNix(Path program, const Strings & args, const std::optional & input) +{ + if (runNixPtr) + (*runNixPtr)(program, args, input); + else + throw Error("Cannot run '%s' because no method of calling the Nix CLI was provided. This is a configuration problem pertaining to how this program was built. See Nix 2.25 release notes", program); +} + + std::unique_ptr AbstractNixRepl::create( const LookupPath & lookupPath, nix::ref store, ref state, - std::function getValues) + std::function getValues, RunNix * runNix) { return std::make_unique( lookupPath, diff --git a/src/libcmd/repl.hh b/src/libcmd/repl.hh index 3fd4b2c39..11d1820f5 100644 --- a/src/libcmd/repl.hh +++ b/src/libcmd/repl.hh @@ -19,9 +19,19 @@ struct AbstractNixRepl typedef std::vector> AnnotatedValues; + using RunNix = void(Path program, const Strings & args, const std::optional & input); + + /** + * @param runNix Function to run the nix CLI to support various + * `:` commands. Optional; if not provided, + * everything else will still work fine, but those commands won't. + */ static std::unique_ptr create( - const LookupPath & lookupPath, nix::ref store, ref state, - std::function getValues); + const LookupPath & lookupPath, + nix::ref store, + ref state, + std::function getValues, + RunNix * runNix = nullptr); static ReplExitStatus runSimple( ref evalState, diff --git a/src/libexpr-c/.version b/src/libexpr-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libexpr-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libexpr-c/build-utils-meson b/src/libexpr-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libexpr-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr-c/local.mk b/src/libexpr-c/local.mk index 51b02562e..227a4095b 100644 --- a/src/libexpr-c/local.mk +++ b/src/libexpr-c/local.mk @@ -15,7 +15,7 @@ libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \ $(INCLUDE_libstore) $(INCLUDE_libstorec) \ $(INCLUDE_libexpr) $(INCLUDE_libexprc) -libexprc_LIBS = libutil libutilc libstore libstorec libexpr +libexprc_LIBS = libutil libutilc libstore libstorec libfetchers libexpr libexprc_LDFLAGS += $(THREAD_LDFLAGS) diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build new file mode 100644 index 000000000..6db5b83b8 --- /dev/null +++ b/src/libexpr-c/meson.build @@ -0,0 +1,93 @@ +project('nix-expr-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-expr'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-expr.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-expr.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_expr.cc', + 'nix_api_external.cc', + 'nix_api_value.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_expr.h', + 'nix_api_external.h', + 'nix_api_value.h', +) + +# TODO move this header to libexpr, maybe don't use it in tests? +headers += files('nix_api_expr_internal.h') + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixexprc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index a29c3425e..333e99460 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -1,12 +1,11 @@ #include -#include #include #include -#include "config.hh" #include "eval.hh" +#include "eval-gc.hh" #include "globals.hh" -#include "util.hh" +#include "eval-settings.hh" #include "nix_api_expr.h" #include "nix_api_expr_internal.h" @@ -15,10 +14,8 @@ #include "nix_api_util.h" #include "nix_api_util_internal.h" -#ifdef HAVE_BOEHMGC -#include -#define GC_INCLUDE_NEW 1 -#include "gc_cpp.h" +#if HAVE_BOEHMGC +# include #endif nix_err nix_libexpr_init(nix_c_context * context) @@ -42,45 +39,56 @@ nix_err nix_libexpr_init(nix_c_context * context) } nix_err nix_expr_eval_from_string( - nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value) + nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value) { if (context) context->last_err_code = NIX_OK; try { nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path))); - state->state.eval(parsedExpr, *(nix::Value *) value); - state->state.forceValue(*(nix::Value *) value, nix::noPos); + state->state.eval(parsedExpr, value->value); + state->state.forceValue(value->value, nix::noPos); } NIXC_CATCH_ERRS } -nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value) +nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, nix_value * arg, nix_value * value) { if (context) context->last_err_code = NIX_OK; try { - state->state.callFunction(*(nix::Value *) fn, *(nix::Value *) arg, *(nix::Value *) value, nix::noPos); - state->state.forceValue(*(nix::Value *) value, nix::noPos); + state->state.callFunction(fn->value, arg->value, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); } NIXC_CATCH_ERRS } -nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value) +nix_err nix_value_call_multi(nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value) { if (context) context->last_err_code = NIX_OK; try { - state->state.forceValue(*(nix::Value *) value, nix::noPos); + state->state.callFunction(fn->value, nargs, (nix::Value * *)args, value->value, nix::noPos); + state->state.forceValue(value->value, nix::noPos); } NIXC_CATCH_ERRS } -nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value) +nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value) { if (context) context->last_err_code = NIX_OK; try { - state->state.forceValueDeep(*(nix::Value *) value); + state->state.forceValue(value->value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValueDeep(value->value); } NIXC_CATCH_ERRS } @@ -95,7 +103,23 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c for (size_t i = 0; lookupPath_c[i] != nullptr; i++) lookupPath.push_back(lookupPath_c[i]); - return new EvalState{nix::EvalState(nix::LookupPath::parse(lookupPath), store->ptr)}; + void * p = ::operator new( + sizeof(EvalState), + static_cast(alignof(EvalState))); + auto * p2 = static_cast(p); + new (p) EvalState { + .fetchSettings = nix::fetchers::Settings{}, + .settings = nix::EvalSettings{ + nix::settings.readOnlyMode, + }, + .state = nix::EvalState( + nix::LookupPath::parse(lookupPath), + store->ptr, + p2->fetchSettings, + p2->settings), + }; + loadConfFile(p2->settings); + return p2; } NIXC_CATCH_ERRS_NULL } @@ -105,7 +129,7 @@ void nix_state_free(EvalState * state) delete state; } -#ifdef HAVE_BOEHMGC +#if HAVE_BOEHMGC std::unordered_map< const void *, unsigned int, @@ -170,9 +194,18 @@ nix_err nix_gc_decref(nix_c_context * context, const void *) void nix_gc_now() {} #endif +nix_err nix_value_incref(nix_c_context * context, nix_value *x) +{ + return nix_gc_incref(context, (const void *) x); +} +nix_err nix_value_decref(nix_c_context * context, nix_value *x) +{ + return nix_gc_decref(context, (const void *) x); +} + void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd)) { -#ifdef HAVE_BOEHMGC +#if HAVE_BOEHMGC GC_REGISTER_FINALIZER(obj, finalizer, cd, 0, 0); #endif } diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h index 04fc92f0f..1764b49f3 100644 --- a/src/libexpr-c/nix_api_expr.h +++ b/src/libexpr-c/nix_api_expr.h @@ -3,25 +3,7 @@ /** @defgroup libexpr libexpr * @brief Bindings to the Nix language evaluator * - * Example (without error handling): - * @code{.c} - * int main() { - * nix_libexpr_init(NULL); - * - * Store* store = nix_store_open(NULL, "dummy", NULL); - * EvalState* state = nix_state_create(NULL, NULL, store); // empty nix path - * Value *value = nix_alloc_value(NULL, state); - * - * nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value); - * nix_value_force(NULL, state, value); - * printf("nix version: %s\n", nix_get_string(NULL, value)); - * - * nix_gc_decref(NULL, value); - * nix_state_free(state); - * nix_store_free(store); - * return 0; - * } - * @endcode + * See *[Embedding the Nix Evaluator](@ref nix_evaluator_example)* for an example. * @{ */ /** @file @@ -30,6 +12,17 @@ #include "nix_api_store.h" #include "nix_api_util.h" +#include + +#ifndef __has_c_attribute +# define __has_c_attribute(x) 0 +#endif + +#if __has_c_attribute(deprecated) +# define NIX_DEPRECATED(msg) [[deprecated(msg)]] +#else +# define NIX_DEPRECATED(msg) +#endif #ifdef __cplusplus extern "C" { @@ -46,14 +39,23 @@ extern "C" { * @see nix_state_create */ typedef struct EvalState EvalState; // nix::EvalState -/** - * @brief Represents a value in the Nix language. + +/** @brief A Nix language value, or thunk that may evaluate to a value. + * + * Values are the primary objects manipulated in the Nix language. + * They are considered to be immutable from a user's perspective, but the process of evaluating a value changes its + * ValueType if it was a thunk. After a value has been evaluated, its ValueType does not change. + * + * Evaluation in this context refers to the process of evaluating a single value object, also called "forcing" the + * value; see `nix_value_force`. + * + * The evaluator manages its own memory, but your use of the C API must follow the reference counting rules. * - * Owned by the garbage collector. - * @struct Value * @see value_manip + * @see nix_value_incref, nix_value_decref */ -typedef void Value; // nix::Value +typedef struct nix_value nix_value; +NIX_DEPRECATED("use nix_value instead") typedef nix_value Value; // Function prototypes /** @@ -82,7 +84,7 @@ nix_err nix_libexpr_init(nix_c_context * context); * @return NIX_OK if the evaluation was successful, an error code otherwise. */ nix_err nix_expr_eval_from_string( - nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value); + nix_c_context * context, EvalState * state, const char * expr, const char * path, nix_value * value); /** * @brief Calls a Nix function with an argument. @@ -96,20 +98,55 @@ nix_err nix_expr_eval_from_string( * @see nix_init_apply() for a similar function that does not performs the call immediately, but stores it as a thunk. * Note the different argument order. */ -nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value); +nix_err nix_value_call(nix_c_context * context, EvalState * state, nix_value * fn, nix_value * arg, nix_value * value); + +/** + * @brief Calls a Nix function with multiple arguments. + * + * Technically these are functions that return functions. It is common for Nix + * functions to be curried, so this function is useful for calling them. + * + * @param[out] context Optional, stores error information + * @param[in] state The state of the evaluation. + * @param[in] fn The Nix function to call. + * @param[in] nargs The number of arguments. + * @param[in] args The arguments to pass to the function. + * @param[out] value The result of the function call. + * + * @see nix_value_call For the single argument primitive. + * @see NIX_VALUE_CALL For a macro that wraps this function for convenience. + */ +nix_err nix_value_call_multi( + nix_c_context * context, EvalState * state, nix_value * fn, size_t nargs, nix_value ** args, nix_value * value); + +/** + * @brief Calls a Nix function with multiple arguments. + * + * Technically these are functions that return functions. It is common for Nix + * functions to be curried, so this function is useful for calling them. + * + * @param[out] context Optional, stores error information + * @param[in] state The state of the evaluation. + * @param[out] value The result of the function call. + * @param[in] fn The Nix function to call. + * @param[in] args The arguments to pass to the function. + * + * @see nix_value_call_multi + */ +#define NIX_VALUE_CALL(context, state, value, fn, ...) \ + do { \ + nix_value * args_array[] = {__VA_ARGS__}; \ + size_t nargs = sizeof(args_array) / sizeof(args_array[0]); \ + nix_value_call_multi(context, state, fn, nargs, args_array, value); \ + } while (0) /** * @brief Forces the evaluation of a Nix value. * - * The Nix interpreter is lazy, and not-yet-evaluated Values can be + * The Nix interpreter is lazy, and not-yet-evaluated values can be * of type NIX_TYPE_THUNK instead of their actual value. * - * This function converts these Values into their final type. - * - * @note You don't need this function for basic API usage, since all functions - * that return a value call it for you. The only place you will see a - * NIX_TYPE_THUNK is in the arguments that are passed to a PrimOp function - * you supplied to nix_alloc_primop. + * This function mutates such a `nix_value`, so that, if successful, it has its final type. * * @param[out] context Optional, stores error information * @param[in] state The state of the evaluation. @@ -118,7 +155,7 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, V * @return NIX_OK if the force operation was successful, an error code * otherwise. */ -nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value); +nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * value); /** * @brief Forces the deep evaluation of a Nix value. @@ -134,13 +171,13 @@ nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * valu * @return NIX_OK if the deep force operation was successful, an error code * otherwise. */ -nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value); +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_value * value); /** * @brief Create a new Nix language evaluator state. * * @param[out] context Optional, stores error information - * @param[in] lookupPath Array of strings corresponding to entries in NIX_PATH. + * @param[in] lookupPath Null-terminated array of strings corresponding to entries in NIX_PATH. * @param[in] store The Nix store to use. * @return A new Nix state or NULL on failure. */ @@ -168,6 +205,11 @@ void nix_state_free(EvalState * state); * you're done with a value returned by the evaluator. * @{ */ + +// TODO: Deprecate nix_gc_incref in favor of the type-specific reference counting functions? +// e.g. nix_value_incref. +// It gives implementors more flexibility, and adds safety, so that generated +// bindings can be used without fighting the host type system (where applicable). /** * @brief Increment the garbage collector reference counter for the given object. * diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index 7743849fd..12f24b6eb 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -1,12 +1,16 @@ #ifndef NIX_API_EXPR_INTERNAL_H #define NIX_API_EXPR_INTERNAL_H +#include "fetch-settings.hh" #include "eval.hh" +#include "eval-settings.hh" #include "attr-set.hh" #include "nix_api_value.h" struct EvalState { + nix::fetchers::Settings fetchSettings; + nix::EvalSettings settings; nix::EvalState state; }; @@ -20,6 +24,11 @@ struct ListBuilder nix::ListBuilder builder; }; +struct nix_value +{ + nix::Value value; +}; + struct nix_string_return { std::string str; diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc index 3c3dd6ca9..d673bcb0b 100644 --- a/src/libexpr-c/nix_api_external.cc +++ b/src/libexpr-c/nix_api_external.cc @@ -14,12 +14,6 @@ #include -#ifdef HAVE_BOEHMGC -# include "gc/gc.h" -# define GC_INCLUDE_NEW 1 -# include "gc_cpp.h" -#endif - void nix_set_string_return(nix_string_return * str, const char * c) { str->str = c; @@ -115,7 +109,7 @@ public: /** * Compare to another value of the same type. */ - virtual bool operator==(const ExternalValueBase & b) const override + virtual bool operator==(const ExternalValueBase & b) const noexcept override { if (!desc.equal) { return false; @@ -174,7 +168,7 @@ ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalV context->last_err_code = NIX_OK; try { auto ret = new -#ifdef HAVE_BOEHMGC +#if HAVE_BOEHMGC (GC) #endif NixCExternalValue(*desc, v); diff --git a/src/libexpr-c/nix_api_external.h b/src/libexpr-c/nix_api_external.h index 12ea00407..6c524b975 100644 --- a/src/libexpr-c/nix_api_external.h +++ b/src/libexpr-c/nix_api_external.h @@ -48,7 +48,7 @@ void nix_set_string_return(nix_string_return * str, const char * c); * Print to the nix_printer * * @param[out] context Optional, stores error information - * @param printer The nix_printer to print to + * @param[out] printer The nix_printer to print to * @param[in] str The string to print * @returns NIX_OK if everything worked */ @@ -136,7 +136,7 @@ typedef struct NixCExternalValueDesc * or setting it to the empty string, will make the conversion throw an error. */ void (*printValueAsJSON)( - void * self, EvalState *, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res); + void * self, EvalState * state, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res); /** * @brief Convert the external value to XML * @@ -155,7 +155,7 @@ typedef struct NixCExternalValueDesc */ void (*printValueAsXML)( void * self, - EvalState *, + EvalState * state, int strict, int location, void * doc, diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index 0366e5020..bae078d31 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -14,56 +14,55 @@ #include "nix_api_value.h" #include "value/context.hh" -#ifdef HAVE_BOEHMGC -# include "gc/gc.h" -# define GC_INCLUDE_NEW 1 -# include "gc_cpp.h" -#endif - // Internal helper functions to check [in] and [out] `Value *` parameters -static const nix::Value & check_value_not_null(const Value * value) +static const nix::Value & check_value_not_null(const nix_value * value) { if (!value) { - throw std::runtime_error("Value is null"); + throw std::runtime_error("nix_value is null"); } return *((const nix::Value *) value); } -static nix::Value & check_value_not_null(Value * value) +static nix::Value & check_value_not_null(nix_value * value) { if (!value) { - throw std::runtime_error("Value is null"); + throw std::runtime_error("nix_value is null"); } - return *((nix::Value *) value); + return value->value; } -static const nix::Value & check_value_in(const Value * value) +static const nix::Value & check_value_in(const nix_value * value) { auto & v = check_value_not_null(value); if (!v.isValid()) { - throw std::runtime_error("Uninitialized Value"); + throw std::runtime_error("Uninitialized nix_value"); } return v; } -static nix::Value & check_value_in(Value * value) +static nix::Value & check_value_in(nix_value * value) { auto & v = check_value_not_null(value); if (!v.isValid()) { - throw std::runtime_error("Uninitialized Value"); + throw std::runtime_error("Uninitialized nix_value"); } return v; } -static nix::Value & check_value_out(Value * value) +static nix::Value & check_value_out(nix_value * value) { auto & v = check_value_not_null(value); if (v.isValid()) { - throw std::runtime_error("Value already initialized. Variables are immutable"); + throw std::runtime_error("nix_value already initialized. Variables are immutable"); } return v; } +static inline nix_value * as_nix_value_ptr(nix::Value * v) +{ + return reinterpret_cast(v); +} + /** * Helper function to convert calls from nix into C API. * @@ -73,10 +72,43 @@ static void nix_c_primop_wrapper( PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v) { nix_c_context ctx; - f(userdata, &ctx, (EvalState *) &state, (Value **) args, (Value *) &v); - /* TODO: In the future, this should throw different errors depending on the error code */ - if (ctx.last_err_code != NIX_OK) - state.error("Error from builtin function: %s", *ctx.last_err).atPos(pos).debugThrow(); + + // v currently has a thunk, but the C API initializers require an uninitialized value. + // + // We can't destroy the thunk, because that makes it impossible to retry, + // which is needed for tryEval and for evaluation drivers that evaluate more + // than one value (e.g. an attrset with two derivations, both of which + // reference v). + // + // Instead we create a temporary value, and then assign the result to v. + // This does not give the primop definition access to the thunk, but that's + // ok because we don't see a need for this yet (e.g. inspecting thunks, + // or maybe something to make blackholes work better; we don't know). + nix::Value vTmp; + + f(userdata, &ctx, (EvalState *) &state, (nix_value **) args, (nix_value *) &vTmp); + + if (ctx.last_err_code != NIX_OK) { + /* TODO: Throw different errors depending on the error code */ + state.error("Error from custom function: %s", *ctx.last_err).atPos(pos).debugThrow(); + } + + if (!vTmp.isValid()) { + state.error("Implementation error in custom function: return value was not initialized") + .atPos(pos) + .debugThrow(); + } + + if (vTmp.type() == nix::nThunk) { + // We might allow this in the future if it makes sense for the evaluator + // e.g. implementing tail recursion by returning a thunk to the next + // "iteration". Until then, this is most likely a mistake or misunderstanding. + state.error("Implementation error in custom function: return value must not be a thunk") + .atPos(pos) + .debugThrow(); + } + + v = vTmp; } PrimOp * nix_alloc_primop( @@ -93,7 +125,7 @@ PrimOp * nix_alloc_primop( try { using namespace std::placeholders; auto p = new -#ifdef HAVE_BOEHMGC +#if HAVE_BOEHMGC (GC) #endif nix::PrimOp{ @@ -121,19 +153,19 @@ nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp) NIXC_CATCH_ERRS } -Value * nix_alloc_value(nix_c_context * context, EvalState * state) +nix_value * nix_alloc_value(nix_c_context * context, EvalState * state) { if (context) context->last_err_code = NIX_OK; try { - Value * res = state->state.allocValue(); + nix_value * res = as_nix_value_ptr(state->state.allocValue()); nix_gc_incref(nullptr, res); return res; } NIXC_CATCH_ERRS_NULL } -ValueType nix_get_type(nix_c_context * context, const Value * value) +ValueType nix_get_type(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -169,7 +201,7 @@ ValueType nix_get_type(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(NIX_TYPE_NULL); } -const char * nix_get_typename(nix_c_context * context, const Value * value) +const char * nix_get_typename(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -181,7 +213,7 @@ const char * nix_get_typename(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_NULL } -bool nix_get_bool(nix_c_context * context, const Value * value) +bool nix_get_bool(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -193,7 +225,8 @@ bool nix_get_bool(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(false); } -nix_err nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data) +nix_err +nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_callback callback, void * user_data) { if (context) context->last_err_code = NIX_OK; @@ -205,7 +238,7 @@ nix_err nix_get_string(nix_c_context * context, const Value * value, nix_get_str NIXC_CATCH_ERRS } -const char * nix_get_path_string(nix_c_context * context, const Value * value) +const char * nix_get_path_string(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -224,7 +257,7 @@ const char * nix_get_path_string(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_NULL } -unsigned int nix_get_list_size(nix_c_context * context, const Value * value) +unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -236,7 +269,7 @@ unsigned int nix_get_list_size(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(0); } -unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value) +unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -248,7 +281,7 @@ unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(0); } -double nix_get_float(nix_c_context * context, const Value * value) +double nix_get_float(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -260,19 +293,19 @@ double nix_get_float(nix_c_context * context, const Value * value) NIXC_CATCH_ERRS_RES(0.0); } -int64_t nix_get_int(nix_c_context * context, const Value * value) +int64_t nix_get_int(nix_c_context * context, const nix_value * value) { if (context) context->last_err_code = NIX_OK; try { auto & v = check_value_in(value); assert(v.type() == nix::nInt); - return v.integer(); + return v.integer().value; } NIXC_CATCH_ERRS_RES(0); } -ExternalValue * nix_get_external(nix_c_context * context, Value * value) +ExternalValue * nix_get_external(nix_c_context * context, nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -284,7 +317,7 @@ ExternalValue * nix_get_external(nix_c_context * context, Value * value) NIXC_CATCH_ERRS_NULL; } -Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix) +nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix) { if (context) context->last_err_code = NIX_OK; @@ -295,12 +328,12 @@ Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalSta nix_gc_incref(nullptr, p); if (p != nullptr) state->state.forceValue(*p, nix::noPos); - return (Value *) p; + return as_nix_value_ptr(p); } NIXC_CATCH_ERRS_NULL } -Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name) +nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name) { if (context) context->last_err_code = NIX_OK; @@ -312,7 +345,7 @@ Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalSt if (attr) { nix_gc_incref(nullptr, attr->value); state->state.forceValue(*attr->value, nix::noPos); - return attr->value; + return as_nix_value_ptr(attr->value); } nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute"); return nullptr; @@ -320,7 +353,7 @@ Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalSt NIXC_CATCH_ERRS_NULL } -bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name) +bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name) { if (context) context->last_err_code = NIX_OK; @@ -336,35 +369,36 @@ bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState NIXC_CATCH_ERRS_RES(false); } -Value * -nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name) +nix_value * nix_get_attr_byidx( + nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name) { if (context) context->last_err_code = NIX_OK; try { auto & v = check_value_in(value); const nix::Attr & a = (*v.attrs())[i]; - *name = ((const std::string &) (state->state.symbols[a.name])).c_str(); + *name = state->state.symbols[a.name].c_str(); nix_gc_incref(nullptr, a.value); state->state.forceValue(*a.value, nix::noPos); - return a.value; + return as_nix_value_ptr(a.value); } NIXC_CATCH_ERRS_NULL } -const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i) +const char * +nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i) { if (context) context->last_err_code = NIX_OK; try { auto & v = check_value_in(value); const nix::Attr & a = (*v.attrs())[i]; - return ((const std::string &) (state->state.symbols[a.name])).c_str(); + return state->state.symbols[a.name].c_str(); } NIXC_CATCH_ERRS_NULL } -nix_err nix_init_bool(nix_c_context * context, Value * value, bool b) +nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b) { if (context) context->last_err_code = NIX_OK; @@ -376,7 +410,7 @@ nix_err nix_init_bool(nix_c_context * context, Value * value, bool b) } // todo string context -nix_err nix_init_string(nix_c_context * context, Value * value, const char * str) +nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * str) { if (context) context->last_err_code = NIX_OK; @@ -387,7 +421,7 @@ nix_err nix_init_string(nix_c_context * context, Value * value, const char * str NIXC_CATCH_ERRS } -nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str) +nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * value, const char * str) { if (context) context->last_err_code = NIX_OK; @@ -398,7 +432,7 @@ nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * val NIXC_CATCH_ERRS } -nix_err nix_init_float(nix_c_context * context, Value * value, double d) +nix_err nix_init_float(nix_c_context * context, nix_value * value, double d) { if (context) context->last_err_code = NIX_OK; @@ -409,7 +443,7 @@ nix_err nix_init_float(nix_c_context * context, Value * value, double d) NIXC_CATCH_ERRS } -nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i) +nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i) { if (context) context->last_err_code = NIX_OK; @@ -420,7 +454,7 @@ nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i) NIXC_CATCH_ERRS } -nix_err nix_init_null(nix_c_context * context, Value * value) +nix_err nix_init_null(nix_c_context * context, nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -431,7 +465,7 @@ nix_err nix_init_null(nix_c_context * context, Value * value) NIXC_CATCH_ERRS } -nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value * arg) +nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * fn, nix_value * arg) { if (context) context->last_err_code = NIX_OK; @@ -444,7 +478,7 @@ nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value NIXC_CATCH_ERRS } -nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val) +nix_err nix_init_external(nix_c_context * context, nix_value * value, ExternalValue * val) { if (context) context->last_err_code = NIX_OK; @@ -471,7 +505,8 @@ ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, NIXC_CATCH_ERRS_NULL } -nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, Value * value) +nix_err +nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -491,7 +526,7 @@ void nix_list_builder_free(ListBuilder * list_builder) #endif } -nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value * value) +nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -502,7 +537,7 @@ nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value NIXC_CATCH_ERRS } -nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * p) +nix_err nix_init_primop(nix_c_context * context, nix_value * value, PrimOp * p) { if (context) context->last_err_code = NIX_OK; @@ -513,7 +548,7 @@ nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * p) NIXC_CATCH_ERRS } -nix_err nix_copy_value(nix_c_context * context, Value * value, const Value * source) +nix_err nix_copy_value(nix_c_context * context, nix_value * value, const nix_value * source) { if (context) context->last_err_code = NIX_OK; @@ -525,7 +560,7 @@ nix_err nix_copy_value(nix_c_context * context, Value * value, const Value * sou NIXC_CATCH_ERRS } -nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b) +nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuilder * b) { if (context) context->last_err_code = NIX_OK; @@ -551,7 +586,7 @@ BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * NIXC_CATCH_ERRS_NULL } -nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * bb, const char * name, Value * value) +nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * bb, const char * name, nix_value * value) { if (context) context->last_err_code = NIX_OK; @@ -572,7 +607,7 @@ void nix_bindings_builder_free(BindingsBuilder * bb) #endif } -nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD) +nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, nix_value * value, bool isIFD) { if (context) context->last_err_code = NIX_OK; diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h index b2b3439ef..044f68c9e 100644 --- a/src/libexpr-c/nix_api_value.h +++ b/src/libexpr-c/nix_api_value.h @@ -35,8 +35,11 @@ typedef enum { } ValueType; // forward declarations -typedef void Value; +typedef struct nix_value nix_value; typedef struct EvalState EvalState; + +[[deprecated("use nix_value instead")]] typedef nix_value Value; + // type defs /** @brief Stores an under-construction set of bindings * @ingroup value_manip @@ -79,6 +82,7 @@ typedef struct nix_realised_string nix_realised_string; * @{ */ /** @brief Function pointer for primops + * * When you want to return an error, call nix_set_err_msg(context, NIX_ERR_UNKNOWN, "your error message here"). * * @param[in] user_data Arbitrary data that was initially supplied to nix_alloc_primop @@ -89,7 +93,8 @@ typedef struct nix_realised_string nix_realised_string; * @param[out] ret return value * @see nix_alloc_primop, nix_init_primop */ -typedef void (*PrimOpFun)(void * user_data, nix_c_context * context, EvalState * state, Value ** args, Value * ret); +typedef void (*PrimOpFun)( + void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret); /** @brief Allocate a PrimOp * @@ -141,13 +146,33 @@ nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp); * @return value, or null in case of errors * */ -Value * nix_alloc_value(nix_c_context * context, EvalState * state); +nix_value * nix_alloc_value(nix_c_context * context, EvalState * state); + +/** + * @brief Increment the garbage collector reference counter for the given `nix_value`. + * + * The Nix language evaluator C API keeps track of alive objects by reference counting. + * When you're done with a refcounted pointer, call nix_value_decref(). + * + * @param[out] context Optional, stores error information + * @param[in] value The object to keep alive + */ +nix_err nix_value_incref(nix_c_context * context, nix_value * value); + +/** + * @brief Decrement the garbage collector reference counter for the given object + * + * @param[out] context Optional, stores error information + * @param[in] value The object to stop referencing + */ +nix_err nix_value_decref(nix_c_context * context, nix_value * value); /** @addtogroup value_manip Manipulating values - * @brief Functions to inspect and change Nix language values, represented by Value. + * @brief Functions to inspect and change Nix language values, represented by nix_value. * @{ */ -/** @name Getters +/** @anchor getters + * @name Getters */ /**@{*/ /** @brief Get value type @@ -155,7 +180,7 @@ Value * nix_alloc_value(nix_c_context * context, EvalState * state); * @param[in] value Nix value to inspect * @return type of nix value */ -ValueType nix_get_type(nix_c_context * context, const Value * value); +ValueType nix_get_type(nix_c_context * context, const nix_value * value); /** @brief Get type name of value as defined in the evaluator * @param[out] context Optional, stores error information @@ -163,14 +188,14 @@ ValueType nix_get_type(nix_c_context * context, const Value * value); * @return type name, owned string * @todo way to free the result */ -const char * nix_get_typename(nix_c_context * context, const Value * value); +const char * nix_get_typename(nix_c_context * context, const nix_value * value); /** @brief Get boolean value * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return true or false, error info via context */ -bool nix_get_bool(nix_c_context * context, const Value * value); +bool nix_get_bool(nix_c_context * context, const nix_value * value); /** @brief Get the raw string * @@ -184,7 +209,7 @@ bool nix_get_bool(nix_c_context * context, const Value * value); * @return error code, NIX_OK on success. */ nix_err -nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data); +nix_get_string(nix_c_context * context, const nix_value * value, nix_get_string_callback callback, void * user_data); /** @brief Get path as string * @param[out] context Optional, stores error information @@ -192,42 +217,42 @@ nix_get_string(nix_c_context * context, const Value * value, nix_get_string_call * @return string * @return NULL in case of error. */ -const char * nix_get_path_string(nix_c_context * context, const Value * value); +const char * nix_get_path_string(nix_c_context * context, const nix_value * value); /** @brief Get the length of a list * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return length of list, error info via context */ -unsigned int nix_get_list_size(nix_c_context * context, const Value * value); +unsigned int nix_get_list_size(nix_c_context * context, const nix_value * value); /** @brief Get the element count of an attrset * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return attrset element count, error info via context */ -unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value); +unsigned int nix_get_attrs_size(nix_c_context * context, const nix_value * value); /** @brief Get float value in 64 bits * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return float contents, error info via context */ -double nix_get_float(nix_c_context * context, const Value * value); +double nix_get_float(nix_c_context * context, const nix_value * value); /** @brief Get int value * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return int contents, error info via context */ -int64_t nix_get_int(nix_c_context * context, const Value * value); +int64_t nix_get_int(nix_c_context * context, const nix_value * value); /** @brief Get external reference * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @return reference to external, NULL in case of error */ -ExternalValue * nix_get_external(nix_c_context * context, Value *); +ExternalValue * nix_get_external(nix_c_context * context, nix_value *); /** @brief Get the ix'th element of a list * @@ -238,7 +263,7 @@ ExternalValue * nix_get_external(nix_c_context * context, Value *); * @param[in] ix list element to get * @return value, NULL in case of errors */ -Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix); +nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix); /** @brief Get an attr by name * @@ -249,7 +274,7 @@ Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalSta * @param[in] name attribute name * @return value, NULL in case of errors */ -Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name); +nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name); /** @brief Check if an attribute name exists on a value * @param[out] context Optional, stores error information @@ -258,7 +283,7 @@ Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalSt * @param[in] name attribute name * @return value, error info via context */ -bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name); +bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name); /** @brief Get an attribute by index in the sorted bindings * @@ -272,8 +297,8 @@ bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * @param[out] name will store a pointer to the attribute name * @return value, NULL in case of errors */ -Value * -nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name); +nix_value * nix_get_attr_byidx( + nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name); /** @brief Get an attribute name by index in the sorted bindings * @@ -286,7 +311,8 @@ nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * sta * @param[in] i attribute index * @return name, NULL in case of errors */ -const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i); +const char * +nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i); /**@}*/ /** @name Initializers @@ -303,7 +329,7 @@ const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * valu * @param[in] b the boolean value * @return error code, NIX_OK on success. */ -nix_err nix_init_bool(nix_c_context * context, Value * value, bool b); +nix_err nix_init_bool(nix_c_context * context, nix_value * value, bool b); /** @brief Set a string * @param[out] context Optional, stores error information @@ -311,7 +337,7 @@ nix_err nix_init_bool(nix_c_context * context, Value * value, bool b); * @param[in] str the string, copied * @return error code, NIX_OK on success. */ -nix_err nix_init_string(nix_c_context * context, Value * value, const char * str); +nix_err nix_init_string(nix_c_context * context, nix_value * value, const char * str); /** @brief Set a path * @param[out] context Optional, stores error information @@ -319,7 +345,7 @@ nix_err nix_init_string(nix_c_context * context, Value * value, const char * str * @param[in] str the path string, copied * @return error code, NIX_OK on success. */ -nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str); +nix_err nix_init_path_string(nix_c_context * context, EvalState * s, nix_value * value, const char * str); /** @brief Set a float * @param[out] context Optional, stores error information @@ -327,7 +353,7 @@ nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * val * @param[in] d the float, 64-bits * @return error code, NIX_OK on success. */ -nix_err nix_init_float(nix_c_context * context, Value * value, double d); +nix_err nix_init_float(nix_c_context * context, nix_value * value, double d); /** @brief Set an int * @param[out] context Optional, stores error information @@ -336,13 +362,13 @@ nix_err nix_init_float(nix_c_context * context, Value * value, double d); * @return error code, NIX_OK on success. */ -nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i); +nix_err nix_init_int(nix_c_context * context, nix_value * value, int64_t i); /** @brief Set null * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @return error code, NIX_OK on success. */ -nix_err nix_init_null(nix_c_context * context, Value * value); +nix_err nix_init_null(nix_c_context * context, nix_value * value); /** @brief Set the value to a thunk that will perform a function application when needed. * @@ -358,7 +384,7 @@ nix_err nix_init_null(nix_c_context * context, Value * value); * @see nix_value_call() for a similar function that performs the call immediately and only stores the return value. * Note the different argument order. */ -nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value * arg); +nix_err nix_init_apply(nix_c_context * context, nix_value * value, nix_value * fn, nix_value * arg); /** @brief Set an external value * @param[out] context Optional, stores error information @@ -366,7 +392,7 @@ nix_err nix_init_apply(nix_c_context * context, Value * value, Value * fn, Value * @param[in] val the external value to set. Will be GC-referenced by the value. * @return error code, NIX_OK on success. */ -nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val); +nix_err nix_init_external(nix_c_context * context, nix_value * value, ExternalValue * val); /** @brief Create a list from a list builder * @param[out] context Optional, stores error information @@ -374,7 +400,7 @@ nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * @param[out] value Nix value to modify * @return error code, NIX_OK on success. */ -nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value * value); +nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, nix_value * value); /** @brief Create a list builder * @param[out] context Optional, stores error information @@ -391,7 +417,8 @@ ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, * @param[in] value value to insert * @return error code, NIX_OK on success. */ -nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, Value * value); +nix_err +nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, nix_value * value); /** @brief Free a list builder * @@ -406,7 +433,7 @@ void nix_list_builder_free(ListBuilder * list_builder); * @param[in] b bindings builder to use. Make sure to unref this afterwards. * @return error code, NIX_OK on success. */ -nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b); +nix_err nix_make_attrs(nix_c_context * context, nix_value * value, BindingsBuilder * b); /** @brief Set primop * @param[out] context Optional, stores error information @@ -415,14 +442,14 @@ nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * * @see nix_alloc_primop * @return error code, NIX_OK on success. */ -nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * op); +nix_err nix_init_primop(nix_c_context * context, nix_value * value, PrimOp * op); /** @brief Copy from another value * @param[out] context Optional, stores error information * @param[out] value Nix value to modify * @param[in] source value to copy from * @return error code, NIX_OK on success. */ -nix_err nix_copy_value(nix_c_context * context, Value * value, const Value * source); +nix_err nix_copy_value(nix_c_context * context, nix_value * value, const nix_value * source); /**@}*/ /** @brief Create a bindings builder @@ -442,7 +469,7 @@ BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * * @return error code, NIX_OK on success. */ nix_err -nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, const char * name, Value * value); +nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, const char * name, nix_value * value); /** @brief Free a bindings builder * @@ -469,7 +496,7 @@ void nix_bindings_builder_free(BindingsBuilder * builder); You should set this to false when building for your application's purpose. * @return NULL if failed, are a new nix_realised_string, which must be freed with nix_realised_string_free */ -nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD); +nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, nix_value * value, bool isIFD); /** @brief Start of the string * @param[in] realised_string diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix new file mode 100644 index 000000000..eb42195a4 --- /dev/null +++ b/src/libexpr-c/package.nix @@ -0,0 +1,74 @@ +{ lib +, stdenv +, mkMesonDerivation + +, meson +, ninja +, pkg-config + +, nix-store-c +, nix-expr + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-expr-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-store-c + nix-expr + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libexpr/.version b/src/libexpr/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libexpr/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 9ad201b63..2f67260c5 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -76,7 +76,7 @@ std::pair findAlongAttrPath(EvalState & state, const std::strin if (!a) { std::set attrNames; for (auto & attr : *v->attrs()) - attrNames.insert(state.symbols[attr.name]); + attrNames.insert(std::string(state.symbols[attr.name])); auto suggestions = Suggestions::bestMatches(attrNames, attr); throw AttrPathNotFound(suggestions, "attribute '%1%' in selection path '%2%' not found", attr, attrPath); @@ -134,7 +134,7 @@ std::pair findPackageFilename(EvalState & state, Value & v return {SourcePath{path.accessor, CanonPath(fn.substr(0, colon))}, lineno}; } catch (std::invalid_argument & e) { fail(); - abort(); + unreachable(); } } diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh index ba798196d..4df9a1acd 100644 --- a/src/libexpr/attr-set.hh +++ b/src/libexpr/attr-set.hh @@ -5,7 +5,6 @@ #include "symbol-table.hh" #include -#include namespace nix { @@ -28,9 +27,9 @@ struct Attr Attr(Symbol name, Value * value, PosIdx pos = noPos) : name(name), pos(pos), value(value) { }; Attr() { }; - bool operator < (const Attr & a) const + auto operator <=> (const Attr & a) const { - return name < a.name; + return name <=> a.name; } }; diff --git a/src/libexpr/build-utils-meson b/src/libexpr/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libexpr/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/call-flake.nix similarity index 100% rename from src/libexpr/flake/call-flake.nix rename to src/libexpr/call-flake.nix diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index d60967a14..ea3319f99 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -4,9 +4,30 @@ #include "eval.hh" #include "eval-inline.hh" #include "store-api.hh" +// Need specialization involving `SymbolStr` just in this one module. +#include "strings-inline.hh" namespace nix::eval_cache { +CachedEvalError::CachedEvalError(ref cursor, Symbol attr) + : EvalError(cursor->root->state, "cached failure of attribute '%s'", cursor->getAttrPathStr(attr)) + , cursor(cursor), attr(attr) +{ } + +void CachedEvalError::force() +{ + auto & v = cursor->forceValue(); + + if (v.type() == nAttrs) { + auto a = v.attrs()->get(this->attr); + + state.forceValue(*a->value, a->pos); + } + + // Shouldn't happen. + throw EvalError(state, "evaluation of cached failed attribute '%s' unexpectedly succeeded", cursor->getAttrPathStr(attr)); +} + static const char * schema = R"sql( create table if not exists Attributes ( parent integer not null, @@ -48,7 +69,7 @@ struct AttrDb { auto state(_state->lock()); - Path cacheDir = getCacheDir() + "/nix/eval-cache-v5"; + Path cacheDir = getCacheDir() + "/eval-cache-v5"; createDirs(cacheDir); Path dbPath = cacheDir + "/" + fingerprint.to_string(HashFormat::Base16, false) + ".sqlite"; @@ -76,11 +97,11 @@ struct AttrDb { try { auto state(_state->lock()); - if (!failed) + if (!failed && state->txn->active) state->txn->commit(); state->txn.reset(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -91,7 +112,7 @@ struct AttrDb try { return fun(); } catch (SQLiteError &) { - ignoreException(); + ignoreExceptionExceptInterrupt(); failed = true; return 0; } @@ -206,7 +227,7 @@ struct AttrDb (key.first) (symbols[key.second]) (AttrType::ListOfStrings) - (concatStringsSep("\t", l)).exec(); + (dropEmptyInitThenConcatStringsSep("\t", l)).exec(); return state->db.getLastInsertedRowId(); }); @@ -307,7 +328,7 @@ struct AttrDb case AttrType::Bool: return {{rowId, queryAttribute.getInt(2) != 0}}; case AttrType::Int: - return {{rowId, int_t{queryAttribute.getInt(2)}}}; + return {{rowId, int_t{NixInt{queryAttribute.getInt(2)}}}}; case AttrType::ListOfStrings: return {{rowId, tokenizeString>(queryAttribute.getStr(2), "\t")}}; case AttrType::Missing: @@ -330,7 +351,7 @@ static std::shared_ptr makeAttrDb( try { return std::make_shared(cfg, fingerprint, symbols); } catch (SQLiteError &) { - ignoreException(); + ignoreExceptionExceptInterrupt(); return nullptr; } } @@ -416,12 +437,12 @@ std::vector AttrCursor::getAttrPath(Symbol name) const std::string AttrCursor::getAttrPathStr() const { - return concatStringsSep(".", root->state.symbols.resolve(getAttrPath())); + return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath())); } std::string AttrCursor::getAttrPathStr(Symbol name) const { - return concatStringsSep(".", root->state.symbols.resolve(getAttrPath(name))); + return dropEmptyInitThenConcatStringsSep(".", root->state.symbols.resolve(getAttrPath(name))); } Value & AttrCursor::forceValue() @@ -450,7 +471,7 @@ Value & AttrCursor::forceValue() else if (v.type() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean()), v.boolean()}; else if (v.type() == nInt) - cachedValue = {root->db->setInt(getKey(), v.integer()), int_t{v.integer()}}; + cachedValue = {root->db->setInt(getKey(), v.integer().value), int_t{v.integer()}}; else if (v.type() == nAttrs) ; // FIXME: do something? else @@ -465,12 +486,12 @@ Suggestions AttrCursor::getSuggestionsForAttr(Symbol name) auto attrNames = getAttrs(); std::set strAttrNames; for (auto & name : attrNames) - strAttrNames.insert(root->state.symbols[name]); + strAttrNames.insert(std::string(root->state.symbols[name])); return Suggestions::bestMatches(strAttrNames, root->state.symbols[name]); } -std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErrors) +std::shared_ptr AttrCursor::maybeGetAttr(Symbol name) { if (root->db) { if (!cachedValue) @@ -487,12 +508,9 @@ std::shared_ptr AttrCursor::maybeGetAttr(Symbol name, bool forceErro if (attr) { if (std::get_if(&attr->second)) return nullptr; - else if (std::get_if(&attr->second)) { - if (forceErrors) - debug("reevaluating failed cached attribute '%s'", getAttrPathStr(name)); - else - throw CachedEvalError(root->state, "cached failure of attribute '%s'", getAttrPathStr(name)); - } else + else if (std::get_if(&attr->second)) + throw CachedEvalError(ref(shared_from_this()), name); + else return std::make_shared(root, std::make_pair(shared_from_this(), name), nullptr, std::move(attr)); } @@ -537,9 +555,9 @@ std::shared_ptr AttrCursor::maybeGetAttr(std::string_view name) return maybeGetAttr(root->state.symbols.create(name)); } -ref AttrCursor::getAttr(Symbol name, bool forceErrors) +ref AttrCursor::getAttr(Symbol name) { - auto p = maybeGetAttr(name, forceErrors); + auto p = maybeGetAttr(name); if (!p) throw Error("attribute '%s' does not exist", getAttrPathStr(name)); return ref(p); @@ -550,11 +568,11 @@ ref AttrCursor::getAttr(std::string_view name) return getAttr(root->state.symbols.create(name)); } -OrSuggestions> AttrCursor::findAlongAttrPath(const std::vector & attrPath, bool force) +OrSuggestions> AttrCursor::findAlongAttrPath(const std::vector & attrPath) { auto res = shared_from_this(); for (auto & attr : attrPath) { - auto child = res->maybeGetAttr(attr, force); + auto child = res->maybeGetAttr(attr); if (!child) { auto suggestions = res->getSuggestionsForAttr(attr); return OrSuggestions>::failed(suggestions); @@ -751,8 +769,9 @@ bool AttrCursor::isDerivation() StorePath AttrCursor::forceDerivation() { - auto aDrvPath = getAttr(root->state.sDrvPath, true); + auto aDrvPath = getAttr(root->state.sDrvPath); auto drvPath = root->state.store->parseStorePath(aDrvPath->getString()); + drvPath.requireDerivation(); if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) { /* The eval cache contains 'drvPath', but the actual path has been garbage-collected. So force it to be regenerated. */ diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh index 46c4999c8..b1911e3a4 100644 --- a/src/libexpr/eval-cache.hh +++ b/src/libexpr/eval-cache.hh @@ -10,14 +10,28 @@ namespace nix::eval_cache { -MakeError(CachedEvalError, EvalError); - struct AttrDb; class AttrCursor; +struct CachedEvalError : EvalError +{ + const ref cursor; + const Symbol attr; + + CachedEvalError(ref cursor, Symbol attr); + + /** + * Evaluate this attribute, which should result in a regular + * `EvalError` exception being thrown. + */ + [[noreturn]] + void force(); +}; + class EvalCache : public std::enable_shared_from_this { friend class AttrCursor; + friend struct CachedEvalError; std::shared_ptr db; EvalState & state; @@ -73,6 +87,7 @@ typedef std::variant< class AttrCursor : public std::enable_shared_from_this { friend class EvalCache; + friend struct CachedEvalError; ref root; typedef std::optional, Symbol>> Parent; @@ -102,11 +117,11 @@ public: Suggestions getSuggestionsForAttr(Symbol name); - std::shared_ptr maybeGetAttr(Symbol name, bool forceErrors = false); + std::shared_ptr maybeGetAttr(Symbol name); std::shared_ptr maybeGetAttr(std::string_view name); - ref getAttr(Symbol name, bool forceErrors = false); + ref getAttr(Symbol name); ref getAttr(std::string_view name); @@ -114,7 +129,7 @@ public: * Get an attribute along a chain of attrsets. Note that this does * not auto-call functors or functions. */ - OrSuggestions> findAlongAttrPath(const std::vector & attrPath, bool force = false); + OrSuggestions> findAlongAttrPath(const std::vector & attrPath); std::string getString(); diff --git a/src/libexpr/eval-error.cc b/src/libexpr/eval-error.cc index 8db03610b..cdb0b4772 100644 --- a/src/libexpr/eval-error.cc +++ b/src/libexpr/eval-error.cc @@ -27,8 +27,7 @@ EvalErrorBuilder & EvalErrorBuilder::atPos(Value & value, PosIdx fallback) template EvalErrorBuilder & EvalErrorBuilder::withTrace(PosIdx pos, const std::string_view text) { - error.err.traces.push_front( - Trace{.pos = error.state.positions[pos], .hint = HintFmt(std::string(text))}); + error.addTrace(error.state.positions[pos], text); return *this; } @@ -71,15 +70,17 @@ EvalErrorBuilder::addTrace(PosIdx pos, std::string_view formatString, const A return *this; } +template +EvalErrorBuilder & EvalErrorBuilder::setIsFromExpr() +{ + error.err.isFromExpr = true; + return *this; +} + template void EvalErrorBuilder::debugThrow() { - if (error.state.debugRepl && !error.state.debugTraces.empty()) { - const DebugTrace & last = error.state.debugTraces.front(); - const Env * env = &last.env; - const Expr * expr = &last.expr; - error.state.runDebugRepl(&error, *env, *expr); - } + error.state.runDebugRepl(&error); // `EvalState` is the only class that can construct an `EvalErrorBuilder`, // and it does so in dynamic storage. This is the final method called on @@ -91,6 +92,15 @@ void EvalErrorBuilder::debugThrow() throw error; } +template +void EvalErrorBuilder::panic() +{ + logError(error.info()); + printError("This is a bug! An unexpected condition occurred, causing the Nix evaluator to have to stop. If you could share a reproducible example or a core dump, please open an issue at https://github.com/NixOS/nix/issues"); + abort(); +} + +template class EvalErrorBuilder; template class EvalErrorBuilder; template class EvalErrorBuilder; template class EvalErrorBuilder; @@ -99,7 +109,6 @@ template class EvalErrorBuilder; template class EvalErrorBuilder; template class EvalErrorBuilder; template class EvalErrorBuilder; -template class EvalErrorBuilder; template class EvalErrorBuilder; } diff --git a/src/libexpr/eval-error.hh b/src/libexpr/eval-error.hh index 7e0cbe982..ed004eb53 100644 --- a/src/libexpr/eval-error.hh +++ b/src/libexpr/eval-error.hh @@ -1,7 +1,5 @@ #pragma once -#include - #include "error.hh" #include "pos-idx.hh" @@ -15,27 +13,39 @@ class EvalState; template class EvalErrorBuilder; -class EvalError : public Error +/** + * Base class for all errors that occur during evaluation. + * + * Most subclasses should inherit from `EvalError` instead of this class. + */ +class EvalBaseError : public Error { template friend class EvalErrorBuilder; public: EvalState & state; - EvalError(EvalState & state, ErrorInfo && errorInfo) + EvalBaseError(EvalState & state, ErrorInfo && errorInfo) : Error(errorInfo) , state(state) { } template - explicit EvalError(EvalState & state, const std::string & formatString, const Args &... formatArgs) + explicit EvalBaseError(EvalState & state, const std::string & formatString, const Args &... formatArgs) : Error(formatString, formatArgs...) , state(state) { } }; +/** + * `EvalError` is the base class for almost all errors that occur during evaluation. + * + * All instances of `EvalError` should show a degree of purity that allows them to be + * cached in pure mode. This means that they should not depend on the configuration or the overall environment. + */ +MakeError(EvalError, EvalBaseError); MakeError(ParseError, Error); MakeError(AssertionError, EvalError); MakeError(ThrownError, AssertionError); @@ -43,7 +53,6 @@ MakeError(Abort, EvalError); MakeError(TypeError, EvalError); MakeError(UndefinedVarError, EvalError); MakeError(MissingArgumentError, EvalError); -MakeError(CachedEvalError, EvalError); MakeError(InfiniteRecursionError, EvalError); struct InvalidPathError : public EvalError @@ -91,6 +100,8 @@ public: [[nodiscard, gnu::noinline]] EvalErrorBuilder & addTrace(PosIdx pos, HintFmt hint); + [[nodiscard, gnu::noinline]] EvalErrorBuilder & setIsFromExpr(); + template [[nodiscard, gnu::noinline]] EvalErrorBuilder & addTrace(PosIdx pos, std::string_view formatString, const Args &... formatArgs); @@ -99,6 +110,12 @@ public: * Delete the `EvalErrorBuilder` and throw the underlying exception. */ [[gnu::noinline, gnu::noreturn]] void debugThrow(); + + /** + * A programming error or fatal condition occurred. Abort the process for core dump and debugging. + * This does not print a proper backtrace, because unwinding the stack is destructive. + */ + [[gnu::noinline, gnu::noreturn]] void panic(); }; } diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc new file mode 100644 index 000000000..07ce05a2c --- /dev/null +++ b/src/libexpr/eval-gc.cc @@ -0,0 +1,117 @@ +#include "error.hh" +#include "environment-variables.hh" +#include "eval-settings.hh" +#include "config-global.hh" +#include "serialise.hh" +#include "eval-gc.hh" + +#if HAVE_BOEHMGC + +# include +# if __FreeBSD__ +# include +# endif + +# include +# include +# include + +# include +# include +# include + +#endif + +namespace nix { + +#if HAVE_BOEHMGC +/* Called when the Boehm GC runs out of memory. */ +static void * oomHandler(size_t requested) +{ + /* Convert this to a proper C++ exception. */ + throw std::bad_alloc(); +} + +static inline void initGCReal() +{ + /* Initialise the Boehm garbage collector. */ + + /* Don't look for interior pointers. This reduces the odds of + misdetection a bit. */ + GC_set_all_interior_pointers(0); + + /* We don't have any roots in data segments, so don't scan from + there. */ + GC_set_no_dls(1); + + /* Enable perf measurements. This is just a setting; not much of a + start of something. */ + GC_start_performance_measurement(); + + GC_INIT(); + + GC_set_oom_fn(oomHandler); + + /* Set the initial heap size to something fairly big (25% of + physical RAM, up to a maximum of 384 MiB) so that in most cases + we don't need to garbage collect at all. (Collection has a + fairly significant overhead.) The heap size can be overridden + through libgc's GC_INITIAL_HEAP_SIZE environment variable. We + should probably also provide a nix.conf setting for this. Note + that GC_expand_hp() causes a lot of virtual, but not physical + (resident) memory to be allocated. This might be a problem on + systems that don't overcommit. */ + if (!getEnv("GC_INITIAL_HEAP_SIZE")) { + size_t size = 32 * 1024 * 1024; +# if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES) + size_t maxSize = 384 * 1024 * 1024; + long pageSize = sysconf(_SC_PAGESIZE); + long pages = sysconf(_SC_PHYS_PAGES); + if (pageSize != -1) + size = (pageSize * pages) / 4; // 25% of RAM + if (size > maxSize) + size = maxSize; +# endif + debug("setting initial heap size to %1% bytes", size); + GC_expand_hp(size); + } +} + +static size_t gcCyclesAfterInit = 0; + +size_t getGCCycles() +{ + assertGCInitialized(); + return static_cast(GC_get_gc_no()) - gcCyclesAfterInit; +} + +#endif + +static bool gcInitialised = false; + +void initGC() +{ + if (gcInitialised) + return; + +#if HAVE_BOEHMGC + initGCReal(); + + gcCyclesAfterInit = GC_get_gc_no(); +#endif + + // NIX_PATH must override the regular setting + // See the comment in applyConfig + if (auto nixPathEnv = getEnv("NIX_PATH")) { + globalConfig.set("nix-path", concatStringsSep(" ", EvalSettings::parseNixPath(nixPathEnv.value()))); + } + + gcInitialised = true; +} + +void assertGCInitialized() +{ + assert(gcInitialised); +} + +} // namespace nix diff --git a/src/libexpr/eval-gc.hh b/src/libexpr/eval-gc.hh new file mode 100644 index 000000000..f3b699b54 --- /dev/null +++ b/src/libexpr/eval-gc.hh @@ -0,0 +1,53 @@ +#pragma once +///@file + +#include + +#if HAVE_BOEHMGC + +# define GC_INCLUDE_NEW + +# include +# include +# include + +#else + +# include + +/* Some dummy aliases for Boehm GC definitions to reduce the number of + #ifdefs. */ + +template +using traceable_allocator = std::allocator; + +template +using gc_allocator = std::allocator; + +# define GC_MALLOC_ATOMIC std::malloc + +struct gc +{}; + +#endif + +namespace nix { + +/** + * Initialise the Boehm GC, if applicable. + */ +void initGC(); + +/** + * Make sure `initGC` has already been called. + */ +void assertGCInitialized(); + +#ifdef HAVE_BOEHMGC +/** + * The number of GC cycles since initGC(). + */ +size_t getGCCycles(); +#endif + +} // namespace nix diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 6fa34b062..d5ce238b2 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -4,6 +4,7 @@ #include "print.hh" #include "eval.hh" #include "eval-error.hh" +#include "eval-settings.hh" namespace nix { @@ -138,5 +139,12 @@ inline void EvalState::forceList(Value & v, const PosIdx pos, std::string_view e } } +[[gnu::always_inline]] +inline CallDepth EvalState::addCallDepth(const PosIdx pos) { + if (callDepth > settings.maxCallDepth) + error("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow(); + + return CallDepth(callDepth); +}; } diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 2ccbe327f..4cbcb39b9 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -8,7 +8,7 @@ namespace nix { /* Very hacky way to parse $NIX_PATH, which is colon-separated, but can contain URLs (e.g. "nixpkgs=https://bla...:foo=https://"). */ -static Strings parseNixPath(const std::string & s) +Strings EvalSettings::parseNixPath(const std::string & s) { Strings res; @@ -44,10 +44,13 @@ static Strings parseNixPath(const std::string & s) return res; } -EvalSettings::EvalSettings() +EvalSettings::EvalSettings(bool & readOnlyMode, EvalSettings::LookupPathHooks lookupPathHooks) + : readOnlyMode{readOnlyMode} + , lookupPathHooks{lookupPathHooks} { - auto var = getEnv("NIX_PATH"); - if (var) nixPath = parseNixPath(*var); + auto var = getEnv("NIX_ABORT_ON_WARN"); + if (var && (var == "1" || var == "yes" || var == "true")) + builtinsAbortOnWarn = true; } Strings EvalSettings::getDefaultNixPath() @@ -63,11 +66,9 @@ Strings EvalSettings::getDefaultNixPath() } }; - if (!evalSettings.restrictEval && !evalSettings.pureEval) { - add(getNixDefExpr() + "/channels"); - add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); - add(rootChannelsDir()); - } + add(getNixDefExpr() + "/channels"); + add(rootChannelsDir() + "/nixpkgs", "nixpkgs"); + add(rootChannelsDir()); return res; } @@ -89,20 +90,16 @@ std::string EvalSettings::resolvePseudoUrl(std::string_view url) return std::string(url); } -const std::string & EvalSettings::getCurrentSystem() +const std::string & EvalSettings::getCurrentSystem() const { const auto & evalSystem = currentSystem.get(); return evalSystem != "" ? evalSystem : settings.thisSystem.get(); } -EvalSettings evalSettings; - -static GlobalConfig::Register rEvalSettings(&evalSettings); - Path getNixDefExpr() { return settings.useXDGBaseDirectories - ? getStateDir() + "/nix/defexpr" + ? getStateDir() + "/defexpr" : getHome() + "/.nix-defexpr"; } diff --git a/src/libexpr/eval-settings.hh b/src/libexpr/eval-settings.hh index 60d3a6f25..115e3ee50 100644 --- a/src/libexpr/eval-settings.hh +++ b/src/libexpr/eval-settings.hh @@ -2,49 +2,117 @@ ///@file #include "config.hh" +#include "ref.hh" namespace nix { +class Store; + struct EvalSettings : Config { - EvalSettings(); + /** + * Function used to interpet look path entries of a given scheme. + * + * The argument is the non-scheme part of the lookup path entry (see + * `LookupPathHooks` below). + * + * The return value is (a) whether the entry was valid, and, if so, + * what does it map to. + * + * @todo Return (`std::optional` of) `SourceAccssor` or something + * more structured instead of mere `std::string`? + */ + using LookupPathHook = std::optional(ref store, std::string_view); + + /** + * Map from "scheme" to a `LookupPathHook`. + * + * Given a lookup path value (i.e. either the whole thing, or after + * the `=`) in the form of: + * + * ``` + * : + * ``` + * + * if `` is a key in this map, then `` is + * passed to the hook that is the value in this map. + */ + using LookupPathHooks = std::map>; + + EvalSettings(bool & readOnlyMode, LookupPathHooks lookupPathHooks = {}); + + bool & readOnlyMode; static Strings getDefaultNixPath(); static bool isPseudoUrl(std::string_view s); + static Strings parseNixPath(const std::string & s); + static std::string resolvePseudoUrl(std::string_view url); - Setting enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", - "Whether builtin functions that allow executing native code should be enabled."}; + LookupPathHooks lookupPathHooks; + + Setting enableNativeCode{this, false, "allow-unsafe-native-code-during-evaluation", R"( + Enable built-in functions that allow executing native code. + + In particular, this adds: + - `builtins.importNative` *path* *symbol* + + Opens dynamic shared object (DSO) at *path*, loads the function with the symbol name *symbol* from it and runs it. + The loaded function must have the following signature: + ```cpp + extern "C" typedef void (*ValueInitialiser) (EvalState & state, Value & v); + ``` + + The [Nix C++ API documentation](@docroot@/development/documentation.md#api-documentation) has more details on evaluator internals. + + - `builtins.exec` *arguments* + + Execute a program, where *arguments* are specified as a list of strings, and parse its output as a Nix expression. + )"}; Setting nixPath{ - this, getDefaultNixPath(), "nix-path", + this, {}, "nix-path", R"( List of search paths to use for [lookup path](@docroot@/language/constructs/lookup-path.md) resolution. This setting determines the value of - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtin-constants.md#builtins-findFile). + [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) and can be used with [`builtins.findFile`](@docroot@/language/builtins.md#builtins-findFile). - The default value is + - The configuration setting is overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) + environment variable. + - `NIX_PATH` is overridden by [specifying the setting as the command line flag](@docroot@/command-ref/conf-file.md#command-line-flags) `--nix-path`. + - Any current value is extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path`. - ``` - $HOME/.nix-defexpr/channels - nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs - $NIX_STATE_DIR/profiles/per-user/root/channels - ``` + If the respective paths are accessible, the default values are: - It can be overridden with the [`NIX_PATH` environment variable](@docroot@/command-ref/env-common.md#env-NIX_PATH) or the [`-I` command line option](@docroot@/command-ref/opt-common.md#opt-I). + - `$HOME/.nix-defexpr/channels` + + The [user channel link](@docroot@/command-ref/files/default-nix-expression.md#user-channel-link), pointing to the current state of [channels](@docroot@/command-ref/files/channels.md) for the current user. + + - `nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs` + + The current state of the `nixpkgs` channel for the `root` user. + + - `$NIX_STATE_DIR/profiles/per-user/root/channels` + + The current state of all channels for the `root` user. + + These files are set up by the [Nix installer](@docroot@/installation/installing-binary.md). + See [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) for details on the environment variable. > **Note** > - > If [pure evaluation](#conf-pure-eval) is enabled, `nixPath` evaluates to the empty list `[ ]`. + > If [restricted evaluation](@docroot@/command-ref/conf-file.md#conf-restrict-eval) is enabled, the default value is empty. + > + > If [pure evaluation](#conf-pure-eval) is enabled, `builtins.nixPath` *always* evaluates to the empty list `[ ]`. )", {}, false}; Setting currentSystem{ this, "", "eval-system", R"( This option defines - [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) + [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) in the Nix language if it is set as a non-empty string. Otherwise, if it is defined as the empty string (the default), the value of the [`system` ](#conf-system) @@ -58,14 +126,14 @@ struct EvalSettings : Config * Implements the `eval-system` vs `system` defaulting logic * described for `eval-system`. */ - const std::string & getCurrentSystem(); + const std::string & getCurrentSystem() const; Setting restrictEval{ this, false, "restrict-eval", R"( If set to `true`, the Nix evaluator will not allow access to any files outside of - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath), + [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath), or to URIs outside of [`allowed-uris`](@docroot@/command-ref/conf-file.md#conf-allowed-uris). )"}; @@ -76,10 +144,10 @@ struct EvalSettings : Config - Restrict file system and network access to files specified by cryptographic hash - Disable impure constants: - - [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) - - [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) - - [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath) - - [`builtins.storePath`](@docroot@/language/builtin-constants.md#builtins-storePath) + - [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) + - [`builtins.currentTime`](@docroot@/language/builtins.md#builtins-currentTime) + - [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) + - [`builtins.storePath`](@docroot@/language/builtins.md#builtins-storePath) )" }; @@ -126,7 +194,11 @@ struct EvalSettings : Config )"}; Setting useEvalCache{this, true, "eval-cache", - "Whether to use the flake evaluation cache."}; + R"( + Whether to use the flake evaluation cache. + Certain commands won't have to evaluate when invoked for the second time with a particular version of a flake. + Intermediate results are not cached. + )"}; Setting ignoreExceptionsDuringTry{this, false, "ignore-try", R"( @@ -142,16 +214,40 @@ struct EvalSettings : Config Setting builtinsTraceDebugger{this, false, "debugger-on-trace", R"( - If set to true and the `--debugger` flag is given, - [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace) will - enter the debugger like - [`builtins.break`](@docroot@/language/builtins.md#builtins-break). + If set to true and the `--debugger` flag is given, the following functions + will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break). + + * [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace) + * [`builtins.traceVerbose`](@docroot@/language/builtins.md#builtins-traceVerbose) + if [`trace-verbose`](#conf-trace-verbose) is set to true. + * [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) This is useful for debugging warnings in third-party Nix code. )"}; -}; -extern EvalSettings evalSettings; + Setting builtinsDebuggerOnWarn{this, false, "debugger-on-warn", + R"( + If set to true and the `--debugger` flag is given, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) + will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break). + + This is useful for debugging warnings in third-party Nix code. + + Use [`debugger-on-trace`](#conf-debugger-on-trace) to also enter the debugger on legacy warnings that are logged with [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace). + )"}; + + Setting builtinsAbortOnWarn{this, false, "abort-on-warn", + R"( + If set to true, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) will throw an error when logging a warning. + + This will give you a stack trace that leads to the location of the warning. + + This is useful for finding information about warnings in third-party Nix code when you can not start the interactive debugger, such as when Nix is called from a non-interactive script. See [`debugger-on-warn`](#conf-debugger-on-warn). + + Currently, a stack trace can only be produced when the debugger is enabled, or when evaluation is aborted. + + This option can be enabled by setting `NIX_ABORT_ON_WARN=1` in the environment. + )"}; +}; /** * Conventionally part of the default nix path in impure mode. diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index d7e3a2cdb..f17753415 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1,15 +1,13 @@ #include "eval.hh" #include "eval-settings.hh" -#include "hash.hh" #include "primops.hh" #include "print-options.hh" -#include "shared.hh" +#include "exit.hh" #include "types.hh" #include "util.hh" #include "store-api.hh" #include "derivations.hh" #include "downstream-placeholder.hh" -#include "globals.hh" #include "eval-inline.hh" #include "filetransfer.hh" #include "function-trace.hh" @@ -17,45 +15,30 @@ #include "print.hh" #include "filtering-source-accessor.hh" #include "memory-source-accessor.hh" -#include "signals.hh" #include "gc-small-vector.hh" #include "url.hh" #include "fetch-to-store.hh" #include "tarball.hh" -#include "flake/flakeref.hh" #include "parser-tab.hh" #include -#include #include +#include #include #include #include #include #include #include -#include #include #include #ifndef _WIN32 // TODO use portable implementation -# include +# include #endif -#if HAVE_BOEHMGC - -#define GC_INCLUDE_NEW - -#include -#include -#include - -#include -#include -#include - -#endif +#include "strings-inline.hh" using json = nlohmann::json; @@ -64,24 +47,7 @@ namespace nix { static char * allocString(size_t size) { char * t; -#if HAVE_BOEHMGC t = (char *) GC_MALLOC_ATOMIC(size); -#else - t = (char *) malloc(size); -#endif - if (!t) throw std::bad_alloc(); - return t; -} - - -static char * dupString(const char * s) -{ - char * t; -#if HAVE_BOEHMGC - t = GC_STRDUP(s); -#else - t = strdup(s); -#endif if (!t) throw std::bad_alloc(); return t; } @@ -105,11 +71,7 @@ static const char * makeImmutableString(std::string_view s) RootValue allocRootValue(Value * v) { -#if HAVE_BOEHMGC return std::allocate_shared(traceable_allocator(), v); -#else - return std::make_shared(v); -#endif } // Pretty print types for assertion errors @@ -155,7 +117,7 @@ std::string_view showType(ValueType type, bool withArticle) case nFloat: return WA("a", "float"); case nThunk: return WA("a", "thunk"); } - abort(); + unreachable(); } @@ -206,53 +168,6 @@ bool Value::isTrivial() const } -#if HAVE_BOEHMGC -/* Called when the Boehm GC runs out of memory. */ -static void * oomHandler(size_t requested) -{ - /* Convert this to a proper C++ exception. */ - throw std::bad_alloc(); -} - -class BoehmGCStackAllocator : public StackAllocator { - boost::coroutines2::protected_fixedsize_stack stack { - // We allocate 8 MB, the default max stack size on NixOS. - // A smaller stack might be quicker to allocate but reduces the stack - // depth available for source filter expressions etc. - std::max(boost::context::stack_traits::default_size(), static_cast(8 * 1024 * 1024)) - }; - - // This is specific to boost::coroutines2::protected_fixedsize_stack. - // The stack protection page is included in sctx.size, so we have to - // subtract one page size from the stack size. - std::size_t pfss_usable_stack_size(boost::context::stack_context &sctx) { - return sctx.size - boost::context::stack_traits::page_size(); - } - - public: - boost::context::stack_context allocate() override { - auto sctx = stack.allocate(); - - // Stacks generally start at a high address and grow to lower addresses. - // Architectures that do the opposite are rare; in fact so rare that - // boost_routine does not implement it. - // So we subtract the stack size. - GC_add_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); - return sctx; - } - - void deallocate(boost::context::stack_context sctx) override { - GC_remove_roots(static_cast(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); - stack.deallocate(sctx); - } - -}; - -static BoehmGCStackAllocator boehmGCStackAllocator; - -#endif - - static Symbol getName(const AttrName & name, EvalState & state, Env & env) { if (name.symbol) { @@ -265,90 +180,17 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) } } -#if HAVE_BOEHMGC -/* Disable GC while this object lives. Used by CoroutineContext. - * - * Boehm keeps a count of GC_disable() and GC_enable() calls, - * and only enables GC when the count matches. - */ -class BoehmDisableGC { -public: - BoehmDisableGC() { - GC_disable(); - }; - ~BoehmDisableGC() { - GC_enable(); - }; -}; -#endif - -static bool gcInitialised = false; - -void initGC() -{ - if (gcInitialised) return; - -#if HAVE_BOEHMGC - /* Initialise the Boehm garbage collector. */ - - /* Don't look for interior pointers. This reduces the odds of - misdetection a bit. */ - GC_set_all_interior_pointers(0); - - /* We don't have any roots in data segments, so don't scan from - there. */ - GC_set_no_dls(1); - - GC_INIT(); - - GC_set_oom_fn(oomHandler); - - StackAllocator::defaultAllocator = &boehmGCStackAllocator; - - -#if NIX_BOEHM_PATCH_VERSION != 1 - printTalkative("Unpatched BoehmGC, disabling GC inside coroutines"); - /* Used to disable GC when entering coroutines on macOS */ - create_coro_gc_hook = []() -> std::shared_ptr { - return std::make_shared(); - }; -#endif - - /* Set the initial heap size to something fairly big (25% of - physical RAM, up to a maximum of 384 MiB) so that in most cases - we don't need to garbage collect at all. (Collection has a - fairly significant overhead.) The heap size can be overridden - through libgc's GC_INITIAL_HEAP_SIZE environment variable. We - should probably also provide a nix.conf setting for this. Note - that GC_expand_hp() causes a lot of virtual, but not physical - (resident) memory to be allocated. This might be a problem on - systems that don't overcommit. */ - if (!getEnv("GC_INITIAL_HEAP_SIZE")) { - size_t size = 32 * 1024 * 1024; -#if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES) - size_t maxSize = 384 * 1024 * 1024; - long pageSize = sysconf(_SC_PAGESIZE); - long pages = sysconf(_SC_PHYS_PAGES); - if (pageSize != -1) - size = (pageSize * pages) / 4; // 25% of RAM - if (size > maxSize) size = maxSize; -#endif - debug("setting initial heap size to %1% bytes", size); - GC_expand_hp(size); - } - -#endif - - gcInitialised = true; -} - static constexpr size_t BASE_ENV_SIZE = 128; EvalState::EvalState( - const LookupPath & _lookupPath, + const LookupPath & lookupPathFromArguments, ref store, + const fetchers::Settings & fetchSettings, + const EvalSettings & settings, std::shared_ptr buildStore) - : sWith(symbols.create("")) + : fetchSettings{fetchSettings} + , settings{settings} + , sWith(symbols.create("")) , sOutPath(symbols.create("outPath")) , sDrvPath(symbols.create("drvPath")) , sType(symbols.create("type")) @@ -368,6 +210,12 @@ EvalState::EvalState( , sRight(symbols.create("right")) , sWrong(symbols.create("wrong")) , sStructuredAttrs(symbols.create("__structuredAttrs")) + , sAllowedReferences(symbols.create("allowedReferences")) + , sAllowedRequisites(symbols.create("allowedRequisites")) + , sDisallowedReferences(symbols.create("disallowedReferences")) + , sDisallowedRequisites(symbols.create("disallowedRequisites")) + , sMaxSize(symbols.create("maxSize")) + , sMaxClosureSize(symbols.create("maxClosureSize")) , sBuilder(symbols.create("builder")) , sArgs(symbols.create("args")) , sContentAddressed(symbols.create("__contentAddressed")) @@ -398,10 +246,10 @@ EvalState::EvalState( , repair(NoRepair) , emptyBindings(0) , rootFS( - evalSettings.restrictEval || evalSettings.pureEval + settings.restrictEval || settings.pureEval ? ref(AllowListSourceAccessor::create(getFSSourceAccessor(), {}, - [](const CanonPath & path) -> RestrictedPathError { - auto modeInformation = evalSettings.pureEval + [&settings](const CanonPath & path) -> RestrictedPathError { + auto modeInformation = settings.pureEval ? "in pure evaluation mode (use '--impure' to override)" : "in restricted mode"; throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); @@ -415,7 +263,7 @@ EvalState::EvalState( )} , callFlakeInternal{internalFS->addFile( CanonPath("call-flake.nix"), - #include "flake/call-flake.nix.gen.hh" + #include "call-flake.nix.gen.hh" )} , store(store) , buildStore(buildStore ? buildStore : store) @@ -438,7 +286,7 @@ EvalState::EvalState( countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; - assert(gcInitialised); + assertGCInitialized(); static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); @@ -451,12 +299,21 @@ EvalState::EvalState( vStringSymlink.mkString("symlink"); vStringUnknown.mkString("unknown"); - /* Initialise the Nix expression search path. */ - if (!evalSettings.pureEval) { - for (auto & i : _lookupPath.elements) + /* Construct the Nix expression search path. */ + assert(lookupPath.elements.empty()); + if (!settings.pureEval) { + for (auto & i : lookupPathFromArguments.elements) { lookupPath.elements.emplace_back(LookupPath::Elem {i}); - for (auto & i : evalSettings.nixPath.get()) + } + /* $NIX_PATH overriding regular settings is implemented as a hack in `initGC()` */ + for (auto & i : settings.nixPath.get()) { lookupPath.elements.emplace_back(LookupPath::Elem::parse(i)); + } + if (!settings.restrictEval) { + for (auto & i : EvalSettings::getDefaultNixPath()) { + lookupPath.elements.emplace_back(LookupPath::Elem::parse(i)); + } + } } /* Allow access to all paths in the search path. */ @@ -533,9 +390,9 @@ bool isAllowedURI(std::string_view uri, const Strings & allowedUris) void EvalState::checkURI(const std::string & uri) { - if (!evalSettings.restrictEval) return; + if (!settings.restrictEval) return; - if (isAllowedURI(uri, evalSettings.allowedUris.get())) return; + if (isAllowedURI(uri, settings.allowedUris.get())) return; /* If the URI is a path, then check it against allowedPaths as well. */ @@ -580,7 +437,7 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info) constantInfos.push_back({name2, info}); - if (!(evalSettings.pureEval && info.impureOnly)) { + if (!(settings.pureEval && info.impureOnly)) { /* Check the type, if possible. We might know the type of a thunk in advance, so be allowed @@ -679,6 +536,68 @@ std::optional EvalState::getDoc(Value & v) .doc = doc, }; } + if (v.isLambda()) { + auto exprLambda = v.payload.lambda.fun; + + std::ostringstream s; + std::string name; + auto pos = positions[exprLambda->getPos()]; + std::string docStr; + + if (exprLambda->name) { + name = symbols[exprLambda->name]; + } + + if (exprLambda->docComment) { + docStr = exprLambda->docComment.getInnerText(positions); + } + + if (name.empty()) { + s << "Function "; + } + else { + s << "Function `" << name << "`"; + if (pos) + s << "\\\n … " ; + else + s << "\\\n"; + } + if (pos) { + s << "defined at " << pos; + } + if (!docStr.empty()) { + s << "\n\n"; + } + + s << docStr; + + return Doc { + .pos = pos, + .name = name, + .arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though... + .args = {}, + .doc = makeImmutableString(toView(s)), // NOTE: memory leak when compiled without GC + }; + } + if (isFunctor(v)) { + try { + Value & functor = *v.attrs()->find(sFunctor)->value; + Value * vp = &v; + Value partiallyApplied; + // The first paramater is not user-provided, and may be + // handled by code that is opaque to the user, like lib.const = x: y: y; + // So preferably we show docs that are relevant to the + // "partially applied" function returned by e.g. `const`. + // We apply the first argument: + callFunction(functor, 1, &vp, partiallyApplied, noPos); + auto _level = addCallDepth(noPos); + return getDoc(partiallyApplied); + } + catch (Error & e) { + e.addTrace(nullptr, "while partially calling '%1%' to retrieve documentation", "__functor"); + throw; + } + } return {}; } @@ -755,11 +674,11 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En if (se.isWith && !env.values[0]->isThunk()) { // add 'with' bindings. for (auto & j : *env.values[0]->attrs()) - vm[st[j.name]] = j.value; + vm.insert_or_assign(std::string(st[j.name]), j.value); } else { // iterate through staticenv bindings and add them. for (auto & i : se.vars) - vm[st[i.first]] = env.values[i.second]; + vm.insert_or_assign(std::string(st[i.first]), env.values[i.second]); } } } @@ -785,6 +704,24 @@ public: } }; +bool EvalState::canDebug() +{ + return debugRepl && !debugTraces.empty(); +} + +void EvalState::runDebugRepl(const Error * error) +{ + if (!canDebug()) + return; + + assert(!debugTraces.empty()); + const DebugTrace & last = debugTraces.front(); + const Env & env = last.env; + const Expr & expr = last.expr; + + runDebugRepl(error, env, expr); +} + void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & expr) { // Make sure we have a debugger to run and we're not already in a debugger. @@ -825,7 +762,7 @@ void EvalState::runDebugRepl(const Error * error, const Env & env, const Expr & case ReplExitStatus::Continue: break; default: - abort(); + unreachable(); } } } @@ -881,9 +818,10 @@ static const char * * encodeContext(const NixStringContext & context) size_t n = 0; auto ctx = (const char * *) allocBytes((context.size() + 1) * sizeof(char *)); - for (auto & i : context) - ctx[n++] = dupString(i.to_string().c_str()); - ctx[n] = 0; + for (auto & i : context) { + ctx[n++] = makeImmutableString({i.to_string()}); + } + ctx[n] = nullptr; return ctx; } else return nullptr; @@ -1115,7 +1053,7 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) if (!e) e = parseExprFromFile(resolvedPath); - fileParseCache[resolvedPath] = e; + fileParseCache.emplace(resolvedPath, e); try { auto dts = debugRepl @@ -1138,8 +1076,8 @@ void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) throw; } - fileEvalCache[resolvedPath] = v; - if (path != resolvedPath) fileEvalCache[path] = v; + fileEvalCache.emplace(resolvedPath, v); + if (path != resolvedPath) fileEvalCache.emplace(path, v); } @@ -1194,7 +1132,7 @@ inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v, const PosIdx po void Expr::eval(EvalState & state, Env & env, Value & v) { - abort(); + unreachable(); } @@ -1442,7 +1380,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) if (!(j = vAttrs->attrs()->get(name))) { std::set allAttrNames; for (auto & attr : *vAttrs->attrs()) - allAttrNames.insert(state.symbols[attr.name]); + allAttrNames.insert(std::string(state.symbols[attr.name])); auto suggestions = Suggestions::bestMatches(allAttrNames, state.symbols[name]); state.error("attribute '%1%' missing", state.symbols[name]) .atPos(pos).withSuggestions(suggestions).withFrame(env, *this).debugThrow(); @@ -1469,6 +1407,22 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) v = *vAttrs; } +Symbol ExprSelect::evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs) +{ + Value vTmp; + Symbol name = getName(attrPath[attrPath.size() - 1], state, env); + + if (attrPath.size() == 1) { + e->eval(state, env, vTmp); + } else { + ExprSelect init(*this); + init.attrPath.pop_back(); + init.eval(state, env, vTmp); + } + attrs = vTmp; + return name; +} + void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) { @@ -1500,28 +1454,11 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v) v.mkLambda(&env, this); } -namespace { -/** Increments a count on construction and decrements on destruction. - */ -class CallDepth { - size_t & count; -public: - CallDepth(size_t & count) : count(count) { - ++count; - } - ~CallDepth() { - --count; - } -}; -}; - void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & vRes, const PosIdx pos) { - if (callDepth > evalSettings.maxCallDepth) - error("stack overflow; max-call-depth exceeded").atPos(pos).debugThrow(); - CallDepth _level(callDepth); + auto _level = addCallDepth(pos); - auto trace = evalSettings.traceFunctionCalls + auto trace = settings.traceFunctionCalls ? std::make_unique(positions[pos]) : nullptr; @@ -1600,7 +1537,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & if (!lambda.formals->has(i.name)) { std::set formalNames; for (auto & formal : lambda.formals->formals) - formalNames.insert(symbols[formal.name]); + formalNames.insert(std::string(symbols[formal.name])); auto suggestions = Suggestions::bestMatches(formalNames, symbols[i.name]); error("function '%1%' called with unexpected argument '%2%'", (lambda.name ? std::string(symbols[lambda.name]) : "anonymous lambda"), @@ -1611,7 +1548,7 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & .withFrame(*fun.payload.lambda.env, lambda) .debugThrow(); } - abort(); // can't happen + unreachable(); } } @@ -1865,7 +1802,20 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) { std::ostringstream out; cond->show(state.symbols, out); - state.error("assertion '%1%' failed", out.str()).atPos(pos).withFrame(env, *this).debugThrow(); + auto exprStr = toView(out); + + if (auto eq = dynamic_cast(cond)) { + try { + Value v1; eq->e1->eval(state, env, v1); + Value v2; eq->e2->eval(state, env, v2); + state.assertEqValues(v1, v2, eq->pos, "in an equality assertion"); + } catch (AssertionError & e) { + e.addTrace(state.positions[pos], "while evaluating the condition of the assertion '%s'", exprStr); + throw; + } + } + + state.error("assertion '%1%' failed", exprStr).atPos(pos).withFrame(env, *this).debugThrow(); } body->eval(state, env, v); } @@ -1993,7 +1943,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) NixStringContext context; std::vector s; size_t sSize = 0; - NixInt n = 0; + NixInt n{0}; NixFloat nf = 0; bool first = !forceString; @@ -2037,17 +1987,22 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) if (firstType == nInt) { if (vTmp.type() == nInt) { - n += vTmp.integer(); + auto newN = n + vTmp.integer(); + if (auto checked = newN.valueChecked(); checked.has_value()) { + n = NixInt(*checked); + } else { + state.error("integer overflow in adding %1% + %2%", n, vTmp.integer()).atPos(i_pos).debugThrow(); + } } else if (vTmp.type() == nFloat) { // Upgrade the type from int to float; firstType = nFloat; - nf = n; + nf = n.value; nf += vTmp.fpoint(); } else state.error("cannot add %1% to an integer", showType(vTmp)).atPos(i_pos).withFrame(env, *this).debugThrow(); } else if (firstType == nFloat) { if (vTmp.type() == nInt) { - nf += vTmp.integer(); + nf += vTmp.integer().value; } else if (vTmp.type() == nFloat) { nf += vTmp.fpoint(); } else @@ -2172,7 +2127,7 @@ NixFloat EvalState::forceFloat(Value & v, const PosIdx pos, std::string_view err try { forceValue(v, pos); if (v.type() == nInt) - return v.integer(); + return v.integer().value; else if (v.type() != nFloat) error( "expected a float but found %1%: %2%", @@ -2359,7 +2314,7 @@ BackedStringView EvalState::coerceToString( shell scripting convenience, just like `null'. */ if (v.type() == nBool && v.boolean()) return "1"; if (v.type() == nBool && !v.boolean()) return ""; - if (v.type() == nInt) return std::to_string(v.integer()); + if (v.type() == nInt) return std::to_string(v.integer().value); if (v.type() == nFloat) return std::to_string(v.fpoint()); if (v.type() == nNull) return ""; @@ -2397,21 +2352,21 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat if (nix::isDerivation(path.path.abs())) error("file names are not allowed to end in '%1%'", drvExtension).debugThrow(); - auto i = srcToStore.find(path); + auto dstPathCached = get(*srcToStore.lock(), path); - auto dstPath = i != srcToStore.end() - ? i->second + auto dstPath = dstPathCached + ? *dstPathCached : [&]() { auto dstPath = fetchToStore( *store, path.resolveSymlinks(), settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, path.baseName(), - FileIngestionMethod::Recursive, + ContentAddressMethod::Raw::NixArchive, nullptr, repair); allowPath(dstPath); - srcToStore.insert_or_assign(path, dstPath); + srcToStore.lock()->try_emplace(path, dstPath); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); return dstPath; }(); @@ -2522,6 +2477,214 @@ SingleDerivedPath EvalState::coerceToSingleDerivedPath(const PosIdx pos, Value & } + +// NOTE: This implementation must match eqValues! +// We accept this burden because informative error messages for +// `assert a == b; x` are critical for our users' testing UX. +void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) +{ + // This implementation must match eqValues. + forceValue(v1, pos); + forceValue(v2, pos); + + if (&v1 == &v2) + return; + + // Special case type-compatibility between float and int + if ((v1.type() == nInt || v1.type() == nFloat) && (v2.type() == nInt || v2.type() == nFloat)) { + if (eqValues(v1, v2, pos, errorCtx)) { + return; + } else { + error( + "%s with value '%s' is not equal to %s with value '%s'", + showType(v1), + ValuePrinter(*this, v1, errorPrintOptions), + showType(v2), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + } + + if (v1.type() != v2.type()) { + error( + "%s of value '%s' is not equal to %s of value '%s'", + showType(v1), + ValuePrinter(*this, v1, errorPrintOptions), + showType(v2), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + + switch (v1.type()) { + case nInt: + if (v1.integer() != v2.integer()) { + error("integer '%d' is not equal to integer '%d'", v1.integer(), v2.integer()).debugThrow(); + } + return; + + case nBool: + if (v1.boolean() != v2.boolean()) { + error( + "boolean '%s' is not equal to boolean '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + return; + + case nString: + if (strcmp(v1.c_str(), v2.c_str()) != 0) { + error( + "string '%s' is not equal to string '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + return; + + case nPath: + if (v1.payload.path.accessor != v2.payload.path.accessor) { + error( + "path '%s' is not equal to path '%s' because their accessors are different", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + if (strcmp(v1.payload.path.path, v2.payload.path.path) != 0) { + error( + "path '%s' is not equal to path '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + return; + + case nNull: + return; + + case nList: + if (v1.listSize() != v2.listSize()) { + error( + "list of size '%d' is not equal to list of size '%d', left hand side is '%s', right hand side is '%s'", + v1.listSize(), + v2.listSize(), + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + for (size_t n = 0; n < v1.listSize(); ++n) { + try { + assertEqValues(*v1.listElems()[n], *v2.listElems()[n], pos, errorCtx); + } catch (Error & e) { + e.addTrace(positions[pos], "while comparing list element %d", n); + throw; + } + } + return; + + case nAttrs: { + if (isDerivation(v1) && isDerivation(v2)) { + auto i = v1.attrs()->get(sOutPath); + auto j = v2.attrs()->get(sOutPath); + if (i && j) { + try { + assertEqValues(*i->value, *j->value, pos, errorCtx); + return; + } catch (Error & e) { + e.addTrace(positions[pos], "while comparing a derivation by its '%s' attribute", "outPath"); + throw; + } + assert(false); + } + } + + if (v1.attrs()->size() != v2.attrs()->size()) { + error( + "attribute names of attribute set '%s' differs from attribute set '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + + // Like normal comparison, we compare the attributes in non-deterministic Symbol index order. + // This function is called when eqValues has found a difference, so to reliably + // report about its result, we should follow in its literal footsteps and not + // try anything fancy that could lead to an error. + Bindings::const_iterator i, j; + for (i = v1.attrs()->begin(), j = v2.attrs()->begin(); i != v1.attrs()->end(); ++i, ++j) { + if (i->name != j->name) { + // A difference in a sorted list means that one attribute is not contained in the other, but we don't + // know which. Let's find out. Could use <, but this is more clear. + if (!v2.attrs()->get(i->name)) { + error( + "attribute name '%s' is contained in '%s', but not in '%s'", + symbols[i->name], + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + if (!v1.attrs()->get(j->name)) { + error( + "attribute name '%s' is missing in '%s', but is contained in '%s'", + symbols[j->name], + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + assert(false); + } + try { + assertEqValues(*i->value, *j->value, pos, errorCtx); + } catch (Error & e) { + // The order of traces is reversed, so this presents as + // where left hand side is + // at + // where right hand side is + // at + // while comparing attribute '' + if (j->pos != noPos) + e.addTrace(positions[j->pos], "where right hand side is"); + if (i->pos != noPos) + e.addTrace(positions[i->pos], "where left hand side is"); + e.addTrace(positions[pos], "while comparing attribute '%s'", symbols[i->name]); + throw; + } + } + return; + } + + case nFunction: + error("distinct functions and immediate comparisons of identical functions compare as unequal") + .debugThrow(); + + case nExternal: + if (!(*v1.external() == *v2.external())) { + error( + "external value '%s' is not equal to external value '%s'", + ValuePrinter(*this, v1, errorPrintOptions), + ValuePrinter(*this, v2, errorPrintOptions)) + .debugThrow(); + } + return; + + case nFloat: + // !!! + if (!(v1.fpoint() == v2.fpoint())) { + error("float '%f' is not equal to float '%f'", v1.fpoint(), v2.fpoint()).debugThrow(); + } + return; + + case nThunk: // Must not be left by forceValue + assert(false); + default: // Note that we pass compiler flags that should make `default:` unreachable. + // Also note that this probably ran after `eqValues`, which implements + // the same logic more efficiently (without having to unwind stacks), + // so maybe `assertEqValues` and `eqValues` are out of sync. Check it for solutions. + error("assertEqValues: cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).panic(); + } +} + +// This implementation must match assertEqValues bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) { forceValue(v1, pos); @@ -2534,9 +2697,9 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v // Special case type-compatibility between float and int if (v1.type() == nInt && v2.type() == nFloat) - return v1.integer() == v2.fpoint(); + return v1.integer().value == v2.fpoint(); if (v1.type() == nFloat && v2.type() == nInt) - return v1.fpoint() == v2.integer(); + return v1.fpoint() == v2.integer().value; // All other types are not compatible with each other. if (v1.type() != v2.type()) return false; @@ -2595,11 +2758,13 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return *v1.external() == *v2.external(); case nFloat: + // !!! return v1.fpoint() == v2.fpoint(); case nThunk: // Must not be left by forceValue - default: - error("cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).debugThrow(); + assert(false); + default: // Note that we pass compiler flags that should make `default:` unreachable. + error("eqValues: cannot compare %1% with %2%", showType(v1), showType(v2)).withTrace(pos, errorCtx).panic(); } } @@ -2648,6 +2813,11 @@ void EvalState::printStatistics() #if HAVE_BOEHMGC GC_word heapSize, totalBytes; GC_get_heap_usage_safe(&heapSize, 0, 0, 0, &totalBytes); + double gcFullOnlyTime = ({ + auto ms = GC_get_full_gc_total_time(); + ms * 0.001; + }); + auto gcCycles = getGCCycles(); #endif auto outPath = getEnv("NIX_SHOW_STATS_PATH").value_or("-"); @@ -2658,6 +2828,15 @@ void EvalState::printStatistics() #ifndef _WIN32 // TODO implement topObj["cpuTime"] = cpuTime; #endif + topObj["time"] = { +#ifndef _WIN32 // TODO implement + {"cpu", cpuTime}, +#endif +#if HAVE_BOEHMGC + {GC_is_incremental_mode() ? "gcNonIncremental" : "gc", gcFullOnlyTime}, + {GC_is_incremental_mode() ? "gcNonIncrementalFraction" : "gcFraction", gcFullOnlyTime / cpuTime}, +#endif + }; topObj["envs"] = { {"number", nrEnvs}, {"elements", nrValuesInEnvs}, @@ -2699,6 +2878,7 @@ void EvalState::printStatistics() topObj["gc"] = { {"heapSize", heapSize}, {"totalBytes", totalBytes}, + {"cycles", gcCycles}, }; #endif @@ -2754,7 +2934,7 @@ void EvalState::printStatistics() } -SourcePath resolveExprPath(SourcePath path) +SourcePath resolveExprPath(SourcePath path, bool addDefaultNix) { unsigned int followCount = 0, maxFollow = 1024; @@ -2770,7 +2950,7 @@ SourcePath resolveExprPath(SourcePath path) } /* If `path' refers to a directory, append `/default.nix'. */ - if (path.resolveSymlinks().lstat().type == SourceAccessor::tDirectory) + if (addDefaultNix && path.resolveSymlinks().lstat().type == SourceAccessor::tDirectory) return path / "default.nix"; return path; @@ -2849,7 +3029,7 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ return {corepkgsFS, CanonPath(path.substr(3))}; error( - evalSettings.pureEval + settings.pureEval ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" : "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)", path @@ -2863,14 +3043,20 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pa auto i = lookupPathResolved.find(value); if (i != lookupPathResolved.end()) return i->second; - std::optional res; + auto finish = [&](std::string res) { + debug("resolved search path element '%s' to '%s'", value, res); + lookupPathResolved.emplace(value, res); + return res; + }; if (EvalSettings::isPseudoUrl(value)) { try { auto accessor = fetchers::downloadTarball( - EvalSettings::resolvePseudoUrl(value)).accessor; + store, + fetchSettings, + EvalSettings::resolvePseudoUrl(value)); auto storePath = fetchToStore(*store, SourcePath(accessor), FetchMode::Copy); - res = { store->toRealPath(storePath) }; + return finish(store->toRealPath(storePath)); } catch (Error & e) { logWarning({ .msg = HintFmt("Nix search path entry '%1%' cannot be downloaded, ignoring", value) @@ -2878,15 +3064,17 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pa } } - else if (hasPrefix(value, "flake:")) { - experimentalFeatureSettings.require(Xp::Flakes); - auto flakeRef = parseFlakeRef(value.substr(6), {}, true, false); - debug("fetching flake search path element '%s''", value); - auto storePath = flakeRef.resolve(store).fetchTree(store).first; - res = { store->toRealPath(storePath) }; + if (auto colPos = value.find(':'); colPos != value.npos) { + auto scheme = value.substr(0, colPos); + auto rest = value.substr(colPos + 1); + if (auto * hook = get(settings.lookupPathHooks, scheme)) { + auto res = (*hook)(store, rest); + if (res) + return finish(std::move(*res)); + } } - else { + { auto path = absPath(value); /* Allow access to paths in the search path. */ @@ -2903,22 +3091,17 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pa } if (pathExists(path)) - res = { path }; + return finish(std::move(path)); else { logWarning({ .msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value) }); - res = std::nullopt; } } - if (res) - debug("resolved search path element '%s' to '%s'", value, *res); - else - debug("failed to resolve search path element '%s'", value); + debug("failed to resolve search path element '%s'", value); + return std::nullopt; - lookupPathResolved.emplace(value, res); - return res; } @@ -2929,13 +3112,37 @@ Expr * EvalState::parse( const SourcePath & basePath, std::shared_ptr & staticEnv) { - auto result = parseExprFromBuf(text, length, origin, basePath, symbols, positions, rootFS, exprSymbols); + DocCommentMap tmpDocComments; // Only used when not origin is not a SourcePath + DocCommentMap *docComments = &tmpDocComments; + + if (auto sourcePath = std::get_if(&origin)) { + auto [it, _] = positionToDocComment.try_emplace(*sourcePath); + docComments = &it->second; + } + + auto result = parseExprFromBuf(text, length, origin, basePath, symbols, settings, positions, *docComments, rootFS, exprSymbols); result->bindVars(*this, staticEnv); return result; } +DocComment EvalState::getDocCommentForPos(PosIdx pos) +{ + auto pos2 = positions[pos]; + auto path = pos2.getSourcePath(); + if (!path) + return {}; + + auto table = positionToDocComment.find(*path); + if (table == positionToDocComment.end()) + return {}; + + auto it = table->second.find(pos); + if (it == table->second.end()) + return {}; + return it->second; +} std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { @@ -2945,7 +3152,7 @@ std::string ExternalValueBase::coerceToString(EvalState & state, const PosIdx & } -bool ExternalValueBase::operator==(const ExternalValueBase & b) const +bool ExternalValueBase::operator==(const ExternalValueBase & b) const noexcept { return false; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 7ca2d6227..f7ed6be83 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -9,14 +9,15 @@ #include "symbol-table.hh" #include "config.hh" #include "experimental-features.hh" +#include "position.hh" +#include "pos-table.hh" #include "source-accessor.hh" #include "search-path.hh" #include "repl-exit-status.hh" +#include "ref.hh" #include #include -#include -#include #include namespace nix { @@ -29,6 +30,8 @@ namespace nix { constexpr size_t maxPrimOpArity = 8; class Store; +namespace fetchers { struct Settings; } +struct EvalSettings; class EvalState; class StorePath; struct SingleDerivedPath; @@ -38,11 +41,25 @@ namespace eval_cache { class EvalCache; } +/** + * Increments a count on construction and decrements on destruction. + */ +class CallDepth { + size_t & count; + +public: + CallDepth(size_t & count) : count(count) { + ++count; + } + ~CallDepth() { + --count; + } +}; /** * Function that implements a primop. */ -typedef void (* PrimOpFun) (EvalState & state, const PosIdx pos, Value * * args, Value & v); +using PrimOpFun = void(EvalState & state, const PosIdx pos, Value * * args, Value & v); /** * Info about a primitive operation, and its implementation @@ -83,7 +100,7 @@ struct PrimOp /** * Implementation of the primop. */ - std::function::type> fun; + std::function fun; /** * Optional experimental for this to be gated on. @@ -122,11 +139,9 @@ struct Constant bool impureOnly = false; }; -#if HAVE_BOEHMGC - typedef std::map, traceable_allocator > > ValMap; -#else - typedef std::map ValMap; -#endif +typedef std::map, traceable_allocator > > ValMap; + +typedef std::unordered_map DocCommentMap; struct Env { @@ -146,12 +161,6 @@ std::string printValue(EvalState & state, Value & v); std::ostream & operator << (std::ostream & os, const ValueType t); -/** - * Initialise the Boehm GC, if applicable. - */ -void initGC(); - - struct RegexCache; std::shared_ptr makeRegexCache(); @@ -167,13 +176,18 @@ struct DebugTrace { class EvalState : public std::enable_shared_from_this { public: + const fetchers::Settings & fetchSettings; + const EvalSettings & settings; SymbolTable symbols; PosTable positions; const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, - sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, + sRight, sWrong, sStructuredAttrs, + sAllowedReferences, sAllowedRequisites, sDisallowedReferences, sDisallowedRequisites, + sMaxSize, sMaxClosureSize, + sBuilder, sArgs, sContentAddressed, sImpure, sOutputHash, sOutputHashAlgo, sOutputHashMode, sRecurseForDerivations, @@ -276,6 +290,18 @@ public: return std::shared_ptr();; } + /** Whether a debug repl can be started. If `false`, `runDebugRepl(error)` will return without starting a repl. */ + bool canDebug(); + + /** Use front of `debugTraces`; see `runDebugRepl(error,env,expr)` */ + void runDebugRepl(const Error * error); + + /** + * Run a debug repl with the given error, environment and expression. + * @param error The error to debug, may be nullptr. + * @param env The environment to debug, matching the expression. + * @param expr The expression to debug, matching the environment. + */ void runDebugRepl(const Error * error, const Env & env, const Expr & expr); template @@ -294,28 +320,26 @@ private: /* Cache for calls to addToStore(); maps source paths to the store paths. */ - std::map srcToStore; + Sync> srcToStore; /** * A cache from path names to parse trees. */ -#if HAVE_BOEHMGC - typedef std::map, traceable_allocator>> FileParseCache; -#else - typedef std::map FileParseCache; -#endif + typedef std::unordered_map, std::equal_to, traceable_allocator>> FileParseCache; FileParseCache fileParseCache; /** * A cache from path names to values. */ -#if HAVE_BOEHMGC - typedef std::map, traceable_allocator>> FileEvalCache; -#else - typedef std::map FileEvalCache; -#endif + typedef std::unordered_map, std::equal_to, traceable_allocator>> FileEvalCache; FileEvalCache fileEvalCache; + /** + * Associate source positions of certain AST nodes with their preceding doc comment, if they have one. + * Grouped by file. + */ + std::unordered_map positionToDocComment; + LookupPath lookupPath; std::map> lookupPathResolved; @@ -342,6 +366,8 @@ public: EvalState( const LookupPath & _lookupPath, ref store, + const fetchers::Settings & fetchSettings, + const EvalSettings & settings, std::shared_ptr buildStore = nullptr); ~EvalState(); @@ -602,6 +628,12 @@ public: const char * doc; }; + /** + * Retrieve the documentation for a value. This will evaluate the value if + * it is a thunk, and it will partially apply __functor if applicable. + * + * @param v The value to get the documentation for. + */ std::optional getDoc(Value & v); private: @@ -626,12 +658,26 @@ private: public: + /** + * Check that the call depth is within limits, and increment it, until the returned object is destroyed. + */ + inline CallDepth addCallDepth(const PosIdx pos); + /** * Do a deep equality test between two values. That is, list * elements and attributes are compared recursively. */ bool eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx); + /** + * Like `eqValues`, but throws an `AssertionError` if not equal. + * + * WARNING: + * Callers should call `eqValues` first and report if `assertEqValues` behaves + * incorrectly. (e.g. if it doesn't throw if eqValues returns false or vice versa) + */ + void assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx); + bool isFunctor(Value & fun); // FIXME: use std::span @@ -756,6 +802,8 @@ public: std::string_view pathArg, PosIdx pos); + DocComment getDocCommentForPos(PosIdx pos); + private: /** @@ -838,8 +886,10 @@ std::string showType(const Value & v); /** * If `path` refers to a directory, then append "/default.nix". + * + * @param addDefaultNix Whether to append "/default.nix" after resolving symlinks. */ -SourcePath resolveExprPath(SourcePath path); +SourcePath resolveExprPath(SourcePath path, bool addDefaultNix = true); /** * Whether a URI is allowed, assuming restrictEval is enabled diff --git a/src/libexpr/gc-small-vector.hh b/src/libexpr/gc-small-vector.hh index 7f4f08fc7..8330dd2dc 100644 --- a/src/libexpr/gc-small-vector.hh +++ b/src/libexpr/gc-small-vector.hh @@ -2,28 +2,15 @@ #include -#if HAVE_BOEHMGC - -#include -#include -#include - -#endif +#include "value.hh" namespace nix { -struct Value; - /** * A GC compatible vector that may used a reserved portion of `nItems` on the stack instead of allocating on the heap. */ -#if HAVE_BOEHMGC template using SmallVector = boost::container::small_vector>; -#else -template -using SmallVector = boost::container::small_vector; -#endif /** * A vector of value pointers. See `SmallVector`. @@ -39,4 +26,4 @@ using SmallValueVector = SmallVector; template using SmallTemporaryValueVector = SmallVector; -} \ No newline at end of file +} diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index cf10ed84a..1ac13fcd2 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -69,10 +69,17 @@ std::string PackageInfo::querySystem() const std::optional PackageInfo::queryDrvPath() const { if (!drvPath && attrs) { - NixStringContext context; - if (auto i = attrs->get(state->sDrvPath)) - drvPath = {state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation")}; - else + if (auto i = attrs->get(state->sDrvPath)) { + NixStringContext context; + auto found = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation"); + try { + found.requireDerivation(); + } catch (Error & e) { + e.addTrace(state->positions[i->pos], "while evaluating the 'drvPath' attribute of a derivation"); + throw; + } + drvPath = {std::move(found)}; + } else drvPath = {std::nullopt}; } return drvPath.value_or(std::nullopt); @@ -239,8 +246,8 @@ NixInt PackageInfo::queryMetaInt(const std::string & name, NixInt def) if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - if (auto n = string2Int(v->c_str())) - return *n; + if (auto n = string2Int(v->c_str())) + return NixInt{*n}; } return def; } @@ -335,9 +342,9 @@ std::optional getDerivation(EvalState & state, Value & v, } -static std::string addToPath(const std::string & s1, const std::string & s2) +static std::string addToPath(const std::string & s1, std::string_view s2) { - return s1.empty() ? s2 : s1 + "." + s2; + return s1.empty() ? std::string(s2) : s1 + "." + s2; } @@ -367,21 +374,27 @@ static void getDerivations(EvalState & state, Value & vIn, bound to the attribute with the "lower" name should take precedence). */ for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) { - debug("evaluating attribute '%1%'", state.symbols[i->name]); - if (!std::regex_match(std::string(state.symbols[i->name]), attrRegex)) - continue; - std::string pathPrefix2 = addToPath(pathPrefix, state.symbols[i->name]); - if (combineChannels) - getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); - else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) { - /* If the value of this attribute is itself a set, - should we recurse into it? => Only if it has a - `recurseForDerivations = true' attribute. */ - if (i->value->type() == nAttrs) { - auto j = i->value->attrs()->get(state.sRecurseForDerivations); - if (j && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`")) - getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + std::string_view symbol{state.symbols[i->name]}; + try { + debug("evaluating attribute '%1%'", symbol); + if (!std::regex_match(symbol.begin(), symbol.end(), attrRegex)) + continue; + std::string pathPrefix2 = addToPath(pathPrefix, symbol); + if (combineChannels) + getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + else if (getDerivation(state, *i->value, pathPrefix2, drvs, done, ignoreAssertionFailures)) { + /* If the value of this attribute is itself a set, + should we recurse into it? => Only if it has a + `recurseForDerivations = true' attribute. */ + if (i->value->type() == nAttrs) { + auto j = i->value->attrs()->get(state.sRecurseForDerivations); + if (j && state.forceBool(*j->value, j->pos, "while evaluating the attribute `recurseForDerivations`")) + getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); + } } + } catch (Error & e) { + e.addTrace(state.positions[i->pos], "while evaluating the attribute '%s'", symbol); + throw; } } } diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh index db3eedb05..e4e277af8 100644 --- a/src/libexpr/get-drvs.hh +++ b/src/libexpr/get-drvs.hh @@ -83,11 +83,7 @@ public: }; -#if HAVE_BOEHMGC typedef std::list> PackageInfos; -#else -typedef std::list PackageInfos; -#endif /** diff --git a/src/libexpr/json-to-value.cc b/src/libexpr/json-to-value.cc index 20bee193f..9ac56541a 100644 --- a/src/libexpr/json-to-value.cc +++ b/src/libexpr/json-to-value.cc @@ -2,6 +2,7 @@ #include "value.hh" #include "eval.hh" +#include #include #include @@ -42,7 +43,7 @@ class JSONSax : nlohmann::json_sax { auto attrs2 = state.buildBindings(attrs.size()); for (auto & i : attrs) attrs2.insert(i.first, i.second); - parent->value(state).mkAttrs(attrs2.alreadySorted()); + parent->value(state).mkAttrs(attrs2); return std::move(parent); } void add() override { v = nullptr; } @@ -80,42 +81,46 @@ class JSONSax : nlohmann::json_sax { public: JSONSax(EvalState & state, Value & v) : state(state), rs(new JSONState(&v)) {}; - bool null() + bool null() override { rs->value(state).mkNull(); rs->add(); return true; } - bool boolean(bool val) + bool boolean(bool val) override { rs->value(state).mkBool(val); rs->add(); return true; } - bool number_integer(number_integer_t val) + bool number_integer(number_integer_t val) override { rs->value(state).mkInt(val); rs->add(); return true; } - bool number_unsigned(number_unsigned_t val) + bool number_unsigned(number_unsigned_t val_) override { + if (val_ > std::numeric_limits::max()) { + throw Error("unsigned json number %1% outside of Nix integer range", val_); + } + NixInt::Inner val = val_; rs->value(state).mkInt(val); rs->add(); return true; } - bool number_float(number_float_t val, const string_t & s) + bool number_float(number_float_t val, const string_t & s) override { rs->value(state).mkFloat(val); rs->add(); return true; } - bool string(string_t & val) + bool string(string_t & val) override { rs->value(state).mkString(val); rs->add(); @@ -123,7 +128,7 @@ public: } #if NLOHMANN_JSON_VERSION_MAJOR >= 3 && NLOHMANN_JSON_VERSION_MINOR >= 8 - bool binary(binary_t&) + bool binary(binary_t&) override { // This function ought to be unreachable assert(false); @@ -131,35 +136,35 @@ public: } #endif - bool start_object(std::size_t len) + bool start_object(std::size_t len) override { rs = std::make_unique(std::move(rs)); return true; } - bool key(string_t & name) + bool key(string_t & name) override { dynamic_cast(rs.get())->key(name, state); return true; } - bool end_object() { + bool end_object() override { rs = rs->resolve(state); rs->add(); return true; } - bool end_array() { + bool end_array() override { return end_object(); } - bool start_array(size_t len) { + bool start_array(size_t len) override { rs = std::make_unique(std::move(rs), len != std::numeric_limits::max() ? len : 128); return true; } - bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) { + bool parse_error(std::size_t, const std::string&, const nlohmann::detail::exception& ex) override { throw JSONParseError("%s", ex.what()); } }; diff --git a/src/libexpr/lexer-helpers.cc b/src/libexpr/lexer-helpers.cc new file mode 100644 index 000000000..d9eeb73e2 --- /dev/null +++ b/src/libexpr/lexer-helpers.cc @@ -0,0 +1,30 @@ +#include "lexer-tab.hh" +#include "lexer-helpers.hh" +#include "parser-tab.hh" + +void nix::lexer::internal::initLoc(YYLTYPE * loc) +{ + loc->beginOffset = loc->endOffset = 0; +} + +void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len) +{ + loc->stash(); + + LexerState & lexerState = *yyget_extra(yyscanner); + + if (lexerState.docCommentDistance == 1) { + // Preceding token was a doc comment. + ParserLocation doc; + doc.beginOffset = lexerState.lastDocCommentLoc.beginOffset; + ParserLocation docEnd; + docEnd.beginOffset = lexerState.lastDocCommentLoc.endOffset; + DocComment docComment{lexerState.at(doc), lexerState.at(docEnd)}; + PosIdx locPos = lexerState.at(*loc); + lexerState.positionToDocComment.emplace(locPos, docComment); + } + lexerState.docCommentDistance++; + + loc->beginOffset = loc->endOffset; + loc->endOffset += len; +} diff --git a/src/libexpr/lexer-helpers.hh b/src/libexpr/lexer-helpers.hh new file mode 100644 index 000000000..caba6e18f --- /dev/null +++ b/src/libexpr/lexer-helpers.hh @@ -0,0 +1,9 @@ +#pragma once + +namespace nix::lexer::internal { + +void initLoc(YYLTYPE * loc); + +void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len); + +} // namespace nix::lexer diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index ee2b6b807..a7e44cb72 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -5,7 +5,7 @@ %option stack %option nodefault %option nounput noyy_top_state - +%option extra-type="::nix::LexerState *" %s DEFAULT %x STRING @@ -14,38 +14,31 @@ %x INPATH_SLASH %x PATH_START +%top { +#include "parser-tab.hh" // YYSTYPE +#include "parser-state.hh" +} %{ #ifdef __clang__ #pragma clang diagnostic ignored "-Wunneeded-internal-declaration" #endif -#include - #include "nixexpr.hh" #include "parser-tab.hh" +#include "lexer-helpers.hh" + +namespace nix { + struct LexerState; +} using namespace nix; +using namespace nix::lexer::internal; namespace nix { #define CUR_POS state->at(*yylloc) -static void initLoc(YYLTYPE * loc) -{ - loc->first_line = loc->last_line = 0; - loc->first_column = loc->last_column = 0; -} - -static void adjustLoc(YYLTYPE * loc, const char * s, size_t len) -{ - loc->stash(); - - loc->first_column = loc->last_column; - loc->last_column += len; -} - - // we make use of the fact that the parser receives a private copy of the input // string and can munge around in it. static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) @@ -74,6 +67,14 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) return {result, size_t(t - result)}; } +static void requireExperimentalFeature(const ExperimentalFeature & feature, const Pos & pos) +{ + if (!experimentalFeatureSettings.isEnabled(feature)) + throw ParseError(ErrorInfo{ + .msg = HintFmt("experimental Nix feature '%1%' is disabled; add '--extra-experimental-features %1%' to enable it", showExperimentalFeature(feature)), + .pos = pos, + }); +} } @@ -81,7 +82,7 @@ static StringToken unescapeStr(SymbolTable & symbols, char * s, size_t length) #pragma GCC diagnostic ignored "-Wimplicit-fallthrough" #define YY_USER_INIT initLoc(yylloc) -#define YY_USER_ACTION adjustLoc(yylloc, yytext, yyleng); +#define YY_USER_ACTION adjustLoc(yyscanner, yylloc, yytext, yyleng); #define PUSH_STATE(state) yy_push_state(state, yyscanner) #define POP_STATE() yy_pop_state(yyscanner) @@ -126,12 +127,19 @@ or { return OR_KW; } \-\> { return IMPL; } \/\/ { return UPDATE; } \+\+ { return CONCAT; } +\<\| { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]); + return PIPE_FROM; + } +\|\> { requireExperimentalFeature(Xp::PipeOperators, state->positions[CUR_POS]); + return PIPE_INTO; + } {ID} { yylval->id = {yytext, (size_t) yyleng}; return ID; } {INT} { errno = 0; - try { - yylval->n = boost::lexical_cast(yytext); - } catch (const boost::bad_lexical_cast &) { + std::optional numMay = string2Int(yytext); + if (numMay.has_value()) { + yylval->n = NixInt{*numMay}; + } else { throw ParseError(ErrorInfo{ .msg = HintFmt("invalid integer '%1%'", yytext), .pos = state->positions[CUR_POS], @@ -280,9 +288,32 @@ or { return OR_KW; } {SPATH} { yylval->path = {yytext, (size_t) yyleng}; return SPATH; } {URI} { yylval->uri = {yytext, (size_t) yyleng}; return URI; } -[ \t\r\n]+ /* eat up whitespace */ -\#[^\r\n]* /* single-line comments */ -\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ +%{ +// Doc comment rule +// +// \/\*\* /** +// [^/*] reject /**/ (empty comment) and /*** +// ([^*]|\*+[^*/])*\*+\/ same as the long comment rule +// ( )* zero or more non-ending sequences +// \* end(1) +// \/ end(2) +%} +\/\*\*[^/*]([^*]|\*+[^*/])*\*+\/ /* doc comments */ { + LexerState & lexerState = *yyget_extra(yyscanner); + lexerState.docCommentDistance = 0; + lexerState.lastDocCommentLoc.beginOffset = yylloc->beginOffset; + lexerState.lastDocCommentLoc.endOffset = yylloc->endOffset; +} + + +%{ +// The following rules have docCommentDistance-- +// This compensates for the docCommentDistance++ which happens by default to +// make all the other rules invalidate the doc comment. +%} +[ \t\r\n]+ /* eat up whitespace */ { yyget_extra(yyscanner)->docCommentDistance--; } +\#[^\r\n]* /* single-line comments */ { yyget_extra(yyscanner)->docCommentDistance--; } +\/\*([^*]|\*+[^*/])*\*+\/ /* long comments */ { yyget_extra(yyscanner)->docCommentDistance--; } {ANY} { /* Don't return a negative number, as this will cause diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index ecadc5e5d..68518e184 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -8,14 +8,15 @@ libexpr_SOURCES := \ $(wildcard $(d)/*.cc) \ $(wildcard $(d)/value/*.cc) \ $(wildcard $(d)/primops/*.cc) \ - $(wildcard $(d)/flake/*.cc) \ $(d)/lexer-tab.cc \ $(d)/parser-tab.cc # Not just for this library itself, but also for downstream libraries using this library INCLUDE_libexpr := -I $(d) -libexpr_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libmain) $(INCLUDE_libexpr) +libexpr_CXXFLAGS += \ + $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) \ + -DGC_THREADS libexpr_LIBS = libutil libstore libfetchers @@ -43,11 +44,7 @@ $(eval $(call install-file-in, $(buildprefix)$(d)/nix-expr.pc, $(libdir)/pkgconf $(foreach i, $(wildcard src/libexpr/value/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/value, 0644))) -$(foreach i, $(wildcard src/libexpr/flake/*.hh), \ - $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh -$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/call-flake.nix.gen.hh - -$(buildprefix)src/libexpr/primops/fromTOML.o: ERROR_SWITCH_ENUM = +$(d)/eval.cc: $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/call-flake.nix.gen.hh diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build new file mode 100644 index 000000000..4d8a38b43 --- /dev/null +++ b/src/libexpr/meson.build @@ -0,0 +1,210 @@ +project('nix-expr', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +boost = dependency( + 'boost', + modules : ['container', 'context'], + include_type: 'system', +) +# boost is a public dependency, but not a pkg-config dependency unfortunately, so we +# put in `deps_other`. +deps_other += boost + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +bdw_gc = dependency('bdw-gc', required : get_option('gc')) +if bdw_gc.found() + deps_public += bdw_gc + foreach funcspec : [ + 'pthread_attr_get_np', + 'pthread_getattr_np', + ] + define_name = 'HAVE_' + funcspec.underscorify().to_upper() + define_value = cxx.has_function(funcspec).to_int() + configdata.set(define_name, define_value) + endforeach + configdata.set('GC_THREADS', 1) +endif +configdata.set('HAVE_BOEHMGC', bdw_gc.found().to_int()) + +toml11 = dependency( + 'toml11', + version : '>=3.7.0', + method : 'cmake', + include_type: 'system', +) +deps_other += toml11 + +config_h = configure_file( + configuration : configdata, + output : 'config-expr.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +parser_tab = custom_target( + input : 'parser.y', + output : [ + 'parser-tab.cc', + 'parser-tab.hh', + ], + command : [ + 'bison', + '-v', + '-o', + '@OUTPUT0@', + '@INPUT@', + '-d', + ], + # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add + # an install script below which removes parser-tab.cc. + install : true, + install_dir : get_option('includedir') / 'nix', +) + +lexer_tab = custom_target( + input : [ + 'lexer.l', + parser_tab, + ], + output : [ + 'lexer-tab.cc', + 'lexer-tab.hh', + ], + command : [ + 'flex', + '--outfile', + '@OUTPUT0@', + '--header-file=' + '@OUTPUT1@', + '@INPUT0@', + ], + # NOTE(Qyriad): Meson doesn't support installing only part of a custom target, so we add + # an install script below which removes lexer-tab.cc. + install : true, + install_dir : get_option('includedir') / 'nix', +) + +subdir('build-utils-meson/generate-header') + +generated_headers = [] +foreach header : [ + 'imported-drv-to-derivation.nix', + 'fetchurl.nix', + 'call-flake.nix', +] + generated_headers += gen_header.process(header) +endforeach + +sources = files( + 'attr-path.cc', + 'attr-set.cc', + 'eval-cache.cc', + 'eval-error.cc', + 'eval-gc.cc', + 'eval-settings.cc', + 'eval.cc', + 'function-trace.cc', + 'get-drvs.cc', + 'json-to-value.cc', + 'lexer-helpers.cc', + 'nixexpr.cc', + 'paths.cc', + 'primops.cc', + 'print-ambiguous.cc', + 'print.cc', + 'search-path.cc', + 'value-to-json.cc', + 'value-to-xml.cc', + 'value/context.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'attr-path.hh', + 'attr-set.hh', + 'eval-cache.hh', + 'eval-error.hh', + 'eval-gc.hh', + 'eval-inline.hh', + 'eval-settings.hh', + 'eval.hh', + 'function-trace.hh', + 'gc-small-vector.hh', + 'get-drvs.hh', + 'json-to-value.hh', + # internal: 'lexer-helpers.hh', + 'nixexpr.hh', + 'parser-state.hh', + 'pos-idx.hh', + 'pos-table.hh', + 'primops.hh', + 'print-ambiguous.hh', + 'print-options.hh', + 'print.hh', + 'repl-exit-status.hh', + 'search-path.hh', + 'symbol-table.hh', + 'value-to-json.hh', + 'value-to-xml.hh', + 'value.hh', + 'value/context.hh', +) + +subdir('primops') + +this_library = library( + 'nixexpr', + sources, + parser_tab, + lexer_tab, + generated_headers, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libexpr/meson.options b/src/libexpr/meson.options new file mode 100644 index 000000000..242d30ea7 --- /dev/null +++ b/src/libexpr/meson.options @@ -0,0 +1,3 @@ +option('gc', type : 'feature', + description : 'enable garbage collection in the Nix expression evaluator (requires Boehm GC)', +) diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index c1e2b0448..063ff0753 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -1,11 +1,13 @@ #include "nixexpr.hh" -#include "derivations.hh" #include "eval.hh" #include "symbol-table.hh" #include "util.hh" #include "print.hh" #include +#include + +#include "strings-inline.hh" namespace nix { @@ -23,7 +25,7 @@ std::ostream & operator <<(std::ostream & str, const SymbolStr & symbol) void Expr::show(const SymbolTable & symbols, std::ostream & str) const { - abort(); + unreachable(); } void ExprInt::show(const SymbolTable & symbols, std::ostream & str) const @@ -80,7 +82,9 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co return sa < sb; }); std::vector inherits; - std::map> inheritsFrom; + // We can use the displacement as a proxy for the order in which the symbols were parsed. + // The assignment of displacements should be deterministic, so that showBindings is deterministic. + std::map> inheritsFrom; for (auto & i : sorted) { switch (i->second.kind) { case AttrDef::Kind::Plain: @@ -91,7 +95,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co case AttrDef::Kind::InheritedFrom: { auto & select = dynamic_cast(*i->second.e); auto & from = dynamic_cast(*select.e); - inheritsFrom[&from].push_back(i->first); + inheritsFrom[from.displ].push_back(i->first); break; } } @@ -103,7 +107,7 @@ void ExprAttrs::showBindings(const SymbolTable & symbols, std::ostream & str) co } for (const auto & [from, syms] : inheritsFrom) { str << "inherit ("; - (*inheritFromExprs)[from->displ]->show(symbols, str); + (*inheritFromExprs)[from]->show(symbols, str); str << ")"; for (auto sym : syms) str << " " << symbols[sym]; str << "; "; @@ -267,7 +271,7 @@ std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) void Expr::bindVars(EvalState & es, const std::shared_ptr & env) { - abort(); + unreachable(); } void ExprInt::bindVars(EvalState & es, const std::shared_ptr & env) @@ -581,6 +585,22 @@ std::string ExprLambda::showNamePos(const EvalState & state) const return fmt("%1% at %2%", id, state.positions[pos]); } +void ExprLambda::setDocComment(DocComment docComment) { + // RFC 145 specifies that the innermost doc comment wins. + // See https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md#ambiguous-placement + if (!this->docComment) { + this->docComment = docComment; + + // Curried functions are defined by putting a function directly + // in the body of another function. To render docs for those, we + // need to propagate the doc comment to the innermost function. + // + // If we have our own comment, we've already propagated it, so this + // belongs in the same conditional. + body->setDocComment(docComment); + } +}; + /* Position table. */ @@ -625,4 +645,50 @@ size_t SymbolTable::totalSize() const return n; } +std::string DocComment::getInnerText(const PosTable & positions) const { + auto beginPos = positions[begin]; + auto endPos = positions[end]; + auto docCommentStr = beginPos.getSnippetUpTo(endPos).value_or(""); + + // Strip "/**" and "*/" + constexpr size_t prefixLen = 3; + constexpr size_t suffixLen = 2; + std::string docStr = docCommentStr.substr(prefixLen, docCommentStr.size() - prefixLen - suffixLen); + if (docStr.empty()) + return {}; + // Turn the now missing "/**" into indentation + docStr = " " + docStr; + // Strip indentation (for the whole, potentially multi-line string) + docStr = stripIndentation(docStr); + return docStr; +} + + + +/* ‘Cursed or’ handling. + * + * In parser.y, every use of expr_select in a production must call one of the + * two below functions. + * + * To be removed by https://github.com/NixOS/nix/pull/11121 + */ + +void ExprCall::resetCursedOr() +{ + cursedOrEndPos.reset(); +} + +void ExprCall::warnIfCursedOr(const SymbolTable & symbols, const PosTable & positions) +{ + if (cursedOrEndPos.has_value()) { + std::ostringstream out; + out << "at " << positions[pos] << ": " + "This expression uses `or` as an identifier in a way that will change in a future Nix release.\n" + "Wrap this entire expression in parentheses to preserve its current meaning:\n" + " (" << positions[pos].getSnippetUpTo(positions[*cursedOrEndPos]).value_or("could not read expression") << ")\n" + "Give feedback at https://github.com/NixOS/nix/pull/11121"; + warn(out.str()); + } +} + } diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index e37e3bdd1..bdf4e214a 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -6,21 +6,58 @@ #include "value.hh" #include "symbol-table.hh" -#include "error.hh" -#include "position.hh" #include "eval-error.hh" #include "pos-idx.hh" -#include "pos-table.hh" namespace nix { - -struct Env; -struct Value; class EvalState; +class PosTable; +struct Env; struct ExprWith; struct StaticEnv; +struct Value; +/** + * A documentation comment, in the sense of [RFC 145](https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md) + * + * Note that this does not implement the following: + * - argument attribute names ("formals"): TBD + * - argument names: these are internal to the function and their names may not be optimal for documentation + * - function arity (degree of currying or number of ':'s): + * - Functions returning partially applied functions have a higher arity + * than can be determined locally and without evaluation. + * We do not want to present false data. + * - Some functions should be thought of as transformations of other + * functions. For instance `overlay -> overlay -> overlay` is the simplest + * way to understand `composeExtensions`, but its implementation looks like + * `f: g: final: prev: <...>`. The parameters `final` and `prev` are part + * of the overlay concept, while distracting from the function's purpose. + */ +struct DocComment { + + /** + * Start of the comment, including the opening, ie `/` and `**`. + */ + PosIdx begin; + + /** + * Position right after the final asterisk and `/` that terminate the comment. + */ + PosIdx end; + + /** + * Whether the comment is set. + * + * A `DocComment` is small enough that it makes sense to pass by value, and + * therefore baking optionality into it is also useful, to avoiding the memory + * overhead of `std::optional`. + */ + operator bool() const { return static_cast(begin); } + + std::string getInnerText(const PosTable & positions) const; + +}; /** * An attribute path is a sequence of attribute names. @@ -57,7 +94,12 @@ struct Expr virtual void eval(EvalState & state, Env & env, Value & v); virtual Value * maybeThunk(EvalState & state, Env & env); virtual void setName(Symbol name); + virtual void setDocComment(DocComment docComment) { }; virtual PosIdx getPos() const { return noPos; } + + // These are temporary methods to be used only in parser.y + virtual void resetCursedOr() { }; + virtual void warnIfCursedOr(const SymbolTable & symbols, const PosTable & positions) { }; }; #define COMMON_METHODS \ @@ -69,6 +111,7 @@ struct ExprInt : Expr { Value v; ExprInt(NixInt n) { v.mkInt(n); }; + ExprInt(NixInt::Inner n) { v.mkInt(n); }; Value * maybeThunk(EvalState & state, Env & env) override; COMMON_METHODS }; @@ -148,7 +191,7 @@ struct ExprInheritFrom : ExprVar this->fromWith = nullptr; } - void bindVars(EvalState & es, const std::shared_ptr & env); + void bindVars(EvalState & es, const std::shared_ptr & env) override; }; struct ExprSelect : Expr @@ -159,6 +202,17 @@ struct ExprSelect : Expr ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) : pos(pos), e(e), def(def), attrPath(std::move(attrPath)) { }; ExprSelect(const PosIdx & pos, Expr * e, Symbol name) : pos(pos), e(e), def(0) { attrPath.push_back(AttrName(name)); }; PosIdx getPos() const override { return pos; } + + /** + * Evaluate the `a.b.c` part of `a.b.c.d`. This exists mostly for the purpose of :doc in the repl. + * + * @param[out] v The attribute set that should contain the last attribute name (if it exists). + * @return The last attribute name in `attrPath` + * + * @note This does *not* evaluate the final attribute, and does not fail if that's the only attribute that does not exist. + */ + Symbol evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs); + COMMON_METHODS }; @@ -281,6 +335,8 @@ struct ExprLambda : Expr Symbol arg; Formals * formals; Expr * body; + DocComment docComment; + ExprLambda(PosIdx pos, Symbol arg, Formals * formals, Expr * body) : pos(pos), arg(arg), formals(formals), body(body) { @@ -293,6 +349,7 @@ struct ExprLambda : Expr std::string showNamePos(const EvalState & state) const; inline bool hasFormals() const { return formals != nullptr; } PosIdx getPos() const override { return pos; } + virtual void setDocComment(DocComment docComment) override; COMMON_METHODS }; @@ -301,10 +358,16 @@ struct ExprCall : Expr Expr * fun; std::vector args; PosIdx pos; + std::optional cursedOrEndPos; // used during parsing to warn about https://github.com/NixOS/nix/issues/11118 ExprCall(const PosIdx & pos, Expr * fun, std::vector && args) - : fun(fun), args(args), pos(pos) + : fun(fun), args(args), pos(pos), cursedOrEndPos({}) + { } + ExprCall(const PosIdx & pos, Expr * fun, std::vector && args, PosIdx && cursedOrEndPos) + : fun(fun), args(args), pos(pos), cursedOrEndPos(cursedOrEndPos) { } PosIdx getPos() const override { return pos; } + virtual void resetCursedOr() override; + virtual void warnIfCursedOr(const SymbolTable & symbols, const PosTable & positions) override; COMMON_METHODS }; diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix new file mode 100644 index 000000000..4d10079ff --- /dev/null +++ b/src/libexpr/package.nix @@ -0,0 +1,113 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config +, bison +, flex +, cmake # for resolving toml11 dep + +, nix-util +, nix-store +, nix-fetchers +, boost +, boehmgc +, nlohmann_json +, toml11 + +# Configuration Options + +, version + +# Whether to use garbage collection for the Nix language evaluator. +# +# If it is disabled, we just leak memory, but this is not as bad as it +# sounds so long as evaluation just takes places within short-lived +# processes. (When the process exits, the memory is reclaimed; it is +# only leaked *within* the process.) +# +# Temporarily disabled on Windows because the `GC_throw_bad_alloc` +# symbol is missing during linking. +, enableGC ? !stdenv.hostPlatform.isWindows +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-expr"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + ./primops/meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ./lexer.l + ./parser.y + (fileset.fileFilter (file: file.hasExt "nix") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + bison + flex + cmake + ]; + + buildInputs = [ + toml11 + ]; + + propagatedBuildInputs = [ + nix-util + nix-store + nix-fetchers + boost + nlohmann_json + ] ++ lib.optional enableGC boehmgc; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "gc" enableGC) + ]; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libexpr/parser-state.hh b/src/libexpr/parser-state.hh index 5a928e9aa..8ad0d9ad7 100644 --- a/src/libexpr/parser-state.hh +++ b/src/libexpr/parser-state.hh @@ -1,6 +1,8 @@ #pragma once ///@file +#include + #include "eval.hh" namespace nix { @@ -18,27 +20,62 @@ struct StringToken operator std::string_view() const { return {p, l}; } }; +// This type must be trivially copyable; see YYLTYPE_IS_TRIVIAL in parser.y. struct ParserLocation { - int first_line, first_column; - int last_line, last_column; + int beginOffset; + int endOffset; // backup to recover from yyless(0) - int stashed_first_column, stashed_last_column; + int stashedBeginOffset, stashedEndOffset; void stash() { - stashed_first_column = first_column; - stashed_last_column = last_column; + stashedBeginOffset = beginOffset; + stashedEndOffset = endOffset; } void unstash() { - first_column = stashed_first_column; - last_column = stashed_last_column; + beginOffset = stashedBeginOffset; + endOffset = stashedEndOffset; } + + /** Latest doc comment position, or 0. */ + int doc_comment_first_column, doc_comment_last_column; +}; + +struct LexerState +{ + /** + * Tracks the distance to the last doc comment, in terms of lexer tokens. + * + * The lexer sets this to 0 when reading a doc comment, and increments it + * for every matched rule; see `lexer-helpers.cc`. + * Whitespace and comment rules decrement the distance, so that they result + * in a net 0 change in distance. + */ + int docCommentDistance = std::numeric_limits::max(); + + /** + * The location of the last doc comment. + * + * (stashing fields are not used) + */ + ParserLocation lastDocCommentLoc; + + /** + * @brief Maps some positions to a DocComment, where the comment is relevant to the location. + */ + std::unordered_map & positionToDocComment; + + PosTable & positions; + PosTable::Origin origin; + + PosIdx at(const ParserLocation & loc); }; struct ParserState { + const LexerState & lexerState; SymbolTable & symbols; PosTable & positions; Expr * result; @@ -46,10 +83,11 @@ struct ParserState PosTable::Origin origin; const ref rootFS; const Expr::AstSymbols & s; + const EvalSettings & settings; void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); void dupAttr(Symbol attr, const PosIdx pos, const PosIdx prevPos); - void addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos); + void addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc); Formals * validateFormals(Formals * formals, PosIdx pos = noPos, Symbol arg = {}); Expr * stripIndentation(const PosIdx pos, std::vector>> && es); @@ -73,11 +111,12 @@ inline void ParserState::dupAttr(Symbol attr, const PosIdx pos, const PosIdx pre }); } -inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * e, const PosIdx pos) +inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, const ParserLocation & loc, Expr * e, const ParserLocation & exprLoc) { AttrPath::iterator i; // All attrpaths have at least one attr assert(!attrPath.empty()); + auto pos = at(loc); // Checking attrPath validity. // =========================== for (i = attrPath.begin(); i + 1 < attrPath.end(); i++) { @@ -142,6 +181,12 @@ inline void ParserState::addAttr(ExprAttrs * attrs, AttrPath && attrPath, Expr * } else { attrs->dynamicAttrs.push_back(ExprAttrs::DynamicAttrDef(i->expr, e, pos)); } + + auto it = lexerState.positionToDocComment.find(pos); + if (it != lexerState.positionToDocComment.end()) { + e->setDocComment(it->second); + lexerState.positionToDocComment.emplace(at(exprLoc), it->second); + } } inline Formals * ParserState::validateFormals(Formals * formals, PosIdx pos, Symbol arg) @@ -254,12 +299,23 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos, s2 = std::string(s2, 0, p + 1); } - es2->emplace_back(i->first, new ExprString(std::move(s2))); + // Ignore empty strings for a minor optimisation and AST simplification + if (s2 != "") { + es2->emplace_back(i->first, new ExprString(std::move(s2))); + } }; for (; i != es.end(); ++i, --n) { std::visit(overloaded { trimExpr, trimString }, i->second); } + // If there is nothing at all, return the empty string directly. + // This also ensures that equivalent empty strings result in the same ast, which is helpful when testing formatters. + if (es2->size() == 0) { + auto *const result = new ExprString(""); + delete es2; + return result; + } + /* If this is a single string, then don't do a concatenation. */ if (es2->size() == 1 && dynamic_cast((*es2)[0].second)) { auto *const result = (*es2)[0].second; @@ -269,9 +325,14 @@ inline Expr * ParserState::stripIndentation(const PosIdx pos, return new ExprConcatStrings(pos, true, es2); } +inline PosIdx LexerState::at(const ParserLocation & loc) +{ + return positions.add(origin, loc.beginOffset); +} + inline PosIdx ParserState::at(const ParserLocation & loc) { - return positions.add(origin, loc.first_column); + return positions.add(origin, loc.beginOffset); } } diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 00300449f..944c7b1af 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -1,4 +1,4 @@ -%glr-parser +%define api.location.type { ::nix::ParserLocation } %define api.pure %locations %define parse.error verbose @@ -8,8 +8,7 @@ %parse-param { nix::ParserState * state } %lex-param { void * scanner } %lex-param { nix::ParserState * state } -%expect 1 -%expect-rr 1 +%expect 0 %code requires { @@ -25,22 +24,50 @@ #include "nixexpr.hh" #include "eval.hh" #include "eval-settings.hh" -#include "globals.hh" #include "parser-state.hh" -#define YYLTYPE ::nix::ParserLocation +// Bison seems to have difficulty growing the parser stack when using C++ with +// a custom location type. This undocumented macro tells Bison that our +// location type is "trivially copyable" in C++-ese, so it is safe to use the +// same memcpy macro it uses to grow the stack that it uses with its own +// default location type. Without this, we get "error: memory exhausted" when +// parsing some large Nix files. Our other options are to increase the initial +// stack size (200 by default) to be as large as we ever want to support (so +// that growing the stack is unnecessary), or redefine the stack-relocation +// macro ourselves (which is also undocumented). +#define YYLTYPE_IS_TRIVIAL 1 + #define YY_DECL int yylex \ (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state) +// For efficiency, we only track offsets; not line,column coordinates +# define YYLLOC_DEFAULT(Current, Rhs, N) \ + do \ + if (N) \ + { \ + (Current).beginOffset = YYRHSLOC (Rhs, 1).beginOffset; \ + (Current).endOffset = YYRHSLOC (Rhs, N).endOffset; \ + } \ + else \ + { \ + (Current).beginOffset = (Current).endOffset = \ + YYRHSLOC (Rhs, 0).endOffset; \ + } \ + while (0) + namespace nix { +typedef std::unordered_map DocCommentMap; + Expr * parseExprFromBuf( char * text, size_t length, Pos::Origin origin, const SourcePath & basePath, SymbolTable & symbols, + const EvalSettings & settings, PosTable & positions, + DocCommentMap & docComments, const ref rootFS, const Expr::AstSymbols & astSymbols); @@ -59,14 +86,13 @@ YY_DECL; using namespace nix; -#define CUR_POS state->at(*yylocp) +#define CUR_POS state->at(yyloc) void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error) { if (std::string_view(error).starts_with("syntax error, unexpected end of file")) { - loc->first_column = loc->last_column; - loc->first_line = loc->last_line; + loc->beginOffset = loc->endOffset; } throw ParseError({ .msg = HintFmt(error), @@ -74,6 +100,22 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * }); } +#define SET_DOC_POS(lambda, pos) setDocPosition(state->lexerState, lambda, state->at(pos)) +static void setDocPosition(const LexerState & lexerState, ExprLambda * lambda, PosIdx start) { + auto it = lexerState.positionToDocComment.find(start); + if (it != lexerState.positionToDocComment.end()) { + lambda->setDocComment(it->second); + } +} + +static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) { + if (auto e2 = dynamic_cast(fn)) { + e2->args.push_back(arg); + return fn; + } + return new ExprCall(pos, fn, {arg}); +} + %} @@ -98,9 +140,10 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * %type start expr expr_function expr_if expr_op %type expr_select expr_simple expr_app +%type expr_pipe_from expr_pipe_into %type expr_list -%type binds -%type formals +%type binds binds1 +%type formals formal_set %type formal %type attrpath %type attrs @@ -115,10 +158,12 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * %token PATH HPATH SPATH PATH_END %token URI %token IF THEN ELSE ASSERT WITH LET IN_KW REC INHERIT EQ NEQ AND OR IMPL OR_KW +%token PIPE_FROM PIPE_INTO /* <| and |> */ %token DOLLAR_CURLY /* == ${ */ %token IND_STRING_OPEN IND_STRING_CLOSE %token ELLIPSIS + %right IMPL %left OR %left AND @@ -140,18 +185,28 @@ expr: expr_function; expr_function : ID ':' expr_function - { $$ = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); } - | '{' formals '}' ':' expr_function - { $$ = new ExprLambda(CUR_POS, state->validateFormals($2), $5); } - | '{' formals '}' '@' ID ':' expr_function - { - auto arg = state->symbols.create($5); - $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($2, CUR_POS, arg), $7); + { auto me = new ExprLambda(CUR_POS, state->symbols.create($1), 0, $3); + $$ = me; + SET_DOC_POS(me, @1); } - | ID '@' '{' formals '}' ':' expr_function + | formal_set ':' expr_function[body] + { auto me = new ExprLambda(CUR_POS, state->validateFormals($formal_set), $body); + $$ = me; + SET_DOC_POS(me, @1); + } + | formal_set '@' ID ':' expr_function[body] { - auto arg = state->symbols.create($1); - $$ = new ExprLambda(CUR_POS, arg, state->validateFormals($4, CUR_POS, arg), $7); + auto arg = state->symbols.create($ID); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body); + $$ = me; + SET_DOC_POS(me, @1); + } + | ID '@' formal_set ':' expr_function[body] + { + auto arg = state->symbols.create($ID); + auto me = new ExprLambda(CUR_POS, arg, state->validateFormals($formal_set, CUR_POS, arg), $body); + $$ = me; + SET_DOC_POS(me, @1); } | ASSERT expr ';' expr_function { $$ = new ExprAssert(CUR_POS, $2, $4); } @@ -170,9 +225,21 @@ expr_function expr_if : IF expr THEN expr ELSE expr { $$ = new ExprIf(CUR_POS, $2, $4, $6); } + | expr_pipe_from + | expr_pipe_into | expr_op ; +expr_pipe_from + : expr_op PIPE_FROM expr_pipe_from { $$ = makeCall(state->at(@2), $1, $3); } + | expr_op PIPE_FROM expr_op { $$ = makeCall(state->at(@2), $1, $3); } + ; + +expr_pipe_into + : expr_pipe_into PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); } + | expr_op PIPE_INTO expr_op { $$ = makeCall(state->at(@2), $3, $1); } + ; + expr_op : '!' expr_op %prec NOT { $$ = new ExprOpNot($2); } | '-' expr_op %prec NEGATE { $$ = new ExprCall(CUR_POS, new ExprVar(state->s.sub), {new ExprInt(0), $2}); } @@ -197,25 +264,28 @@ expr_op ; expr_app - : expr_app expr_select { - if (auto e2 = dynamic_cast($1)) { - e2->args.push_back($2); - $$ = $1; - } else - $$ = new ExprCall(CUR_POS, $1, {$2}); - } - | expr_select + : expr_app expr_select { $$ = makeCall(CUR_POS, $1, $2); $2->warnIfCursedOr(state->symbols, state->positions); } + | /* Once a ‘cursed or’ reaches this nonterminal, it is no longer cursed, + because the uncursed parse would also produce an expr_app. But we need + to remove the cursed status in order to prevent valid things like + `f (g or)` from triggering the warning. */ + expr_select { $$ = $1; $$->resetCursedOr(); } ; expr_select : expr_simple '.' attrpath { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), nullptr); delete $3; } | expr_simple '.' attrpath OR_KW expr_select - { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), $5); delete $3; } - | /* Backwards compatibility: because Nixpkgs has a rarely used - function named ‘or’, allow stuff like ‘map or [...]’. */ + { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), $5); delete $3; $5->warnIfCursedOr(state->symbols, state->positions); } + | /* Backwards compatibility: because Nixpkgs has a function named ‘or’, + allow stuff like ‘map or [...]’. This production is problematic (see + https://github.com/NixOS/nix/issues/11118) and will be refactored in the + future by treating `or` as a regular identifier. The refactor will (in + very rare cases, we think) change the meaning of expressions, so we mark + the ExprCall with data (establishing that it is a ‘cursed or’) that can + be used to emit a warning when an affected expression is parsed. */ expr_simple OR_KW - { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, state->s.or_)}); } + { $$ = new ExprCall(CUR_POS, $1, {new ExprVar(CUR_POS, state->s.or_)}, state->positions.add(state->origin, @$.endOffset)); } | expr_simple ; @@ -259,11 +329,13 @@ expr_simple /* Let expressions `let {..., body = ...}' are just desugared into `(rec {..., body = ...}).body'. */ | LET '{' binds '}' - { $3->recursive = true; $$ = new ExprSelect(noPos, $3, state->s.body); } + { $3->recursive = true; $3->pos = CUR_POS; $$ = new ExprSelect(noPos, $3, state->s.body); } | REC '{' binds '}' - { $3->recursive = true; $$ = $3; } - | '{' binds '}' - { $$ = $2; } + { $3->recursive = true; $3->pos = CUR_POS; $$ = $3; } + | '{' binds1 '}' + { $2->pos = CUR_POS; $$ = $2; } + | '{' '}' + { $$ = new ExprAttrs(CUR_POS); } | '[' expr_list ']' { $$ = $2; } ; @@ -287,14 +359,14 @@ string_parts_interpolated path_start : PATH { - Path path(absPath({$1.p, $1.l}, state->basePath.path.abs())); + Path path(absPath(std::string_view{$1.p, $1.l}, state->basePath.path.abs())); /* add back in the trailing '/' to the first segment */ if ($1.p[$1.l-1] == '/' && $1.l > 1) path += "/"; $$ = new ExprPath(ref(state->rootFS), std::move(path)); } | HPATH { - if (evalSettings.pureEval) { + if (state->settings.pureEval) { throw Error( "the path '%s' can not be resolved in pure mode", std::string_view($1.p, $1.l) @@ -312,37 +384,50 @@ ind_string_parts ; binds - : binds attrpath '=' expr ';' { $$ = $1; state->addAttr($$, std::move(*$2), $4, state->at(@2)); delete $2; } - | binds INHERIT attrs ';' - { $$ = $1; - for (auto & [i, iPos] : *$3) { - if ($$->attrs.find(i.symbol) != $$->attrs.end()) - state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos); - $$->attrs.emplace( + : binds1 + | { $$ = new ExprAttrs; } + ; + +binds1 + : binds1[accum] attrpath '=' expr ';' + { $$ = $accum; + state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr); + delete $attrpath; + } + | binds[accum] INHERIT attrs ';' + { $$ = $accum; + for (auto & [i, iPos] : *$attrs) { + if ($accum->attrs.find(i.symbol) != $accum->attrs.end()) + state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos); + $accum->attrs.emplace( i.symbol, ExprAttrs::AttrDef(new ExprVar(iPos, i.symbol), iPos, ExprAttrs::AttrDef::Kind::Inherited)); } - delete $3; + delete $attrs; } - | binds INHERIT '(' expr ')' attrs ';' - { $$ = $1; - if (!$$->inheritFromExprs) - $$->inheritFromExprs = std::make_unique>(); - $$->inheritFromExprs->push_back($4); - auto from = new nix::ExprInheritFrom(state->at(@4), $$->inheritFromExprs->size() - 1); - for (auto & [i, iPos] : *$6) { - if ($$->attrs.find(i.symbol) != $$->attrs.end()) - state->dupAttr(i.symbol, iPos, $$->attrs[i.symbol].pos); - $$->attrs.emplace( + | binds[accum] INHERIT '(' expr ')' attrs ';' + { $$ = $accum; + if (!$accum->inheritFromExprs) + $accum->inheritFromExprs = std::make_unique>(); + $accum->inheritFromExprs->push_back($expr); + auto from = new nix::ExprInheritFrom(state->at(@expr), $accum->inheritFromExprs->size() - 1); + for (auto & [i, iPos] : *$attrs) { + if ($accum->attrs.find(i.symbol) != $accum->attrs.end()) + state->dupAttr(i.symbol, iPos, $accum->attrs[i.symbol].pos); + $accum->attrs.emplace( i.symbol, ExprAttrs::AttrDef( new ExprSelect(iPos, from, i.symbol), iPos, ExprAttrs::AttrDef::Kind::InheritedFrom)); } - delete $6; + delete $attrs; + } + | attrpath '=' expr ';' + { $$ = new ExprAttrs; + state->addAttr($$, std::move(*$attrpath), @attrpath, $expr, @expr); + delete $attrpath; } - | { $$ = new ExprAttrs(state->at(@0)); } ; attrs @@ -396,19 +481,23 @@ string_attr ; expr_list - : expr_list expr_select { $$ = $1; $1->elems.push_back($2); /* !!! dangerous */ } + : expr_list expr_select { $$ = $1; $1->elems.push_back($2); /* !!! dangerous */; $2->warnIfCursedOr(state->symbols, state->positions); } | { $$ = new ExprList; } ; +formal_set + : '{' formals ',' ELLIPSIS '}' { $$ = $formals; $$->ellipsis = true; } + | '{' ELLIPSIS '}' { $$ = new Formals; $$->ellipsis = true; } + | '{' formals ',' '}' { $$ = $formals; $$->ellipsis = false; } + | '{' formals '}' { $$ = $formals; $$->ellipsis = false; } + | '{' '}' { $$ = new Formals; $$->ellipsis = false; } + ; + formals - : formal ',' formals - { $$ = $3; $$->formals.emplace_back(*$1); delete $1; } + : formals[accum] ',' formal + { $$ = $accum; $$->formals.emplace_back(*$formal); delete $formal; } | formal - { $$ = new Formals; $$->formals.emplace_back(*$1); $$->ellipsis = false; delete $1; } - | - { $$ = new Formals; $$->ellipsis = false; } - | ELLIPSIS - { $$ = new Formals; $$->ellipsis = true; } + { $$ = new Formals; $$->formals.emplace_back(*$formal); delete $formal; } ; formal @@ -429,21 +518,30 @@ Expr * parseExprFromBuf( Pos::Origin origin, const SourcePath & basePath, SymbolTable & symbols, + const EvalSettings & settings, PosTable & positions, + DocCommentMap & docComments, const ref rootFS, const Expr::AstSymbols & astSymbols) { yyscan_t scanner; + LexerState lexerState { + .positionToDocComment = docComments, + .positions = positions, + .origin = positions.addOrigin(origin, length), + }; ParserState state { + .lexerState = lexerState, .symbols = symbols, .positions = positions, .basePath = basePath, - .origin = positions.addOrigin(origin, length), + .origin = lexerState.origin, .rootFS = rootFS, .s = astSymbols, + .settings = settings, }; - yylex_init(&scanner); + yylex_init_extra(&lexerState, &scanner); Finally _destroy([&] { yylex_destroy(scanner); }); yy_scan_buffer(text, length, scanner); diff --git a/src/libexpr/pos-idx.hh b/src/libexpr/pos-idx.hh index e94fd85c6..2faa6b7fe 100644 --- a/src/libexpr/pos-idx.hh +++ b/src/libexpr/pos-idx.hh @@ -1,6 +1,7 @@ #pragma once #include +#include namespace nix { @@ -8,6 +9,7 @@ class PosIdx { friend struct LazyPosAcessors; friend class PosTable; + friend class std::hash; private: uint32_t id; @@ -28,9 +30,9 @@ public: return id > 0; } - bool operator<(const PosIdx other) const + auto operator<=>(const PosIdx other) const { - return id < other.id; + return id <=> other.id; } bool operator==(const PosIdx other) const @@ -38,12 +40,25 @@ public: return id == other.id; } - bool operator!=(const PosIdx other) const + size_t hash() const noexcept { - return id != other.id; + return std::hash{}(id); } }; inline PosIdx noPos = {}; } + +namespace std { + +template<> +struct hash +{ + std::size_t operator()(nix::PosIdx pos) const noexcept + { + return pos.hash(); + } +}; + +} // namespace std diff --git a/src/libexpr/pos-table.hh b/src/libexpr/pos-table.hh index 8a0a3ba86..ba2b91cf3 100644 --- a/src/libexpr/pos-table.hh +++ b/src/libexpr/pos-table.hh @@ -1,10 +1,8 @@ #pragma once -#include -#include +#include #include -#include "chunked-vector.hh" #include "pos-idx.hh" #include "position.hh" #include "sync.hh" diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 24c09e747..a3c8a0c9c 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1,11 +1,9 @@ -#include "archive.hh" #include "derivations.hh" #include "downstream-placeholder.hh" #include "eval-inline.hh" #include "eval.hh" #include "eval-settings.hh" #include "gc-small-vector.hh" -#include "globals.hh" #include "json-to-value.hh" #include "names.hh" #include "path-references.hh" @@ -26,6 +24,7 @@ #include #include +#include #include #ifndef _WIN32 @@ -41,6 +40,13 @@ namespace nix { * Miscellaneous *************************************************************/ +static inline Value * mkString(EvalState & state, const std::csub_match & match) +{ + Value * v = state.allocValue(); + v->mkString({match.first, match.second}); + return v; +} + StringMap EvalState::realiseContext(const NixStringContext & context, StorePathSet * maybePathsOut, bool isIFD) { std::vector drvs; @@ -77,8 +83,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS if (drvs.empty()) return {}; - if (isIFD && !evalSettings.enableImportFromDerivation) - error( + if (isIFD && !settings.enableImportFromDerivation) + error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", drvs.begin()->to_string(*store) ).debugThrow(); @@ -427,7 +433,7 @@ static void prim_typeOf(EvalState & state, const PosIdx pos, Value * * args, Val t = args[0]->external()->typeOf(); break; case nFloat: t = "float"; break; - case nThunk: abort(); + case nThunk: unreachable(); } v.mkString(t); } @@ -588,9 +594,9 @@ struct CompareValues { try { if (v1->type() == nFloat && v2->type() == nInt) - return v1->fpoint() < v2->integer(); + return v1->fpoint() < v2->integer().value; if (v1->type() == nInt && v2->type() == nFloat) - return v1->integer() < v2->fpoint(); + return v1->integer().value < v2->fpoint(); if (v1->type() != v2->type()) state.error("cannot compare %s with %s", showType(*v1), showType(*v2)).debugThrow(); // Allow selecting a subset of enum values @@ -632,11 +638,7 @@ struct CompareValues }; -#if HAVE_BOEHMGC typedef std::list> ValueList; -#else -typedef std::list ValueList; -#endif static Bindings::const_iterator getAttr( @@ -720,7 +722,7 @@ static RegisterPrimOp primop_genericClosure(PrimOp { .doc = R"( `builtins.genericClosure` iteratively computes the transitive closure over an arbitrary relation defined by a function. - It takes *attrset* with two attributes named `startSet` and `operator`, and returns a list of attrbute sets: + It takes *attrset* with two attributes named `startSet` and `operator`, and returns a list of attribute sets: - `startSet`: The initial list of attribute sets. @@ -732,11 +734,12 @@ static RegisterPrimOp primop_genericClosure(PrimOp { Each attribute set in the list `startSet` and the list returned by `operator` must have an attribute `key`, which must support equality comparison. The value of `key` can be one of the following types: - - [Number](@docroot@/language/values.md#type-number) - - [Boolean](@docroot@/language/values.md#type-boolean) - - [String](@docroot@/language/values.md#type-string) - - [Path](@docroot@/language/values.md#type-path) - - [List](@docroot@/language/values.md#list) + - [Int](@docroot@/language/types.md#type-int) + - [Float](@docroot@/language/types.md#type-float) + - [Boolean](@docroot@/language/types.md#type-boolean) + - [String](@docroot@/language/types.md#type-string) + - [Path](@docroot@/language/types.md#type-path) + - [List](@docroot@/language/types.md#list) The result is produced by calling the `operator` on each `item` that has not been called yet, including newly added items, until no new items are added. Items are compared by their `key` attribute. @@ -779,15 +782,14 @@ static RegisterPrimOp primop_break({ )", .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { - if (state.debugRepl && !state.debugTraces.empty()) { + if (state.canDebug()) { auto error = Error(ErrorInfo { .level = lvlInfo, .msg = HintFmt("breakpoint reached"), .pos = state.positions[pos], }); - auto & dt = state.debugTraces.front(); - state.runDebugRepl(&error, dt.env, dt.expr); + state.runDebugRepl(&error); } // Return the value we were passed. @@ -806,7 +808,7 @@ static RegisterPrimOp primop_abort({ NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.abort").toOwned(); - state.error("evaluation aborted with the following error message: '%1%'", s).debugThrow(); + state.error("evaluation aborted with the following error message: '%1%'", s).setIsFromExpr().debugThrow(); } }); @@ -825,7 +827,7 @@ static RegisterPrimOp primop_throw({ NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtin.throw").toOwned(); - state.error(s).debugThrow(); + state.error(s).setIsFromExpr().debugThrow(); } }); @@ -901,7 +903,7 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value * * args, Va MaintainCount trylevel(state.trylevel); ReplExitStatus (* savedDebugRepl)(ref es, const ValMap & extraEnv) = nullptr; - if (state.debugRepl && evalSettings.ignoreExceptionsDuringTry) + if (state.debugRepl && state.settings.ignoreExceptionsDuringTry) { /* to prevent starting the repl from exceptions withing a tryEval, null it. */ savedDebugRepl = state.debugRepl; @@ -950,7 +952,7 @@ static RegisterPrimOp primop_tryEval({ static void prim_getEnv(EvalState & state, const PosIdx pos, Value * * args, Value & v) { std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.getEnv")); - v.mkString(evalSettings.restrictEval || evalSettings.pureEval ? "" : getEnv(name).value_or("")); + v.mkString(state.settings.restrictEval || state.settings.pureEval ? "" : getEnv(name).value_or("")); } static RegisterPrimOp primop_getEnv({ @@ -1017,9 +1019,8 @@ static void prim_trace(EvalState & state, const PosIdx pos, Value * * args, Valu printError("trace: %1%", args[0]->string_view()); else printError("trace: %1%", ValuePrinter(state, *args[0])); - if (evalSettings.builtinsTraceDebugger && state.debugRepl && !state.debugTraces.empty()) { - const DebugTrace & last = state.debugTraces.front(); - state.runDebugRepl(nullptr, last.env, last.expr); + if (state.settings.builtinsTraceDebugger) { + state.runDebugRepl(nullptr); } state.forceValue(*args[1], pos); v = *args[1]; @@ -1042,6 +1043,55 @@ static RegisterPrimOp primop_trace({ .fun = prim_trace, }); +static void prim_warn(EvalState & state, const PosIdx pos, Value * * args, Value & v) +{ + // We only accept a string argument for now. The use case for pretty printing a value is covered by `trace`. + // By rejecting non-strings we allow future versions to add more features without breaking existing code. + auto msgStr = state.forceString(*args[0], pos, "while evaluating the first argument; the message passed to builtins.warn"); + + { + BaseError msg(std::string{msgStr}); + msg.atPos(state.positions[pos]); + auto info = msg.info(); + info.level = lvlWarn; + info.isFromExpr = true; + logWarning(info); + } + + if (state.settings.builtinsAbortOnWarn) { + // Not an EvalError or subclass, which would cause the error to be stored in the eval cache. + state.error("aborting to reveal stack trace of warning, as abort-on-warn is set").setIsFromExpr().debugThrow(); + } + if (state.settings.builtinsTraceDebugger || state.settings.builtinsDebuggerOnWarn) { + state.runDebugRepl(nullptr); + } + state.forceValue(*args[1], pos); + v = *args[1]; +} + +static RegisterPrimOp primop_warn({ + .name = "__warn", + .args = {"e1", "e2"}, + .doc = R"( + Evaluate *e1*, which must be a string and print iton standard error as a warning. + Then return *e2*. + This function is useful for non-critical situations where attention is advisable. + + If the + [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace) + or [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn) + option is set to `true` and the `--debugger` flag is given, the + interactive debugger will be started when `warn` is called (like + [`break`](@docroot@/language/builtins.md#builtins-break)). + + If the + [`abort-on-warn`](@docroot@/command-ref/conf-file.md#conf-abort-on-warn) + option is set, the evaluation will be aborted after the warning is printed. + This is useful to reveal the stack trace of the warning, when the context is non-interactive and a debugger can not be launched. + )", + .fun = prim_warn, +}); + /* Takes two arguments and evaluates to the second one. Used as the * builtins.traceVerbose implementation when --trace-verbose is not enabled @@ -1115,12 +1165,34 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value * * } } +/** + * Early validation for the derivation name, for better error message. + * It is checked again when constructing store paths. + * + * @todo Check that the `.drv` suffix also fits. + */ +static void checkDerivationName(EvalState & state, std::string_view drvName) +{ + try { + checkName(drvName); + } catch (BadStorePathName & e) { + // "Please pass a different name": Users may not be aware that they can + // pass a different one, in functions like `fetchurl` where the name + // is optional. + // Note that Nixpkgs generally won't trigger this, because `mkDerivation` + // sanitizes the name. + state.error("invalid derivation name: %s. Please pass a different '%s'.", Uncolored(e.message()), "name").debugThrow(); + } +} + static void derivationStrictInternal( EvalState & state, const std::string & drvName, const Bindings * attrs, Value & v) { + checkDerivationName(state, drvName); + /* Check whether attributes should be passed as a JSON file. */ using nlohmann::json; std::optional jsonObject; @@ -1155,13 +1227,13 @@ static void derivationStrictInternal( for (auto & i : attrs->lexicographicOrder(state.symbols)) { if (i->name == state.sIgnoreNulls) continue; - const std::string & key = state.symbols[i->name]; + auto key = state.symbols[i->name]; vomit("processing attribute '%1%'", key); auto handleHashMode = [&](const std::string_view s) { if (s == "recursive") { // back compat, new name is "nar" - ingestionMethod = FileIngestionMethod::Recursive; + ingestionMethod = ContentAddressMethod::Raw::NixArchive; } else try { ingestionMethod = ContentAddressMethod::parse(s); } catch (UsageError &) { @@ -1169,9 +1241,9 @@ static void derivationStrictInternal( "invalid value '%s' for 'outputHashMode' attribute", s ).atPos(v).debugThrow(); } - if (ingestionMethod == TextIngestionMethod {}) + if (ingestionMethod == ContentAddressMethod::Raw::Text) experimentalFeatureSettings.require(Xp::DynamicDerivations); - if (ingestionMethod == FileIngestionMethod::Git) + if (ingestionMethod == ContentAddressMethod::Raw::Git) experimentalFeatureSettings.require(Xp::GitHashing); }; @@ -1239,7 +1311,7 @@ static void derivationStrictInternal( if (i->name == state.sStructuredAttrs) continue; - (*jsonObject)[key] = printValueAsJSON(state, true, *i->value, pos, context); + jsonObject->emplace(key, printValueAsJSON(state, true, *i->value, pos, context)); if (i->name == state.sBuilder) drv.builder = state.forceString(*i->value, context, pos, context_below); @@ -1260,6 +1332,20 @@ static void derivationStrictInternal( handleOutputs(ss); } + if (i->name == state.sAllowedReferences) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks..allowedReferences' instead", drvName); + if (i->name == state.sAllowedRequisites) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks..allowedRequisites' instead", drvName); + if (i->name == state.sDisallowedReferences) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks..disallowedReferences' instead", drvName); + if (i->name == state.sDisallowedRequisites) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks..disallowedRequisites' instead", drvName); + if (i->name == state.sMaxSize) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks..maxSize' instead", drvName); + if (i->name == state.sMaxClosureSize) + warn("In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks..maxClosureSize' instead", drvName); + + } else { auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned(); drv.env.emplace(key, s); @@ -1329,7 +1415,7 @@ static void derivationStrictInternal( /* Check whether the derivation name is valid. */ if (isDerivation(drvName) && - !(ingestionMethod == ContentAddressMethod { TextIngestionMethod { } } && + !(ingestionMethod == ContentAddressMethod::Raw::Text && outputs.size() == 1 && *(outputs.begin()) == "out")) { @@ -1351,7 +1437,7 @@ static void derivationStrictInternal( auto h = newHashAllowEmpty(*outputHash, outputHashAlgo); - auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); + auto method = ingestionMethod.value_or(ContentAddressMethod::Raw::Flat); DerivationOutput::CAFixed dof { .ca = ContentAddress { @@ -1370,7 +1456,7 @@ static void derivationStrictInternal( .atPos(v).debugThrow(); auto ha = outputHashAlgo.value_or(HashAlgorithm::SHA256); - auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); + auto method = ingestionMethod.value_or(ContentAddressMethod::Raw::NixArchive); for (auto & i : outputs) { drv.env[i] = hashPlaceholder(i); @@ -1516,7 +1602,7 @@ static RegisterPrimOp primop_toPath({ corner cases. */ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - if (evalSettings.pureEval) + if (state.settings.pureEval) state.error( "'%s' is not allowed in pure evaluation mode", "builtins.storePath" @@ -1626,7 +1712,7 @@ static RegisterPrimOp primop_baseNameOf({ .name = "baseNameOf", .args = {"x"}, .doc = R"( - Return the *base name* of either a [path value](@docroot@/language/values.md#type-path) *x* or a string *x*, depending on which type is passed, and according to the following rules. + Return the *base name* of either a [path value](@docroot@/language/types.md#type-path) *x* or a string *x*, depending on which type is passed, and according to the following rules. For a path value, the *base name* is considered to be the part of the path after the last directory separator, including any file extensions. This is the simple case, as path values don't have trailing slashes. @@ -1760,35 +1846,7 @@ static RegisterPrimOp primop_findFile(PrimOp { .doc = R"( Find *lookup-path* in *search-path*. - A search path is represented list of [attribute sets](./values.md#attribute-set) with two attributes: - - `prefix` is a relative path. - - `path` denotes a file system location - The exact syntax depends on the command line interface. - - Examples of search path attribute sets: - - - ``` - { - prefix = "nixos-config"; - path = "/etc/nixos/configuration.nix"; - } - ``` - - - ``` - { - prefix = ""; - path = "/nix/var/nix/profiles/per-user/root/channels"; - } - ``` - - The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/values.html#type-path) of the match: - - - If *lookup-path* matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. - Note that the `path` may need to be downloaded at this point to look inside. - - If the suffix is found inside that directory, then the entry is a match. - The combined absolute path of the directory (now downloaded if need be) and the suffix is returned. - - [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](@docroot@/language/builtin-constants.md#builtins-nixPath): + [Lookup path](@docroot@/language/constructs/lookup-path.md) expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.nixPath`](#builtins-nixPath): ```nix @@ -1799,6 +1857,119 @@ static RegisterPrimOp primop_findFile(PrimOp { ```nix builtins.findFile builtins.nixPath "nixpkgs" ``` + + A search path is represented as a list of [attribute sets](./types.md#attribute-set) with two attributes: + - `prefix` is a relative path. + - `path` denotes a file system location + + Examples of search path attribute sets: + + - ``` + { + prefix = ""; + path = "/nix/var/nix/profiles/per-user/root/channels"; + } + ``` + - ``` + { + prefix = "nixos-config"; + path = "/etc/nixos/configuration.nix"; + } + ``` + - ``` + { + prefix = "nixpkgs"; + path = "https://github.com/NixOS/nixpkgs/tarballs/master"; + } + ``` + - ``` + { + prefix = "nixpkgs"; + path = "channel:nixpkgs-unstable"; + } + ``` + - ``` + { + prefix = "flake-compat"; + path = "flake:github:edolstra/flake-compat"; + } + ``` + + The lookup algorithm checks each entry until a match is found, returning a [path value](@docroot@/language/types.md#type-path) of the match: + + - If a prefix of `lookup-path` matches `prefix`, then the remainder of *lookup-path* (the "suffix") is searched for within the directory denoted by `path`. + The contents of `path` may need to be downloaded at this point to look inside. + + - If the suffix is found inside that directory, then the entry is a match. + The combined absolute path of the directory (now downloaded if need be) and the suffix is returned. + + > **Example** + > + > A *search-path* value + > + > ``` + > [ + > { + > prefix = ""; + > path = "/home/eelco/Dev"; + > } + > { + > prefix = "nixos-config"; + > path = "/etc/nixos"; + > } + > ] + > ``` + > + > and a *lookup-path* value `"nixos-config"` will cause Nix to try `/home/eelco/Dev/nixos-config` and `/etc/nixos` in that order and return the first path that exists. + + If `path` starts with `http://` or `https://`, it is interpreted as the URL of a tarball that will be downloaded and unpacked to a temporary location. + The tarball must consist of a single top-level directory. + + The URLs of the tarballs from the official `nixos.org` channels can be abbreviated as `channel:`. + See [documentation on `nix-channel`](@docroot@/command-ref/nix-channel.md) for details about channels. + + > **Example** + > + > These two search path entries are equivalent: + > + > - ``` + > { + > prefix = "nixpkgs"; + > path = "channel:nixpkgs-unstable"; + > } + > ``` + > - ``` + > { + > prefix = "nixpkgs"; + > path = "https://nixos.org/channels/nixos-unstable/nixexprs.tar.xz"; + > } + > ``` + + Search paths can also point to source trees using [flake URLs](@docroot@/command-ref/new-cli/nix3-flake.md#url-like-syntax). + + + > **Example** + > + > The search path entry + > + > ``` + > { + > prefix = "nixpkgs"; + > path = "flake:nixpkgs"; + > } + > ``` + > specifies that the prefix `nixpkgs` shall refer to the source tree downloaded from the `nixpkgs` entry in the flake registry. + > + > Similarly + > + > ``` + > { + > prefix = "nixpkgs"; + > path = "flake:github:nixos/nixpkgs/nixos-22.05"; + > } + > ``` + > + > makes `` refer to a particular branch of the `NixOS/nixpkgs` repository on GitHub. )", .fun = prim_findFile, }); @@ -1965,7 +2136,7 @@ static void prim_toXML(EvalState & state, const PosIdx pos, Value * * args, Valu std::ostringstream out; NixStringContext context; printValueAsXML(state, true, false, *args[0], out, context, pos); - v.mkString(out.str(), context); + v.mkString(toView(out), context); } static RegisterPrimOp primop_toXML({ @@ -2073,7 +2244,7 @@ static void prim_toJSON(EvalState & state, const PosIdx pos, Value * * args, Val std::ostringstream out; NixStringContext context; printValueAsJSON(state, true, *args[0], pos, out, context); - v.mkString(out.str(), context); + v.mkString(toView(out), context); } static RegisterPrimOp primop_toJSON({ @@ -2146,7 +2317,7 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Val }) : ({ StringSource s { contents }; - state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, state.repair); + state.store->addToStoreFromDump(s, name, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, state.repair); }); /* Note: we don't need to add `context' to the context of the @@ -2209,7 +2380,7 @@ static RegisterPrimOp primop_toFile({ ``` Note that `${configFile}` is a - [string interpolation](@docroot@/language/values.md#type-string), so the result of the + [string interpolation](@docroot@/language/types.md#type-string), so the result of the expression `configFile` (i.e., a path like `/nix/store/m7p7jfny445k...-foo.conf`) will be spliced into the resulting string. @@ -2261,7 +2432,7 @@ static void addPath( std::string_view name, SourcePath path, Value * filterFun, - FileIngestionMethod method, + ContentAddressMethod method, const std::optional expectedHash, Value & v, const NixStringContext & context) @@ -2293,11 +2464,10 @@ static void addPath( std::optional expectedStorePath; if (expectedHash) - expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo { - .method = method, - .hash = *expectedHash, - .references = {}, - }); + expectedStorePath = state.store->makeFixedOutputPathFromCA(name, ContentAddressWithReferences::fromParts( + method, + *expectedHash, + {})); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { auto dstPath = fetchToStore( @@ -2330,7 +2500,7 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * arg "while evaluating the second argument (the path to filter) passed to 'builtins.filterSource'"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); - addPath(state, pos, path.baseName(), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); + addPath(state, pos, path.baseName(), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ @@ -2393,7 +2563,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value std::optional path; std::string name; Value * filterFun = nullptr; - auto method = FileIngestionMethod::Recursive; + auto method = ContentAddressMethod::Raw::NixArchive; std::optional expectedHash; NixStringContext context; @@ -2408,7 +2578,9 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value else if (n == "filter") state.forceFunction(*(filterFun = attr.value), attr.pos, "while evaluating the `filter` parameter passed to builtins.path"); else if (n == "recursive") - method = FileIngestionMethod { state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") }; + method = state.forceBool(*attr.value, attr.pos, "while evaluating the `recursive` attribute passed to builtins.path") + ? ContentAddressMethod::Raw::NixArchive + : ContentAddressMethod::Raw::Flat; else if (n == "sha256") expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `sha256` attribute passed to builtins.path"), HashAlgorithm::SHA256); else @@ -2593,13 +2765,13 @@ static struct LazyPosAcessors { PrimOp primop_lineOfPos{ .arity = 1, .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.positions[PosIdx(args[0]->integer())].line); + v.mkInt(state.positions[PosIdx(args[0]->integer().value)].line); } }; PrimOp primop_columnOfPos{ .arity = 1, .fun = [] (EvalState & state, PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.positions[PosIdx(args[0]->integer())].column); + v.mkInt(state.positions[PosIdx(args[0]->integer().value)].column); } }; @@ -2967,7 +3139,7 @@ static void prim_zipAttrsWith(EvalState & state, const PosIdx pos, Value * * arg std::optional list; }; - std::map attrsSeen; + std::map, traceable_allocator>> attrsSeen; state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.zipAttrsWith"); state.forceList(*args[1], pos, "while evaluating the second argument passed to builtins.zipAttrsWith"); @@ -3075,7 +3247,8 @@ static void elemAt(EvalState & state, const PosIdx pos, Value & list, int n, Val /* Return the n-1'th element of a list. */ static void prim_elemAt(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - elemAt(state, pos, *args[0], state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt"), v); + NixInt::Inner elem = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.elemAt").value; + elemAt(state, pos, *args[0], elem, v); } static RegisterPrimOp primop_elemAt({ @@ -3369,10 +3542,12 @@ static RegisterPrimOp primop_all({ static void prim_genList(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - auto len = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList"); + auto len_ = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.genList").value; - if (len < 0) - state.error("cannot create list of size %1%", len).atPos(pos).debugThrow(); + if (len_ < 0) + state.error("cannot create list of size %1%", len_).atPos(pos).debugThrow(); + + size_t len = size_t(len_); // More strict than striclty (!) necessary, but acceptable // as evaluating map without accessing any values makes little sense. @@ -3629,9 +3804,17 @@ static void prim_add(EvalState & state, const PosIdx pos, Value * * args, Value if (args[0]->type() == nFloat || args[1]->type() == nFloat) v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the addition") + state.forceFloat(*args[1], pos, "while evaluating the second argument of the addition")); - else - v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the addition") - + state.forceInt(*args[1], pos, "while evaluating the second argument of the addition")); + else { + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the addition"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the addition"); + + auto result_ = i1 + i2; + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in adding %1% + %2%", i1, i2).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_add({ @@ -3650,9 +3833,18 @@ static void prim_sub(EvalState & state, const PosIdx pos, Value * * args, Value if (args[0]->type() == nFloat || args[1]->type() == nFloat) v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first argument of the subtraction") - state.forceFloat(*args[1], pos, "while evaluating the second argument of the subtraction")); - else - v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the subtraction") - - state.forceInt(*args[1], pos, "while evaluating the second argument of the subtraction")); + else { + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the subtraction"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the subtraction"); + + auto result_ = i1 - i2; + + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in subtracting %1% - %2%", i1, i2).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_sub({ @@ -3671,9 +3863,18 @@ static void prim_mul(EvalState & state, const PosIdx pos, Value * * args, Value if (args[0]->type() == nFloat || args[1]->type() == nFloat) v.mkFloat(state.forceFloat(*args[0], pos, "while evaluating the first of the multiplication") * state.forceFloat(*args[1], pos, "while evaluating the second argument of the multiplication")); - else - v.mkInt( state.forceInt(*args[0], pos, "while evaluating the first argument of the multiplication") - * state.forceInt(*args[1], pos, "while evaluating the second argument of the multiplication")); + else { + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument of the multiplication"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument of the multiplication"); + + auto result_ = i1 * i2; + + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in multiplying %1% * %2%", i1, i2).atPos(pos).debugThrow(); + } + } } static RegisterPrimOp primop_mul({ @@ -3700,10 +3901,12 @@ static void prim_div(EvalState & state, const PosIdx pos, Value * * args, Value NixInt i1 = state.forceInt(*args[0], pos, "while evaluating the first operand of the division"); NixInt i2 = state.forceInt(*args[1], pos, "while evaluating the second operand of the division"); /* Avoid division overflow as it might raise SIGFPE. */ - if (i1 == std::numeric_limits::min() && i2 == -1) - state.error("overflow in integer division").atPos(pos).debugThrow(); - - v.mkInt(i1 / i2); + auto result_ = i1 / i2; + if (auto result = result_.valueChecked(); result.has_value()) { + v.mkInt(*result); + } else { + state.error("integer overflow in dividing %1% / %2%", i1, i2).atPos(pos).debugThrow(); + } } } @@ -3718,8 +3921,9 @@ static RegisterPrimOp primop_div({ static void prim_bitAnd(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitAnd") - & state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitAnd")); + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitAnd"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitAnd"); + v.mkInt(i1.value & i2.value); } static RegisterPrimOp primop_bitAnd({ @@ -3733,8 +3937,10 @@ static RegisterPrimOp primop_bitAnd({ static void prim_bitOr(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitOr") - | state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitOr")); + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitOr"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitOr"); + + v.mkInt(i1.value | i2.value); } static RegisterPrimOp primop_bitOr({ @@ -3748,8 +3954,10 @@ static RegisterPrimOp primop_bitOr({ static void prim_bitXor(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - v.mkInt(state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitXor") - ^ state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitXor")); + auto i1 = state.forceInt(*args[0], pos, "while evaluating the first argument passed to builtins.bitXor"); + auto i2 = state.forceInt(*args[1], pos, "while evaluating the second argument passed to builtins.bitXor"); + + v.mkInt(i1.value ^ i2.value); } static RegisterPrimOp primop_bitXor({ @@ -3829,13 +4037,19 @@ static RegisterPrimOp primop_toString({ non-negative. */ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring"); + NixInt::Inner start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring").value; if (start < 0) state.error("negative start position in 'substring'").atPos(pos).debugThrow(); - int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring"); + NixInt::Inner len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring").value; + + // Negative length may be idiomatically passed to builtins.substring to get + // the tail of the string. + if (len < 0) { + len = std::numeric_limits::max(); + } // Special-case on empty substring to avoid O(n) strlen // This allows for the use of empty substrings to efficently capture string context @@ -3878,7 +4092,7 @@ static void prim_stringLength(EvalState & state, const PosIdx pos, Value * * arg { NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.stringLength"); - v.mkInt(s->size()); + v.mkInt(NixInt::Inner(s->size())); } static RegisterPrimOp primop_stringLength({ @@ -4013,17 +4227,23 @@ static RegisterPrimOp primop_convertHash({ struct RegexCache { - // TODO use C++20 transparent comparison when available - std::unordered_map cache; - std::list keys; + struct State + { + // TODO use C++20 transparent comparison when available + std::unordered_map cache; + std::list keys; + }; + + Sync state_; std::regex get(std::string_view re) { - auto it = cache.find(re); - if (it != cache.end()) + auto state(state_.lock()); + auto it = state->cache.find(re); + if (it != state->cache.end()) return it->second; - keys.emplace_back(re); - return cache.emplace(keys.back(), std::regex(keys.back(), std::regex::extended)).first->second; + state->keys.emplace_back(re); + return state->cache.emplace(state->keys.back(), std::regex(state->keys.back(), std::regex::extended)).first->second; } }; @@ -4055,7 +4275,7 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) if (!match[i + 1].matched) v2 = &state.vNull; else - (v2 = state.allocValue())->mkString(match[i + 1].str()); + v2 = mkString(state, match[i + 1]); v.mkList(list); } catch (std::regex_error & e) { @@ -4139,7 +4359,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto match = *i; // Add a string for non-matched characters. - (list[idx++] = state.allocValue())->mkString(match.prefix().str()); + list[idx++] = mkString(state, match.prefix()); // Add a list for matched substrings. const size_t slen = match.size() - 1; @@ -4150,14 +4370,14 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) if (!match[si + 1].matched) v2 = &state.vNull; else - (v2 = state.allocValue())->mkString(match[si + 1].str()); + v2 = mkString(state, match[si + 1]); } (list[idx++] = state.allocValue())->mkList(list2); // Add a string for non-matched suffix characters. if (idx == 2 * len) - (list[idx++] = state.allocValue())->mkString(match.suffix().str()); + list[idx++] = mkString(state, match.suffix()); } assert(idx == 2 * len + 1); @@ -4356,7 +4576,8 @@ static void prim_compareVersions(EvalState & state, const PosIdx pos, Value * * { auto version1 = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.compareVersions"); auto version2 = state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.compareVersions"); - v.mkInt(compareVersions(version1, version2)); + auto result = compareVersions(version1, version2); + v.mkInt(result < 0 ? -1 : result > 0 ? 1 : 0); } static RegisterPrimOp primop_compareVersions({ @@ -4428,7 +4649,7 @@ void EvalState::createBaseEnv() addConstant("builtins", v, { .type = nAttrs, .doc = R"( - Contains all the [built-in functions](@docroot@/language/builtins.md) and values. + Contains all the built-in functions and values. Since built-in functions were added over time, [testing for attributes](./operators.md#has-attribute) in `builtins` can be used for graceful fallback on older Nix installations: @@ -4448,7 +4669,7 @@ void EvalState::createBaseEnv() It can be returned by [comparison operators](@docroot@/language/operators.md#Comparison) and used in - [conditional expressions](@docroot@/language/constructs.md#Conditionals). + [conditional expressions](@docroot@/language/syntax.md#Conditionals). The name `true` is not special, and can be shadowed: @@ -4468,7 +4689,7 @@ void EvalState::createBaseEnv() It can be returned by [comparison operators](@docroot@/language/operators.md#Comparison) and used in - [conditional expressions](@docroot@/language/constructs.md#Conditionals). + [conditional expressions](@docroot@/language/syntax.md#Conditionals). The name `false` is not special, and can be shadowed: @@ -4493,7 +4714,7 @@ void EvalState::createBaseEnv() )", }); - if (!evalSettings.pureEval) { + if (!settings.pureEval) { v.mkInt(time(0)); } addConstant("__currentTime", v, { @@ -4520,8 +4741,8 @@ void EvalState::createBaseEnv() .impureOnly = true, }); - if (!evalSettings.pureEval) - v.mkString(evalSettings.getCurrentSystem()); + if (!settings.pureEval) + v.mkString(settings.getCurrentSystem()); addConstant("__currentSystem", v, { .type = nString, .doc = R"( @@ -4601,7 +4822,7 @@ void EvalState::createBaseEnv() #ifndef _WIN32 // TODO implement on Windows // Miscellaneous - if (evalSettings.enableNativeCode) { + if (settings.enableNativeCode) { addPrimOp({ .name = "__importNative", .arity = 2, @@ -4624,7 +4845,7 @@ void EvalState::createBaseEnv() error if `--trace-verbose` is enabled. Then return *e2*. This function is useful for debugging. )", - .fun = evalSettings.traceVerbose ? prim_trace : prim_second, + .fun = settings.traceVerbose ? prim_trace : prim_second, }); /* Add a value containing the current Nix expression search path. */ @@ -4639,7 +4860,17 @@ void EvalState::createBaseEnv() addConstant("__nixPath", v, { .type = nList, .doc = R"( - The value of the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path): a list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). + A list of search path entries used to resolve [lookup paths](@docroot@/language/constructs/lookup-path.md). + Its value is primarily determined by the [`nix-path` configuration setting](@docroot@/command-ref/conf-file.md#conf-nix-path), which are + - Overridden by the [`NIX_PATH`](@docroot@/command-ref/env-common.md#env-NIX_PATH) environment variable or the `--nix-path` option + - Extended by the [`-I` option](@docroot@/command-ref/opt-common.md#opt-I) or `--extra-nix-path` + + > **Example** + > + > ```bash + > $ NIX_PATH= nix-instantiate --eval --expr "builtins.nixPath" -I foo=bar --no-pure-eval + > [ { path = "bar"; prefix = "foo"; } ] + > ``` Lookup path expressions are [desugared](https://en.wikipedia.org/wiki/Syntactic_sugar) using this and [`builtins.findFile`](./builtins.html#builtins-findFile): diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index 8c3f1b4e8..02683b173 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -86,7 +86,7 @@ static RegisterPrimOp primop_unsafeDiscardOutputDependency({ This is the opposite of [`builtins.addDrvOutputDependencies`](#builtins-addDrvOutputDependencies). - This is unsafe because it allows us to "forget" store objects we would have otherwise refered to with the string context, + This is unsafe because it allows us to "forget" store objects we would have otherwise referred to with the string context, whereas Nix normally tracks all dependencies consistently. Safe operations "grow" but never "shrink" string contexts. [`builtins.addDrvOutputDependencies`] in contrast is safe because "derivation deep" string context element always refers to the underlying derivation (among many more things). diff --git a/src/libexpr/primops/derivation.nix b/src/libexpr/primops/derivation.nix index c0fbe8082..f329ff71e 100644 --- a/src/libexpr/primops/derivation.nix +++ b/src/libexpr/primops/derivation.nix @@ -1,6 +1,31 @@ -/* This is the implementation of the ‘derivation’ builtin function. - It's actually a wrapper around the ‘derivationStrict’ primop. */ +# This is the implementation of the ‘derivation’ builtin function. +# It's actually a wrapper around the ‘derivationStrict’ primop. +# Note that the following comment will be shown in :doc in the repl, but not in the manual. +/** + Create a derivation. + + # Inputs + + The single argument is an attribute set that describes what to build and how to build it. + See https://nix.dev/manual/nix/2.23/language/derivations + + # Output + + The result is an attribute set that describes the derivation. + Notably it contains the outputs, which in the context of the Nix language are special strings that refer to the output paths, which may not yet exist. + The realisation of these outputs only occurs when needed; for example + + * When `nix-build` or a similar command is run, it realises the outputs that were requested on its command line. + See https://nix.dev/manual/nix/2.23/command-ref/nix-build + + * When `import`, `readFile`, `readDir` or some other functions are called, they have to realise the outputs they depend on. + This is referred to as "import from derivation". + See https://nix.dev/manual/nix/2.23/language/import-from-derivation + + Note that `derivation` is very bare-bones, and provides almost no commands during the build. + Most likely, you'll want to use functions like `stdenv.mkDerivation` in Nixpkgs to set up a basic environment. +*/ drvAttrs @ { outputs ? [ "out" ], ... }: let diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index d9ba6aa97..64e3abf2d 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -53,7 +53,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a // whitelist. Ah well. state.checkURI(url); - if (evalSettings.pureEval && !rev) + if (state.settings.pureEval && !rev) throw Error("in pure evaluation mode, 'fetchMercurial' requires a Mercurial revision"); fetchers::Attrs attrs; @@ -62,7 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a attrs.insert_or_assign("name", std::string(name)); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); - auto input = fetchers::Input::fromAttrs(std::move(attrs)); + auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); auto [storePath, input2] = input.fetchToStore(state.store); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index fa462dc33..d2266e2bc 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -1,4 +1,4 @@ -#include "libfetchers/attrs.hh" +#include "attrs.hh" #include "primops.hh" #include "eval-inline.hh" #include "eval-settings.hh" @@ -11,6 +11,8 @@ #include "value-to-json.hh" #include "fetch-to-store.hh" +#include + #include #include #include @@ -85,7 +87,7 @@ static void fetchTree( Value & v, const FetchTreeParams & params = FetchTreeParams{} ) { - fetchers::Input input; + fetchers::Input input { state.fetchSettings }; NixStringContext context; std::optional type; if (params.isFetchGit) type = "git"; @@ -122,9 +124,15 @@ static void fetchTree( } else if (attr.value->type() == nBool) attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean()}); - else if (attr.value->type() == nInt) - attrs.emplace(state.symbols[attr.name], uint64_t(attr.value->integer())); - else if (state.symbols[attr.name] == "publicKeys") { + else if (attr.value->type() == nInt) { + auto intValue = attr.value->integer().value; + + if (intValue < 0) { + state.error("negative value given for fetchTree attr %1%: %2%", state.symbols[attr.name], intValue).atPos(pos).debugThrow(); + } + + attrs.emplace(state.symbols[attr.name], uint64_t(intValue)); + } else if (state.symbols[attr.name] == "publicKeys") { experimentalFeatureSettings.require(Xp::VerifiedFetches); attrs.emplace(state.symbols[attr.name], printValueAsJSON(state, true, *attr.value, pos, context).dump()); } @@ -137,13 +145,18 @@ static void fetchTree( attrs.emplace("exportIgnore", Explicit{true}); } + // fetchTree should fetch git repos with shallow = true by default + if (type == "git" && !params.isFetchGit && !attrs.contains("shallow")) { + attrs.emplace("shallow", Explicit{true}); + } + if (!params.allowNameArgument) if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) state.error( "attribute 'name' isn’t supported in call to 'fetchTree'" ).atPos(pos).debugThrow(); - input = fetchers::Input::fromAttrs(std::move(attrs)); + input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); } else { auto url = state.coerceToString(pos, *args[0], context, "while evaluating the first argument passed to the fetcher", @@ -156,20 +169,20 @@ static void fetchTree( if (!attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) { attrs.emplace("exportIgnore", Explicit{true}); } - input = fetchers::Input::fromAttrs(std::move(attrs)); + input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); } else { if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) state.error( "passing a string argument to 'fetchTree' requires the 'flakes' experimental feature" ).atPos(pos).debugThrow(); - input = fetchers::Input::fromURL(url); + input = fetchers::Input::fromURL(state.fetchSettings, url); } } - if (!evalSettings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) + if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) input = lookupInRegistries(state.store, input).first; - if (evalSettings.pureEval && !input.isLocked()) { + if (state.settings.pureEval && !input.isLocked()) { auto fetcher = "fetchTree"; if (params.isFetchGit) fetcher = "fetchGit"; @@ -235,7 +248,7 @@ static RegisterPrimOp primop_fetchTree({ The following source types and associated input attributes are supported. - `"file"` @@ -320,6 +333,8 @@ static RegisterPrimOp primop_fetchTree({ - `ref` (String, optional) + By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled. + A [Git reference](https://git-scm.com/book/en/v2/Git-Internals-Git-References), such as a branch or tag name. Default: `"HEAD"` @@ -333,8 +348,9 @@ static RegisterPrimOp primop_fetchTree({ - `shallow` (Bool, optional) Make a shallow clone when fetching the Git tree. + When this is enabled, the options `ref` and `allRefs` have no effect anymore. - Default: `false` + Default: `true` - `submodules` (Bool, optional) @@ -344,8 +360,11 @@ static RegisterPrimOp primop_fetchTree({ - `allRefs` (Bool, optional) - If set to `true`, always fetch the entire repository, even if the latest commit is still in the cache. - Otherwise, only the latest commit is fetched if it is not already cached. + By default, this has no effect. This becomes relevant only once `shallow` cloning is disabled. + + Whether to fetch all references (eg. branches and tags) of the repository. + With this argument being true, it's possible to load a `rev` from *any* `ref`. + (Without setting this option, only `rev`s from the specified `ref` are supported). Default: `false` @@ -372,7 +391,7 @@ static RegisterPrimOp primop_fetchTree({ - `"mercurial"` *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references). - The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-flakes) to be enabled. + The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled. > **Example** > @@ -420,7 +439,10 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v state.forceValue(*args[0], pos); - if (args[0]->type() == nAttrs) { + bool isArgAttrs = args[0]->type() == nAttrs; + bool nameAttrPassed = false; + + if (isArgAttrs) { for (auto & attr : *args[0]->attrs()) { std::string_view n(state.symbols[attr.name]); @@ -428,8 +450,10 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v url = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the url we should fetch"); else if (n == "sha256") expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the sha256 of the content we should fetch"), HashAlgorithm::SHA256); - else if (n == "name") + else if (n == "name") { + nameAttrPassed = true; name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the name of the content we should fetch"); + } else state.error("unsupported argument '%s' to '%s'", n, who) .atPos(pos).debugThrow(); @@ -442,14 +466,27 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v url = state.forceStringNoCtx(*args[0], pos, "while evaluating the url we should fetch"); if (who == "fetchTarball") - url = evalSettings.resolvePseudoUrl(*url); + url = state.settings.resolvePseudoUrl(*url); state.checkURI(*url); if (name == "") name = baseNameOf(*url); - if (evalSettings.pureEval && !expectedHash) + try { + checkName(name); + } catch (BadStorePathName & e) { + auto resolution = + nameAttrPassed ? HintFmt("Please change the value for the 'name' attribute passed to '%s', so that it can create a valid store path.", who) : + isArgAttrs ? HintFmt("Please add a valid 'name' attribute to the argument for '%s', so that it can create a valid store path.", who) : + HintFmt("Please pass an attribute set with 'url' and 'name' attributes to '%s', so that it can create a valid store path.", who); + + state.error( + std::string("invalid store path name when fetching URL '%s': %s. %s"), *url, Uncolored(e.message()), Uncolored(resolution.str())) + .atPos(pos).debugThrow(); + } + + if (state.settings.pureEval && !expectedHash) state.error("in pure evaluation mode, '%s' requires a 'sha256' argument", who).atPos(pos).debugThrow(); // early exit if pinned and already in the store @@ -457,7 +494,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto expectedPath = state.store->makeFixedOutputPath( name, FixedOutputInfo { - .method = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat, + .method = unpack ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat, .hash = *expectedHash, .references = {} }); @@ -472,7 +509,11 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v // https://github.com/NixOS/nix/issues/4313 auto storePath = unpack - ? fetchToStore(*state.store, fetchers::downloadTarball(*url).accessor, FetchMode::Copy, name) + ? fetchToStore( + *state.store, + fetchers::downloadTarball(state.store, state.fetchSettings, *url), + FetchMode::Copy, + name) : fetchers::downloadFile(state.store, *url, name).storePath; if (expectedHash) { @@ -500,9 +541,19 @@ static void prim_fetchurl(EvalState & state, const PosIdx pos, Value * * args, V static RegisterPrimOp primop_fetchurl({ .name = "__fetchurl", - .args = {"url"}, + .args = {"arg"}, .doc = R"( Download the specified URL and return the path of the downloaded file. + `arg` can be either a string denoting the URL, or an attribute set with the following attributes: + + - `url` + + The URL of the file to download. + + - `name` (default: the last path component of the URL) + + A name for the file in the store. This can be useful if the URL has any + characters that are invalid for the store. Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval). )", @@ -520,11 +571,11 @@ static RegisterPrimOp primop_fetchTarball({ .doc = R"( Download the specified URL, unpack it and return the path of the unpacked tree. The file must be a tape archive (`.tar`) compressed - with `gzip`, `bzip2` or `xz`. The top-level path component of the - files in the tarball is removed, so it is best if the tarball - contains a single directory at top level. The typical use of the - function is to obtain external Nix expression dependencies, such as - a particular version of Nixpkgs, e.g. + with `gzip`, `bzip2` or `xz`. If the tarball consists of a + single directory, then the top-level path component of the files + in the tarball is removed. The typical use of the function is to + obtain external Nix expression dependencies, such as a + particular version of Nixpkgs, e.g. ```nix with import (fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-14.12.tar.gz) {}; @@ -599,6 +650,8 @@ static RegisterPrimOp primop_fetchGit({ [Git reference]: https://git-scm.com/book/en/v2/Git-Internals-Git-References + This option has no effect once `shallow` cloning is enabled. + By default, the `ref` value is prefixed with `refs/heads/`. As of 2.3.0, Nix will not prefix `refs/heads/` if `ref` starts with `refs/`. @@ -616,23 +669,25 @@ static RegisterPrimOp primop_fetchGit({ - `shallow` (default: `false`) Make a shallow clone when fetching the Git tree. - + When this is enabled, the options `ref` and `allRefs` have no effect anymore. - `allRefs` - Whether to fetch all references of the repository. - With this argument being true, it's possible to load a `rev` from *any* `ref` + Whether to fetch all references (eg. branches and tags) of the repository. + With this argument being true, it's possible to load a `rev` from *any* `ref`. (by default only `rev`s from the specified `ref` are supported). + This option has no effect once `shallow` cloning is enabled. + - `verifyCommit` (default: `true` if `publicKey` or `publicKeys` are provided, otherwise `false`) Whether to check `rev` for a signature matching `publicKey` or `publicKeys`. If `verifyCommit` is enabled, then `fetchGit` cannot use a local repository with uncommitted changes. - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - `publicKey` The public key against which `rev` is verified if `verifyCommit` is enabled. - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - `keytype` (default: `"ssh-ed25519"`) @@ -644,7 +699,7 @@ static RegisterPrimOp primop_fetchGit({ - `"ssh-ed25519"` - `"ssh-ed25519-sk"` - `"ssh-rsa"` - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). - `publicKeys` @@ -658,7 +713,7 @@ static RegisterPrimOp primop_fetchGit({ } ``` - Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). + Requires the [`verified-fetches` experimental feature](@docroot@/development/experimental-features.md#xp-feature-verified-fetches). Here are some examples of how to use `fetchGit`. diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 9bee8ca38..264046711 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -1,10 +1,10 @@ #include "primops.hh" #include "eval-inline.hh" -#include "../../toml11/toml.hpp" - #include +#include + namespace nix { static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, Value & val) @@ -66,7 +66,7 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V attrs.alloc("_type").mkString("timestamp"); std::ostringstream s; s << t; - attrs.alloc("value").mkString(s.str()); + attrs.alloc("value").mkString(toView(s)); v.mkAttrs(attrs); } else { throw std::runtime_error("Dates and times are not supported"); diff --git a/src/libexpr/primops/meson.build b/src/libexpr/primops/meson.build new file mode 100644 index 000000000..f910fe237 --- /dev/null +++ b/src/libexpr/primops/meson.build @@ -0,0 +1,12 @@ +generated_headers += gen_header.process( + 'derivation.nix', + preserve_path_from: meson.project_source_root(), +) + +sources += files( + 'context.cc', + 'fetchClosure.cc', + 'fetchMercurial.cc', + 'fetchTree.cc', + 'fromTOML.cc', +) diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc index 5d55b45da..a40c98643 100644 --- a/src/libexpr/print-ambiguous.cc +++ b/src/libexpr/print-ambiguous.cc @@ -94,7 +94,7 @@ void printAmbiguous( break; default: printError("Nix evaluator internal error: printAmbiguous: invalid value type"); - abort(); + unreachable(); } } diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index 7799a0bbe..d62aaf25f 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -1,5 +1,6 @@ #include #include +#include #include "print.hh" #include "ansicolor.hh" @@ -162,8 +163,8 @@ private: EvalState & state; PrintOptions options; std::optional seen; - size_t attrsPrinted = 0; - size_t listItemsPrinted = 0; + size_t totalAttrsPrinted = 0; + size_t totalListItemsPrinted = 0; std::string indent; void increaseIndent() @@ -271,22 +272,36 @@ private: void printDerivation(Value & v) { - NixStringContext context; - std::string storePath; - if (auto i = v.attrs()->get(state.sDrvPath)) - storePath = state.store->printStorePath(state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation")); + std::optional storePath; + if (auto i = v.attrs()->get(state.sDrvPath)) { + NixStringContext context; + storePath = state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation"); + } + + /* This unfortunately breaks printing nested values because of + how the pretty printer is used (when pretting printing and warning + to same terminal / std stream). */ +#if 0 + if (storePath && !storePath->isDerivation()) + warn( + "drvPath attribute '%s' is not a valid store path to a derivation, this value not work properly", + state.store->printStorePath(*storePath)); +#endif if (options.ansiColors) output << ANSI_GREEN; output << "«derivation"; - if (!storePath.empty()) { - output << " " << storePath; + if (storePath) { + output << " " << state.store->printStorePath(*storePath); } output << "»"; if (options.ansiColors) output << ANSI_NORMAL; } + /** + * @note This may force items. + */ bool shouldPrettyPrintAttrs(AttrVec & v) { if (!options.shouldPrettyPrint() || v.empty()) { @@ -303,6 +318,9 @@ private: return true; } + // It is ok to force the item(s) here, because they will be printed anyway. + state.forceValue(*item, item->determinePos(noPos)); + // Pretty-print single-item attrsets only if they contain nested // structures. auto itemType = item->type(); @@ -333,11 +351,13 @@ private: auto prettyPrint = shouldPrettyPrintAttrs(sorted); + size_t currentAttrsPrinted = 0; + for (auto & i : sorted) { printSpace(prettyPrint); - if (attrsPrinted >= options.maxAttrs) { - printElided(sorted.size() - attrsPrinted, "attribute", "attributes"); + if (totalAttrsPrinted >= options.maxAttrs) { + printElided(sorted.size() - currentAttrsPrinted, "attribute", "attributes"); break; } @@ -345,7 +365,8 @@ private: output << " = "; print(*i.second, depth + 1); output << ";"; - attrsPrinted++; + totalAttrsPrinted++; + currentAttrsPrinted++; } decreaseIndent(); @@ -356,6 +377,9 @@ private: } } + /** + * @note This may force items. + */ bool shouldPrettyPrintList(std::span list) { if (!options.shouldPrettyPrint() || list.empty()) { @@ -372,6 +396,9 @@ private: return true; } + // It is ok to force the item(s) here, because they will be printed anyway. + state.forceValue(*item, item->determinePos(noPos)); + // Pretty-print single-item lists only if they contain nested // structures. auto itemType = item->type(); @@ -390,11 +417,14 @@ private: output << "["; auto listItems = v.listItems(); auto prettyPrint = shouldPrettyPrintList(listItems); + + size_t currentListItemsPrinted = 0; + for (auto elem : listItems) { printSpace(prettyPrint); - if (listItemsPrinted >= options.maxListItems) { - printElided(listItems.size() - listItemsPrinted, "item", "items"); + if (totalListItemsPrinted >= options.maxListItems) { + printElided(listItems.size() - currentListItemsPrinted, "item", "items"); break; } @@ -403,7 +433,8 @@ private: } else { printNullptr(); } - listItemsPrinted++; + totalListItemsPrinted++; + currentListItemsPrinted++; } decreaseIndent(); @@ -429,7 +460,7 @@ private: std::ostringstream s; s << state.positions[v.payload.lambda.fun->pos]; - output << " @ " << filterANSIEscapes(s.str()); + output << " @ " << filterANSIEscapes(toView(s)); } } else if (v.isPrimOp()) { if (v.primOp()) @@ -444,7 +475,7 @@ private: else output << "primop"; } else { - abort(); + unreachable(); } output << "»"; @@ -473,7 +504,7 @@ private: if (options.ansiColors) output << ANSI_NORMAL; } else { - abort(); + unreachable(); } } @@ -576,8 +607,8 @@ public: void print(Value & v) { - attrsPrinted = 0; - listItemsPrinted = 0; + totalAttrsPrinted = 0; + totalListItemsPrinted = 0; indent.clear(); if (options.trackRepeated) { diff --git a/src/libexpr/symbol-table.hh b/src/libexpr/symbol-table.hh index 967a186dd..be12f6248 100644 --- a/src/libexpr/symbol-table.hh +++ b/src/libexpr/symbol-table.hh @@ -7,6 +7,7 @@ #include "types.hh" #include "chunked-vector.hh" +#include "error.hh" namespace nix { @@ -30,9 +31,9 @@ public: return *s == s2; } - operator const std::string & () const + const char * c_str() const { - return *s; + return s->c_str(); } operator const std::string_view () const @@ -41,6 +42,11 @@ public: } friend std::ostream & operator <<(std::ostream & os, const SymbolStr & symbol); + + bool empty() const + { + return s->empty(); + } }; /** @@ -62,9 +68,10 @@ public: explicit operator bool() const { return id > 0; } - bool operator<(const Symbol other) const { return id < other.id; } + auto operator<=>(const Symbol other) const { return id <=> other.id; } bool operator==(const Symbol other) const { return id == other.id; } - bool operator!=(const Symbol other) const { return id != other.id; } + + friend class std::hash; }; /** @@ -109,7 +116,7 @@ public: SymbolStr operator[](Symbol s) const { if (s.id == 0 || s.id > store.size()) - abort(); + unreachable(); return SymbolStr(store[s.id - 1]); } @@ -128,3 +135,12 @@ public: }; } + +template<> +struct std::hash +{ + std::size_t operator()(const nix::Symbol & s) const noexcept + { + return std::hash{}(s.id); + } +}; diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 936ecf078..8044fe347 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -22,7 +22,7 @@ json printValueAsJSON(EvalState & state, bool strict, switch (v.type()) { case nInt: - out = v.integer(); + out = v.integer().value; break; case nBool: @@ -58,7 +58,7 @@ json printValueAsJSON(EvalState & state, bool strict, out = json::object(); for (auto & a : v.attrs()->lexicographicOrder(state.symbols)) { try { - out[state.symbols[a->name]] = printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore); + out.emplace(state.symbols[a->name], printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore)); } catch (Error & e) { e.addTrace(state.positions[a->pos], HintFmt("while evaluating attribute '%1%'", state.symbols[a->name])); diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 1de8cdf84..9734ebec4 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -9,7 +9,7 @@ namespace nix { -static XMLAttrs singletonAttrs(const std::string & name, const std::string & value) +static XMLAttrs singletonAttrs(const std::string & name, std::string_view value) { XMLAttrs attrs; attrs[name] = value; diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 61cf2d310..0ffe74dab 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -2,17 +2,15 @@ ///@file #include -#include #include +#include "eval-gc.hh" #include "symbol-table.hh" #include "value/context.hh" #include "source-path.hh" #include "print-options.hh" +#include "checked-arithmetic.hh" -#if HAVE_BOEHMGC -#include -#endif #include namespace nix { @@ -74,8 +72,8 @@ class EvalState; class XMLWriter; class Printer; -typedef int64_t NixInt; -typedef double NixFloat; +using NixInt = checked::Checked; +using NixFloat = double; /** * External values must descend from ExternalValueBase, so that @@ -112,7 +110,7 @@ class ExternalValueBase * Compare to another value of the same type. Defaults to uncomparable, * i.e. always false. */ - virtual bool operator ==(const ExternalValueBase & b) const; + virtual bool operator ==(const ExternalValueBase & b) const noexcept; /** * Print the value as JSON. Defaults to unconvertable, i.e. throws an error @@ -286,7 +284,7 @@ public: if (invalidIsThunk) return nThunk; else - abort(); + unreachable(); } inline void finishValue(InternalType newType, Payload newPayload) @@ -305,6 +303,11 @@ public: return internalType != tUninitialized; } + inline void mkInt(NixInt::Inner n) + { + mkInt(NixInt{n}); + } + inline void mkInt(NixInt n) { finishValue(tInt, { .integer = n }); @@ -326,9 +329,9 @@ public: void mkStringMove(const char * s, const NixStringContext & context); - inline void mkString(const Symbol & s) + inline void mkString(const SymbolStr & s) { - mkString(((const std::string &) s).c_str()); + mkString(s.c_str()); } void mkPath(const SourcePath & path); @@ -449,7 +452,7 @@ public: return std::string_view(payload.string.c_str); } - const char * const c_str() const + const char * c_str() const { assert(internalType == tString); return payload.string.c_str; @@ -493,15 +496,9 @@ void Value::mkBlackhole() } -#if HAVE_BOEHMGC typedef std::vector> ValueVector; -typedef std::map, traceable_allocator>> ValueMap; +typedef std::unordered_map, std::equal_to, traceable_allocator>> ValueMap; typedef std::map, traceable_allocator>> ValueVectorMap; -#else -typedef std::vector ValueVector; -typedef std::map ValueMap; -typedef std::map ValueVectorMap; -#endif /** diff --git a/src/libfetchers/.version b/src/libfetchers/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libfetchers/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index b788c5948..25d04cdc9 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -33,7 +33,7 @@ nlohmann::json attrsToJSON(const Attrs & attrs) json[attr.first] = *v; } else if (auto v = std::get_if>(&attr.second)) { json[attr.first] = v->t; - } else abort(); + } else unreachable(); } return json; } @@ -99,7 +99,7 @@ std::map attrsToQuery(const Attrs & attrs) query.insert_or_assign(attr.first, *v); } else if (auto v = std::get_if>(&attr.second)) { query.insert_or_assign(attr.first, v->t ? "1" : "0"); - } else abort(); + } else unreachable(); } return query; } diff --git a/src/libfetchers/build-utils-meson b/src/libfetchers/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libfetchers/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 7019b0325..b0b6cb887 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -36,7 +36,7 @@ struct CacheImpl : Cache { auto state(_state.lock()); - auto dbPath = getCacheDir() + "/nix/fetcher-cache-v2.sqlite"; + auto dbPath = getCacheDir() + "/fetcher-cache-v2.sqlite"; createDirs(dirOf(dbPath)); state->db = SQLite(dbPath); diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc index e7d5244dc..c7ed4c7af 100644 --- a/src/libfetchers/fetch-settings.cc +++ b/src/libfetchers/fetch-settings.cc @@ -1,13 +1,9 @@ #include "fetch-settings.hh" -namespace nix { +namespace nix::fetchers { -FetchSettings::FetchSettings() +Settings::Settings() { } -FetchSettings fetchSettings; - -static GlobalConfig::Register rFetchSettings(&fetchSettings); - } diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index 50cd4d161..f7cb34a02 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -9,11 +9,11 @@ #include -namespace nix { +namespace nix::fetchers { -struct FetchSettings : public Config +struct Settings : public Config { - FetchSettings(); + Settings(); Setting accessTokens{this, {}, "access-tokens", R"( @@ -70,30 +70,6 @@ struct FetchSettings : public Config Setting warnDirty{this, true, "warn-dirty", "Whether to warn about dirty Git/Mercurial trees."}; - Setting flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry", - R"( - Path or URI of the global flake registry. - - When empty, disables the global flake registry. - )", - {}, true, Xp::Flakes}; - - Setting useRegistries{this, true, "use-registries", - "Whether to use flake registries to resolve flake references.", - {}, true, Xp::Flakes}; - - Setting acceptFlakeConfig{this, false, "accept-flake-config", - "Whether to accept nix configuration from a flake without prompting.", - {}, true, Xp::Flakes}; - - Setting commitLockFileSummary{ - this, "", "commit-lock-file-summary", - R"( - The commit summary to use when committing changed flake lock files. If - empty, the summary is generated based on the action performed. - )", - {"commit-lockfile-summary"}, true, Xp::Flakes}; - Setting trustTarballsFromGitForges{ this, true, "trust-tarballs-from-git-forges", R"( @@ -109,9 +85,13 @@ struct FetchSettings : public Config e.g. `github:NixOS/patchelf/7c2f768bf9601268a4e71c2ebe91e2011918a70f?narHash=sha256-PPXqKY2hJng4DBVE0I4xshv/vGLUskL7jl53roB8UdU%3D`. )"}; + Setting flakeRegistry{this, "https://channels.nixos.org/flake-registry.json", "flake-registry", + R"( + Path or URI of the global flake registry. + + When empty, disables the global flake registry. + )", + {}, true, Xp::Flakes}; }; -// FIXME: don't use a global variable. -extern FetchSettings fetchSettings; - } diff --git a/src/libfetchers/fetch-to-store.hh b/src/libfetchers/fetch-to-store.hh index 81af1e240..c762629f3 100644 --- a/src/libfetchers/fetch-to-store.hh +++ b/src/libfetchers/fetch-to-store.hh @@ -18,7 +18,7 @@ StorePath fetchToStore( const SourcePath & path, FetchMode mode, std::string_view name = "source", - ContentAddressMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, PathFilter * filter = nullptr, RepairFlag repair = NoRepair); diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 73923907c..b07e8cb6e 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -35,9 +35,11 @@ nlohmann::json dumpRegisterInputSchemeInfo() { return res; } -Input Input::fromURL(const std::string & url, bool requireTree) +Input Input::fromURL( + const Settings & settings, + const std::string & url, bool requireTree) { - return fromURL(parseURL(url), requireTree); + return fromURL(settings, parseURL(url), requireTree); } static void fixupInput(Input & input) @@ -49,10 +51,12 @@ static void fixupInput(Input & input) input.getLastModified(); } -Input Input::fromURL(const ParsedURL & url, bool requireTree) +Input Input::fromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) { for (auto & [_, inputScheme] : *inputSchemes) { - auto res = inputScheme->inputFromURL(url, requireTree); + auto res = inputScheme->inputFromURL(settings, url, requireTree); if (res) { experimentalFeatureSettings.require(inputScheme->experimentalFeature()); res->scheme = inputScheme; @@ -64,7 +68,7 @@ Input Input::fromURL(const ParsedURL & url, bool requireTree) throw Error("input '%s' is unsupported", url.url); } -Input Input::fromAttrs(Attrs && attrs) +Input Input::fromAttrs(const Settings & settings, Attrs && attrs) { auto schemeName = ({ auto schemeNameOpt = maybeGetStrAttr(attrs, "type"); @@ -78,7 +82,7 @@ Input Input::fromAttrs(Attrs && attrs) // but not all of them. Doing this is to support those other // operations which are supposed to be robust on // unknown/uninterpretable inputs. - Input input; + Input input { settings }; input.attrs = attrs; fixupInput(input); return input; @@ -99,7 +103,7 @@ Input Input::fromAttrs(Attrs && attrs) if (name != "type" && allowedAttrs.count(name) == 0) throw Error("input attribute '%s' not supported by scheme '%s'", name, schemeName); - auto res = inputScheme->inputFromAttrs(attrs); + auto res = inputScheme->inputFromAttrs(settings, attrs); if (!res) return raw(); res->scheme = inputScheme; fixupInput(*res); @@ -146,7 +150,7 @@ Attrs Input::toAttrs() const return attrs; } -bool Input::operator ==(const Input & other) const +bool Input::operator ==(const Input & other) const noexcept { return attrs == other.attrs; } @@ -166,24 +170,6 @@ std::pair Input::fetchToStore(ref store) const if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); - /* The tree may already be in the Nix store, or it could be - substituted (which is often faster than fetching from the - original source). So check that. */ - if (getNarHash()) { - try { - auto storePath = computeStorePath(*store); - - store->ensurePath(storePath); - - debug("using substituted/cached input '%s' in '%s'", - to_string(), store->printStorePath(storePath)); - - return {std::move(storePath), *this}; - } catch (Error & e) { - debug("substitution of input '%s' failed: %s", to_string(), e.what()); - } - } - auto [storePath, input] = [&]() -> std::pair { try { auto [accessor, final] = getAccessorUnchecked(store); @@ -260,6 +246,7 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto auto [accessor, final] = scheme->getAccessor(store, *this); + assert(!accessor->fingerprint); accessor->fingerprint = scheme->getFingerprint(store, final); return {accessor, std::move(final)}; @@ -305,7 +292,7 @@ StorePath Input::computeStorePath(Store & store) const if (!narHash) throw Error("cannot compute store path for unlocked input '%s'", to_string()); return store.makeFixedOutputPath(getName(), FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = *narHash, .references = {}, }); @@ -418,7 +405,7 @@ namespace nlohmann { using namespace nix; fetchers::PublicKey adl_serializer::from_json(const json & json) { - fetchers::PublicKey res = { }; + fetchers::PublicKey res = { }; if (auto type = optionalValueAt(json, "type")) res.type = getString(*type); diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 551be9a1f..a5f9bdcc6 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -11,12 +11,16 @@ #include #include +#include "ref.hh" + namespace nix { class Store; class StorePath; struct SourceAccessor; } namespace nix::fetchers { struct InputScheme; +struct Settings; + /** * The `Input` object is generated by a specific fetcher, based on * user-supplied information, and contains @@ -28,6 +32,12 @@ struct Input { friend struct InputScheme; + const Settings * settings; + + Input(const Settings & settings) + : settings{&settings} + { } + std::shared_ptr scheme; // note: can be null Attrs attrs; @@ -42,16 +52,22 @@ public: * * The URL indicate which sort of fetcher, and provides information to that fetcher. */ - static Input fromURL(const std::string & url, bool requireTree = true); + static Input fromURL( + const Settings & settings, + const std::string & url, bool requireTree = true); - static Input fromURL(const ParsedURL & url, bool requireTree = true); + static Input fromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree = true); /** * Create an `Input` from a an `Attrs`. * * The URL indicate which sort of fetcher, and provides information to that fetcher. */ - static Input fromAttrs(Attrs && attrs); + static Input fromAttrs( + const Settings & settings, + Attrs && attrs); ParsedURL toURL() const; @@ -73,7 +89,7 @@ public: */ bool isLocked() const; - bool operator ==(const Input & other) const; + bool operator ==(const Input & other) const noexcept; bool contains(const Input & other) const; @@ -146,9 +162,13 @@ struct InputScheme virtual ~InputScheme() { } - virtual std::optional inputFromURL(const ParsedURL & url, bool requireTree) const = 0; + virtual std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const = 0; - virtual std::optional inputFromAttrs(const Attrs & attrs) const = 0; + virtual std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const = 0; /** * What is the name of the scheme? diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index dfd9e536d..d4557b6d4 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -2,6 +2,12 @@ namespace nix { +std::optional FilteringSourceAccessor::getPhysicalPath(const CanonPath & path) +{ + checkAccess(path); + return next->getPhysicalPath(prefix / path); +} + std::string FilteringSourceAccessor::readFile(const CanonPath & path) { checkAccess(path); diff --git a/src/libfetchers/filtering-source-accessor.hh b/src/libfetchers/filtering-source-accessor.hh index 9ec7bc21f..1f8d84e53 100644 --- a/src/libfetchers/filtering-source-accessor.hh +++ b/src/libfetchers/filtering-source-accessor.hh @@ -30,6 +30,8 @@ struct FilteringSourceAccessor : SourceAccessor displayPrefix.clear(); } + std::optional getPhysicalPath(const CanonPath & path) override; + std::string readFile(const CanonPath & path) override; bool pathExists(const CanonPath & path) override; diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index 2ea1e15ed..95ee33089 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -13,13 +13,17 @@ #include #include #include +#include #include +#include #include #include #include #include #include #include +#include +#include #include #include @@ -76,6 +80,9 @@ typedef std::unique_ptr> StatusLi typedef std::unique_ptr> Remote; typedef std::unique_ptr> GitConfig; typedef std::unique_ptr> ConfigIterator; +typedef std::unique_ptr> ObjectDb; +typedef std::unique_ptr> PackBuilder; +typedef std::unique_ptr> Indexer; // A helper to ensure that we don't leak objects returned by libgit2. template @@ -115,10 +122,10 @@ git_oid hashToOID(const Hash & hash) return oid; } -Object lookupObject(git_repository * repo, const git_oid & oid) +Object lookupObject(git_repository * repo, const git_oid & oid, git_object_t type = GIT_OBJECT_ANY) { Object obj; - if (git_object_lookup(Setter(obj), repo, &oid, GIT_OBJECT_ANY)) { + if (git_object_lookup(Setter(obj), repo, &oid, type)) { auto err = git_error_last(); throw Error("getting Git object '%s': %s", oid, err->message); } @@ -126,34 +133,134 @@ Object lookupObject(git_repository * repo, const git_oid & oid) } template -T peelObject(git_repository * repo, git_object * obj, git_object_t type) +T peelObject(git_object * obj, git_object_t type) { T obj2; if (git_object_peel((git_object * *) (typename T::pointer *) Setter(obj2), obj, type)) { auto err = git_error_last(); - throw Error("peeling Git object '%s': %s", git_object_id(obj), err->message); + throw Error("peeling Git object '%s': %s", *git_object_id(obj), err->message); } return obj2; } +template +T dupObject(typename T::pointer obj) +{ + T obj2; + if (git_object_dup((git_object * *) (typename T::pointer *) Setter(obj2), (git_object *) obj)) + throw Error("duplicating object '%s': %s", *git_object_id((git_object *) obj), git_error_last()->message); + return obj2; +} + +/** + * Peel the specified object (i.e. follow tag and commit objects) to + * either a blob or a tree. + */ +static Object peelToTreeOrBlob(git_object * obj) +{ + /* git_object_peel() doesn't handle blob objects, so handle those + specially. */ + if (git_object_type(obj) == GIT_OBJECT_BLOB) + return dupObject(obj); + else + return peelObject(obj, GIT_OBJECT_TREE); +} + +struct PackBuilderContext { + std::exception_ptr exception; + + void handleException(const char * activity, int errCode) + { + switch (errCode) { + case GIT_OK: + break; + case GIT_EUSER: + if (!exception) + panic("PackBuilderContext::handleException: user error, but exception was not set"); + + std::rethrow_exception(exception); + default: + throw Error("%s: %i, %s", Uncolored(activity), errCode, git_error_last()->message); + } + } +}; + +extern "C" { + +/** + * A `git_packbuilder_progress` implementation that aborts the pack building if needed. + */ +static int packBuilderProgressCheckInterrupt(int stage, uint32_t current, uint32_t total, void *payload) +{ + PackBuilderContext & args = * (PackBuilderContext *) payload; + try { + checkInterrupt(); + return GIT_OK; + } catch (const std::exception & e) { + args.exception = std::current_exception(); + return GIT_EUSER; + } +}; +static git_packbuilder_progress PACKBUILDER_PROGRESS_CHECK_INTERRUPT = &packBuilderProgressCheckInterrupt; + +} // extern "C" + +static void initRepoAtomically(std::filesystem::path &path, bool bare) { + if (pathExists(path.string())) return; + + Path tmpDir = createTempDir(os_string_to_string(PathViewNG { std::filesystem::path(path).parent_path() })); + AutoDelete delTmpDir(tmpDir, true); + Repository tmpRepo; + + if (git_repository_init(Setter(tmpRepo), tmpDir.c_str(), bare)) + throw Error("creating Git repository %s: %s", path, git_error_last()->message); + try { + std::filesystem::rename(tmpDir, path); + } catch (std::filesystem::filesystem_error & e) { + if (e.code() == std::errc::file_exists) // Someone might race us to create the repository. + return; + else + throw SysError("moving temporary git repository from %s to %s", tmpDir, path); + } + // we successfully moved the repository, so the temporary directory no longer exists. + delTmpDir.cancel(); +} + struct GitRepoImpl : GitRepo, std::enable_shared_from_this { /** Location of the repository on disk. */ std::filesystem::path path; + /** + * libgit2 repository. Note that new objects are not written to disk, + * because we are using a mempack backend. For writing to disk, see + * `flush()`, which is also called by `GitFileSystemObjectSink::sync()`. + */ Repository repo; + /** + * In-memory object store for efficient batched writing to packfiles. + * Owned by `repo`. + */ + git_odb_backend * mempack_backend; GitRepoImpl(std::filesystem::path _path, bool create, bool bare) : path(std::move(_path)) { initLibGit2(); - if (pathExists(path.string())) { - if (git_repository_open(Setter(repo), path.string().c_str())) - throw Error("opening Git repository '%s': %s", path, git_error_last()->message); - } else { - if (git_repository_init(Setter(repo), path.string().c_str(), bare)) - throw Error("creating Git repository '%s': %s", path, git_error_last()->message); - } + initRepoAtomically(path, bare); + if (git_repository_open(Setter(repo), path.string().c_str())) + throw Error("opening Git repository %s: %s", path, git_error_last()->message); + + ObjectDb odb; + if (git_repository_odb(Setter(odb), repo.get())) + throw Error("getting Git object database: %s", git_error_last()->message); + + // mempack_backend will be owned by the repository, so we are not expected to free it ourselves. + if (git_mempack_new(&mempack_backend)) + throw Error("creating mempack backend: %s", git_error_last()->message); + + if (git_odb_add_backend(odb.get(), mempack_backend, 999)) + throw Error("adding mempack backend to Git object database: %s", git_error_last()->message); } operator git_repository * () @@ -161,12 +268,68 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return repo.get(); } + void flush() override { + checkInterrupt(); + + git_buf buf = GIT_BUF_INIT; + Finally _disposeBuf { [&] { git_buf_dispose(&buf); } }; + PackBuilder packBuilder; + PackBuilderContext packBuilderContext; + git_packbuilder_new(Setter(packBuilder), *this); + git_packbuilder_set_callbacks(packBuilder.get(), PACKBUILDER_PROGRESS_CHECK_INTERRUPT, &packBuilderContext); + git_packbuilder_set_threads(packBuilder.get(), 0 /* autodetect */); + + packBuilderContext.handleException( + "preparing packfile", + git_mempack_write_thin_pack(mempack_backend, packBuilder.get()) + ); + checkInterrupt(); + packBuilderContext.handleException( + "writing packfile", + git_packbuilder_write_buf(&buf, packBuilder.get()) + ); + checkInterrupt(); + + std::string repo_path = std::string(git_repository_path(repo.get())); + while (!repo_path.empty() && repo_path.back() == '/') + repo_path.pop_back(); + std::string pack_dir_path = repo_path + "/objects/pack"; + + // TODO (performance): could the indexing be done in a separate thread? + // we'd need a more streaming variation of + // git_packbuilder_write_buf, or incur the cost of + // copying parts of the buffer to a separate thread. + // (synchronously on the git_packbuilder_write_buf thread) + Indexer indexer; + git_indexer_progress stats; + if (git_indexer_new(Setter(indexer), pack_dir_path.c_str(), 0, nullptr, nullptr)) + throw Error("creating git packfile indexer: %s", git_error_last()->message); + + // TODO: provide index callback for checkInterrupt() termination + // though this is about an order of magnitude faster than the packbuilder + // expect up to 1 sec latency due to uninterruptible git_indexer_append. + constexpr size_t chunkSize = 128 * 1024; + for (size_t offset = 0; offset < buf.size; offset += chunkSize) { + if (git_indexer_append(indexer.get(), buf.ptr + offset, std::min(chunkSize, buf.size - offset), &stats)) + throw Error("appending to git packfile index: %s", git_error_last()->message); + checkInterrupt(); + } + + if (git_indexer_commit(indexer.get(), &stats)) + throw Error("committing git packfile index: %s", git_error_last()->message); + + if (git_mempack_reset(mempack_backend)) + throw Error("resetting git mempack backend: %s", git_error_last()->message); + + checkInterrupt(); + } + uint64_t getRevCount(const Hash & rev) override { std::unordered_set done; std::queue todo; - todo.push(peelObject(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT)); + todo.push(peelObject(lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT)); while (auto commit = pop(todo)) { if (!done.insert(*git_commit_id(commit->get())).second) continue; @@ -184,7 +347,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this uint64_t getLastModified(const Hash & rev) override { - auto commit = peelObject(*this, lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT); + auto commit = peelObject(lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT); return git_commit_time(commit.get()); } @@ -437,7 +600,13 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this std::string re = R"(Good "git" signature for \* with .* key SHA256:[)"; for (const fetchers::PublicKey & k : publicKeys){ // Calculate sha256 fingerprint from public key and escape the regex symbol '+' to match the key literally - auto fingerprint = trim(hashString(HashAlgorithm::SHA256, base64Decode(k.key)).to_string(nix::HashFormat::Base64, false), "="); + std::string keyDecoded; + try { + keyDecoded = base64Decode(k.key); + } catch (Error & e) { + e.addTrace({}, "while decoding public key '%s' used for git signature", k.key); + } + auto fingerprint = trim(hashString(HashAlgorithm::SHA256, keyDecoded).to_string(nix::HashFormat::Base64, false), "="); auto escaped_fingerprint = std::regex_replace(fingerprint, std::regex("\\+"), "\\+" ); re += "(" + escaped_fingerprint + ")"; } @@ -463,6 +632,23 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return narHash; } + + Hash dereferenceSingletonDirectory(const Hash & oid_) override + { + auto oid = hashToOID(oid_); + + auto _tree = lookupObject(*this, oid, GIT_OBJECT_TREE); + auto tree = (const git_tree *) &*_tree; + + if (git_tree_entrycount(tree) == 1) { + auto entry = git_tree_entry_byindex(tree, 0); + auto mode = git_tree_entry_filemode(entry); + if (mode == GIT_FILEMODE_TREE) + oid = *git_tree_entry_id(entry); + } + + return toHash(oid); + } }; ref GitRepo::openRepo(const std::filesystem::path & path, bool create, bool bare) @@ -476,11 +662,11 @@ ref GitRepo::openRepo(const std::filesystem::path & path, bool create, struct GitSourceAccessor : SourceAccessor { ref repo; - Tree root; + Object root; GitSourceAccessor(ref repo_, const Hash & rev) : repo(repo_) - , root(peelObject(*repo, lookupObject(*repo, hashToOID(rev)).get(), GIT_OBJECT_TREE)) + , root(peelToTreeOrBlob(lookupObject(*repo, hashToOID(rev)).get())) { } @@ -506,7 +692,7 @@ struct GitSourceAccessor : SourceAccessor std::optional maybeLstat(const CanonPath & path) override { if (path.isRoot()) - return Stat { .type = tDirectory }; + return Stat { .type = git_object_type(root.get()) == GIT_OBJECT_TREE ? tDirectory : tRegular }; auto entry = lookup(path); if (!entry) @@ -561,12 +747,16 @@ struct GitSourceAccessor : SourceAccessor return readBlob(path, true); } - Hash getSubmoduleRev(const CanonPath & path) + /** + * If `path` exists and is a submodule, return its + * revision. Otherwise return nothing. + */ + std::optional getSubmoduleRev(const CanonPath & path) { - auto entry = need(path); + auto entry = lookup(path); - if (git_tree_entry_type(entry) != GIT_OBJECT_COMMIT) - throw Error("'%s' is not a submodule", showPath(path)); + if (!entry || git_tree_entry_type(entry) != GIT_OBJECT_COMMIT) + return std::nullopt; return toHash(*git_tree_entry_id(entry)); } @@ -616,10 +806,10 @@ struct GitSourceAccessor : SourceAccessor std::optional lookupTree(const CanonPath & path) { if (path.isRoot()) { - Tree tree; - if (git_tree_dup(Setter(tree), root.get())) - throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message); - return tree; + if (git_object_type(root.get()) == GIT_OBJECT_TREE) + return dupObject((git_tree *) &*root); + else + return std::nullopt; } auto entry = lookup(path); @@ -646,10 +836,10 @@ struct GitSourceAccessor : SourceAccessor std::variant getTree(const CanonPath & path) { if (path.isRoot()) { - Tree tree; - if (git_tree_dup(Setter(tree), root.get())) - throw Error("duplicating directory '%s': %s", showPath(path), git_error_last()->message); - return tree; + if (git_object_type(root.get()) == GIT_OBJECT_TREE) + return dupObject((git_tree *) &*root); + else + throw Error("Git root object '%s' is not a directory", *git_object_id(root.get())); } auto entry = need(path); @@ -669,6 +859,9 @@ struct GitSourceAccessor : SourceAccessor Blob getBlob(const CanonPath & path, bool expectSymlink) { + if (!expectSymlink && git_object_type(root.get()) == GIT_OBJECT_BLOB) + return dupObject((git_blob *) &*root); + auto notExpected = [&]() { throw Error( @@ -782,12 +975,26 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink std::vector pendingDirs; - size_t componentsToStrip = 1; - void pushBuilder(std::string name) { + const git_tree_entry * entry; + Tree prevTree = nullptr; + + if (!pendingDirs.empty() && + (entry = git_treebuilder_get(pendingDirs.back().builder.get(), name.c_str()))) + { + /* Clone a tree that we've already finished. This happens + if a tarball has directory entries that are not + contiguous. */ + if (git_tree_entry_type(entry) != GIT_OBJECT_TREE) + throw Error("parent of '%s' is not a directory", name); + + if (git_tree_entry_to_object((git_object * *) (git_tree * *) Setter(prevTree), *repo, entry)) + throw Error("looking up parent of '%s': %s", name, git_error_last()->message); + } + git_treebuilder * b; - if (git_treebuilder_new(&b, *repo, nullptr)) + if (git_treebuilder_new(&b, *repo, prevTree.get())) throw Error("creating a tree builder: %s", git_error_last()->message); pendingDirs.push_back({ .name = std::move(name), .builder = TreeBuilder(b) }); }; @@ -839,9 +1046,6 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink { std::span pathComponents2{pathComponents}; - if (pathComponents2.size() <= componentsToStrip) return false; - pathComponents2 = pathComponents2.subspan(componentsToStrip); - updateBuilders( isDir ? pathComponents2 @@ -851,10 +1055,10 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink } void createRegularFile( - const Path & path, + const CanonPath & path, std::function func) override { - auto pathComponents = tokenizeString>(path, "/"); + auto pathComponents = tokenizeString>(path.rel(), "/"); if (!prepareDirs(pathComponents, false)) return; git_writestream * stream = nullptr; @@ -862,11 +1066,11 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink throw Error("creating a blob stream object: %s", git_error_last()->message); struct CRF : CreateRegularFileSink { - const Path & path; + const CanonPath & path; GitFileSystemObjectSinkImpl & back; git_writestream * stream; bool executable = false; - CRF(const Path & path, GitFileSystemObjectSinkImpl & back, git_writestream * stream) + CRF(const CanonPath & path, GitFileSystemObjectSinkImpl & back, git_writestream * stream) : path(path), back(back), stream(stream) {} void operator () (std::string_view data) override @@ -891,15 +1095,15 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink : GIT_FILEMODE_BLOB); } - void createDirectory(const Path & path) override + void createDirectory(const CanonPath & path) override { - auto pathComponents = tokenizeString>(path, "/"); + auto pathComponents = tokenizeString>(path.rel(), "/"); (void) prepareDirs(pathComponents, true); } - void createSymlink(const Path & path, const std::string & target) override + void createSymlink(const CanonPath & path, const std::string & target) override { - auto pathComponents = tokenizeString>(path, "/"); + auto pathComponents = tokenizeString>(path.rel(), "/"); if (!prepareDirs(pathComponents, false)) return; git_oid oid; @@ -909,11 +1113,69 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK); } - Hash sync() override { + void createHardlink(const CanonPath & path, const CanonPath & target) override + { + std::vector pathComponents; + for (auto & c : path) + pathComponents.emplace_back(c); + + if (!prepareDirs(pathComponents, false)) return; + + // We can't just look up the path from the start of the root, since + // some parent directories may not have finished yet, so we compute + // a relative path that helps us find the right git_tree_builder or object. + auto relTarget = CanonPath(path).parent()->makeRelative(target); + + auto dir = pendingDirs.rbegin(); + + // For each ../ component at the start, go up one directory. + // CanonPath::makeRelative() always puts all .. elements at the start, + // so they're all handled by this loop: + std::string_view relTargetLeft(relTarget); + while (hasPrefix(relTargetLeft, "../")) { + if (dir == pendingDirs.rend()) + throw Error("invalid hard link target '%s' for path '%s'", target, path); + ++dir; + relTargetLeft = relTargetLeft.substr(3); + } + if (dir == pendingDirs.rend()) + throw Error("invalid hard link target '%s' for path '%s'", target, path); + + // Look up the remainder of the target, starting at the + // top-most `git_treebuilder`. + std::variant curDir{dir->builder.get()}; + Object tree; // needed to keep `entry` alive + const git_tree_entry * entry = nullptr; + + for (auto & c : CanonPath(relTargetLeft)) { + if (auto builder = std::get_if(&curDir)) { + assert(*builder); + if (!(entry = git_treebuilder_get(*builder, std::string(c).c_str()))) + throw Error("cannot find hard link target '%s' for path '%s'", target, path); + curDir = *git_tree_entry_id(entry); + } else if (auto oid = std::get_if(&curDir)) { + tree = lookupObject(*repo, *oid, GIT_OBJECT_TREE); + if (!(entry = git_tree_entry_byname((const git_tree *) &*tree, std::string(c).c_str()))) + throw Error("cannot find hard link target '%s' for path '%s'", target, path); + curDir = *git_tree_entry_id(entry); + } + } + + assert(entry); + + addToTree(*pathComponents.rbegin(), + *git_tree_entry_id(entry), + git_tree_entry_filemode(entry)); + } + + Hash flush() override + { updateBuilders({}); auto [oid, _name] = popBuilder(); + repo->flush(); + return toHash(oid); } }; @@ -980,8 +1242,10 @@ std::vector> GitRepoImpl::getSubmodules auto rawAccessor = getRawAccessor(rev); for (auto & submodule : parseSubmodules(pathTemp)) { - auto rev = rawAccessor->getSubmoduleRev(submodule.path); - result.push_back({std::move(submodule), rev}); + /* Filter out .gitmodules entries that don't exist or are not + submodules. */ + if (auto rev = rawAccessor->getSubmoduleRev(submodule.path)) + result.push_back({std::move(submodule), *rev}); } return result; @@ -989,7 +1253,7 @@ std::vector> GitRepoImpl::getSubmodules ref getTarballCache() { - static auto repoDir = std::filesystem::path(getCacheDir()) / "nix" / "tarball-cache"; + static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache"; return GitRepo::openRepo(repoDir, true, true); } diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index 29d799554..f45b5a504 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -7,12 +7,16 @@ namespace nix { namespace fetchers { struct PublicKey; } -struct GitFileSystemObjectSink : FileSystemObjectSink +/** + * A sink that writes into a Git repository. Note that nothing may be written + * until `flush()` is called. + */ +struct GitFileSystemObjectSink : ExtendedFileSystemObjectSink { /** * Flush builder and return a final Git hash. */ - virtual Hash sync() = 0; + virtual Hash flush() = 0; }; struct GitRepo @@ -80,6 +84,8 @@ struct GitRepo virtual ref getFileSystemObjectSink() = 0; + virtual void flush() = 0; + virtual void fetch( const std::string & url, const std::string & refspec, @@ -98,6 +104,13 @@ struct GitRepo * serialisation. This is memoised on-disk. */ virtual Hash treeHashToNarHash(const Hash & treeHash) = 0; + + /** + * If the specified Git object is a directory with a single entry + * that is a directory, return the ID of that object. + * Otherwise, return the passed ID unchanged. + */ + virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; }; ref getTarballCache(); diff --git a/src/libfetchers/unix/git.cc b/src/libfetchers/git.cc similarity index 95% rename from src/libfetchers/unix/git.cc rename to src/libfetchers/git.cc index fa7ef3621..99d91919e 100644 --- a/src/libfetchers/unix/git.cc +++ b/src/libfetchers/git.cc @@ -13,13 +13,16 @@ #include "git-utils.hh" #include "logging.hh" #include "finally.hh" - #include "fetch-settings.hh" +#include "json-utils.hh" #include #include #include -#include + +#ifndef _WIN32 +# include +#endif using namespace std::string_literals; @@ -38,21 +41,10 @@ bool isCacheFileWithinTtl(time_t now, const struct stat & st) return st.st_mtime + settings.tarballTtl > now; } -bool touchCacheFile(const Path & path, time_t touch_time) -{ - struct timeval times[2]; - times[0].tv_sec = touch_time; - times[0].tv_usec = 0; - times[1].tv_sec = touch_time; - times[1].tv_usec = 0; - - return lutimes(path.c_str(), times) == 0; -} - Path getCachePath(std::string_view key, bool shallow) { return getCacheDir() - + "/nix/gitv3/" + + "/gitv3/" + hashString(HashAlgorithm::SHA256, key).to_string(HashFormat::Nix32, false) + (shallow ? "-shallow" : ""); } @@ -98,7 +90,15 @@ bool storeCachedHead(const std::string & actualUrl, const std::string & headRef) try { runProgram("git", true, { "-C", cacheDir, "--git-dir", ".", "symbolic-ref", "--", "HEAD", headRef }); } catch (ExecError &e) { - if (!WIFEXITED(e.status)) throw; + if ( +#ifndef WIN32 // TODO abstract over exit status handling on Windows + !WIFEXITED(e.status) +#else + e.status != 0 +#endif + ) + throw; + return false; } /* No need to touch refs/HEAD, because `git symbolic-ref` updates the mtime. */ @@ -164,7 +164,9 @@ static const Hash nullRev{HashAlgorithm::SHA1}; struct GitInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "git" && url.scheme != "git+http" && @@ -190,7 +192,7 @@ struct GitInputScheme : InputScheme attrs.emplace("url", url2.to_string()); - return inputFromAttrs(attrs); + return inputFromAttrs(settings, attrs); } @@ -222,7 +224,9 @@ struct GitInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { for (auto & [name, _] : attrs) if (name == "verifyCommit" @@ -238,7 +242,7 @@ struct GitInputScheme : InputScheme throw BadURL("invalid Git branch/tag name '%s'", *ref); } - Input input; + Input input{settings}; input.attrs = attrs; auto url = fixGitURL(getStrAttr(attrs, "url")); parseURL(url); @@ -329,7 +333,13 @@ struct GitInputScheme : InputScheme .program = "git", .args = {"-C", repoInfo.url, "--git-dir", repoInfo.gitDir, "check-ignore", "--quiet", std::string(path.rel())}, }); - auto exitCode = WEXITSTATUS(result.first); + auto exitCode = +#ifndef WIN32 // TODO abstract over exit status handling on Windows + WEXITSTATUS(result.first) +#else + result.first +#endif + ; if (exitCode != 0) { // The path is not `.gitignore`d, we can add the file. @@ -360,13 +370,13 @@ struct GitInputScheme : InputScheme /* URL of the repo, or its path if isLocal. Never a `file` URL. */ std::string url; - void warnDirty() const + void warnDirty(const Settings & settings) const { if (workdirInfo.isDirty) { - if (!fetchSettings.allowDirty) + if (!settings.allowDirty) throw Error("Git tree '%s' is dirty", url); - if (fetchSettings.warnDirty) + if (settings.warnDirty) warn("Git tree '%s' is dirty", url); } } @@ -573,8 +583,12 @@ struct GitInputScheme : InputScheme warn("could not update local clone of Git repository '%s'; continuing with the most recent version", repoInfo.url); } - if (!touchCacheFile(localRefFile, now)) - warn("could not update mtime for file '%s': %s", localRefFile, strerror(errno)); + try { + if (!input.getRev()) + setWriteTime(localRefFile, now, now); + } catch (Error & e) { + warn("could not update mtime for file '%s': %s", localRefFile, e.info().msg); + } if (!originalRef && !storeCachedHead(repoInfo.url, ref)) warn("could not update cached head '%s' for '%s'", ref, repoInfo.url); } @@ -644,7 +658,7 @@ struct GitInputScheme : InputScheme attrs.insert_or_assign("exportIgnore", Explicit{ exportIgnore }); attrs.insert_or_assign("submodules", Explicit{ true }); attrs.insert_or_assign("allRefs", Explicit{ true }); - auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); + auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store); submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); @@ -702,7 +716,7 @@ struct GitInputScheme : InputScheme // TODO: fall back to getAccessorFromCommit-like fetch when submodules aren't checked out // attrs.insert_or_assign("allRefs", Explicit{ true }); - auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); + auto submoduleInput = fetchers::Input::fromAttrs(*input.settings, std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store); submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); @@ -734,7 +748,7 @@ struct GitInputScheme : InputScheme verifyCommit(input, repo); } else { - repoInfo.warnDirty(); + repoInfo.warnDirty(*input.settings); if (repoInfo.workdirInfo.headRev) { input.attrs.insert_or_assign("dirtyRev", diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index d62a7482e..308cff33a 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -31,7 +31,9 @@ struct GitArchiveInputScheme : InputScheme { virtual std::optional> accessHeaderFromToken(const std::string & token) const = 0; - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const fetchers::Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != schemeName()) return {}; @@ -90,7 +92,7 @@ struct GitArchiveInputScheme : InputScheme if (ref && rev) throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev()); - Input input; + Input input{settings}; input.attrs.insert_or_assign("type", std::string { schemeName() }); input.attrs.insert_or_assign("owner", path[0]); input.attrs.insert_or_assign("repo", path[1]); @@ -119,12 +121,14 @@ struct GitArchiveInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const fetchers::Settings & settings, + const Attrs & attrs) const override { getStrAttr(attrs, "owner"); getStrAttr(attrs, "repo"); - Input input; + Input input{settings}; input.attrs = attrs; return input; } @@ -168,18 +172,20 @@ struct GitArchiveInputScheme : InputScheme return input; } - std::optional getAccessToken(const std::string & host) const + std::optional getAccessToken(const fetchers::Settings & settings, const std::string & host) const { - auto tokens = fetchSettings.accessTokens.get(); + auto tokens = settings.accessTokens.get(); if (auto token = get(tokens, host)) return *token; return {}; } - Headers makeHeadersWithAuthTokens(const std::string & host) const + Headers makeHeadersWithAuthTokens( + const fetchers::Settings & settings, + const std::string & host) const { Headers headers; - auto accessToken = getAccessToken(host); + auto accessToken = getAccessToken(settings, host); if (accessToken) { auto hdr = accessHeaderFromToken(*accessToken); if (hdr) @@ -248,12 +254,19 @@ struct GitArchiveInputScheme : InputScheme getFileTransfer()->download(std::move(req), sink); }); + auto act = std::make_unique(*logger, lvlInfo, actUnknown, + fmt("unpacking '%s' into the Git cache", input.to_string())); + TarArchive archive { *source }; - auto parseSink = getTarballCache()->getFileSystemObjectSink(); + auto tarballCache = getTarballCache(); + auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); + auto tree = parseSink->flush(); + + act.reset(); TarballInfo tarballInfo { - .treeHash = parseSink->sync(), + .treeHash = tarballCache->dereferenceSingletonDirectory(tree), .lastModified = lastModified }; @@ -295,7 +308,7 @@ struct GitArchiveInputScheme : InputScheme locking. FIXME: in the future, we may want to require a Git tree hash instead of a NAR hash. */ return input.getRev().has_value() - && (fetchSettings.trustTarballsFromGitForges || + && (input.settings->trustTarballsFromGitForges || input.getNarHash().has_value()); } @@ -352,7 +365,7 @@ struct GitHubInputScheme : GitArchiveInputScheme : "https://%s/api/v3/repos/%s/%s/commits/%s", host, getOwner(input), getRepo(input), *input.getRef()); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); auto json = nlohmann::json::parse( readFile( @@ -369,7 +382,7 @@ struct GitHubInputScheme : GitArchiveInputScheme { auto host = getHost(input); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); // If we have no auth headers then we default to the public archive // urls so we do not run into rate limits. @@ -389,7 +402,7 @@ struct GitHubInputScheme : GitArchiveInputScheme void clone(const Input & input, const Path & destDir) const override { auto host = getHost(input); - Input::fromURL(fmt("git+https://%s/%s/%s.git", + Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s.git", host, getOwner(input), getRepo(input))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); @@ -426,16 +439,22 @@ struct GitLabInputScheme : GitArchiveInputScheme auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef()); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); auto json = nlohmann::json::parse( readFile( store->toRealPath( downloadFile(store, url, "source", headers).storePath))); - return RefInfo { - .rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1) - }; + if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) { + return RefInfo { + .rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1) + }; + } if (json.is_array() && json.size() == 0) { + throw Error("No commits returned by GitLab API -- does the git ref really exist?"); + } else { + throw Error("Unexpected response received from GitLab: %s", json); + } } DownloadUrl getDownloadUrl(const Input & input) const override @@ -450,7 +469,7 @@ struct GitLabInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(HashFormat::Base16, false)); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); return DownloadUrl { url, headers }; } @@ -458,7 +477,7 @@ struct GitLabInputScheme : GitArchiveInputScheme { auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); // FIXME: get username somewhere - Input::fromURL(fmt("git+https://%s/%s/%s.git", + Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); @@ -490,7 +509,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme auto base_url = fmt("https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); std::string refUri; if (ref == "HEAD") { @@ -537,14 +556,14 @@ struct SourceHutInputScheme : GitArchiveInputScheme host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), input.getRev()->to_string(HashFormat::Base16, false)); - Headers headers = makeHeadersWithAuthTokens(host); + Headers headers = makeHeadersWithAuthTokens(*input.settings, host); return DownloadUrl { url, headers }; } void clone(const Input & input, const Path & destDir) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); - Input::fromURL(fmt("git+https://%s/%s/%s", + Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef(), input.getRev()) .clone(destDir); diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index ba5078631..2e5cd82c7 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -8,7 +8,9 @@ std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); struct IndirectInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "flake") return {}; @@ -41,7 +43,7 @@ struct IndirectInputScheme : InputScheme // FIXME: forbid query params? - Input input; + Input input{settings}; input.attrs.insert_or_assign("type", "indirect"); input.attrs.insert_or_assign("id", id); if (rev) input.attrs.insert_or_assign("rev", rev->gitRev()); @@ -65,13 +67,15 @@ struct IndirectInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { auto id = getStrAttr(attrs, "id"); if (!std::regex_match(id, flakeRegex)) throw BadURL("'%s' is not a valid flake ID", id); - Input input; + Input input{settings}; input.attrs = attrs; return input; } diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk index 0fef1466b..e229a0993 100644 --- a/src/libfetchers/local.mk +++ b/src/libfetchers/local.mk @@ -5,16 +5,10 @@ libfetchers_NAME = libnixfetchers libfetchers_DIR := $(d) libfetchers_SOURCES := $(wildcard $(d)/*.cc) -ifdef HOST_UNIX - libfetchers_SOURCES += $(wildcard $(d)/unix/*.cc) -endif # Not just for this library itself, but also for downstream libraries using this library INCLUDE_libfetchers := -I $(d) -ifdef HOST_UNIX - INCLUDE_libfetchers += -I $(d)/unix -endif libfetchers_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) diff --git a/src/libfetchers/unix/mercurial.cc b/src/libfetchers/mercurial.cc similarity index 94% rename from src/libfetchers/unix/mercurial.cc rename to src/libfetchers/mercurial.cc index 7bdf1e937..2c987f79d 100644 --- a/src/libfetchers/unix/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -45,7 +45,9 @@ static std::string runHg(const Strings & args, const std::optional struct MercurialInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "hg+http" && url.scheme != "hg+https" && @@ -68,7 +70,7 @@ struct MercurialInputScheme : InputScheme attrs.emplace("url", url2.to_string()); - return inputFromAttrs(attrs); + return inputFromAttrs(settings, attrs); } std::string_view schemeName() const override @@ -88,7 +90,9 @@ struct MercurialInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { parseURL(getStrAttr(attrs, "url")); @@ -97,7 +101,7 @@ struct MercurialInputScheme : InputScheme throw BadURL("invalid Mercurial branch/tag name '%s'", *ref); } - Input input; + Input input{settings}; input.attrs = attrs; return input; } @@ -182,10 +186,10 @@ struct MercurialInputScheme : InputScheme /* This is an unclean working tree. So copy all tracked files. */ - if (!fetchSettings.allowDirty) + if (!input.settings->allowDirty) throw Error("Mercurial tree '%s' is unclean", actualUrl); - if (fetchSettings.warnDirty) + if (input.settings->warnDirty) warn("Mercurial tree '%s' is unclean", actualUrl); input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl }))); @@ -213,7 +217,7 @@ struct MercurialInputScheme : InputScheme auto storePath = store->addToStore( input.getName(), {getFSSourceAccessor(), CanonPath(actualPath)}, - FileIngestionMethod::Recursive, HashAlgorithm::SHA256, {}, + ContentAddressMethod::Raw::NixArchive, HashAlgorithm::SHA256, {}, filter); return storePath; @@ -259,7 +263,7 @@ struct MercurialInputScheme : InputScheme return makeResult(res->value, res->storePath); } - Path cacheDir = fmt("%s/nix/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Nix32, false)); + Path cacheDir = fmt("%s/hg/%s", getCacheDir(), hashString(HashAlgorithm::SHA256, actualUrl).to_string(HashFormat::Nix32, false)); /* If this is a commit hash that we already have, we don't have to pull again. */ diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build new file mode 100644 index 000000000..d4f202796 --- /dev/null +++ b/src/libfetchers/meson.build @@ -0,0 +1,95 @@ +project('nix-fetchers', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +libgit2 = dependency('libgit2') +deps_private += libgit2 + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'attrs.cc', + 'cache.cc', + 'fetch-settings.cc', + 'fetch-to-store.cc', + 'fetchers.cc', + 'filtering-source-accessor.cc', + 'git.cc', + 'git-utils.cc', + 'github.cc', + 'indirect.cc', + 'mercurial.cc', + 'mounted-source-accessor.cc', + 'path.cc', + 'store-path-accessor.cc', + 'registry.cc', + 'tarball.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'attrs.hh', + 'cache.hh', + 'fetch-settings.hh', + 'fetch-to-store.hh', + 'filtering-source-accessor.hh', + 'git-utils.hh', + 'mounted-source-accessor.hh', + 'fetchers.hh', + 'registry.hh', + 'store-path-accessor.hh', + 'tarball.hh', +) + +this_library = library( + 'nixfetchers', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix new file mode 100644 index 000000000..9b5d8bff7 --- /dev/null +++ b/src/libfetchers/package.nix @@ -0,0 +1,78 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-store +, nlohmann_json +, libgit2 +, man + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-fetchers"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + libgit2 + ]; + + propagatedBuildInputs = [ + nix-store + nix-util + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index 68958d559..fca0df84b 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -7,14 +7,16 @@ namespace nix::fetchers { struct PathInputScheme : InputScheme { - std::optional inputFromURL(const ParsedURL & url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & url, bool requireTree) const override { if (url.scheme != "path") return {}; if (url.authority && *url.authority != "") throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority); - Input input; + Input input{settings}; input.attrs.insert_or_assign("type", "path"); input.attrs.insert_or_assign("path", url.path); @@ -54,11 +56,13 @@ struct PathInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { getStrAttr(attrs, "path"); - Input input; + Input input{settings}; input.attrs = attrs; return input; } diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index e00b9de46..7f7a09053 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -1,3 +1,4 @@ +#include "fetch-settings.hh" #include "registry.hh" #include "tarball.hh" #include "users.hh" @@ -5,19 +6,18 @@ #include "store-api.hh" #include "local-fs-store.hh" -#include "fetch-settings.hh" - #include namespace nix::fetchers { std::shared_ptr Registry::read( + const Settings & settings, const Path & path, RegistryType type) { - auto registry = std::make_shared(type); + auto registry = std::make_shared(settings, type); if (!pathExists(path)) - return std::make_shared(type); + return std::make_shared(settings, type); try { @@ -37,8 +37,8 @@ std::shared_ptr Registry::read( auto exact = i.find("exact"); registry->entries.push_back( Entry { - .from = Input::fromAttrs(jsonToAttrs(i["from"])), - .to = Input::fromAttrs(std::move(toAttrs)), + .from = Input::fromAttrs(settings, jsonToAttrs(i["from"])), + .to = Input::fromAttrs(settings, std::move(toAttrs)), .extraAttrs = extraAttrs, .exact = exact != i.end() && exact.value() }); @@ -107,37 +107,36 @@ static Path getSystemRegistryPath() return settings.nixConfDir + "/registry.json"; } -static std::shared_ptr getSystemRegistry() +static std::shared_ptr getSystemRegistry(const Settings & settings) { static auto systemRegistry = - Registry::read(getSystemRegistryPath(), Registry::System); + Registry::read(settings, getSystemRegistryPath(), Registry::System); return systemRegistry; } Path getUserRegistryPath() { - return getConfigDir() + "/nix/registry.json"; + return getConfigDir() + "/registry.json"; } -std::shared_ptr getUserRegistry() +std::shared_ptr getUserRegistry(const Settings & settings) { static auto userRegistry = - Registry::read(getUserRegistryPath(), Registry::User); + Registry::read(settings, getUserRegistryPath(), Registry::User); return userRegistry; } -std::shared_ptr getCustomRegistry(const Path & p) +std::shared_ptr getCustomRegistry(const Settings & settings, const Path & p) { static auto customRegistry = - Registry::read(p, Registry::Custom); + Registry::read(settings, p, Registry::Custom); return customRegistry; } -static std::shared_ptr flagRegistry = - std::make_shared(Registry::Flag); - -std::shared_ptr getFlagRegistry() +std::shared_ptr getFlagRegistry(const Settings & settings) { + static auto flagRegistry = + std::make_shared(settings, Registry::Flag); return flagRegistry; } @@ -146,37 +145,37 @@ void overrideRegistry( const Input & to, const Attrs & extraAttrs) { - flagRegistry->add(from, to, extraAttrs); + getFlagRegistry(*from.settings)->add(from, to, extraAttrs); } -static std::shared_ptr getGlobalRegistry(ref store) +static std::shared_ptr getGlobalRegistry(const Settings & settings, ref store) { static auto reg = [&]() { - auto path = fetchSettings.flakeRegistry.get(); + auto path = settings.flakeRegistry.get(); if (path == "") { - return std::make_shared(Registry::Global); // empty registry + return std::make_shared(settings, Registry::Global); // empty registry } if (!hasPrefix(path, "/")) { auto storePath = downloadFile(store, path, "flake-registry.json").storePath; if (auto store2 = store.dynamic_pointer_cast()) - store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json"); + store2->addPermRoot(storePath, getCacheDir() + "/flake-registry.json"); path = store->toRealPath(storePath); } - return Registry::read(path, Registry::Global); + return Registry::read(settings, path, Registry::Global); }(); return reg; } -Registries getRegistries(ref store) +Registries getRegistries(const Settings & settings, ref store) { Registries registries; - registries.push_back(getFlagRegistry()); - registries.push_back(getUserRegistry()); - registries.push_back(getSystemRegistry()); - registries.push_back(getGlobalRegistry(store)); + registries.push_back(getFlagRegistry(settings)); + registries.push_back(getUserRegistry(settings)); + registries.push_back(getSystemRegistry(settings)); + registries.push_back(getGlobalRegistry(settings, store)); return registries; } @@ -193,7 +192,7 @@ std::pair lookupInRegistries( n++; if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string()); - for (auto & registry : getRegistries(store)) { + for (auto & registry : getRegistries(*input.settings, store)) { // FIXME: O(n) for (auto & entry : registry->entries) { if (entry.exact) { diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index f57ab1e6b..0d68ac395 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -10,6 +10,8 @@ namespace nix::fetchers { struct Registry { + const Settings & settings; + enum RegistryType { Flag = 0, User = 1, @@ -29,11 +31,13 @@ struct Registry std::vector entries; - Registry(RegistryType type) - : type(type) + Registry(const Settings & settings, RegistryType type) + : settings{settings} + , type{type} { } static std::shared_ptr read( + const Settings & settings, const Path & path, RegistryType type); void write(const Path & path); @@ -48,13 +52,13 @@ struct Registry typedef std::vector> Registries; -std::shared_ptr getUserRegistry(); +std::shared_ptr getUserRegistry(const Settings & settings); -std::shared_ptr getCustomRegistry(const Path & p); +std::shared_ptr getCustomRegistry(const Settings & settings, const Path & p); Path getUserRegistryPath(); -Registries getRegistries(ref store); +Registries getRegistries(const Settings & settings, ref store); void overrideRegistry( const Input & from, diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index d03ff82ce..28574e7b1 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -2,12 +2,10 @@ #include "fetchers.hh" #include "cache.hh" #include "filetransfer.hh" -#include "globals.hh" #include "store-api.hh" #include "archive.hh" #include "tarfile.hh" #include "types.hh" -#include "split.hh" #include "store-path-accessor.hh" #include "store-api.hh" #include "git-utils.hh" @@ -92,6 +90,7 @@ DownloadFileResult downloadFile( /* Cache metadata for all URLs in the redirect chain. */ for (auto & url : res.urls) { key.second.insert_or_assign("url", url); + assert(!res.urls.empty()); infoAttrs.insert_or_assign("url", *res.urls.rbegin()); getCache()->upsert(key, *store, infoAttrs, *storePath); } @@ -104,7 +103,7 @@ DownloadFileResult downloadFile( }; } -DownloadTarballResult downloadTarball( +static DownloadTarballResult downloadTarball_( const std::string & url, const Headers & headers) { @@ -145,6 +144,9 @@ DownloadTarballResult downloadTarball( // TODO: fall back to cached value if download fails. + auto act = std::make_unique(*logger, lvlInfo, actUnknown, + fmt("unpacking '%s' into the Git cache", url)); + AutoDelete cleanupTemp; /* Note: if the download is cached, `importTarball()` will receive @@ -166,8 +168,12 @@ DownloadTarballResult downloadTarball( TarArchive{path}; }) : TarArchive{*source}; - auto parseSink = getTarballCache()->getFileSystemObjectSink(); + auto tarballCache = getTarballCache(); + auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); + auto tree = parseSink->flush(); + + act.reset(); auto res(_res->lock()); @@ -179,7 +185,8 @@ DownloadTarballResult downloadTarball( infoAttrs = cached->value; } else { infoAttrs.insert_or_assign("etag", res->etag); - infoAttrs.insert_or_assign("treeHash", parseSink->sync().gitRev()); + infoAttrs.insert_or_assign("treeHash", + tarballCache->dereferenceSingletonDirectory(tree).gitRev()); infoAttrs.insert_or_assign("lastModified", uint64_t(lastModified)); if (res->immutableUrl) infoAttrs.insert_or_assign("immutableUrl", *res->immutableUrl); @@ -197,12 +204,28 @@ DownloadTarballResult downloadTarball( return attrsToResult(infoAttrs); } +ref downloadTarball( + ref store, + const Settings & settings, + const std::string & url) +{ + /* Go through Input::getAccessor() to ensure that the resulting + accessor has a fingerprint. */ + fetchers::Attrs attrs; + attrs.insert_or_assign("type", "tarball"); + attrs.insert_or_assign("url", url); + + auto input = Input::fromAttrs(settings, std::move(attrs)); + + return input.getAccessor(store).first; +} + // An input scheme corresponding to a curl-downloadable resource. struct CurlInputScheme : InputScheme { const std::set transportUrlSchemes = {"file", "http", "https"}; - const bool hasTarballExtension(std::string_view path) const + bool hasTarballExtension(std::string_view path) const { return hasSuffix(path, ".zip") || hasSuffix(path, ".tar") || hasSuffix(path, ".tgz") || hasSuffix(path, ".tar.gz") @@ -214,12 +237,14 @@ struct CurlInputScheme : InputScheme static const std::set specialParams; - std::optional inputFromURL(const ParsedURL & _url, bool requireTree) const override + std::optional inputFromURL( + const Settings & settings, + const ParsedURL & _url, bool requireTree) const override { if (!isValidURL(_url, requireTree)) return std::nullopt; - Input input; + Input input{settings}; auto url = _url; @@ -267,9 +292,11 @@ struct CurlInputScheme : InputScheme }; } - std::optional inputFromAttrs(const Attrs & attrs) const override + std::optional inputFromAttrs( + const Settings & settings, + const Attrs & attrs) const override { - Input input; + Input input{settings}; input.attrs = attrs; //input.locked = (bool) maybeGetStrAttr(input.attrs, "hash"); @@ -344,12 +371,12 @@ struct TarballInputScheme : CurlInputScheme { auto input(_input); - auto result = downloadTarball(getStrAttr(input.attrs, "url"), {}); + auto result = downloadTarball_(getStrAttr(input.attrs, "url"), {}); result.accessor->setPathDisplay("«" + input.to_string() + "»"); if (result.immutableUrl) { - auto immutableInput = Input::fromURL(*result.immutableUrl); + auto immutableInput = Input::fromURL(*input.settings, *result.immutableUrl); // FIXME: would be nice to support arbitrary flakerefs // here, e.g. git flakes. if (immutableInput.getType() != "tarball") @@ -365,6 +392,16 @@ struct TarballInputScheme : CurlInputScheme return {result.accessor, input}; } + + std::optional getFingerprint(ref store, const Input & input) const override + { + if (auto narHash = input.getNarHash()) + return narHash->to_string(HashFormat::SRI, true); + else if (auto rev = input.getRev()) + return rev->gitRev(); + else + return std::nullopt; + } }; static auto rTarballInputScheme = OnStartup([] { registerInputScheme(std::make_unique()); }); diff --git a/src/libfetchers/tarball.hh b/src/libfetchers/tarball.hh index ba0dfd623..2042041d5 100644 --- a/src/libfetchers/tarball.hh +++ b/src/libfetchers/tarball.hh @@ -1,11 +1,12 @@ #pragma once -#include "types.hh" -#include "path.hh" -#include "hash.hh" - #include +#include "hash.hh" +#include "path.hh" +#include "ref.hh" +#include "types.hh" + namespace nix { class Store; struct SourceAccessor; @@ -13,6 +14,8 @@ struct SourceAccessor; namespace nix::fetchers { +struct Settings; + struct DownloadFileResult { StorePath storePath; @@ -39,8 +42,9 @@ struct DownloadTarballResult * Download and import a tarball into the Git cache. The result is the * Git tree hash of the root directory. */ -DownloadTarballResult downloadTarball( - const std::string & url, - const Headers & headers = {}); +ref downloadTarball( + ref store, + const Settings & settings, + const std::string & url); } diff --git a/src/libflake/.version b/src/libflake/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libflake/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libflake/build-utils-meson b/src/libflake/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libflake/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libexpr/flake/config.cc b/src/libflake/flake/config.cc similarity index 89% rename from src/libexpr/flake/config.cc rename to src/libflake/flake/config.cc index e0c5d4512..4879de463 100644 --- a/src/libexpr/flake/config.cc +++ b/src/libflake/flake/config.cc @@ -1,6 +1,6 @@ #include "users.hh" -#include "globals.hh" -#include "fetch-settings.hh" +#include "config-global.hh" +#include "flake/settings.hh" #include "flake.hh" #include @@ -12,7 +12,7 @@ typedef std::map> TrustedList; Path trustedListPath() { - return getDataDir() + "/nix/trusted-settings.json"; + return getDataDir() + "/trusted-settings.json"; } static TrustedList readTrustedList() @@ -30,7 +30,7 @@ static void writeTrustedList(const TrustedList & trustedList) writeFile(path, nlohmann::json(trustedList).dump()); } -void ConfigFile::apply() +void ConfigFile::apply(const Settings & flakeSettings) { std::set whitelist{"bash-prompt", "bash-prompt-prefix", "bash-prompt-suffix", "flake-registry", "commit-lock-file-summary", "commit-lockfile-summary"}; @@ -47,11 +47,11 @@ void ConfigFile::apply() else if (auto* b = std::get_if>(&value)) valueS = b->t ? "true" : "false"; else if (auto ss = std::get_if>(&value)) - valueS = concatStringsSep(" ", *ss); // FIXME: evil + valueS = dropEmptyInitThenConcatStringsSep(" ", *ss); // FIXME: evil else assert(false); - if (!whitelist.count(baseName) && !nix::fetchSettings.acceptFlakeConfig) { + if (!whitelist.count(baseName) && !flakeSettings.acceptFlakeConfig) { bool trusted = false; auto trustedList = readTrustedList(); auto tlname = get(trustedList, name); diff --git a/src/libexpr/flake/flake.cc b/src/libflake/flake/flake.cc similarity index 87% rename from src/libexpr/flake/flake.cc rename to src/libflake/flake/flake.cc index 3af9ef14e..d18e01464 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libflake/flake/flake.cc @@ -9,9 +9,12 @@ #include "fetchers.hh" #include "finally.hh" #include "fetch-settings.hh" +#include "flake/settings.hh" #include "value-to-json.hh" #include "local-fs-store.hh" +#include + namespace nix { using namespace flake; @@ -97,7 +100,7 @@ static std::map parseFlakeInputs( const std::optional & baseDir, InputPath lockRootPath); static FlakeInput parseFlakeInput(EvalState & state, - const std::string & inputName, Value * value, const PosIdx pos, + std::string_view inputName, Value * value, const PosIdx pos, const std::optional & baseDir, InputPath lockRootPath) { expectType(state, nAttrs, *value, pos); @@ -139,9 +142,16 @@ static FlakeInput parseFlakeInput(EvalState & state, case nBool: attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean() }); break; - case nInt: - attrs.emplace(state.symbols[attr.name], (long unsigned int) attr.value->integer()); + case nInt: { + auto intValue = attr.value->integer().value; + + if (intValue < 0) { + state.error("negative value given for flake input attribute %1%: %2%", state.symbols[attr.name], intValue).debugThrow(); + } + + attrs.emplace(state.symbols[attr.name], uint64_t(intValue)); break; + } default: if (attr.name == state.symbols.create("publicKeys")) { experimentalFeatureSettings.require(Xp::VerifiedFetches); @@ -163,7 +173,7 @@ static FlakeInput parseFlakeInput(EvalState & state, if (attrs.count("type")) try { - input.ref = FlakeRef::fromAttrs(attrs); + input.ref = FlakeRef::fromAttrs(state.fetchSettings, attrs); } catch (Error & e) { e.addTrace(state.positions[pos], HintFmt("while evaluating flake input")); throw; @@ -173,11 +183,11 @@ static FlakeInput parseFlakeInput(EvalState & state, if (!attrs.empty()) throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, state.positions[pos]); if (url) - input.ref = parseFlakeRef(*url, baseDir, true, input.isFlake); + input.ref = parseFlakeRef(state.fetchSettings, *url, baseDir, true, input.isFlake); } if (!input.follows && !input.ref) - input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}}); + input.ref = FlakeRef::fromAttrs(state.fetchSettings, {{"type", "indirect"}, {"id", std::string(inputName)}}); return input; } @@ -243,7 +253,7 @@ static Flake readFlake( for (auto & formal : outputs->value->payload.lambda.fun->formals->formals) { if (formal.name != state.sSelf) flake.inputs.emplace(state.symbols[formal.name], FlakeInput { - .ref = parseFlakeRef(state.symbols[formal.name]) + .ref = parseFlakeRef(state.fetchSettings, std::string(state.symbols[formal.name])) }); } } @@ -271,7 +281,7 @@ static Flake readFlake( else if (setting.value->type() == nInt) flake.config.settings.emplace( state.symbols[setting.name], - state.forceInt(*setting.value, setting.pos, "")); + state.forceInt(*setting.value, setting.pos, "").value); else if (setting.value->type() == nBool) flake.config.settings.emplace( state.symbols[setting.name], @@ -328,16 +338,19 @@ Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup return getFlake(state, originalRef, allowLookup, flakeCache); } -static LockFile readLockFile(const SourcePath & lockFilePath) +static LockFile readLockFile( + const fetchers::Settings & fetchSettings, + const SourcePath & lockFilePath) { return lockFilePath.pathExists() - ? LockFile(lockFilePath.readFile(), fmt("%s", lockFilePath)) + ? LockFile(fetchSettings, lockFilePath.readFile(), fmt("%s", lockFilePath)) : LockFile(); } /* Compute an in-memory lock file for the specified top-level flake, and optionally write it to file, if the flake is writable. */ LockedFlake lockFlake( + const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags) @@ -346,21 +359,22 @@ LockedFlake lockFlake( FlakeCache flakeCache; - auto useRegistries = lockFlags.useRegistries.value_or(fetchSettings.useRegistries); + auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); auto flake = getFlake(state, topRef, useRegistries, flakeCache); if (lockFlags.applyNixConfig) { - flake.config.apply(); + flake.config.apply(settings); state.store->setOptions(); } try { - if (!fetchSettings.allowDirty && lockFlags.referenceLockFilePath) { + if (!state.fetchSettings.allowDirty && lockFlags.referenceLockFilePath) { throw Error("reference lock file was provided, but the `allow-dirty` setting is set to false"); } auto oldLockFile = readLockFile( + state.fetchSettings, lockFlags.referenceLockFilePath.value_or( flake.lockFilePath())); @@ -596,7 +610,7 @@ LockedFlake lockFlake( inputFlake.inputs, childNode, inputPath, oldLock ? std::dynamic_pointer_cast(oldLock) - : readLockFile(inputFlake.lockFilePath()).root.get_ptr(), + : readLockFile(state.fetchSettings, inputFlake.lockFilePath()).root.get_ptr(), oldLock ? lockRootPath : inputPath, localPath, false); @@ -659,7 +673,7 @@ LockedFlake lockFlake( if (lockFlags.writeLockFile) { if (sourcePath || lockFlags.outputLockFilePath) { if (auto unlockedInput = newLockFile.isUnlocked()) { - if (fetchSettings.warnDirty) + if (state.fetchSettings.warnDirty) warn("will not write lock file of flake '%s' because it has an unlocked input ('%s')", topRef, *unlockedInput); } else { if (!lockFlags.updateLockFile) @@ -691,7 +705,7 @@ LockedFlake lockFlake( if (lockFlags.commitLockFile) { std::string cm; - cm = fetchSettings.commitLockFileSummary.get(); + cm = settings.commitLockFileSummary.get(); if (cm == "") { cm = fmt("%s: %s", relPath, lockFileExists ? "Update" : "Add"); @@ -739,6 +753,21 @@ LockedFlake lockFlake( } } +std::pair sourcePathToStorePath( + ref store, + const SourcePath & _path) +{ + auto path = _path.path.abs(); + + if (auto store2 = store.dynamic_pointer_cast()) { + auto realStoreDir = store2->getRealStoreDir(); + if (isInDir(path, realStoreDir)) + path = store2->storeDir + path.substr(realStoreDir.size()); + } + + return store->toStorePath(path); +} + void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes) @@ -756,17 +785,7 @@ void callFlake(EvalState & state, auto lockedNode = node.dynamic_pointer_cast(); - // FIXME: This is a hack to support chroot stores. Remove this - // once we can pass a sourcePath rather than a storePath to - // call-flake.nix. - auto path = sourcePath.path.abs(); - if (auto store = state.store.dynamic_pointer_cast()) { - auto realStoreDir = store->getRealStoreDir(); - if (isInDir(path, realStoreDir)) - path = store->storeDir + path.substr(realStoreDir.size()); - } - - auto [storePath, subdir] = state.store->toStorePath(path); + auto [storePath, subdir] = sourcePathToStorePath(state.store, sourcePath); emitTreeAttrs( state, @@ -799,46 +818,49 @@ void callFlake(EvalState & state, state.callFunction(*vTmp1, vOverrides, vRes, noPos); } -static void prim_getFlake(EvalState & state, const PosIdx pos, Value * * args, Value & v) +void initLib(const Settings & settings) { - std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); - auto flakeRef = parseFlakeRef(flakeRefS, {}, true); - if (evalSettings.pureEval && !flakeRef.input.isLocked()) - throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, state.positions[pos]); + auto prim_getFlake = [&settings](EvalState & state, const PosIdx pos, Value * * args, Value & v) + { + std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.getFlake")); + auto flakeRef = parseFlakeRef(state.fetchSettings, flakeRefS, {}, true); + if (state.settings.pureEval && !flakeRef.input.isLocked()) + throw Error("cannot call 'getFlake' on unlocked flake reference '%s', at %s (use --impure to override)", flakeRefS, state.positions[pos]); - callFlake(state, - lockFlake(state, flakeRef, - LockFlags { - .updateLockFile = false, - .writeLockFile = false, - .useRegistries = !evalSettings.pureEval && fetchSettings.useRegistries, - .allowUnlocked = !evalSettings.pureEval, - }), - v); + callFlake(state, + lockFlake(settings, state, flakeRef, + LockFlags { + .updateLockFile = false, + .writeLockFile = false, + .useRegistries = !state.settings.pureEval && settings.useRegistries, + .allowUnlocked = !state.settings.pureEval, + }), + v); + }; + + RegisterPrimOp::primOps->push_back({ + .name = "__getFlake", + .args = {"args"}, + .doc = R"( + Fetch a flake from a flake reference, and return its output attributes and some metadata. For example: + + ```nix + (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix + ``` + + Unless impure evaluation is allowed (`--impure`), the flake reference + must be "locked", e.g. contain a Git revision or content hash. An + example of an unlocked usage is: + + ```nix + (builtins.getFlake "github:edolstra/dwarffs").rev + ``` + )", + .fun = prim_getFlake, + .experimentalFeature = Xp::Flakes, + }); } -static RegisterPrimOp r2({ - .name = "__getFlake", - .args = {"args"}, - .doc = R"( - Fetch a flake from a flake reference, and return its output attributes and some metadata. For example: - - ```nix - (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix - ``` - - Unless impure evaluation is allowed (`--impure`), the flake reference - must be "locked", e.g. contain a Git revision or content hash. An - example of an unlocked usage is: - - ```nix - (builtins.getFlake "github:edolstra/dwarffs").rev - ``` - )", - .fun = prim_getFlake, - .experimentalFeature = Xp::Flakes, -}); - static void prim_parseFlakeRef( EvalState & state, const PosIdx pos, @@ -847,7 +869,7 @@ static void prim_parseFlakeRef( { std::string flakeRefS(state.forceStringNoCtx(*args[0], pos, "while evaluating the argument passed to builtins.parseFlakeRef")); - auto attrs = parseFlakeRef(flakeRefS, {}, true).toAttrs(); + auto attrs = parseFlakeRef(state.fetchSettings, flakeRefS, {}, true).toAttrs(); auto binds = state.buildBindings(attrs.size()); for (const auto & [key, value] : attrs) { auto s = state.symbols.create(key); @@ -896,8 +918,13 @@ static void prim_flakeRefToString( for (const auto & attr : *args[0]->attrs()) { auto t = attr.value->type(); if (t == nInt) { - attrs.emplace(state.symbols[attr.name], - (uint64_t) attr.value->integer()); + auto intValue = attr.value->integer().value; + + if (intValue < 0) { + state.error("negative value given for flake ref attr %1%: %2%", state.symbols[attr.name], intValue).atPos(pos).debugThrow(); + } + + attrs.emplace(state.symbols[attr.name], uint64_t(intValue)); } else if (t == nBool) { attrs.emplace(state.symbols[attr.name], Explicit { attr.value->boolean() }); @@ -912,7 +939,7 @@ static void prim_flakeRefToString( showType(*attr.value)).debugThrow(); } } - auto flakeRef = FlakeRef::fromAttrs(attrs); + auto flakeRef = FlakeRef::fromAttrs(state.fetchSettings, attrs); v.mkString(flakeRef.to_string()); } @@ -949,10 +976,20 @@ std::optional LockedFlake::getFingerprint(ref store) const auto fingerprint = flake.lockedRef.input.getFingerprint(store); if (!fingerprint) return std::nullopt; + *fingerprint += fmt(";%s;%s", flake.lockedRef.subdir, lockFile); + + /* Include revCount and lastModified because they're not + necessarily implied by the content fingerprint (e.g. for + tarball flakes) but can influence the evaluation result. */ + if (auto revCount = flake.lockedRef.input.getRevCount()) + *fingerprint += fmt(";revCount=%d", *revCount); + if (auto lastModified = flake.lockedRef.input.getLastModified()) + *fingerprint += fmt(";lastModified=%d", *lastModified); + // FIXME: as an optimization, if the flake contains a lock file // and we haven't changed it, then it's sufficient to use // flake.sourceInfo.storePath for the fingerprint. - return hashString(HashAlgorithm::SHA256, fmt("%s;%s;%s", *fingerprint, flake.lockedRef.subdir, lockFile)); + return hashString(HashAlgorithm::SHA256, *fingerprint); } Flake::~Flake() { } diff --git a/src/libexpr/flake/flake.hh b/src/libflake/flake/flake.hh similarity index 89% rename from src/libexpr/flake/flake.hh rename to src/libflake/flake/flake.hh index 1ba085f0f..496e18673 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libflake/flake/flake.hh @@ -12,6 +12,16 @@ class EvalState; namespace flake { +struct Settings; + +/** + * Initialize `libnixflake` + * + * So far, this registers the `builtins.getFlake` primop, which depends + * on the choice of `flake:Settings`. + */ +void initLib(const Settings & settings); + struct FlakeInput; typedef std::map FlakeInputs; @@ -57,7 +67,7 @@ struct ConfigFile std::map settings; - void apply(); + void apply(const Settings & settings); }; /** @@ -194,6 +204,7 @@ struct LockFlags }; LockedFlake lockFlake( + const Settings & settings, EvalState & state, const FlakeRef & flakeRef, const LockFlags & lockFlags); @@ -203,6 +214,16 @@ void callFlake( const LockedFlake & lockedFlake, Value & v); +/** + * Map a `SourcePath` to the corresponding store path. This is a + * temporary hack to support chroot stores while we don't have full + * lazy trees. FIXME: Remove this once we can pass a sourcePath rather + * than a storePath to call-flake.nix. + */ +std::pair sourcePathToStorePath( + ref store, + const SourcePath & path); + } void emitTreeAttrs( diff --git a/src/libexpr/flake/flakeref.cc b/src/libflake/flake/flakeref.cc similarity index 83% rename from src/libexpr/flake/flakeref.cc rename to src/libflake/flake/flakeref.cc index 6e4aad64d..01fe747f9 100644 --- a/src/libexpr/flake/flakeref.cc +++ b/src/libflake/flake/flakeref.cc @@ -36,11 +36,6 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) return str; } -bool FlakeRef::operator ==(const FlakeRef & other) const -{ - return input == other.input && subdir == other.subdir; -} - FlakeRef FlakeRef::resolve(ref store) const { auto [input2, extraAttrs] = lookupInRegistries(store, input); @@ -48,28 +43,32 @@ FlakeRef FlakeRef::resolve(ref store) const } FlakeRef parseFlakeRef( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool allowMissing, bool isFlake) { - auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); + auto [flakeRef, fragment] = parseFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake); if (fragment != "") throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); return flakeRef; } std::optional maybeParseFlakeRef( - const std::string & url, const std::optional & baseDir) + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir) { try { - return parseFlakeRef(url, baseDir); + return parseFlakeRef(fetchSettings, url, baseDir); } catch (Error &) { return {}; } } std::pair parsePathFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool allowMissing, @@ -89,7 +88,7 @@ std::pair parsePathFlakeRefWithFragment( if (fragmentStart != std::string::npos) { fragment = percentDecode(url.substr(fragmentStart+1)); } - if (pathEnd != std::string::npos && fragmentStart != std::string::npos) { + if (pathEnd != std::string::npos && fragmentStart != std::string::npos && url[pathEnd] == '?') { query = decodeQuery(url.substr(pathEnd+1, fragmentStart-pathEnd-1)); } @@ -166,7 +165,7 @@ std::pair parsePathFlakeRefWithFragment( parsedURL.query.insert_or_assign("shallow", "1"); return std::make_pair( - FlakeRef(fetchers::Input::fromURL(parsedURL), getOr(parsedURL.query, "dir", "")), + FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL), getOr(parsedURL.query, "dir", "")), fragment); } @@ -185,13 +184,14 @@ std::pair parsePathFlakeRefWithFragment( attrs.insert_or_assign("type", "path"); attrs.insert_or_assign("path", path); - return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(std::move(attrs)), ""), fragment); + return std::make_pair(FlakeRef(fetchers::Input::fromAttrs(fetchSettings, std::move(attrs)), ""), fragment); }; /* Check if 'url' is a flake ID. This is an abbreviated syntax for 'flake:?ref=&rev='. */ -std::optional> parseFlakeIdRef( +static std::optional> parseFlakeIdRef( + const fetchers::Settings & fetchSettings, const std::string & url, bool isFlake ) @@ -213,7 +213,7 @@ std::optional> parseFlakeIdRef( }; return std::make_pair( - FlakeRef(fetchers::Input::fromURL(parsedURL, isFlake), ""), + FlakeRef(fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake), ""), percentDecode(match.str(6))); } @@ -221,6 +221,7 @@ std::optional> parseFlakeIdRef( } std::optional> parseURLFlakeRef( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool isFlake @@ -236,7 +237,7 @@ std::optional> parseURLFlakeRef( std::string fragment; std::swap(fragment, parsedURL.fragment); - auto input = fetchers::Input::fromURL(parsedURL, isFlake); + auto input = fetchers::Input::fromURL(fetchSettings, parsedURL, isFlake); input.parent = baseDir; return std::make_pair( @@ -245,6 +246,7 @@ std::optional> parseURLFlakeRef( } std::pair parseFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool allowMissing, @@ -254,31 +256,34 @@ std::pair parseFlakeRefWithFragment( std::smatch match; - if (auto res = parseFlakeIdRef(url, isFlake)) { + if (auto res = parseFlakeIdRef(fetchSettings, url, isFlake)) { return *res; - } else if (auto res = parseURLFlakeRef(url, baseDir, isFlake)) { + } else if (auto res = parseURLFlakeRef(fetchSettings, url, baseDir, isFlake)) { return *res; } else { - return parsePathFlakeRefWithFragment(url, baseDir, allowMissing, isFlake); + return parsePathFlakeRefWithFragment(fetchSettings, url, baseDir, allowMissing, isFlake); } } std::optional> maybeParseFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir) { try { - return parseFlakeRefWithFragment(url, baseDir); + return parseFlakeRefWithFragment(fetchSettings, url, baseDir); } catch (Error & e) { return {}; } } -FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) +FlakeRef FlakeRef::fromAttrs( + const fetchers::Settings & fetchSettings, + const fetchers::Attrs & attrs) { auto attrs2(attrs); attrs2.erase("dir"); return FlakeRef( - fetchers::Input::fromAttrs(std::move(attrs2)), + fetchers::Input::fromAttrs(fetchSettings, std::move(attrs2)), fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); } @@ -289,13 +294,16 @@ std::pair FlakeRef::fetchTree(ref store) const } std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir, bool allowMissing, bool isFlake) { auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(url); - auto [flakeRef, fragment] = parseFlakeRefWithFragment(std::string { prefix }, baseDir, allowMissing, isFlake); + auto [flakeRef, fragment] = parseFlakeRefWithFragment( + fetchSettings, + std::string { prefix }, baseDir, allowMissing, isFlake); return {std::move(flakeRef), fragment, std::move(extendedOutputsSpec)}; } diff --git a/src/libexpr/flake/flakeref.hh b/src/libflake/flake/flakeref.hh similarity index 84% rename from src/libexpr/flake/flakeref.hh rename to src/libflake/flake/flakeref.hh index 04c812ed0..1064538a7 100644 --- a/src/libexpr/flake/flakeref.hh +++ b/src/libflake/flake/flakeref.hh @@ -1,14 +1,12 @@ #pragma once ///@file +#include + #include "types.hh" -#include "hash.hh" #include "fetchers.hh" #include "outputs-spec.hh" -#include -#include - namespace nix { class Store; @@ -48,7 +46,7 @@ struct FlakeRef */ Path subdir; - bool operator==(const FlakeRef & other) const; + bool operator ==(const FlakeRef & other) const = default; FlakeRef(fetchers::Input && input, const Path & subdir) : input(std::move(input)), subdir(subdir) @@ -61,7 +59,9 @@ struct FlakeRef FlakeRef resolve(ref store) const; - static FlakeRef fromAttrs(const fetchers::Attrs & attrs); + static FlakeRef fromAttrs( + const fetchers::Settings & fetchSettings, + const fetchers::Attrs & attrs); std::pair fetchTree(ref store) const; }; @@ -72,6 +72,7 @@ std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ FlakeRef parseFlakeRef( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, @@ -81,12 +82,15 @@ FlakeRef parseFlakeRef( * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ std::optional maybeParseFlake( - const std::string & url, const std::optional & baseDir = {}); + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir = {}); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ std::pair parseFlakeRefWithFragment( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, @@ -96,12 +100,15 @@ std::pair parseFlakeRefWithFragment( * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ std::optional> maybeParseFlakeRefWithFragment( - const std::string & url, const std::optional & baseDir = {}); + const fetchers::Settings & fetchSettings, + const std::string & url, + const std::optional & baseDir = {}); /** * @param baseDir Optional [base directory](https://nixos.org/manual/nix/unstable/glossary#gloss-base-directory) */ std::tuple parseFlakeRefWithFragmentAndExtendedOutputsSpec( + const fetchers::Settings & fetchSettings, const std::string & url, const std::optional & baseDir = {}, bool allowMissing = false, diff --git a/src/libexpr/flake/lockfile.cc b/src/libflake/flake/lockfile.cc similarity index 93% rename from src/libexpr/flake/lockfile.cc rename to src/libflake/flake/lockfile.cc index d252214dd..70b60716f 100644 --- a/src/libexpr/flake/lockfile.cc +++ b/src/libflake/flake/lockfile.cc @@ -1,6 +1,7 @@ +#include + #include "lockfile.hh" #include "store-api.hh" -#include "url-parts.hh" #include #include @@ -8,9 +9,12 @@ #include #include +#include "strings.hh" + namespace nix::flake { -FlakeRef getFlakeRef( +static FlakeRef getFlakeRef( + const fetchers::Settings & fetchSettings, const nlohmann::json & json, const char * attr, const char * info) @@ -26,15 +30,17 @@ FlakeRef getFlakeRef( attrs.insert_or_assign(k.first, k.second); } } - return FlakeRef::fromAttrs(attrs); + return FlakeRef::fromAttrs(fetchSettings, attrs); } throw Error("attribute '%s' missing in lock file", attr); } -LockedNode::LockedNode(const nlohmann::json & json) - : lockedRef(getFlakeRef(json, "locked", "info")) // FIXME: remove "info" - , originalRef(getFlakeRef(json, "original", nullptr)) +LockedNode::LockedNode( + const fetchers::Settings & fetchSettings, + const nlohmann::json & json) + : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info" + , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) { if (!lockedRef.input.isLocked()) @@ -48,12 +54,13 @@ StorePath LockedNode::computeStorePath(Store & store) const } -static std::shared_ptr doFind(const ref& root, const InputPath & path, std::vector& visited) { +static std::shared_ptr doFind(const ref & root, const InputPath & path, std::vector & visited) +{ auto pos = root; auto found = std::find(visited.cbegin(), visited.cend(), path); - if(found != visited.end()) { + if (found != visited.end()) { std::vector cycle; std::transform(found, visited.cend(), std::back_inserter(cycle), printInputPath); cycle.push_back(printInputPath(path)); @@ -84,7 +91,9 @@ std::shared_ptr LockFile::findInput(const InputPath & path) return doFind(root, path, visited); } -LockFile::LockFile(std::string_view contents, std::string_view path) +LockFile::LockFile( + const fetchers::Settings & fetchSettings, + std::string_view contents, std::string_view path) { auto json = nlohmann::json::parse(contents); @@ -113,7 +122,7 @@ LockFile::LockFile(std::string_view contents, std::string_view path) auto jsonNode2 = nodes.find(inputKey); if (jsonNode2 == nodes.end()) throw Error("lock file references missing node '%s'", inputKey); - auto input = make_ref(*jsonNode2); + auto input = make_ref(fetchSettings, *jsonNode2); k = nodeMap.insert_or_assign(inputKey, input).first; getInputs(*input, *jsonNode2); } @@ -243,11 +252,6 @@ bool LockFile::operator ==(const LockFile & other) const return toJSON().first == other.toJSON().first; } -bool LockFile::operator !=(const LockFile & other) const -{ - return !(*this == other); -} - InputPath parseInputPath(std::string_view s) { InputPath path; diff --git a/src/libexpr/flake/lockfile.hh b/src/libflake/flake/lockfile.hh similarity index 86% rename from src/libexpr/flake/lockfile.hh rename to src/libflake/flake/lockfile.hh index 7e62e6d09..841931c11 100644 --- a/src/libexpr/flake/lockfile.hh +++ b/src/libflake/flake/lockfile.hh @@ -45,7 +45,9 @@ struct LockedNode : Node : lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake) { } - LockedNode(const nlohmann::json & json); + LockedNode( + const fetchers::Settings & fetchSettings, + const nlohmann::json & json); StorePath computeStorePath(Store & store) const; }; @@ -55,7 +57,9 @@ struct LockFile ref root = make_ref(); LockFile() {}; - LockFile(std::string_view contents, std::string_view path); + LockFile( + const fetchers::Settings & fetchSettings, + std::string_view contents, std::string_view path); typedef std::map, std::string> KeyMap; @@ -70,9 +74,6 @@ struct LockFile std::optional isUnlocked() const; bool operator ==(const LockFile & other) const; - // Needed for old gcc versions that don't synthesize it (like gcc 8.2.2 - // that is still the default on aarch64-linux) - bool operator !=(const LockFile & other) const; std::shared_ptr findInput(const InputPath & path); diff --git a/src/libflake/flake/nix-flake.pc.in b/src/libflake/flake/nix-flake.pc.in new file mode 100644 index 000000000..10c52f5e9 --- /dev/null +++ b/src/libflake/flake/nix-flake.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Requires: nix-util nix-store nix-expr +Libs: -L${libdir} -lnixflake +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libflake/flake/settings.cc b/src/libflake/flake/settings.cc new file mode 100644 index 000000000..6a0294e62 --- /dev/null +++ b/src/libflake/flake/settings.cc @@ -0,0 +1,7 @@ +#include "flake/settings.hh" + +namespace nix::flake { + +Settings::Settings() {} + +} diff --git a/src/libflake/flake/settings.hh b/src/libflake/flake/settings.hh new file mode 100644 index 000000000..fee247a7d --- /dev/null +++ b/src/libflake/flake/settings.hh @@ -0,0 +1,50 @@ +#pragma once +///@file + +#include "types.hh" +#include "config.hh" +#include "util.hh" + +#include +#include + +#include + +namespace nix::flake { + +struct Settings : public Config +{ + Settings(); + + Setting useRegistries{ + this, + true, + "use-registries", + "Whether to use flake registries to resolve flake references.", + {}, + true, + Xp::Flakes}; + + Setting acceptFlakeConfig{ + this, + false, + "accept-flake-config", + "Whether to accept nix configuration from a flake without prompting.", + {}, + true, + Xp::Flakes}; + + Setting commitLockFileSummary{ + this, + "", + "commit-lock-file-summary", + R"( + The commit summary to use when committing changed flake lock files. If + empty, the summary is generated based on the action performed. + )", + {"commit-lockfile-summary"}, + true, + Xp::Flakes}; +}; + +} diff --git a/src/libexpr/flake/url-name.cc b/src/libflake/flake/url-name.cc similarity index 100% rename from src/libexpr/flake/url-name.cc rename to src/libflake/flake/url-name.cc diff --git a/src/libexpr/flake/url-name.hh b/src/libflake/flake/url-name.hh similarity index 100% rename from src/libexpr/flake/url-name.hh rename to src/libflake/flake/url-name.hh diff --git a/src/libflake/local.mk b/src/libflake/local.mk new file mode 100644 index 000000000..5e604ef3a --- /dev/null +++ b/src/libflake/local.mk @@ -0,0 +1,22 @@ +libraries += libflake + +libflake_NAME = libnixflake + +libflake_DIR := $(d) + +libflake_SOURCES := $(wildcard $(d)/*.cc $(d)/flake/*.cc) + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libflake := -I $(d) + +libflake_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) + +libflake_LDFLAGS += $(THREAD_LDFLAGS) + +libflake_LIBS = libutil libstore libfetchers libexpr + +$(eval $(call install-file-in, $(buildprefix)$(d)/flake/nix-flake.pc, $(libdir)/pkgconfig, 0644)) + +$(foreach i, $(wildcard src/libflake/flake/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) diff --git a/src/libflake/meson.build b/src/libflake/meson.build new file mode 100644 index 000000000..d2bb179df --- /dev/null +++ b/src/libflake/meson.build @@ -0,0 +1,77 @@ +project('nix-flake', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-fetchers'), + dependency('nix-expr'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + '-include', 'config-expr.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'flake/config.cc', + 'flake/flake.cc', + 'flake/flakeref.cc', + 'flake/lockfile.cc', + 'flake/settings.cc', + 'flake/url-name.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'flake/flake.hh', + 'flake/flakeref.hh', + 'flake/lockfile.hh', + 'flake/settings.hh', + 'flake/url-name.hh', +) + +this_library = library( + 'nixflake', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libflake/package.nix b/src/libflake/package.nix new file mode 100644 index 000000000..851adf07e --- /dev/null +++ b/src/libflake/package.nix @@ -0,0 +1,78 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-store +, nix-fetchers +, nix-expr +, nlohmann_json +, libgit2 +, man + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-flake"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-store + nix-util + nix-fetchers + nix-expr + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libmain-c/.version b/src/libmain-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libmain-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libmain-c/build-utils-meson b/src/libmain-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libmain-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build new file mode 100644 index 000000000..345382712 --- /dev/null +++ b/src/libmain-c/meson.build @@ -0,0 +1,86 @@ +project('nix-main-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), + dependency('nix-main'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), + dependency('nix-store-c'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-main.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-main.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-main.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_main.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_main.h', +) + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixmainc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libmain-c/nix_api_main.cc b/src/libmain-c/nix_api_main.cc new file mode 100644 index 000000000..692d53f47 --- /dev/null +++ b/src/libmain-c/nix_api_main.cc @@ -0,0 +1,16 @@ +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +#include "plugin.hh" + +nix_err nix_init_plugins(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::initPlugins(); + } + NIXC_CATCH_ERRS +} diff --git a/src/libmain-c/nix_api_main.h b/src/libmain-c/nix_api_main.h new file mode 100644 index 000000000..3957b992f --- /dev/null +++ b/src/libmain-c/nix_api_main.h @@ -0,0 +1,40 @@ +#ifndef NIX_API_MAIN_H +#define NIX_API_MAIN_H +/** + * @defgroup libmain libmain + * @brief C bindings for nix libmain + * + * libmain has misc utilities for CLI commands + * @{ + */ +/** @file + * @brief Main entry for the libmain C bindings + */ + +#include "nix_api_util.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +/** + * @brief Loads the plugins specified in Nix's plugin-files setting. + * + * Call this once, after calling your desired init functions and setting + * relevant settings. + * + * @param[out] context Optional, stores error information + * @return NIX_OK if the initialization was successful, an error code otherwise. + */ +nix_err nix_init_plugins(nix_c_context * context); + +// cffi end +#ifdef __cplusplus +} +#endif +/** + * @} + */ +#endif // NIX_API_MAIN_H diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix new file mode 100644 index 000000000..ce6f67300 --- /dev/null +++ b/src/libmain-c/package.nix @@ -0,0 +1,79 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util-c +, nix-store +, nix-store-c +, nix-main + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-main-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util-c + nix-store + nix-store-c + nix-main + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libmain/.version b/src/libmain/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libmain/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libmain/build-utils-meson b/src/libmain/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libmain/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 5b49aaabc..768b2177c 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -1,9 +1,11 @@ #include "common-args.hh" #include "args/root.hh" +#include "config-global.hh" #include "globals.hh" #include "logging.hh" #include "loggers.hh" #include "util.hh" +#include "plugin.hh" namespace nix { diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc index 9829859de..a4e0530c8 100644 --- a/src/libmain/loggers.cc +++ b/src/libmain/loggers.cc @@ -36,7 +36,7 @@ Logger * makeDefaultLogger() { return logger; } default: - abort(); + unreachable(); } } diff --git a/src/libmain/meson.build b/src/libmain/meson.build new file mode 100644 index 000000000..7fcadf06d --- /dev/null +++ b/src/libmain/meson.build @@ -0,0 +1,101 @@ +project('nix-main', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +pubsetbuf_test = ''' +#include + +using namespace std; + +char buf[1024]; + +int main() { + cerr.rdbuf()->pubsetbuf(buf, sizeof(buf)); +} +''' + +configdata.set( + 'HAVE_PUBSETBUF', + cxx.compiles(pubsetbuf_test).to_int(), + description: 'Optionally used for buffering on standard error' +) + +config_h = configure_file( + configuration : configdata, + output : 'config-main.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-main.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'common-args.cc', + 'loggers.cc', + 'plugin.cc', + 'progress-bar.cc', + 'shared.cc', +) + +if host_machine.system() != 'windows' + sources += files( + 'unix/stack.cc', + ) +endif + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'common-args.hh', + 'loggers.hh', + 'plugin.hh', + 'progress-bar.hh', + 'shared.hh', +) + +this_library = library( + 'nixmain', + sources, + dependencies : deps_public + deps_private + deps_other, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libmain/package.nix b/src/libmain/package.nix new file mode 100644 index 000000000..47513dbdc --- /dev/null +++ b/src/libmain/package.nix @@ -0,0 +1,73 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, openssl + +, nix-util +, nix-store + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-main"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util + nix-store + openssl + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libmain/plugin.cc b/src/libmain/plugin.cc new file mode 100644 index 000000000..ccfd7f900 --- /dev/null +++ b/src/libmain/plugin.cc @@ -0,0 +1,119 @@ +#ifndef _WIN32 +# include +#endif + +#include + +#include "config-global.hh" +#include "signals.hh" + +namespace nix { + +struct PluginFilesSetting : public BaseSetting +{ + bool pluginsLoaded = false; + + PluginFilesSetting( + Config * options, + const Paths & def, + const std::string & name, + const std::string & description, + const std::set & aliases = {}) + : BaseSetting(def, true, name, description, aliases) + { + options->addSetting(this); + } + + Paths parse(const std::string & str) const override; +}; + +Paths PluginFilesSetting::parse(const std::string & str) const +{ + if (pluginsLoaded) + throw UsageError( + "plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); + return BaseSetting::parse(str); +} + +struct PluginSettings : Config +{ + PluginFilesSetting pluginFiles{ + this, + {}, + "plugin-files", + R"( + A list of plugin files to be loaded by Nix. Each of these files will + be dlopened by Nix. If they contain the symbol `nix_plugin_entry()`, + this symbol will be called. Alternatively, they can affect execution + through static initialization. In particular, these plugins may construct + static instances of RegisterPrimOp to add new primops or constants to the + expression language, RegisterStoreImplementation to add new store + implementations, RegisterCommand to add new subcommands to the `nix` + command, and RegisterSetting to add new nix config settings. See the + constructors for those types for more details. + + Warning! These APIs are inherently unstable and may change from + release to release. + + Since these files are loaded into the same address space as Nix + itself, they must be DSOs compatible with the instance of Nix + running at the time (i.e. compiled against the same headers, not + linked to any incompatible libraries). They should not be linked to + any Nix libs directly, as those will be available already at load + time. + + If an entry in the list is a directory, all files in the directory + are loaded as plugins (non-recursively). + )"}; +}; + +static PluginSettings pluginSettings; + +static GlobalConfig::Register rPluginSettings(&pluginSettings); + +void initPlugins() +{ + assert(!pluginSettings.pluginFiles.pluginsLoaded); + for (const auto & pluginFile : pluginSettings.pluginFiles.get()) { + std::vector pluginFiles; + try { + auto ents = std::filesystem::directory_iterator{pluginFile}; + for (const auto & ent : ents) { + checkInterrupt(); + pluginFiles.emplace_back(ent.path()); + } + } catch (std::filesystem::filesystem_error & e) { + if (e.code() != std::errc::not_a_directory) + throw; + pluginFiles.emplace_back(pluginFile); + } + for (const auto & file : pluginFiles) { + checkInterrupt(); + /* handle is purposefully leaked as there may be state in the + DSO needed by the action of the plugin. */ +#ifndef _WIN32 // TODO implement via DLL loading on Windows + void * handle = dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); + if (!handle) + throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); + + /* Older plugins use a statically initialized object to run their code. + Newer plugins can also export nix_plugin_entry() */ + void (*nix_plugin_entry)() = (void (*)()) dlsym(handle, "nix_plugin_entry"); + if (nix_plugin_entry) + nix_plugin_entry(); +#else + throw Error("could not dynamically open plugin file '%s'", file); +#endif + } + } + + /* Since plugins can add settings, try to re-apply previously + unknown settings. */ + globalConfig.reapplyUnknownSettings(); + globalConfig.warnUnknownSettings(); + + /* Tell the user if they try to set plugin-files after we've already loaded */ + pluginSettings.pluginFiles.pluginsLoaded = true; +} + +} diff --git a/src/libmain/plugin.hh b/src/libmain/plugin.hh new file mode 100644 index 000000000..4221c1b17 --- /dev/null +++ b/src/libmain/plugin.hh @@ -0,0 +1,12 @@ +#pragma once +///@file + +namespace nix { + +/** + * This should be called after settings are initialized, but before + * anything else + */ +void initPlugins(); + +} diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index ce45eae2b..22f890f7d 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -7,6 +7,7 @@ #include #include #include +#include #include #include @@ -76,6 +77,9 @@ private: bool haveUpdate = true; }; + /** Helps avoid unnecessary redraws, see `redraw()` */ + Sync lastOutput_; + Sync state_; std::thread updateThread; @@ -154,10 +158,10 @@ public: { auto state(state_.lock()); - std::stringstream oss; + std::ostringstream oss; showErrorInfo(oss, ei, loggerSettings.showTrace.get()); - log(*state, ei.level, oss.str()); + log(*state, ei.level, toView(oss)); } void log(State & state, Verbosity lvl, std::string_view s) @@ -358,6 +362,22 @@ public: updateCV.notify_one(); } + /** + * Redraw, if the output has changed. + * + * Excessive redrawing is noticable on slow terminals, and it interferes + * with text selection in some terminals, including libvte-based terminal + * emulators. + */ + void redraw(std::string newOutput) + { + auto lastOutput(lastOutput_.lock()); + if (newOutput != *lastOutput) { + writeToStderr(newOutput); + *lastOutput = std::move(newOutput); + } + } + std::chrono::milliseconds draw(State & state) { auto nextWakeup = std::chrono::milliseconds::max(); @@ -411,7 +431,7 @@ public: auto width = getWindowSize().second; if (width <= 0) width = std::numeric_limits::max(); - writeToStderr("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K"); + redraw("\r" + filterANSIEscapes(line, false, width) + ANSI_NORMAL + "\e[K"); return nextWakeup; } diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index c1c936248..50f90bfb3 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -8,7 +8,6 @@ #include "signals.hh" #include -#include #include #include @@ -23,6 +22,8 @@ #include +#include "exit.hh" +#include "strings.hh" namespace nix { @@ -320,6 +321,10 @@ void showManPage(const std::string & name) restoreProcessContext(); setEnv("MANPATH", settings.nixManDir.c_str()); execlp("man", "man", name.c_str(), nullptr); + if (errno == ENOENT) { + // Not SysError because we don't want to suffix the errno, aka No such file or directory. + throw Error("The '%1%' command was not found, but it is needed for '%2%' and some other '%3%' commands' help text. Perhaps you could install the '%1%' command?", "man", name.c_str(), "nix-*"); + } throw SysError("command 'man %1%' failed", name.c_str()); } @@ -411,7 +416,7 @@ RunPager::~RunPager() } #endif } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index aa44e1321..712b404d3 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -8,13 +8,9 @@ #include "common-args.hh" #include "path.hh" #include "derived-path.hh" -#include "exit.hh" #include -#include - - namespace nix { int handleExceptions(const std::string & programName, std::function fun); diff --git a/src/libstore-c/.version b/src/libstore-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libstore-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libstore-c/build-utils-meson b/src/libstore-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libstore-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build new file mode 100644 index 000000000..4bfd944c6 --- /dev/null +++ b/src/libstore-c/meson.build @@ -0,0 +1,85 @@ +project('nix-store-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-store'), +] +deps_public_maybe_subproject = [ + dependency('nix-util-c'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-store.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + '-include', 'config-store.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + '-include', 'config-store.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_store.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_store.h', +) + +# TODO don't install this once tests don't use it and/or move the header into `libstore`, non-`c` +headers += files('nix_api_store_internal.h') + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixstorec', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index 4fe25c7d4..fb7391276 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -29,16 +29,6 @@ nix_err nix_libstore_init_no_load_config(nix_c_context * context) NIXC_CATCH_ERRS } -nix_err nix_init_plugins(nix_c_context * context) -{ - if (context) - context->last_err_code = NIX_OK; - try { - nix::initPlugins(); - } - NIXC_CATCH_ERRS -} - Store * nix_store_open(nix_c_context * context, const char * uri, const char *** params) { if (context) @@ -154,3 +144,15 @@ StorePath * nix_store_path_clone(const StorePath * p) { return new StorePath{p->path}; } + +nix_err nix_store_copy_closure(nix_c_context * context, Store * srcStore, Store * dstStore, StorePath * path) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::RealisedPath::Set paths; + paths.insert(path->path); + nix::copyClosure(*srcStore->ptr, *dstStore->ptr, paths); + } + NIXC_CATCH_ERRS +} diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index 209f91f0d..93208cb7c 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -43,27 +43,20 @@ nix_err nix_libstore_init(nix_c_context * context); nix_err nix_libstore_init_no_load_config(nix_c_context * context); /** - * @brief Loads the plugins specified in Nix's plugin-files setting. + * @brief Open a nix store. * - * Call this once, after calling your desired init functions and setting - * relevant settings. - * - * @param[out] context Optional, stores error information - * @return NIX_OK if the initialization was successful, an error code otherwise. - */ -nix_err nix_init_plugins(nix_c_context * context); - -/** - * @brief Open a nix store * Store instances may share state and resources behind the scenes. + * * @param[out] context Optional, stores error information - * @param[in] uri URI of the nix store, copied - * @param[in] params optional, array of key-value pairs, {{"endpoint", - * "https://s3.local"}} + * @param[in] uri URI of the Nix store, copied. See [*Store URL format* in the Nix Reference + * Manual](https://nixos.org/manual/nix/stable/store/types/#store-url-format). + * @param[in] params optional, null-terminated array of key-value pairs, e.g. {{"endpoint", + * "https://s3.local"}}. See [*Store Types* in the Nix Reference + * Manual](https://nixos.org/manual/nix/stable/store/types). * @return a Store pointer, NULL in case of errors * @see nix_store_free */ -Store * nix_store_open(nix_c_context *, const char * uri, const char *** params); +Store * nix_store_open(nix_c_context * context, const char * uri, const char *** params); /** * @brief Deallocate a nix store and free any resources if not also held by other Store instances. @@ -155,7 +148,9 @@ nix_err nix_store_realise( /** * @brief get the version of a nix store. + * * If the store doesn't have a version (like the dummy store), returns an empty string. + * * @param[out] context Optional, stores error information * @param[in] store nix store reference * @param[in] callback Called with the version. @@ -166,6 +161,16 @@ nix_err nix_store_realise( nix_err nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data); +/** + * @brief Copy the closure of `path` from `srcStore` to `dstStore`. + * + * @param[out] context Optional, stores error information + * @param[in] srcStore nix source store reference + * @param[in] srcStore nix destination store reference + * @param[in] path Path to copy + */ +nix_err nix_store_copy_closure(nix_c_context * context, Store * srcStore, Store * dstStore, StorePath * path); + // cffi end #ifdef __cplusplus } diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix new file mode 100644 index 000000000..e4f372236 --- /dev/null +++ b/src/libstore-c/package.nix @@ -0,0 +1,75 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util-c +, nix-store + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-store-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util-c + nix-store + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libstore/.version b/src/libstore/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libstore/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 5153ca64f..e8c8892b3 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -18,6 +18,7 @@ #include #include #include +#include #include @@ -321,7 +322,7 @@ StorePath BinaryCacheStore::addToStoreFromDump( if (static_cast(dumpMethod) == hashMethod.getFileIngestionMethod()) caHash = hashString(HashAlgorithm::SHA256, dump2.s); switch (dumpMethod) { - case FileSerialisationMethod::Recursive: + case FileSerialisationMethod::NixArchive: // The dump is already NAR in this case, just use it. nar = dump2.s; break; @@ -338,7 +339,7 @@ StorePath BinaryCacheStore::addToStoreFromDump( } else { // Otherwise, we have to do th same hashing as NAR so our single // hash will suffice for both purposes. - if (dumpMethod != FileSerialisationMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) + if (dumpMethod != FileSerialisationMethod::NixArchive || hashAlgo != HashAlgorithm::SHA256) unsupported("addToStoreFromDump"); } StringSource narDump { nar }; @@ -453,7 +454,7 @@ StorePath BinaryCacheStore::addToStore( non-recursive+sha256 so we can just use the default implementation of this method in terms of addToStoreFromDump. */ - auto h = hashPath(path, method.getFileIngestionMethod(), hashAlgo, filter); + auto h = hashPath(path, method.getFileIngestionMethod(), hashAlgo, filter).first; auto source = sinkToSource([&](Sink & sink) { path.dumpPath(sink, filter); diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc index 18f519c5c..96cbfd62f 100644 --- a/src/libstore/build-result.cc +++ b/src/libstore/build-result.cc @@ -2,17 +2,7 @@ namespace nix { -GENERATE_CMP_EXT( - , - BuildResult, - me->status, - me->errorMsg, - me->timesBuilt, - me->isNonDeterministic, - me->builtOutputs, - me->startTime, - me->stopTime, - me->cpuUser, - me->cpuSystem); +bool BuildResult::operator==(const BuildResult &) const noexcept = default; +std::strong_ordering BuildResult::operator<=>(const BuildResult &) const noexcept = default; } diff --git a/src/libstore/build-result.hh b/src/libstore/build-result.hh index 3636ad3a4..8c66cfeb3 100644 --- a/src/libstore/build-result.hh +++ b/src/libstore/build-result.hh @@ -3,7 +3,6 @@ #include "realisation.hh" #include "derived-path.hh" -#include "comparator.hh" #include #include @@ -101,7 +100,8 @@ struct BuildResult */ std::optional cpuUser, cpuSystem; - DECLARE_CMP(BuildResult); + bool operator ==(const BuildResult &) const noexcept; + std::strong_ordering operator <=>(const BuildResult &) const noexcept; bool success() { diff --git a/src/libstore/build-utils-meson b/src/libstore/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libstore/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libstore/unix/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc similarity index 91% rename from src/libstore/unix/build/derivation-goal.cc rename to src/libstore/build/derivation-goal.cc index 89518b055..34ed16a38 100644 --- a/src/libstore/unix/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1,5 +1,9 @@ #include "derivation-goal.hh" -#include "hook-instance.hh" +#ifndef _WIN32 // TODO enable build hook on Windows +# include "hook-instance.hh" +#endif +#include "processes.hh" +#include "config-global.hh" #include "worker.hh" #include "builtins.hh" #include "builtins/buildenv.hh" @@ -19,24 +23,27 @@ #include #include -#include -#include -#include -#include #include -#include #include -#include -#include -#include -#include -#include +#ifndef _WIN32 // TODO abstract over proc exit status +# include +#endif #include +#include "strings.hh" + namespace nix { +Goal::Co DerivationGoal::init() { + if (useDerivation) { + co_return getDerivation(); + } else { + co_return haveDerivation(); + } +} + DerivationGoal::DerivationGoal(const StorePath & drvPath, const OutputsSpec & wantedOutputs, Worker & worker, BuildMode buildMode) : Goal(worker, DerivedPath::Built { .drvPath = makeConstantStorePathRef(drvPath), .outputs = wantedOutputs }) @@ -45,7 +52,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, , wantedOutputs(wantedOutputs) , buildMode(buildMode) { - state = &DerivationGoal::getDerivation; name = fmt( "building of '%s' from .drv file", DerivedPath::Built { makeConstantStorePathRef(drvPath), wantedOutputs }.to_string(worker.store)); @@ -66,7 +72,6 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation { this->drv = std::make_unique(drv); - state = &DerivationGoal::haveDerivation; name = fmt( "building of '%s' from in-memory derivation", DerivedPath::Built { makeConstantStorePathRef(drvPath), drv.outputNames() }.to_string(worker.store)); @@ -85,7 +90,7 @@ DerivationGoal::~DerivationGoal() { /* Careful: we should never ever throw an exception from a destructor. */ - try { closeLogFile(); } catch (...) { ignoreException(); } + try { closeLogFile(); } catch (...) { ignoreExceptionInDestructor(); } } @@ -101,20 +106,18 @@ std::string DerivationGoal::key() void DerivationGoal::killChild() { +#ifndef _WIN32 // TODO enable build hook on Windows hook.reset(); +#endif } void DerivationGoal::timedOut(Error && ex) { killChild(); - done(BuildResult::TimedOut, {}, std::move(ex)); -} - - -void DerivationGoal::work() -{ - (this->*state)(); + // We're not inside a coroutine, hence we can't use co_return here. + // Thus we ignore the return value. + [[maybe_unused]] Done _ = done(BuildResult::TimedOut, {}, std::move(ex)); } void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) @@ -138,7 +141,7 @@ void DerivationGoal::addWantedOutputs(const OutputsSpec & outputs) } -void DerivationGoal::getDerivation() +Goal::Co DerivationGoal::getDerivation() { trace("init"); @@ -146,23 +149,22 @@ void DerivationGoal::getDerivation() exists. If it doesn't, it may be created through a substitute. */ if (buildMode == bmNormal && worker.evalStore.isValidPath(drvPath)) { - loadDerivation(); - return; + co_return loadDerivation(); } addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath))); - state = &DerivationGoal::loadDerivation; + co_await Suspend{}; + co_return loadDerivation(); } -void DerivationGoal::loadDerivation() +Goal::Co DerivationGoal::loadDerivation() { trace("loading derivation"); if (nrFailed != 0) { - done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); - return; + co_return done(BuildResult::MiscFailure, {}, Error("cannot build missing derivation '%s'", worker.store.printStorePath(drvPath))); } /* `drvPath' should already be a root, but let's be on the safe @@ -184,11 +186,11 @@ void DerivationGoal::loadDerivation() } assert(drv); - haveDerivation(); + co_return haveDerivation(); } -void DerivationGoal::haveDerivation() +Goal::Co DerivationGoal::haveDerivation() { trace("have derivation"); @@ -216,8 +218,7 @@ void DerivationGoal::haveDerivation() }); } - gaveUpOnSubstitution(); - return; + co_return gaveUpOnSubstitution(); } for (auto & i : drv->outputsAndOptPaths(worker.store)) @@ -239,8 +240,7 @@ void DerivationGoal::haveDerivation() /* If they are all valid, then we're done. */ if (allValid && buildMode == bmNormal) { - done(BuildResult::AlreadyValid, std::move(validOutputs)); - return; + co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } /* We are first going to try to create the invalid output paths @@ -267,24 +267,21 @@ void DerivationGoal::haveDerivation() } } - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - outputsSubstitutionTried(); - else - state = &DerivationGoal::outputsSubstitutionTried; + if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */ + co_return outputsSubstitutionTried(); } -void DerivationGoal::outputsSubstitutionTried() +Goal::Co DerivationGoal::outputsSubstitutionTried() { trace("all outputs substituted (maybe)"); assert(!drv->type().isImpure()); if (nrFailed > 0 && nrFailed > nrNoSubstituters + nrIncompleteClosure && !settings.tryFallback) { - done(BuildResult::TransientFailure, {}, + co_return done(BuildResult::TransientFailure, {}, Error("some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", worker.store.printStorePath(drvPath))); - return; } /* If the substitutes form an incomplete closure, then we should @@ -318,32 +315,29 @@ void DerivationGoal::outputsSubstitutionTried() if (needRestart == NeedRestartForMoreOutputs::OutputsAddedDoNeed) { needRestart = NeedRestartForMoreOutputs::OutputsUnmodifedDontNeed; - haveDerivation(); - return; + co_return haveDerivation(); } auto [allValid, validOutputs] = checkPathValidity(); if (buildMode == bmNormal && allValid) { - done(BuildResult::Substituted, std::move(validOutputs)); - return; + co_return done(BuildResult::Substituted, std::move(validOutputs)); } if (buildMode == bmRepair && allValid) { - repairClosure(); - return; + co_return repairClosure(); } if (buildMode == bmCheck && !allValid) throw Error("some outputs of '%s' are not valid, so checking is not possible", worker.store.printStorePath(drvPath)); /* Nothing to wait for; tail call */ - gaveUpOnSubstitution(); + co_return gaveUpOnSubstitution(); } /* At least one of the output paths could not be produced using a substitute. So we have to build instead. */ -void DerivationGoal::gaveUpOnSubstitution() +Goal::Co DerivationGoal::gaveUpOnSubstitution() { /* At this point we are building all outputs, so if more are wanted there is no need to restart. */ @@ -404,14 +398,12 @@ void DerivationGoal::gaveUpOnSubstitution() addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i))); } - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - inputsRealised(); - else - state = &DerivationGoal::inputsRealised; + if (!waitees.empty()) co_await Suspend{}; /* to prevent hang (no wake-up event) */ + co_return inputsRealised(); } -void DerivationGoal::repairClosure() +Goal::Co DerivationGoal::repairClosure() { assert(!drv->type().isImpure()); @@ -465,41 +457,39 @@ void DerivationGoal::repairClosure() } if (waitees.empty()) { - done(BuildResult::AlreadyValid, assertPathValidity()); - return; + co_return done(BuildResult::AlreadyValid, assertPathValidity()); + } else { + co_await Suspend{}; + co_return closureRepaired(); } - - state = &DerivationGoal::closureRepaired; } -void DerivationGoal::closureRepaired() +Goal::Co DerivationGoal::closureRepaired() { trace("closure repaired"); if (nrFailed > 0) throw Error("some paths in the output closure of derivation '%s' could not be repaired", worker.store.printStorePath(drvPath)); - done(BuildResult::AlreadyValid, assertPathValidity()); + co_return done(BuildResult::AlreadyValid, assertPathValidity()); } -void DerivationGoal::inputsRealised() +Goal::Co DerivationGoal::inputsRealised() { trace("all inputs realised"); if (nrFailed != 0) { if (!useDerivation) throw Error("some dependencies of '%s' are missing", worker.store.printStorePath(drvPath)); - done(BuildResult::DependencyFailed, {}, Error( + co_return done(BuildResult::DependencyFailed, {}, Error( "%s dependencies of derivation '%s' failed to build", nrFailed, worker.store.printStorePath(drvPath))); - return; } if (retrySubstitution == RetrySubstitution::YesNeed) { retrySubstitution = RetrySubstitution::AlreadyRetried; - haveDerivation(); - return; + co_return haveDerivation(); } /* Gather information necessary for computing the closure and/or @@ -565,8 +555,8 @@ void DerivationGoal::inputsRealised() pathResolved, wantedOutputs, buildMode); addWaitee(resolvedDrvGoal); - state = &DerivationGoal::resolvedFinished; - return; + co_await Suspend{}; + co_return resolvedFinished(); } std::function::ChildNode &)> accumInputPaths; @@ -630,8 +620,9 @@ void DerivationGoal::inputsRealised() /* Okay, try to build. Note that here we don't wait for a build slot to become available, since we don't need one if there is a build hook. */ - state = &DerivationGoal::tryToBuild; worker.wakeUp(shared_from_this()); + co_await Suspend{}; + co_return tryToBuild(); } void DerivationGoal::started() @@ -641,14 +632,22 @@ void DerivationGoal::started() buildMode == bmCheck ? "checking outputs of '%s'" : "building '%s'", worker.store.printStorePath(drvPath)); fmt("building '%s'", worker.store.printStorePath(drvPath)); +#ifndef _WIN32 // TODO enable build hook on Windows if (hook) msg += fmt(" on '%s'", machineName); +#endif act = std::make_unique(*logger, lvlInfo, actBuild, msg, - Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", 1, 1}); + Logger::Fields{worker.store.printStorePath(drvPath), +#ifndef _WIN32 // TODO enable build hook on Windows + hook ? machineName : +#endif + "", + 1, + 1}); mcRunningBuilds = std::make_unique>(worker.runningBuilds); worker.updateProgress(); } -void DerivationGoal::tryToBuild() +Goal::Co DerivationGoal::tryToBuild() { trace("trying to build"); @@ -684,7 +683,8 @@ void DerivationGoal::tryToBuild() actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); worker.waitForAWhile(shared_from_this()); - return; + co_await Suspend{}; + co_return tryToBuild(); } actLock.reset(); @@ -701,8 +701,7 @@ void DerivationGoal::tryToBuild() if (buildMode != bmCheck && allValid) { debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); outputLocks.setDeletion(true); - done(BuildResult::AlreadyValid, std::move(validOutputs)); - return; + co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); } /* If any of the outputs already exist but are not valid, delete @@ -728,9 +727,9 @@ void DerivationGoal::tryToBuild() EOF from the hook. */ actLock.reset(); buildResult.startTime = time(0); // inexact - state = &DerivationGoal::buildDone; started(); - return; + co_await Suspend{}; + co_return buildDone(); case rpPostpone: /* Not now; wait until at least one child finishes or the wake-up timeout expires. */ @@ -739,7 +738,8 @@ void DerivationGoal::tryToBuild() fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); outputLocks.unlock(); - return; + co_await Suspend{}; + co_return tryToBuild(); case rpDecline: /* We should do it ourselves. */ break; @@ -748,11 +748,12 @@ void DerivationGoal::tryToBuild() actLock.reset(); - state = &DerivationGoal::tryLocalBuild; worker.wakeUp(shared_from_this()); + co_await Suspend{}; + co_return tryLocalBuild(); } -void DerivationGoal::tryLocalBuild() { +Goal::Co DerivationGoal::tryLocalBuild() { throw Error( R"( Unable to build with a primary store that isn't a local store; @@ -778,7 +779,13 @@ static void movePath(const Path & src, const Path & dst) { auto st = lstat(src); - bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); + bool changePerm = ( +#ifndef _WIN32 + geteuid() +#else + !isRootUser() +#endif + && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR)); if (changePerm) chmod_(src, st.st_mode | S_IWUSR); @@ -796,7 +803,7 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath) tmpPath (the replacement), so we have to move it out of the way first. We'd better not be interrupted here, because if we're repairing (say) Glibc, we end up with a broken system. */ - Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), random()); + Path oldPath = fmt("%1%.old-%2%-%3%", storePath, getpid(), rand()); if (pathExists(storePath)) movePath(storePath, oldPath); @@ -807,7 +814,7 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath) // attempt to recover movePath(oldPath, storePath); } catch (...) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } throw; } @@ -818,14 +825,20 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath) int DerivationGoal::getChildStatus() { +#ifndef _WIN32 // TODO enable build hook on Windows return hook->pid.kill(); +#else + return 0; +#endif } void DerivationGoal::closeReadPipes() { - hook->builderOut.readSide = -1; - hook->fromHook.readSide = -1; +#ifndef _WIN32 // TODO enable build hook on Windows + hook->builderOut.readSide.close(); + hook->fromHook.readSide.close(); +#endif } @@ -917,7 +930,7 @@ void runPostBuildHook( }); } -void DerivationGoal::buildDone() +Goal::Co DerivationGoal::buildDone() { trace("build done"); @@ -1012,20 +1025,23 @@ void DerivationGoal::buildDone() outputLocks.setDeletion(true); outputLocks.unlock(); - done(BuildResult::Built, std::move(builtOutputs)); + co_return done(BuildResult::Built, std::move(builtOutputs)); } catch (BuildError & e) { outputLocks.unlock(); BuildResult::Status st = BuildResult::MiscFailure; +#ifndef _WIN32 // TODO abstract over proc exit status if (hook && WIFEXITED(status) && WEXITSTATUS(status) == 101) st = BuildResult::TimedOut; else if (hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100)) { } - else { + else +#endif + { assert(derivationType); st = dynamic_cast(&e) ? BuildResult::NotDeterministic : @@ -1034,12 +1050,11 @@ void DerivationGoal::buildDone() BuildResult::PermanentFailure; } - done(st, {}, std::move(e)); - return; + co_return done(st, {}, std::move(e)); } } -void DerivationGoal::resolvedFinished() +Goal::Co DerivationGoal::resolvedFinished() { trace("resolved derivation finished"); @@ -1107,11 +1122,14 @@ void DerivationGoal::resolvedFinished() if (status == BuildResult::AlreadyValid) status = BuildResult::ResolvesToAlreadyValid; - done(status, std::move(builtOutputs)); + co_return done(status, std::move(builtOutputs)); } HookReply DerivationGoal::tryBuildHook() { +#ifdef _WIN32 // TODO enable build hook on Windows + return rpDecline; +#else if (settings.buildHook.get().empty() || !worker.tryBuildHook || !useDerivation) return rpDecline; if (!worker.hook) @@ -1205,17 +1223,18 @@ HookReply DerivationGoal::tryBuildHook() } hook->sink = FdSink(); - hook->toHook.writeSide = -1; + hook->toHook.writeSide.close(); /* Create the log file and pipe. */ Path logFile = openLogFile(); - std::set fds; + std::set fds; fds.insert(hook->fromHook.readSide.get()); fds.insert(hook->builderOut.readSide.get()); worker.childStarted(shared_from_this(), fds, false, false); return rpAccept; +#endif } @@ -1251,7 +1270,11 @@ Path DerivationGoal::openLogFile() Path logFileName = fmt("%s/%s%s", dir, baseName.substr(2), settings.compressLog ? ".bz2" : ""); - fdLogFile = open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC | O_CLOEXEC, 0666); + fdLogFile = toDescriptor(open(logFileName.c_str(), O_CREAT | O_WRONLY | O_TRUNC +#ifndef _WIN32 + | O_CLOEXEC +#endif + , 0666)); if (!fdLogFile) throw SysError("creating log file '%1%'", logFileName); logFileSink = std::make_shared(fdLogFile.get()); @@ -1271,16 +1294,20 @@ void DerivationGoal::closeLogFile() if (logSink2) logSink2->finish(); if (logFileSink) logFileSink->flush(); logSink = logFileSink = 0; - fdLogFile = -1; + fdLogFile.close(); } -bool DerivationGoal::isReadDesc(int fd) +bool DerivationGoal::isReadDesc(Descriptor fd) { +#ifdef _WIN32 // TODO enable build hook on Windows + return false; +#else return fd == hook->builderOut.readSide.get(); +#endif } -void DerivationGoal::handleChildOutput(int fd, std::string_view data) +void DerivationGoal::handleChildOutput(Descriptor fd, std::string_view data) { // local & `ssh://`-builds are dealt with here. auto isWrittenToLog = isReadDesc(fd); @@ -1289,7 +1316,9 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data) logSize += data.size(); if (settings.maxLogSize && logSize > settings.maxLogSize) { killChild(); - done( + // We're not inside a coroutine, hence we can't use co_return here. + // Thus we ignore the return value. + [[maybe_unused]] Done _ = done( BuildResult::LogLimitExceeded, {}, Error("%s killed after writing more than %d bytes of log output", getName(), settings.maxLogSize)); @@ -1310,6 +1339,7 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data) if (logSink) (*logSink)(data); } +#ifndef _WIN32 // TODO enable build hook on Windows if (hook && fd == hook->fromHook.readSide.get()) { for (auto c : data) if (c == '\n') { @@ -1344,10 +1374,11 @@ void DerivationGoal::handleChildOutput(int fd, std::string_view data) } else currentHookLine += c; } +#endif } -void DerivationGoal::handleEOF(int fd) +void DerivationGoal::handleEOF(Descriptor fd) { if (!currentLogLine.empty()) flushLine(); worker.wakeUp(shared_from_this()); @@ -1493,7 +1524,7 @@ SingleDrvOutputs DerivationGoal::assertPathValidity() } -void DerivationGoal::done( +Goal::Done DerivationGoal::done( BuildResult::Status status, SingleDrvOutputs builtOutputs, std::optional ex) @@ -1530,7 +1561,7 @@ void DerivationGoal::done( fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; } - amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); + return amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); } @@ -1538,7 +1569,7 @@ void DerivationGoal::waiteeDone(GoalPtr waitee, ExitCode result) { Goal::waiteeDone(waitee, result); - if (!useDerivation) return; + if (!useDerivation || !drv) return; auto & fullDrv = *dynamic_cast(drv.get()); auto * dg = dynamic_cast(&*waitee); diff --git a/src/libstore/unix/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh similarity index 92% rename from src/libstore/unix/build/derivation-goal.hh rename to src/libstore/build/derivation-goal.hh index ddb5ee1e3..ad3d9ca2a 100644 --- a/src/libstore/unix/build/derivation-goal.hh +++ b/src/libstore/build/derivation-goal.hh @@ -2,7 +2,9 @@ ///@file #include "parsed-derivations.hh" -#include "lock.hh" +#ifndef _WIN32 +# include "user-lock.hh" +#endif #include "outputs-spec.hh" #include "store-api.hh" #include "pathlocks.hh" @@ -12,7 +14,9 @@ namespace nix { using std::map; +#ifndef _WIN32 // TODO enable build hook on Windows struct HookInstance; +#endif typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; @@ -178,19 +182,18 @@ struct DerivationGoal : public Goal std::string currentHookLine; +#ifndef _WIN32 // TODO enable build hook on Windows /** * The build hook. */ std::unique_ptr hook; +#endif /** * The sort of derivation we are building. */ std::optional derivationType; - typedef void (DerivationGoal::*GoalState)(); - GoalState state; - BuildMode buildMode; std::unique_ptr> mcExpectedBuilds, mcRunningBuilds; @@ -221,8 +224,6 @@ struct DerivationGoal : public Goal std::string key() override; - void work() override; - /** * Add wanted outputs to an already existing derivation goal. */ @@ -231,18 +232,19 @@ struct DerivationGoal : public Goal /** * The states. */ - void getDerivation(); - void loadDerivation(); - void haveDerivation(); - void outputsSubstitutionTried(); - void gaveUpOnSubstitution(); - void closureRepaired(); - void inputsRealised(); - void tryToBuild(); - virtual void tryLocalBuild(); - void buildDone(); + Co init() override; + Co getDerivation(); + Co loadDerivation(); + Co haveDerivation(); + Co outputsSubstitutionTried(); + Co gaveUpOnSubstitution(); + Co closureRepaired(); + Co inputsRealised(); + Co tryToBuild(); + virtual Co tryLocalBuild(); + Co buildDone(); - void resolvedFinished(); + Co resolvedFinished(); /** * Is the build hook willing to perform the build? @@ -287,13 +289,13 @@ struct DerivationGoal : public Goal virtual void cleanupPostOutputsRegisteredModeCheck(); virtual void cleanupPostOutputsRegisteredModeNonCheck(); - virtual bool isReadDesc(int fd); + virtual bool isReadDesc(Descriptor fd); /** * Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, std::string_view data) override; - void handleEOF(int fd) override; + void handleChildOutput(Descriptor fd, std::string_view data) override; + void handleEOF(Descriptor fd) override; void flushLine(); /** @@ -323,11 +325,11 @@ struct DerivationGoal : public Goal */ virtual void killChild(); - void repairClosure(); + Co repairClosure(); void started(); - void done( + Done done( BuildResult::Status status, SingleDrvOutputs builtOutputs = {}, std::optional ex = {}); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc new file mode 100644 index 000000000..dedcad2b1 --- /dev/null +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -0,0 +1,161 @@ +#include "drv-output-substitution-goal.hh" +#include "finally.hh" +#include "worker.hh" +#include "substitution-goal.hh" +#include "callback.hh" + +namespace nix { + +DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal( + const DrvOutput & id, + Worker & worker, + RepairFlag repair, + std::optional ca) + : Goal(worker, DerivedPath::Opaque { StorePath::dummy }) + , id(id) +{ + name = fmt("substitution of '%s'", id.to_string()); + trace("created"); +} + + +Goal::Co DrvOutputSubstitutionGoal::init() +{ + trace("init"); + + /* If the derivation already exists, we’re done */ + if (worker.store.queryRealisation(id)) { + co_return amDone(ecSuccess); + } + + auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); + + bool substituterFailed = false; + + for (auto sub : subs) { + trace("trying next substituter"); + + /* The callback of the curl download below can outlive `this` (if + some other error occurs), so it must not touch `this`. So put + the shared state in a separate refcounted object. */ + auto outPipe = std::make_shared(); + #ifndef _WIN32 + outPipe->create(); + #else + outPipe->createAsyncPipe(worker.ioport.get()); + #endif + + auto promise = std::make_shared>>(); + + sub->queryRealisation( + id, + { [outPipe(outPipe), promise(promise)](std::future> res) { + try { + Finally updateStats([&]() { outPipe->writeSide.close(); }); + promise->set_value(res.get()); + } catch (...) { + promise->set_exception(std::current_exception()); + } + } }); + + worker.childStarted(shared_from_this(), { + #ifndef _WIN32 + outPipe->readSide.get() + #else + &*outPipe + #endif + }, true, false); + + co_await Suspend{}; + + worker.childTerminated(this); + + /* + * The realisation corresponding to the given output id. + * Will be filled once we can get it. + */ + std::shared_ptr outputInfo; + + try { + outputInfo = promise->get_future().get(); + } catch (std::exception & e) { + printError(e.what()); + substituterFailed = true; + } + + if (!outputInfo) continue; + + bool failed = false; + + for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { + if (depId != id) { + if (auto localOutputInfo = worker.store.queryRealisation(depId); + localOutputInfo && localOutputInfo->outPath != depPath) { + warn( + "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" + "Local: %s\n" + "Remote: %s", + sub->getUri(), + depId.to_string(), + worker.store.printStorePath(localOutputInfo->outPath), + worker.store.printStorePath(depPath) + ); + failed = true; + break; + } + addWaitee(worker.makeDrvOutputSubstitutionGoal(depId)); + } + } + + if (failed) continue; + + co_return realisationFetched(outputInfo, sub); + } + + /* None left. Terminate this goal and let someone else deal + with it. */ + debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); + + if (substituterFailed) { + worker.failedSubstitutions++; + worker.updateProgress(); + } + + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + co_return amDone(substituterFailed ? ecFailed : ecNoSubstituters); +} + +Goal::Co DrvOutputSubstitutionGoal::realisationFetched(std::shared_ptr outputInfo, nix::ref sub) { + addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath)); + + if (!waitees.empty()) co_await Suspend{}; + + trace("output path substituted"); + + if (nrFailed > 0) { + debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); + co_return amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); + } + + worker.store.registerDrvOutput(*outputInfo); + + trace("finished"); + co_return amDone(ecSuccess); +} + +std::string DrvOutputSubstitutionGoal::key() +{ + /* "a$" ensures substitution goals happen before derivation + goals. */ + return "a$" + std::string(id.to_string()); +} + +void DrvOutputSubstitutionGoal::handleEOF(Descriptor fd) +{ + worker.wakeUp(shared_from_this()); +} + + +} diff --git a/src/libstore/unix/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh similarity index 51% rename from src/libstore/unix/build/drv-output-substitution-goal.hh rename to src/libstore/build/drv-output-substitution-goal.hh index da2426e5e..8c60d0198 100644 --- a/src/libstore/unix/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -1,11 +1,13 @@ #pragma once ///@file +#include +#include + #include "store-api.hh" #include "goal.hh" #include "realisation.hh" -#include -#include +#include "muxable-pipe.hh" namespace nix { @@ -25,53 +27,20 @@ class DrvOutputSubstitutionGoal : public Goal { */ DrvOutput id; - /** - * The realisation corresponding to the given output id. - * Will be filled once we can get it. - */ - std::shared_ptr outputInfo; - - /** - * The remaining substituters. - */ - std::list> subs; - - /** - * The current substituter. - */ - std::shared_ptr sub; - - struct DownloadState - { - Pipe outPipe; - std::promise> promise; - }; - - std::shared_ptr downloadState; - - /** - * Whether a substituter failed. - */ - bool substituterFailed = false; - public: DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); typedef void (DrvOutputSubstitutionGoal::*GoalState)(); GoalState state; - void init(); - void tryNext(); - void realisationFetched(); - void outPathValid(); - void finished(); + Co init() override; + Co realisationFetched(std::shared_ptr outputInfo, nix::ref sub); - void timedOut(Error && ex) override { abort(); }; + void timedOut(Error && ex) override { unreachable(); }; std::string key() override; - void work() override; - void handleEOF(int fd) override; + void handleEOF(Descriptor fd) override; JobCategory jobCategory() const override { return JobCategory::Substitution; diff --git a/src/libstore/unix/build/entry-points.cc b/src/libstore/build/entry-points.cc similarity index 90% rename from src/libstore/unix/build/entry-points.cc rename to src/libstore/build/entry-points.cc index d4bead28e..4c1373bfa 100644 --- a/src/libstore/unix/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -1,7 +1,10 @@ #include "worker.hh" #include "substitution-goal.hh" -#include "derivation-goal.hh" +#ifndef _WIN32 // TODO Enable building on Windows +# include "derivation-goal.hh" +#endif #include "local-store.hh" +#include "strings.hh" namespace nix { @@ -25,9 +28,12 @@ void Store::buildPaths(const std::vector & reqs, BuildMode buildMod ex = std::move(i->ex); } if (i->exitCode != Goal::ecSuccess) { +#ifndef _WIN32 // TODO Enable building on Windows if (auto i2 = dynamic_cast(i.get())) failed.insert(printStorePath(i2->drvPath)); - else if (auto i2 = dynamic_cast(i.get())) + else +#endif + if (auto i2 = dynamic_cast(i.get())) failed.insert(printStorePath(i2->storePath)); } } @@ -74,7 +80,12 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat BuildMode buildMode) { Worker worker(*this, *this); +#ifndef _WIN32 // TODO Enable building on Windows auto goal = worker.makeBasicDerivationGoal(drvPath, drv, OutputsSpec::All {}, buildMode); +#else + std::shared_ptr goal; + throw UnimplementedError("Building derivations not yet implemented on windows."); +#endif try { worker.run(Goals{goal}); diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc new file mode 100644 index 000000000..9a16da145 --- /dev/null +++ b/src/libstore/build/goal.cc @@ -0,0 +1,219 @@ +#include "goal.hh" +#include "worker.hh" + +namespace nix { + +using Co = nix::Goal::Co; +using promise_type = nix::Goal::promise_type; +using handle_type = nix::Goal::handle_type; +using Suspend = nix::Goal::Suspend; + +Co::Co(Co&& rhs) { + this->handle = rhs.handle; + rhs.handle = nullptr; +} +void Co::operator=(Co&& rhs) { + this->handle = rhs.handle; + rhs.handle = nullptr; +} +Co::~Co() { + if (handle) { + handle.promise().alive = false; + handle.destroy(); + } +} + +Co promise_type::get_return_object() { + auto handle = handle_type::from_promise(*this); + return Co{handle}; +}; + +std::coroutine_handle<> promise_type::final_awaiter::await_suspend(handle_type h) noexcept { + auto& p = h.promise(); + auto goal = p.goal; + assert(goal); + goal->trace("in final_awaiter"); + auto c = std::move(p.continuation); + + if (c) { + // We still have a continuation, i.e. work to do. + // We assert that the goal is still busy. + assert(goal->exitCode == ecBusy); + assert(goal->top_co); // Goal must have an active coroutine. + assert(goal->top_co->handle == h); // The active coroutine must be us. + assert(p.alive); // We must not have been destructed. + + // we move continuation to the top, + // note: previous top_co is actually h, so by moving into it, + // we're calling the destructor on h, DON'T use h and p after this! + + // We move our continuation into `top_co`, i.e. the marker for the active continuation. + // By doing this we destruct the old `top_co`, i.e. us, so `h` can't be used anymore. + // Be careful not to access freed memory! + goal->top_co = std::move(c); + + // We resume `top_co`. + return goal->top_co->handle; + } else { + // We have no continuation, i.e. no more work to do, + // so the goal must not be busy anymore. + assert(goal->exitCode != ecBusy); + + // We reset `top_co` for good measure. + p.goal->top_co = {}; + + // We jump to the noop coroutine, which doesn't do anything and immediately suspends. + // This passes control back to the caller of goal.work(). + return std::noop_coroutine(); + } +} + +void promise_type::return_value(Co&& next) { + goal->trace("return_value(Co&&)"); + // Save old continuation. + auto old_continuation = std::move(continuation); + // We set next as our continuation. + continuation = std::move(next); + // We set next's goal, and thus it must not have one already. + assert(!continuation->handle.promise().goal); + continuation->handle.promise().goal = goal; + // Nor can next have a continuation, as we set it to our old one. + assert(!continuation->handle.promise().continuation); + continuation->handle.promise().continuation = std::move(old_continuation); +} + +std::coroutine_handle<> nix::Goal::Co::await_suspend(handle_type caller) { + assert(handle); // we must be a valid coroutine + auto& p = handle.promise(); + assert(!p.continuation); // we must have no continuation + assert(!p.goal); // we must not have a goal yet + auto goal = caller.promise().goal; + assert(goal); + p.goal = goal; + p.continuation = std::move(goal->top_co); // we set our continuation to be top_co (i.e. caller) + goal->top_co = std::move(*this); // we set top_co to ourselves, don't use this anymore after this! + return p.goal->top_co->handle; // we execute ourselves +} + +bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { + std::string s1 = a->key(); + std::string s2 = b->key(); + return s1 < s2; +} + + +BuildResult Goal::getBuildResult(const DerivedPath & req) const { + BuildResult res { buildResult }; + + if (auto pbp = std::get_if(&req)) { + auto & bp = *pbp; + + /* Because goals are in general shared between derived paths + that share the same derivation, we need to filter their + results to get back just the results we care about. + */ + + for (auto it = res.builtOutputs.begin(); it != res.builtOutputs.end();) { + if (bp.outputs.contains(it->first)) + ++it; + else + it = res.builtOutputs.erase(it); + } + } + + return res; +} + + +void addToWeakGoals(WeakGoals & goals, GoalPtr p) +{ + if (goals.find(p) != goals.end()) + return; + goals.insert(p); +} + + +void Goal::addWaitee(GoalPtr waitee) +{ + waitees.insert(waitee); + addToWeakGoals(waitee->waiters, shared_from_this()); +} + + +void Goal::waiteeDone(GoalPtr waitee, ExitCode result) +{ + assert(waitees.count(waitee)); + waitees.erase(waitee); + + trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); + + if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; + + if (result == ecNoSubstituters) ++nrNoSubstituters; + + if (result == ecIncompleteClosure) ++nrIncompleteClosure; + + if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { + + /* If we failed and keepGoing is not set, we remove all + remaining waitees. */ + for (auto & goal : waitees) { + goal->waiters.extract(shared_from_this()); + } + waitees.clear(); + + worker.wakeUp(shared_from_this()); + } +} + +Goal::Done Goal::amDone(ExitCode result, std::optional ex) +{ + trace("done"); + assert(top_co); + assert(exitCode == ecBusy); + assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); + exitCode = result; + + if (ex) { + if (!waiters.empty()) + logError(ex->info()); + else + this->ex = std::move(*ex); + } + + for (auto & i : waiters) { + GoalPtr goal = i.lock(); + if (goal) goal->waiteeDone(shared_from_this(), result); + } + waiters.clear(); + worker.removeGoal(shared_from_this()); + + cleanup(); + + // We drop the continuation. + // In `final_awaiter` this will signal that there is no more work to be done. + top_co->handle.promise().continuation = {}; + + // won't return to caller because of logic in final_awaiter + return Done{}; +} + + +void Goal::trace(std::string_view s) +{ + debug("%1%: %2%", name, s); +} + +void Goal::work() +{ + assert(top_co); + assert(top_co->handle); + assert(top_co->handle.promise().alive); + top_co->handle.resume(); + // We either should be in a state where we can be work()-ed again, + // or we should be done. + assert(top_co || exitCode != ecBusy); +} + + +} diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh new file mode 100644 index 000000000..9c6a40c84 --- /dev/null +++ b/src/libstore/build/goal.hh @@ -0,0 +1,445 @@ +#pragma once +///@file + +#include "store-api.hh" +#include "build-result.hh" + +#include + +namespace nix { + +/** + * Forward definition. + */ +struct Goal; +class Worker; + +/** + * A pointer to a goal. + */ +typedef std::shared_ptr GoalPtr; +typedef std::weak_ptr WeakGoalPtr; + +struct CompareGoalPtrs { + bool operator() (const GoalPtr & a, const GoalPtr & b) const; +}; + +/** + * Set of goals. + */ +typedef std::set Goals; +typedef std::set> WeakGoals; + +/** + * A map of paths to goals (and the other way around). + */ +typedef std::map WeakGoalMap; + +/** + * Used as a hint to the worker on how to schedule a particular goal. For example, + * builds are typically CPU- and memory-bound, while substitutions are I/O bound. + * Using this information, the worker might decide to schedule more or fewer goals + * of each category in parallel. + */ +enum struct JobCategory { + /** + * A build of a derivation; it will use CPU and disk resources. + */ + Build, + /** + * A substitution an arbitrary store object; it will use network resources. + */ + Substitution, +}; + +struct Goal : public std::enable_shared_from_this +{ + typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; + + /** + * Backlink to the worker. + */ + Worker & worker; + + /** + * Goals that this goal is waiting for. + */ + Goals waitees; + + /** + * Goals waiting for this one to finish. Must use weak pointers + * here to prevent cycles. + */ + WeakGoals waiters; + + /** + * Number of goals we are/were waiting for that have failed. + */ + size_t nrFailed = 0; + + /** + * Number of substitution goals we are/were waiting for that + * failed because there are no substituters. + */ + size_t nrNoSubstituters = 0; + + /** + * Number of substitution goals we are/were waiting for that + * failed because they had unsubstitutable references. + */ + size_t nrIncompleteClosure = 0; + + /** + * Name of this goal for debugging purposes. + */ + std::string name; + + /** + * Whether the goal is finished. + */ + ExitCode exitCode = ecBusy; + +protected: + /** + * Build result. + */ + BuildResult buildResult; +public: + + /** + * Suspend our goal and wait until we get @ref work()-ed again. + * `co_await`-able by @ref Co. + */ + struct Suspend {}; + + /** + * Return from the current coroutine and suspend our goal + * if we're not busy anymore, or jump to the next coroutine + * set to be executed/resumed. + */ + struct Return {}; + + /** + * `co_return`-ing this will end the goal. + * If you're not inside a coroutine, you can safely discard this. + */ + struct [[nodiscard]] Done { + private: + Done(){} + + friend Goal; + }; + + // forward declaration of promise_type, see below + struct promise_type; + + /** + * Handle to coroutine using @ref Co and @ref promise_type. + */ + using handle_type = std::coroutine_handle; + + /** + * C++20 coroutine wrapper for use in goal logic. + * Coroutines are functions that use `co_await`/`co_return` (and `co_yield`, but not supported by @ref Co). + * + * @ref Co is meant to be used by methods of subclasses of @ref Goal. + * The main functionality provided by `Co` is + * - `co_await Suspend{}`: Suspends the goal. + * - `co_await f()`: Waits until `f()` finishes. + * - `co_return f()`: Tail-calls `f()`. + * - `co_return Return{}`: Ends coroutine. + * + * The idea is that you implement the goal logic using coroutines, + * and do the core thing a goal can do, suspension, when you have + * children you're waiting for. + * Coroutines allow you to resume the work cleanly. + * + * @note Brief explanation of C++20 coroutines: + * When you `Co f()`, a `std::coroutine_handle` is created, + * alongside its @ref promise_type. + * There are suspension points at the beginning of the coroutine, + * at every `co_await`, and at the final (possibly implicit) `co_return`. + * Once suspended, you can resume the `std::coroutine_handle` by doing `coroutine_handle.resume()`. + * Suspension points are implemented by passing a struct to the compiler + * that implements `await_sus`pend. + * `await_suspend` can either say "cancel suspension", in which case execution resumes, + * "suspend", in which case control is passed back to the caller of `coroutine_handle.resume()` + * or the place where the coroutine function is initially executed in the case of the initial + * suspension, or `await_suspend` can specify another coroutine to jump to, which is + * how tail calls are implemented. + * + * @note Resources: + * - https://lewissbaker.github.io/ + * - https://www.chiark.greenend.org.uk/~sgtatham/quasiblog/coroutines-c++20/ + * - https://www.scs.stanford.edu/~dm/blog/c++-coroutines.html + * + * @todo Allocate explicitly on stack since HALO thing doesn't really work, + * specifically, there's no way to uphold the requirements when trying to do + * tail-calls without using a trampoline AFAICT. + * + * @todo Support returning data natively + */ + struct [[nodiscard]] Co { + /** + * The underlying handle. + */ + handle_type handle; + + explicit Co(handle_type handle) : handle(handle) {}; + void operator=(Co&&); + Co(Co&& rhs); + ~Co(); + + bool await_ready() { return false; }; + /** + * When we `co_await` another @ref Co-returning coroutine, + * we tell the caller of `caller_coroutine.resume()` to switch to our coroutine (@ref handle). + * To make sure we return to the original coroutine, we set it as the continuation of our + * coroutine. In @ref promise_type::final_awaiter we check if it's set and if so we return to it. + * + * To explain in more understandable terms: + * When we `co_await Co_returning_function()`, this function is called on the resultant @ref Co of + * the _called_ function, and C++ automatically passes the caller in. + * + * `goal` field of @ref promise_type is also set here by copying it from the caller. + */ + std::coroutine_handle<> await_suspend(handle_type handle); + void await_resume() {}; + }; + + /** + * Used on initial suspend, does the same as @ref std::suspend_always, + * but asserts that everything has been set correctly. + */ + struct InitialSuspend { + /** + * Handle of coroutine that does the + * initial suspend + */ + handle_type handle; + + bool await_ready() { return false; }; + void await_suspend(handle_type handle_) { + handle = handle_; + } + void await_resume() { + assert(handle); + assert(handle.promise().goal); // goal must be set + assert(handle.promise().goal->top_co); // top_co of goal must be set + assert(handle.promise().goal->top_co->handle == handle); // top_co of goal must be us + } + }; + + /** + * Promise type for coroutines defined using @ref Co. + * Attached to coroutine handle. + */ + struct promise_type { + /** + * Either this is who called us, or it is who we will tail-call. + * It is what we "jump" to once we are done. + */ + std::optional continuation; + + /** + * The goal that we're a part of. + * Set either in @ref Co::await_suspend or in constructor of @ref Goal. + */ + Goal* goal = nullptr; + + /** + * Is set to false when destructed to ensure we don't use a + * destructed coroutine by accident + */ + bool alive = true; + + /** + * The awaiter used by @ref final_suspend. + */ + struct final_awaiter { + bool await_ready() noexcept { return false; }; + /** + * Here we execute our continuation, by passing it back to the caller. + * C++ compiler will create code that takes that and executes it promptly. + * `h` is the handle for the coroutine that is finishing execution, + * thus it must be destroyed. + */ + std::coroutine_handle<> await_suspend(handle_type h) noexcept; + void await_resume() noexcept { assert(false); }; + }; + + /** + * Called by compiler generated code to construct the @ref Co + * that is returned from a @ref Co-returning coroutine. + */ + Co get_return_object(); + + /** + * Called by compiler generated code before body of coroutine. + * We use this opportunity to set the @ref goal field + * and `top_co` field of @ref Goal. + */ + InitialSuspend initial_suspend() { return {}; }; + + /** + * Called on `co_return`. Creates @ref final_awaiter which + * either jumps to continuation or suspends goal. + */ + final_awaiter final_suspend() noexcept { return {}; }; + + /** + * Does nothing, but provides an opportunity for + * @ref final_suspend to happen. + */ + void return_value(Return) {} + + /** + * Does nothing, but provides an opportunity for + * @ref final_suspend to happen. + */ + void return_value(Done) {} + + /** + * When "returning" another coroutine, what happens is that + * we set it as our own continuation, thus once the final suspend + * happens, we transfer control to it. + * The original continuation we had is set as the continuation + * of the coroutine passed in. + * @ref final_suspend is called after this, and @ref final_awaiter will + * pass control off to @ref continuation. + * + * If we already have a continuation, that continuation is set as + * the continuation of the new continuation. Thus, the continuation + * passed to @ref return_value must not have a continuation set. + */ + void return_value(Co&&); + + /** + * If an exception is thrown inside a coroutine, + * we re-throw it in the context of the "resumer" of the continuation. + */ + void unhandled_exception() { throw; }; + + /** + * Allows awaiting a @ref Co. + */ + Co&& await_transform(Co&& co) { return static_cast(co); } + + /** + * Allows awaiting a @ref Suspend. + * Always suspends. + */ + std::suspend_always await_transform(Suspend) { return {}; }; + }; + + /** + * The coroutine being currently executed. + * MUST be updated when switching the coroutine being executed. + * This is used both for memory management and to resume the last + * coroutine executed. + * Destroying this should destroy all coroutines created for this goal. + */ + std::optional top_co; + + /** + * The entry point for the goal + */ + virtual Co init() = 0; + + /** + * Wrapper around @ref init since virtual functions + * can't be used in constructors. + */ + inline Co init_wrapper(); + + /** + * Signals that the goal is done. + * `co_return` the result. If you're not inside a coroutine, you can ignore + * the return value safely. + */ + Done amDone(ExitCode result, std::optional ex = {}); + + virtual void cleanup() { } + + /** + * Project a `BuildResult` with just the information that pertains + * to the given request. + * + * In general, goals may be aliased between multiple requests, and + * the stored `BuildResult` has information for the union of all + * requests. We don't want to leak what the other request are for + * sake of both privacy and determinism, and this "safe accessor" + * ensures we don't. + */ + BuildResult getBuildResult(const DerivedPath &) const; + + /** + * Exception containing an error message, if any. + */ + std::optional ex; + + Goal(Worker & worker, DerivedPath path) + : worker(worker), top_co(init_wrapper()) + { + // top_co shouldn't have a goal already, should be nullptr. + assert(!top_co->handle.promise().goal); + // we set it such that top_co can pass it down to its subcoroutines. + top_co->handle.promise().goal = this; + } + + virtual ~Goal() + { + trace("goal destroyed"); + } + + void work(); + + void addWaitee(GoalPtr waitee); + + virtual void waiteeDone(GoalPtr waitee, ExitCode result); + + virtual void handleChildOutput(Descriptor fd, std::string_view data) + { + unreachable(); + } + + virtual void handleEOF(Descriptor fd) + { + unreachable(); + } + + void trace(std::string_view s); + + std::string getName() const + { + return name; + } + + /** + * Callback in case of a timeout. It should wake up its waiters, + * get rid of any running child processes that are being monitored + * by the worker (important!), etc. + */ + virtual void timedOut(Error && ex) = 0; + + virtual std::string key() = 0; + + /** + * @brief Hint for the scheduler, which concurrency limit applies. + * @see JobCategory + */ + virtual JobCategory jobCategory() const = 0; +}; + +void addToWeakGoals(WeakGoals & goals, GoalPtr p); + +} + +template +struct std::coroutine_traits { + using promise_type = nix::Goal::promise_type; +}; + +nix::Goal::Co nix::Goal::init_wrapper() { + co_return init(); +} diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc new file mode 100644 index 000000000..315500719 --- /dev/null +++ b/src/libstore/build/substitution-goal.cc @@ -0,0 +1,302 @@ +#include "worker.hh" +#include "substitution-goal.hh" +#include "nar-info.hh" +#include "finally.hh" +#include "signals.hh" +#include + +namespace nix { + +PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) + : Goal(worker, DerivedPath::Opaque { storePath }) + , storePath(storePath) + , repair(repair) + , ca(ca) +{ + name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); + trace("created"); + maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); +} + + +PathSubstitutionGoal::~PathSubstitutionGoal() +{ + cleanup(); +} + + +Goal::Done PathSubstitutionGoal::done( + ExitCode result, + BuildResult::Status status, + std::optional errorMsg) +{ + buildResult.status = status; + if (errorMsg) { + debug(*errorMsg); + buildResult.errorMsg = *errorMsg; + } + return amDone(result); +} + + +Goal::Co PathSubstitutionGoal::init() +{ + trace("init"); + + worker.store.addTempRoot(storePath); + + /* If the path already exists we're done. */ + if (!repair && worker.store.isValidPath(storePath)) { + co_return done(ecSuccess, BuildResult::AlreadyValid); + } + + if (settings.readOnlyMode) + throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); + + auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); + + bool substituterFailed = false; + + for (auto sub : subs) { + trace("trying next substituter"); + + cleanup(); + + /* The path the substituter refers to the path as. This will be + * different when the stores have different names. */ + std::optional subPath; + + /* Path info returned by the substituter's query info operation. */ + std::shared_ptr info; + + if (ca) { + subPath = sub->makeFixedOutputPathFromCA( + std::string { storePath.name() }, + ContentAddressWithReferences::withoutRefs(*ca)); + if (sub->storeDir == worker.store.storeDir) + assert(subPath == storePath); + } else if (sub->storeDir != worker.store.storeDir) { + continue; + } + + try { + // FIXME: make async + info = sub->queryPathInfo(subPath ? *subPath : storePath); + } catch (InvalidPath &) { + continue; + } catch (SubstituterDisabled & e) { + if (settings.tryFallback) continue; + else throw e; + } catch (Error & e) { + if (settings.tryFallback) { + logError(e.info()); + continue; + } else throw e; + } + + if (info->path != storePath) { + if (info->isContentAddressed(*sub) && info->references.empty()) { + auto info2 = std::make_shared(*info); + info2->path = storePath; + info = info2; + } else { + printError("asked '%s' for '%s' but got '%s'", + sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); + continue; + } + } + + /* Update the total expected download size. */ + auto narInfo = std::dynamic_pointer_cast(info); + + maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); + + maintainExpectedDownload = + narInfo && narInfo->fileSize + ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) + : nullptr; + + worker.updateProgress(); + + /* Bail out early if this substituter lacks a valid + signature. LocalStore::addToStore() also checks for this, but + only after we've downloaded the path. */ + if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) + { + warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", + worker.store.printStorePath(storePath), sub->getUri()); + continue; + } + + /* To maintain the closure invariant, we first have to realise the + paths referenced by this one. */ + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + addWaitee(worker.makePathSubstitutionGoal(i)); + + if (!waitees.empty()) co_await Suspend{}; + + // FIXME: consider returning boolean instead of passing in reference + bool out = false; // is mutated by tryToRun + co_await tryToRun(subPath ? *subPath : storePath, sub, info, out); + substituterFailed = substituterFailed || out; + } + + /* None left. Terminate this goal and let someone else deal + with it. */ + + if (substituterFailed) { + worker.failedSubstitutions++; + worker.updateProgress(); + } + + /* Hack: don't indicate failure if there were no substituters. + In that case the calling derivation should just do a + build. */ + co_return done( + substituterFailed ? ecFailed : ecNoSubstituters, + BuildResult::NoSubstituters, + fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); +} + + +Goal::Co PathSubstitutionGoal::tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool & substituterFailed) +{ + trace("all references realised"); + + if (nrFailed > 0) { + co_return done( + nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, + BuildResult::DependencyFailed, + fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath))); + } + + for (auto & i : info->references) + if (i != storePath) /* ignore self-references */ + assert(worker.store.isValidPath(i)); + + worker.wakeUp(shared_from_this()); + co_await Suspend{}; + + trace("trying to run"); + + /* Make sure that we are allowed to start a substitution. Note that even + if maxSubstitutionJobs == 0, we still allow a substituter to run. This + prevents infinite waiting. */ + while (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) { + worker.waitForBuildSlot(shared_from_this()); + co_await Suspend{}; + } + + auto maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); + worker.updateProgress(); + +#ifndef _WIN32 + outPipe.create(); +#else + outPipe.createAsyncPipe(worker.ioport.get()); +#endif + + auto promise = std::promise(); + + thr = std::thread([this, &promise, &subPath, &sub]() { + try { + ReceiveInterrupts receiveInterrupts; + + /* Wake up the worker loop when we're done. */ + Finally updateStats([this]() { outPipe.writeSide.close(); }); + + Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); + PushActivity pact(act.id); + + copyStorePath(*sub, worker.store, + subPath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); + + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }); + + worker.childStarted(shared_from_this(), { +#ifndef _WIN32 + outPipe.readSide.get() +#else + &outPipe +#endif + }, true, false); + + co_await Suspend{}; + + trace("substitute finished"); + + thr.join(); + worker.childTerminated(this); + + try { + promise.get_future().get(); + } catch (std::exception & e) { + printError(e.what()); + + /* Cause the parent build to fail unless --fallback is given, + or the substitute has disappeared. The latter case behaves + the same as the substitute never having existed in the + first place. */ + try { + throw; + } catch (SubstituteGone &) { + } catch (...) { + substituterFailed = true; + } + + co_return Return{}; + } + + worker.markContentsGood(storePath); + + printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); + + maintainRunningSubstitutions.reset(); + + maintainExpectedSubstitutions.reset(); + worker.doneSubstitutions++; + + if (maintainExpectedDownload) { + auto fileSize = maintainExpectedDownload->delta; + maintainExpectedDownload.reset(); + worker.doneDownloadSize += fileSize; + } + + assert(maintainExpectedNar); + worker.doneNarSize += maintainExpectedNar->delta; + maintainExpectedNar.reset(); + + worker.updateProgress(); + + co_return done(ecSuccess, BuildResult::Substituted); +} + + +void PathSubstitutionGoal::handleEOF(Descriptor fd) +{ + worker.wakeUp(shared_from_this()); +} + + +void PathSubstitutionGoal::cleanup() +{ + try { + if (thr.joinable()) { + // FIXME: signal worker thread to quit. + thr.join(); + worker.childTerminated(this); + } + + outPipe.close(); + } catch (...) { + ignoreExceptionInDestructor(); + } +} + + +} diff --git a/src/libstore/unix/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh similarity index 57% rename from src/libstore/unix/build/substitution-goal.hh rename to src/libstore/build/substitution-goal.hh index 1d389d328..f2cf797e5 100644 --- a/src/libstore/unix/build/substitution-goal.hh +++ b/src/libstore/build/substitution-goal.hh @@ -1,14 +1,16 @@ #pragma once ///@file -#include "lock.hh" +#include "worker.hh" #include "store-api.hh" #include "goal.hh" +#include "muxable-pipe.hh" +#include +#include +#include namespace nix { -class Worker; - struct PathSubstitutionGoal : public Goal { /** @@ -17,66 +19,29 @@ struct PathSubstitutionGoal : public Goal StorePath storePath; /** - * The path the substituter refers to the path as. This will be - * different when the stores have different names. + * Whether to try to repair a valid path. */ - std::optional subPath; - - /** - * The remaining substituters. - */ - std::list> subs; - - /** - * The current substituter. - */ - std::shared_ptr sub; - - /** - * Whether a substituter failed. - */ - bool substituterFailed = false; - - /** - * Path info returned by the substituter's query info operation. - */ - std::shared_ptr info; + RepairFlag repair; /** * Pipe for the substituter's standard output. */ - Pipe outPipe; + MuxablePipe outPipe; /** * The substituter thread. */ std::thread thr; - std::promise promise; - - /** - * Whether to try to repair a valid path. - */ - RepairFlag repair; - - /** - * Location where we're downloading the substitute. Differs from - * storePath when doing a repair. - */ - Path destPath; - std::unique_ptr> maintainExpectedSubstitutions, maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload; - typedef void (PathSubstitutionGoal::*GoalState)(); - GoalState state; - /** * Content address for recomputing store path */ std::optional ca; - void done( + Done done( ExitCode result, BuildResult::Status status, std::optional errorMsg = {}); @@ -85,7 +50,7 @@ public: PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional ca = std::nullopt); ~PathSubstitutionGoal(); - void timedOut(Error && ex) override { abort(); }; + void timedOut(Error && ex) override { unreachable(); }; /** * We prepend "a$" to the key name to ensure substitution goals @@ -96,23 +61,19 @@ public: return "a$" + std::string(storePath.name()) + "$" + worker.store.printStorePath(storePath); } - void work() override; - /** * The states. */ - void init(); - void tryNext(); - void gotInfo(); - void referencesValid(); - void tryToRun(); - void finished(); + Co init() override; + Co gotInfo(); + Co tryToRun(StorePath subPath, nix::ref sub, std::shared_ptr info, bool & substituterFailed); + Co finished(); /** * Callback used by the worker to write to the log. */ - void handleChildOutput(int fd, std::string_view data) override; - void handleEOF(int fd) override; + void handleChildOutput(Descriptor fd, std::string_view data) override {}; + void handleEOF(Descriptor fd) override; /* Called by destructor, can't be overridden */ void cleanup() override final; diff --git a/src/libstore/unix/build/worker.cc b/src/libstore/build/worker.cc similarity index 80% rename from src/libstore/unix/build/worker.cc rename to src/libstore/build/worker.cc index 03fc280a4..dbe86f43f 100644 --- a/src/libstore/unix/build/worker.cc +++ b/src/libstore/build/worker.cc @@ -1,13 +1,15 @@ +#include "local-store.hh" #include "machines.hh" #include "worker.hh" #include "substitution-goal.hh" #include "drv-output-substitution-goal.hh" -#include "local-derivation-goal.hh" -#include "hook-instance.hh" +#include "derivation-goal.hh" +#ifndef _WIN32 // TODO Enable building on Windows +# include "local-derivation-goal.hh" +# include "hook-instance.hh" +#endif #include "signals.hh" -#include - namespace nix { Worker::Worker(Store & store, Store & evalStore) @@ -17,7 +19,6 @@ Worker::Worker(Store & store, Store & evalStore) , store(store) , evalStore(evalStore) { - /* Debugging: prevent recursive workers. */ nrLocalBuilds = 0; nrSubstitutions = 0; lastWokenUp = steady_time_point::min(); @@ -64,20 +65,27 @@ std::shared_ptr Worker::makeDerivationGoal(const StorePath & drv const OutputsSpec & wantedOutputs, BuildMode buildMode) { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr { - return !dynamic_cast(&store) - ? std::make_shared(drvPath, wantedOutputs, *this, buildMode) - : std::make_shared(drvPath, wantedOutputs, *this, buildMode); + return +#ifndef _WIN32 // TODO Enable building on Windows + dynamic_cast(&store) + ? std::make_shared(drvPath, wantedOutputs, *this, buildMode) + : +#endif + std::make_shared(drvPath, wantedOutputs, *this, buildMode); }); } - std::shared_ptr Worker::makeBasicDerivationGoal(const StorePath & drvPath, const BasicDerivation & drv, const OutputsSpec & wantedOutputs, BuildMode buildMode) { return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr { - return !dynamic_cast(&store) - ? std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode) - : std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); + return +#ifndef _WIN32 // TODO Enable building on Windows + dynamic_cast(&store) + ? std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode) + : +#endif + std::make_shared(drvPath, drv, wantedOutputs, *this, buildMode); }); } @@ -143,7 +151,8 @@ void Worker::removeGoal(GoalPtr goal) { if (auto drvGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(drvGoal, derivationGoals); - else if (auto subGoal = std::dynamic_pointer_cast(goal)) + else + if (auto subGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(subGoal, substitutionGoals); else if (auto subGoal = std::dynamic_pointer_cast(goal)) nix::removeGoal(subGoal, drvOutputSubstitutionGoals); @@ -175,25 +184,25 @@ void Worker::wakeUp(GoalPtr goal) } -unsigned Worker::getNrLocalBuilds() +size_t Worker::getNrLocalBuilds() { return nrLocalBuilds; } -unsigned Worker::getNrSubstitutions() +size_t Worker::getNrSubstitutions() { return nrSubstitutions; } -void Worker::childStarted(GoalPtr goal, const std::set & fds, +void Worker::childStarted(GoalPtr goal, const std::set & channels, bool inBuildSlot, bool respectTimeouts) { Child child; child.goal = goal; child.goal2 = goal.get(); - child.fds = fds; + child.channels = channels; child.timeStarted = child.lastOutput = steady_time_point::clock::now(); child.inBuildSlot = inBuildSlot; child.respectTimeouts = respectTimeouts; @@ -207,7 +216,7 @@ void Worker::childStarted(GoalPtr goal, const std::set & fds, nrLocalBuilds++; break; default: - abort(); + unreachable(); } } } @@ -230,7 +239,7 @@ void Worker::childTerminated(Goal * goal, bool wakeSleepers) nrLocalBuilds--; break; default: - abort(); + unreachable(); } } @@ -286,7 +295,8 @@ void Worker::run(const Goals & _topGoals) .drvPath = makeConstantStorePathRef(goal->drvPath), .outputs = goal->wantedOutputs, }); - } else if (auto goal = dynamic_cast(i.get())) { + } else + if (auto goal = dynamic_cast(i.get())) { topPaths.push_back(DerivedPath::Opaque{goal->storePath}); } } @@ -327,31 +337,27 @@ void Worker::run(const Goals & _topGoals) /* Wait for input. */ if (!children.empty() || !waitingForAWhile.empty()) waitForInput(); - else { - if (awake.empty() && 0U == settings.maxBuildJobs) - { - if (getMachines().empty()) - throw Error( - R"( - Unable to start any build; - either increase '--max-jobs' or enable remote builds. + else if (awake.empty() && 0U == settings.maxBuildJobs) { + if (getMachines().empty()) + throw Error( + R"( + Unable to start any build; + either increase '--max-jobs' or enable remote builds. - For more information run 'man nix.conf' and search for '/machines'. - )" - ); - else - throw Error( - R"( - Unable to start any build; - remote machines may not have all required system features. + For more information run 'man nix.conf' and search for '/machines'. + )" + ); + else + throw Error( + R"( + Unable to start any build; + remote machines may not have all required system features. - For more information run 'man nix.conf' and search for '/machines'. - )" - ); + For more information run 'man nix.conf' and search for '/machines'. + )" + ); - } - assert(!awake.empty()); - } + } else assert(!awake.empty()); } /* If --keep-going is not set, it's possible that the main goal @@ -408,23 +414,25 @@ void Worker::waitForInput() if (useTimeout) vomit("sleeping %d seconds", timeout); + MuxablePipePollState state; + +#ifndef _WIN32 /* Use select() to wait for the input side of any logger pipe to become `available'. Note that `available' (i.e., non-blocking) includes EOF. */ - std::vector pollStatus; - std::map fdToPollStatus; for (auto & i : children) { - for (auto & j : i.fds) { - pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); - fdToPollStatus[j] = pollStatus.size() - 1; + for (auto & j : i.channels) { + state.pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN }); + state.fdToPollStatus[j] = state.pollStatus.size() - 1; } } +#endif - if (poll(pollStatus.data(), pollStatus.size(), - useTimeout ? timeout * 1000 : -1) == -1) { - if (errno == EINTR) return; - throw SysError("waiting for input"); - } + state.poll( +#ifdef _WIN32 + ioport.get(), +#endif + useTimeout ? (std::optional { timeout * 1000 }) : std::nullopt); auto after = steady_time_point::clock::now(); @@ -439,32 +447,18 @@ void Worker::waitForInput() GoalPtr goal = j->goal.lock(); assert(goal); - std::set fds2(j->fds); - std::vector buffer(4096); - for (auto & k : fds2) { - const auto fdPollStatusId = get(fdToPollStatus, k); - assert(fdPollStatusId); - assert(*fdPollStatusId < pollStatus.size()); - if (pollStatus.at(*fdPollStatusId).revents) { - ssize_t rd = ::read(k, buffer.data(), buffer.size()); - // FIXME: is there a cleaner way to handle pt close - // than EIO? Is this even standard? - if (rd == 0 || (rd == -1 && errno == EIO)) { - debug("%1%: got EOF", goal->getName()); - goal->handleEOF(k); - j->fds.erase(k); - } else if (rd == -1) { - if (errno != EINTR) - throw SysError("%s: read failed", goal->getName()); - } else { - printMsg(lvlVomit, "%1%: read %2% bytes", - goal->getName(), rd); - std::string_view data((char *) buffer.data(), rd); - j->lastOutput = after; - goal->handleChildOutput(k, data); - } - } - } + state.iterate( + j->channels, + [&](Descriptor k, std::string_view data) { + printMsg(lvlVomit, "%1%: read %2% bytes", + goal->getName(), data.size()); + j->lastOutput = after; + goal->handleChildOutput(k, data); + }, + [&](Descriptor k) { + debug("%1%: got EOF", goal->getName()); + goal->handleEOF(k); + }); if (goal->exitCode == Goal::ecBusy && 0 != settings.maxSilentTime && @@ -529,9 +523,9 @@ bool Worker::pathContentsGood(const StorePath & path) if (!pathExists(store.printStorePath(path))) res = false; else { - Hash current = hashPath( + auto current = hashPath( {store.getFSAccessor(), CanonPath(store.printStorePath(path))}, - FileIngestionMethod::Recursive, info->narHash.algo); + FileIngestionMethod::NixArchive, info->narHash.algo).first; Hash nullHash(HashAlgorithm::SHA256); res = info->narHash == nullHash || info->narHash == current; } diff --git a/src/libstore/unix/build/worker.hh b/src/libstore/build/worker.hh similarity index 93% rename from src/libstore/unix/build/worker.hh rename to src/libstore/build/worker.hh index ced013ddd..e083dbea6 100644 --- a/src/libstore/unix/build/worker.hh +++ b/src/libstore/build/worker.hh @@ -2,10 +2,10 @@ ///@file #include "types.hh" -#include "lock.hh" #include "store-api.hh" #include "goal.hh" #include "realisation.hh" +#include "muxable-pipe.hh" #include #include @@ -36,14 +36,14 @@ typedef std::chrono::time_point steady_time_point; /** * A mapping used to remember for each child process to what goal it - * belongs, and file descriptors for receiving log data and output + * belongs, and comm channels for receiving log data and output * path creation commands. */ struct Child { WeakGoalPtr goal; Goal * goal2; // ugly hackery - std::set fds; + std::set channels; bool respectTimeouts; bool inBuildSlot; /** @@ -53,11 +53,13 @@ struct Child steady_time_point timeStarted; }; +#ifndef _WIN32 // TODO Enable building on Windows /* Forward definition. */ struct HookInstance; +#endif /** - * The worker class. + * Coordinates one or more realisations and their interdependencies. */ class Worker { @@ -90,12 +92,12 @@ private: * Number of build slots occupied. This includes local builds but does not * include substitutions or remote builds via the build hook. */ - unsigned int nrLocalBuilds; + size_t nrLocalBuilds; /** * Number of substitution slots occupied. */ - unsigned int nrSubstitutions; + size_t nrSubstitutions; /** * Maps used to prevent multiple instantiations of a goal for the @@ -152,10 +154,16 @@ public: */ bool checkMismatch; +#ifdef _WIN32 + AutoCloseFD ioport; +#endif + Store & store; Store & evalStore; +#ifndef _WIN32 // TODO Enable building on Windows std::unique_ptr hook; +#endif uint64_t expectedBuilds = 0; uint64_t doneBuilds = 0; @@ -227,18 +235,18 @@ public: * Return the number of local build processes currently running (but not * remote builds via the build hook). */ - unsigned int getNrLocalBuilds(); + size_t getNrLocalBuilds(); /** * Return the number of substitution processes currently running. */ - unsigned int getNrSubstitutions(); + size_t getNrSubstitutions(); /** * Registers a running child process. `inBuildSlot` means that * the process counts towards the jobs limit. */ - void childStarted(GoalPtr goal, const std::set & fds, + void childStarted(GoalPtr goal, const std::set & channels, bool inBuildSlot, bool respectTimeouts); /** diff --git a/src/libstore/builtins.hh b/src/libstore/builtins.hh index 93558b49e..091946e01 100644 --- a/src/libstore/builtins.hh +++ b/src/libstore/builtins.hh @@ -9,7 +9,8 @@ namespace nix { void builtinFetchurl( const BasicDerivation & drv, const std::map & outputs, - const std::string & netrcData); + const std::string & netrcData, + const std::string & caFileData); void builtinUnpackChannel( const BasicDerivation & drv, diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index ab35c861d..0f7bcd99b 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -1,5 +1,6 @@ #include "buildenv.hh" #include "derivations.hh" +#include "signals.hh" #include #include @@ -30,6 +31,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, } for (const auto & ent : srcFiles) { + checkInterrupt(); auto name = ent.path().filename(); if (name.string()[0] == '.') /* not matched by glob */ diff --git a/src/libstore/unix/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc similarity index 93% rename from src/libstore/unix/builtins/fetchurl.cc rename to src/libstore/builtins/fetchurl.cc index b9dfeba2f..90e58dfdb 100644 --- a/src/libstore/unix/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -9,7 +9,8 @@ namespace nix { void builtinFetchurl( const BasicDerivation & drv, const std::map & outputs, - const std::string & netrcData) + const std::string & netrcData, + const std::string & caFileData) { /* Make the host's netrc data available. Too bad curl requires this to be stored in a file. It would be nice if we could just @@ -19,6 +20,9 @@ void builtinFetchurl( writeFile(settings.netrcFile, netrcData, 0600); } + settings.caFile = "ca-certificates.crt"; + writeFile(settings.caFile, caFileData, 0600); + auto out = get(drv.outputs, "out"); if (!out) throw Error("'builtin:fetchurl' requires an 'out' output"); @@ -38,10 +42,7 @@ void builtinFetchurl( auto source = sinkToSource([&](Sink & sink) { - /* No need to do TLS verification, because we check the hash of - the result anyway. */ FileTransferRequest request(url); - request.verifyTLS = false; request.decompress = false; auto decompressor = makeDecompressionSink( diff --git a/src/libstore/builtins/unpack-channel.cc b/src/libstore/builtins/unpack-channel.cc new file mode 100644 index 000000000..a6369ee1c --- /dev/null +++ b/src/libstore/builtins/unpack-channel.cc @@ -0,0 +1,55 @@ +#include "builtins.hh" +#include "tarfile.hh" + +namespace nix { + +namespace fs { using namespace std::filesystem; } + +void builtinUnpackChannel( + const BasicDerivation & drv, + const std::map & outputs) +{ + auto getAttr = [&](const std::string & name) -> const std::string & { + auto i = drv.env.find(name); + if (i == drv.env.end()) throw Error("attribute '%s' missing", name); + return i->second; + }; + + fs::path out{outputs.at("out")}; + auto & channelName = getAttr("channelName"); + auto & src = getAttr("src"); + + if (fs::path{channelName}.filename().string() != channelName) { + throw Error("channelName is not allowed to contain filesystem separators, got %1%", channelName); + } + + try { + fs::create_directories(out); + } catch (fs::filesystem_error &) { + throw SysError("creating directory '%1%'", out.string()); + } + + unpackTarfile(src, out); + + size_t fileCount; + std::string fileName; + try { + auto entries = fs::directory_iterator{out}; + fileName = entries->path().string(); + fileCount = std::distance(fs::begin(entries), fs::end(entries)); + } catch (fs::filesystem_error &) { + throw SysError("failed to read directory %1%", out.string()); + } + + if (fileCount != 1) + throw Error("channel tarball '%s' contains more than one file", src); + + auto target = out / channelName; + try { + fs::rename(fileName, target); + } catch (fs::filesystem_error &) { + throw SysError("failed to rename %1% to %2%", fileName, target.string()); + } +} + +} diff --git a/src/libstore/common-ssh-store-config.cc b/src/libstore/common-ssh-store-config.cc new file mode 100644 index 000000000..05332b9bb --- /dev/null +++ b/src/libstore/common-ssh-store-config.cc @@ -0,0 +1,43 @@ +#include + +#include "common-ssh-store-config.hh" +#include "ssh.hh" + +namespace nix { + +static std::string extractConnStr(std::string_view scheme, std::string_view _connStr) +{ + if (_connStr.empty()) + throw UsageError("`%s` store requires a valid SSH host as the authority part in Store URI", scheme); + + std::string connStr{_connStr}; + + std::smatch result; + static std::regex v6AddrRegex("^((.*)@)?\\[(.*)\\]$"); + + if (std::regex_match(connStr, result, v6AddrRegex)) { + connStr = result[1].matched ? result.str(1) + result.str(3) : result.str(3); + } + + return connStr; +} + +CommonSSHStoreConfig::CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params) + : StoreConfig(params) + , host(extractConnStr(scheme, host)) +{ +} + +SSHMaster CommonSSHStoreConfig::createSSHMaster(bool useMaster, Descriptor logFD) +{ + return { + host, + sshKey.get(), + sshPublicHostKey.get(), + useMaster, + compress, + logFD, + }; +} + +} diff --git a/src/libstore/common-ssh-store-config.hh b/src/libstore/common-ssh-store-config.hh new file mode 100644 index 000000000..5deb6f4c9 --- /dev/null +++ b/src/libstore/common-ssh-store-config.hh @@ -0,0 +1,62 @@ +#pragma once +///@file + +#include "store-api.hh" + +namespace nix { + +class SSHMaster; + +struct CommonSSHStoreConfig : virtual StoreConfig +{ + using StoreConfig::StoreConfig; + + CommonSSHStoreConfig(std::string_view scheme, std::string_view host, const Params & params); + + const Setting sshKey{this, "", "ssh-key", + "Path to the SSH private key used to authenticate to the remote machine."}; + + const Setting sshPublicHostKey{this, "", "base64-ssh-public-host-key", + "The public host key of the remote machine."}; + + const Setting compress{this, false, "compress", + "Whether to enable SSH compression."}; + + const Setting remoteStore{this, "", "remote-store", + R"( + [Store URL](@docroot@/store/types/index.md#store-url-format) + to be used on the remote machine. The default is `auto` + (i.e. use the Nix daemon or `/nix/store` directly). + )"}; + + /** + * The `parseURL` function supports both IPv6 URIs as defined in + * RFC2732, but also pure addresses. The latter one is needed here to + * connect to a remote store via SSH (it's possible to do e.g. `ssh root@::1`). + * + * When initialized, the following adjustments are made: + * + * - If the URL looks like `root@[::1]` (which is allowed by the URL parser and probably + * needed to pass further flags), it + * will be transformed into `root@::1` for SSH (same for `[::1]` -> `::1`). + * + * - If the URL looks like `root@::1` it will be left as-is. + * + * - In any other case, the string will be left as-is. + * + * Will throw an error if `connStr` is empty too. + */ + std::string host; + + /** + * Small wrapper around `SSHMaster::SSHMaster` that gets most + * arguments from this configuration. + * + * See that constructor for details on the remaining two arguments. + */ + SSHMaster createSSHMaster( + bool useMaster, + Descriptor logFD = INVALID_DESCRIPTOR); +}; + +} diff --git a/src/libstore/content-address.cc b/src/libstore/content-address.cc index 4ed4f2de5..e1cdfece6 100644 --- a/src/libstore/content-address.cc +++ b/src/libstore/content-address.cc @@ -8,98 +8,136 @@ std::string_view makeFileIngestionPrefix(FileIngestionMethod m) { switch (m) { case FileIngestionMethod::Flat: + // Not prefixed for back compat return ""; - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: return "r:"; case FileIngestionMethod::Git: experimentalFeatureSettings.require(Xp::GitHashing); return "git:"; default: - throw Error("impossible, caught both cases"); + assert(false); } } std::string_view ContentAddressMethod::render() const { - return std::visit(overloaded { - [](TextIngestionMethod) -> std::string_view { return "text"; }, - [](FileIngestionMethod m2) { - /* Not prefixed for back compat with things that couldn't produce text before. */ - return renderFileIngestionMethod(m2); - }, - }, raw); + switch (raw) { + case ContentAddressMethod::Raw::Text: + return "text"; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return renderFileIngestionMethod(getFileIngestionMethod()); + default: + assert(false); + } +} + +/** + * **Not surjective** + * + * This is not exposed because `FileIngestionMethod::Flat` maps to + * `ContentAddressMethod::Raw::Flat` and + * `ContentAddressMethod::Raw::Text` alike. We can thus only safely use + * this when the latter is ruled out (e.g. because it is already + * handled). + */ +static ContentAddressMethod fileIngestionMethodToContentAddressMethod(FileIngestionMethod m) +{ + switch (m) { + case FileIngestionMethod::Flat: + return ContentAddressMethod::Raw::Flat; + case FileIngestionMethod::NixArchive: + return ContentAddressMethod::Raw::NixArchive; + case FileIngestionMethod::Git: + return ContentAddressMethod::Raw::Git; + default: + assert(false); + } } ContentAddressMethod ContentAddressMethod::parse(std::string_view m) { if (m == "text") - return TextIngestionMethod {}; + return ContentAddressMethod::Raw::Text; else - return parseFileIngestionMethod(m); + return fileIngestionMethodToContentAddressMethod( + parseFileIngestionMethod(m)); } std::string_view ContentAddressMethod::renderPrefix() const { - return std::visit(overloaded { - [](TextIngestionMethod) -> std::string_view { return "text:"; }, - [](FileIngestionMethod m2) { - /* Not prefixed for back compat with things that couldn't produce text before. */ - return makeFileIngestionPrefix(m2); - }, - }, raw); + switch (raw) { + case ContentAddressMethod::Raw::Text: + return "text:"; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return makeFileIngestionPrefix(getFileIngestionMethod()); + default: + assert(false); + } } ContentAddressMethod ContentAddressMethod::parsePrefix(std::string_view & m) { if (splitPrefix(m, "r:")) { - return FileIngestionMethod::Recursive; + return ContentAddressMethod::Raw::NixArchive; } else if (splitPrefix(m, "git:")) { experimentalFeatureSettings.require(Xp::GitHashing); - return FileIngestionMethod::Git; + return ContentAddressMethod::Raw::Git; } else if (splitPrefix(m, "text:")) { - return TextIngestionMethod {}; + return ContentAddressMethod::Raw::Text; + } + return ContentAddressMethod::Raw::Flat; +} + +/** + * This is slightly more mindful of forward compat in that it uses `fixed:` + * rather than just doing a raw empty prefix or `r:`, which doesn't "save room" + * for future changes very well. + */ +static std::string renderPrefixModern(const ContentAddressMethod & ca) +{ + switch (ca.raw) { + case ContentAddressMethod::Raw::Text: + return "text:"; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return "fixed:" + makeFileIngestionPrefix(ca.getFileIngestionMethod()); + default: + assert(false); } - return FileIngestionMethod::Flat; } std::string ContentAddressMethod::renderWithAlgo(HashAlgorithm ha) const { - return std::visit(overloaded { - [&](const TextIngestionMethod & th) { - return std::string{"text:"} + printHashAlgo(ha); - }, - [&](const FileIngestionMethod & fim) { - return "fixed:" + makeFileIngestionPrefix(fim) + printHashAlgo(ha); - } - }, raw); + return renderPrefixModern(*this) + printHashAlgo(ha); } FileIngestionMethod ContentAddressMethod::getFileIngestionMethod() const { - return std::visit(overloaded { - [&](const TextIngestionMethod & th) { - return FileIngestionMethod::Flat; - }, - [&](const FileIngestionMethod & fim) { - return fim; - } - }, raw); + switch (raw) { + case ContentAddressMethod::Raw::Flat: + return FileIngestionMethod::Flat; + case ContentAddressMethod::Raw::NixArchive: + return FileIngestionMethod::NixArchive; + case ContentAddressMethod::Raw::Git: + return FileIngestionMethod::Git; + case ContentAddressMethod::Raw::Text: + return FileIngestionMethod::Flat; + default: + assert(false); + } } std::string ContentAddress::render() const { - return std::visit(overloaded { - [](const TextIngestionMethod &) -> std::string { - return "text:"; - }, - [](const FileIngestionMethod & method) { - return "fixed:" - + makeFileIngestionPrefix(method); - }, - }, method.raw) - + this->hash.to_string(HashFormat::Nix32, true); + return renderPrefixModern(method) + this->hash.to_string(HashFormat::Nix32, true); } /** @@ -130,17 +168,17 @@ static std::pair parseContentAddressMethodP // No parsing of the ingestion method, "text" only support flat. HashAlgorithm hashAlgo = parseHashAlgorithm_(); return { - TextIngestionMethod {}, + ContentAddressMethod::Raw::Text, std::move(hashAlgo), }; } else if (prefix == "fixed") { // Parse method - auto method = FileIngestionMethod::Flat; + auto method = ContentAddressMethod::Raw::Flat; if (splitPrefix(rest, "r:")) - method = FileIngestionMethod::Recursive; + method = ContentAddressMethod::Raw::NixArchive; else if (splitPrefix(rest, "git:")) { experimentalFeatureSettings.require(Xp::GitHashing); - method = FileIngestionMethod::Git; + method = ContentAddressMethod::Raw::Git; } HashAlgorithm hashAlgo = parseHashAlgorithm_(); return { @@ -201,57 +239,58 @@ size_t StoreReferences::size() const ContentAddressWithReferences ContentAddressWithReferences::withoutRefs(const ContentAddress & ca) noexcept { - return std::visit(overloaded { - [&](const TextIngestionMethod &) -> ContentAddressWithReferences { - return TextInfo { - .hash = ca.hash, - .references = {}, - }; - }, - [&](const FileIngestionMethod & method) -> ContentAddressWithReferences { - return FixedOutputInfo { - .method = method, - .hash = ca.hash, - .references = {}, - }; - }, - }, ca.method.raw); + switch (ca.method.raw) { + case ContentAddressMethod::Raw::Text: + return TextInfo { + .hash = ca.hash, + .references = {}, + }; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return FixedOutputInfo { + .method = ca.method.getFileIngestionMethod(), + .hash = ca.hash, + .references = {}, + }; + default: + assert(false); + } } ContentAddressWithReferences ContentAddressWithReferences::fromParts( ContentAddressMethod method, Hash hash, StoreReferences refs) { - return std::visit(overloaded { - [&](TextIngestionMethod _) -> ContentAddressWithReferences { - if (refs.self) - throw Error("self-reference not allowed with text hashing"); - return ContentAddressWithReferences { - TextInfo { - .hash = std::move(hash), - .references = std::move(refs.others), - } - }; - }, - [&](FileIngestionMethod m2) -> ContentAddressWithReferences { - return ContentAddressWithReferences { - FixedOutputInfo { - .method = m2, - .hash = std::move(hash), - .references = std::move(refs), - } - }; - }, - }, method.raw); + switch (method.raw) { + case ContentAddressMethod::Raw::Text: + if (refs.self) + throw Error("self-reference not allowed with text hashing"); + return TextInfo { + .hash = std::move(hash), + .references = std::move(refs.others), + }; + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + return FixedOutputInfo { + .method = method.getFileIngestionMethod(), + .hash = std::move(hash), + .references = std::move(refs), + }; + default: + assert(false); + } } ContentAddressMethod ContentAddressWithReferences::getMethod() const { return std::visit(overloaded { [](const TextInfo & th) -> ContentAddressMethod { - return TextIngestionMethod {}; + return ContentAddressMethod::Raw::Text; }, [](const FixedOutputInfo & fsh) -> ContentAddressMethod { - return fsh.method; + return fileIngestionMethodToContentAddressMethod( + fsh.method); }, }, raw); } diff --git a/src/libstore/content-address.hh b/src/libstore/content-address.hh index 5925f8e01..bb515013a 100644 --- a/src/libstore/content-address.hh +++ b/src/libstore/content-address.hh @@ -5,7 +5,6 @@ #include "hash.hh" #include "path.hh" #include "file-content-address.hh" -#include "comparator.hh" #include "variant-wrapper.hh" namespace nix { @@ -14,24 +13,6 @@ namespace nix { * Content addressing method */ -/* We only have one way to hash text with references, so this is a single-value - type, mainly useful with std::variant. -*/ - -/** - * The single way we can serialize "text" file system objects. - * - * Somewhat obscure, used by \ref Derivation derivations and - * `builtins.toFile` currently. - * - * TextIngestionMethod is identical to FileIngestionMethod::Fixed except that - * the former may not have self-references and is tagged `text:${algo}:${hash}` - * rather than `fixed:${algo}:${hash}`. The contents of the store path are - * ingested and hashed identically, aside from the slightly different tag and - * restriction on self-references. - */ -struct TextIngestionMethod : std::monostate { }; - /** * Compute the prefix to the hash algorithm which indicates how the * files were ingested. @@ -48,14 +29,52 @@ std::string_view makeFileIngestionPrefix(FileIngestionMethod m); */ struct ContentAddressMethod { - typedef std::variant< - TextIngestionMethod, - FileIngestionMethod - > Raw; + enum struct Raw { + /** + * Calculate a store path using the `FileIngestionMethod::Flat` + * hash of the file system objects, and references. + * + * See `store-object/content-address.md#method-flat` in the + * manual. + */ + Flat, + + /** + * Calculate a store path using the + * `FileIngestionMethod::NixArchive` hash of the file system + * objects, and references. + * + * See `store-object/content-address.md#method-flat` in the + * manual. + */ + NixArchive, + + /** + * Calculate a store path using the `FileIngestionMethod::Git` + * hash of the file system objects, and references. + * + * Part of `ExperimentalFeature::GitHashing`. + * + * See `store-object/content-address.md#method-git` in the + * manual. + */ + Git, + + /** + * Calculate a store path using the `FileIngestionMethod::Flat` + * hash of the file system objects, and references, but in a + * different way than `ContentAddressMethod::Raw::Flat`. + * + * See `store-object/content-address.md#method-text` in the + * manual. + */ + Text, + }; Raw raw; - GENERATE_CMP(ContentAddressMethod, me->raw); + bool operator ==(const ContentAddressMethod &) const = default; + auto operator <=>(const ContentAddressMethod &) const = default; MAKE_WRAPPER_CONSTRUCTOR(ContentAddressMethod); @@ -141,7 +160,8 @@ struct ContentAddress */ Hash hash; - GENERATE_CMP(ContentAddress, me->method, me->hash); + bool operator ==(const ContentAddress &) const = default; + auto operator <=>(const ContentAddress &) const = default; /** * Compute the content-addressability assertion @@ -200,7 +220,9 @@ struct StoreReferences */ size_t size() const; - GENERATE_CMP(StoreReferences, me->self, me->others); + bool operator ==(const StoreReferences &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const StoreReferences &) const = default; }; // This matches the additional info that we need for makeTextPath @@ -217,7 +239,9 @@ struct TextInfo */ StorePathSet references; - GENERATE_CMP(TextInfo, me->hash, me->references); + bool operator ==(const TextInfo &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const TextInfo &) const = default; }; struct FixedOutputInfo @@ -237,7 +261,9 @@ struct FixedOutputInfo */ StoreReferences references; - GENERATE_CMP(FixedOutputInfo, me->hash, me->references); + bool operator ==(const FixedOutputInfo &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const FixedOutputInfo &) const = default; }; /** @@ -254,7 +280,9 @@ struct ContentAddressWithReferences Raw raw; - GENERATE_CMP(ContentAddressWithReferences, me->raw); + bool operator ==(const ContentAddressWithReferences &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=>(const ContentAddressWithReferences &) const = default; MAKE_WRAPPER_CONSTRUCTOR(ContentAddressWithReferences); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 47d6d5541..b921dbe2d 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1,6 +1,7 @@ #include "daemon.hh" #include "signals.hh" #include "worker-protocol.hh" +#include "worker-protocol-connection.hh" #include "worker-protocol-impl.hh" #include "build-result.hh" #include "store-api.hh" @@ -19,6 +20,8 @@ # include "monitor-fd.hh" #endif +#include + namespace nix::daemon { Sink & operator << (Sink & sink, const Logger::Fields & fields) @@ -30,7 +33,7 @@ Sink & operator << (Sink & sink, const Logger::Fields & fields) sink << f.i; else if (f.type == Logger::Field::tString) sink << f.s; - else abort(); + else unreachable(); } return sink; } @@ -87,11 +90,11 @@ struct TunnelLogger : public Logger { if (ei.level > verbosity) return; - std::stringstream oss; + std::ostringstream oss; showErrorInfo(oss, ei, false); StringSink buf; - buf << STDERR_NEXT << oss.str(); + buf << STDERR_NEXT << toView(oss); enqueueMsg(buf.s); } @@ -164,7 +167,7 @@ struct TunnelSink : Sink { Sink & to; TunnelSink(Sink & to) : to(to) { } - void operator () (std::string_view data) + void operator () (std::string_view data) override { to << STDERR_WRITE; writeString(data, to); @@ -243,10 +246,9 @@ struct ClientSettings // the daemon, as that could cause some pretty weird stuff if (parseFeatures(tokenizeString(value)) != experimentalFeatureSettings.experimentalFeatures.get()) debug("Ignoring the client-specified experimental features"); - } else if (name == settings.pluginFiles.name) { - if (tokenizeString(value) != settings.pluginFiles.get()) - warn("Ignoring the client-specified plugin-files.\n" - "The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14."); + } else if (name == "plugin-files") { + warn("Ignoring the client-specified plugin-files.\n" + "The client specifying plugins to the daemon never made sense, and was removed in Nix >=2.14."); } else if (trusted || name == settings.buildTimeout.name @@ -267,26 +269,21 @@ struct ClientSettings }; static void performOp(TunnelLogger * logger, ref store, - TrustedFlag trusted, RecursiveFlag recursive, WorkerProto::Version clientVersion, - Source & from, BufferedSink & to, WorkerProto::Op op) + TrustedFlag trusted, RecursiveFlag recursive, + WorkerProto::BasicServerConnection & conn, + WorkerProto::Op op) { - WorkerProto::ReadConn rconn { - .from = from, - .version = clientVersion, - }; - WorkerProto::WriteConn wconn { - .to = to, - .version = clientVersion, - }; + WorkerProto::ReadConn rconn(conn); + WorkerProto::WriteConn wconn(conn); switch (op) { case WorkerProto::Op::IsValidPath: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); bool result = store->isValidPath(path); logger->stopWork(); - to << result; + conn.to << result; break; } @@ -294,8 +291,8 @@ static void performOp(TunnelLogger * logger, ref store, auto paths = WorkerProto::Serialise::read(*store, rconn); SubstituteFlag substitute = NoSubstitute; - if (GET_PROTOCOL_MINOR(clientVersion) >= 27) { - substitute = readInt(from) ? Substitute : NoSubstitute; + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 27) { + substitute = readInt(conn.from) ? Substitute : NoSubstitute; } logger->startWork(); @@ -309,13 +306,13 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::HasSubstitutes: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); StorePathSet paths; // FIXME paths.insert(path); auto res = store->querySubstitutablePaths(paths); logger->stopWork(); - to << (res.count(path) != 0); + conn.to << (res.count(path) != 0); break; } @@ -329,11 +326,11 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryPathHash: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto hash = store->queryPathInfo(path)->narHash; logger->stopWork(); - to << hash.to_string(HashFormat::Base16, false); + conn.to << hash.to_string(HashFormat::Base16, false); break; } @@ -341,7 +338,7 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::QueryReferrers: case WorkerProto::Op::QueryValidDerivers: case WorkerProto::Op::QueryDerivationOutputs: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); StorePathSet paths; if (op == WorkerProto::Op::QueryReferences) @@ -358,16 +355,16 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryDerivationOutputNames: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto names = store->readDerivation(path).outputNames(); logger->stopWork(); - to << names; + conn.to << names; break; } case WorkerProto::Op::QueryDerivationOutputMap: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto outputs = store->queryPartialDerivationOutputMap(path); logger->stopWork(); @@ -376,48 +373,51 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryDeriver: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); auto info = store->queryPathInfo(path); logger->stopWork(); - to << (info->deriver ? store->printStorePath(*info->deriver) : ""); + conn.to << (info->deriver ? store->printStorePath(*info->deriver) : ""); break; } case WorkerProto::Op::QueryPathFromHashPart: { - auto hashPart = readString(from); + auto hashPart = readString(conn.from); logger->startWork(); auto path = store->queryPathFromHashPart(hashPart); logger->stopWork(); - to << (path ? store->printStorePath(*path) : ""); + conn.to << (path ? store->printStorePath(*path) : ""); break; } case WorkerProto::Op::AddToStore: { - if (GET_PROTOCOL_MINOR(clientVersion) >= 25) { - auto name = readString(from); - auto camStr = readString(from); + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 25) { + auto name = readString(conn.from); + auto camStr = readString(conn.from); auto refs = WorkerProto::Serialise::read(*store, rconn); bool repairBool; - from >> repairBool; + conn.from >> repairBool; auto repair = RepairFlag{repairBool}; logger->startWork(); auto pathInfo = [&]() { // NB: FramedSource must be out of scope before logger->stopWork(); + // FIXME: this means that if there is an error + // half-way through, the client will keep sending + // data, since we haven't sent it the error yet. auto [contentAddressMethod, hashAlgo] = ContentAddressMethod::parseWithAlgo(camStr); - FramedSource source(from); + FramedSource source(conn.from); FileSerialisationMethod dumpMethod; switch (contentAddressMethod.getFileIngestionMethod()) { case FileIngestionMethod::Flat: dumpMethod = FileSerialisationMethod::Flat; break; - case FileIngestionMethod::Recursive: - dumpMethod = FileSerialisationMethod::Recursive; + case FileIngestionMethod::NixArchive: + dumpMethod = FileSerialisationMethod::NixArchive; break; case FileIngestionMethod::Git: // Use NAR; Git is not a serialization method - dumpMethod = FileSerialisationMethod::Recursive; + dumpMethod = FileSerialisationMethod::NixArchive; break; default: assert(false); @@ -432,19 +432,21 @@ static void performOp(TunnelLogger * logger, ref store, } else { HashAlgorithm hashAlgo; std::string baseName; - FileIngestionMethod method; + ContentAddressMethod method; { bool fixed; uint8_t recursive; std::string hashAlgoRaw; - from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; - if (recursive > (uint8_t) FileIngestionMethod::Recursive) + conn.from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw; + if (recursive > true) throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive); - method = FileIngestionMethod { recursive }; + method = recursive + ? ContentAddressMethod::Raw::NixArchive + : ContentAddressMethod::Raw::Flat; /* Compatibility hack. */ if (!fixed) { hashAlgoRaw = "sha256"; - method = FileIngestionMethod::Recursive; + method = ContentAddressMethod::Raw::NixArchive; } hashAlgo = parseHashAlgo(hashAlgoRaw); } @@ -455,33 +457,33 @@ static void performOp(TunnelLogger * logger, ref store, so why all this extra work? We still parse the NAR so that we aren't sending arbitrary data to `saved` unwittingly`, and we know when the NAR ends so we don't - consume the rest of `from` and can't parse another + consume the rest of `conn.from` and can't parse another command. (We don't trust `addToStoreFromDump` to not eagerly consume the entire stream it's given, past the length of the Nar. */ - TeeSource savedNARSource(from, saved); + TeeSource savedNARSource(conn.from, saved); NullFileSystemObjectSink sink; /* just parse the NAR */ parseDump(sink, savedNARSource); }); logger->startWork(); auto path = store->addToStoreFromDump( - *dumpSource, baseName, FileSerialisationMethod::Recursive, method, hashAlgo); + *dumpSource, baseName, FileSerialisationMethod::NixArchive, method, hashAlgo); logger->stopWork(); - to << store->printStorePath(path); + conn.to << store->printStorePath(path); } break; } case WorkerProto::Op::AddMultipleToStore: { bool repair, dontCheckSigs; - from >> repair >> dontCheckSigs; + conn.from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; logger->startWork(); { - FramedSource source(from); + FramedSource source(conn.from); store->addMultipleToStore(source, RepairFlag{repair}, dontCheckSigs ? NoCheckSigs : CheckSigs); @@ -491,47 +493,47 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::AddTextToStore: { - std::string suffix = readString(from); - std::string s = readString(from); + std::string suffix = readString(conn.from); + std::string s = readString(conn.from); auto refs = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); auto path = ({ StringSource source { s }; - store->addToStoreFromDump(source, suffix, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, refs, NoRepair); + store->addToStoreFromDump(source, suffix, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, refs, NoRepair); }); logger->stopWork(); - to << store->printStorePath(path); + conn.to << store->printStorePath(path); break; } case WorkerProto::Op::ExportPath: { - auto path = store->parseStorePath(readString(from)); - readInt(from); // obsolete + auto path = store->parseStorePath(readString(conn.from)); + readInt(conn.from); // obsolete logger->startWork(); - TunnelSink sink(to); + TunnelSink sink(conn.to); store->exportPath(path, sink); logger->stopWork(); - to << 1; + conn.to << 1; break; } case WorkerProto::Op::ImportPaths: { logger->startWork(); - TunnelSource source(from, to); + TunnelSource source(conn.from, conn.to); auto paths = store->importPaths(source, trusted ? NoCheckSigs : CheckSigs); logger->stopWork(); Strings paths2; for (auto & i : paths) paths2.push_back(store->printStorePath(i)); - to << paths2; + conn.to << paths2; break; } case WorkerProto::Op::BuildPaths: { auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; - if (GET_PROTOCOL_MINOR(clientVersion) >= 15) { - mode = (BuildMode) readInt(from); + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 15) { + mode = WorkerProto::Serialise::read(*store, rconn); /* Repairing is not atomic, so disallowed for "untrusted" clients. @@ -548,14 +550,14 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); store->buildPaths(drvs, mode); logger->stopWork(); - to << 1; + conn.to << 1; break; } case WorkerProto::Op::BuildPathsWithResults: { auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; - mode = (BuildMode) readInt(from); + mode = WorkerProto::Serialise::read(*store, rconn); /* Repairing is not atomic, so disallowed for "untrusted" clients. @@ -574,7 +576,7 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::BuildDerivation: { - auto drvPath = store->parseStorePath(readString(from)); + auto drvPath = store->parseStorePath(readString(conn.from)); BasicDerivation drv; /* * Note: unlike wopEnsurePath, this operation reads a @@ -585,8 +587,8 @@ static void performOp(TunnelLogger * logger, ref store, * it cannot be trusted that its outPath was calculated * correctly. */ - readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath)); - BuildMode buildMode = (BuildMode) readInt(from); + readDerivation(conn.from, *store, drv, Derivation::nameFromPath(drvPath)); + auto buildMode = WorkerProto::Serialise::read(*store, rconn); logger->startWork(); auto drvType = drv.type(); @@ -651,20 +653,20 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::EnsurePath: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); store->ensurePath(path); logger->stopWork(); - to << 1; + conn.to << 1; break; } case WorkerProto::Op::AddTempRoot: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); store->addTempRoot(path); logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -674,24 +676,24 @@ static void performOp(TunnelLogger * logger, ref store, "you are not privileged to create perm roots\n\n" "hint: you can just do this client-side without special privileges, and probably want to do that instead."); auto storePath = WorkerProto::Serialise::read(*store, rconn); - Path gcRoot = absPath(readString(from)); + Path gcRoot = absPath(readString(conn.from)); logger->startWork(); auto & localFSStore = require(*store); localFSStore.addPermRoot(storePath, gcRoot); logger->stopWork(); - to << gcRoot; + conn.to << gcRoot; break; } case WorkerProto::Op::AddIndirectRoot: { - Path path = absPath(readString(from)); + Path path = absPath(readString(conn.from)); logger->startWork(); auto & indirectRootStore = require(*store); indirectRootStore.addIndirectRoot(path); logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -699,7 +701,7 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::SyncWithGC: { logger->startWork(); logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -713,24 +715,24 @@ static void performOp(TunnelLogger * logger, ref store, for (auto & i : roots) size += i.second.size(); - to << size; + conn.to << size; for (auto & [target, links] : roots) for (auto & link : links) - to << link << store->printStorePath(target); + conn.to << link << store->printStorePath(target); break; } case WorkerProto::Op::CollectGarbage: { GCOptions options; - options.action = (GCOptions::GCAction) readInt(from); + options.action = (GCOptions::GCAction) readInt(conn.from); options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); - from >> options.ignoreLiveness >> options.maxFreed; + conn.from >> options.ignoreLiveness >> options.maxFreed; // obsolete fields - readInt(from); - readInt(from); - readInt(from); + readInt(conn.from); + readInt(conn.from); + readInt(conn.from); GCResults results; @@ -741,7 +743,7 @@ static void performOp(TunnelLogger * logger, ref store, gcStore.collectGarbage(options, results); logger->stopWork(); - to << results.paths << results.bytesFreed << 0 /* obsolete */; + conn.to << results.paths << results.bytesFreed << 0 /* obsolete */; break; } @@ -750,24 +752,24 @@ static void performOp(TunnelLogger * logger, ref store, ClientSettings clientSettings; - clientSettings.keepFailed = readInt(from); - clientSettings.keepGoing = readInt(from); - clientSettings.tryFallback = readInt(from); - clientSettings.verbosity = (Verbosity) readInt(from); - clientSettings.maxBuildJobs = readInt(from); - clientSettings.maxSilentTime = readInt(from); - readInt(from); // obsolete useBuildHook - clientSettings.verboseBuild = lvlError == (Verbosity) readInt(from); - readInt(from); // obsolete logType - readInt(from); // obsolete printBuildTrace - clientSettings.buildCores = readInt(from); - clientSettings.useSubstitutes = readInt(from); + clientSettings.keepFailed = readInt(conn.from); + clientSettings.keepGoing = readInt(conn.from); + clientSettings.tryFallback = readInt(conn.from); + clientSettings.verbosity = (Verbosity) readInt(conn.from); + clientSettings.maxBuildJobs = readInt(conn.from); + clientSettings.maxSilentTime = readInt(conn.from); + readInt(conn.from); // obsolete useBuildHook + clientSettings.verboseBuild = lvlError == (Verbosity) readInt(conn.from); + readInt(conn.from); // obsolete logType + readInt(conn.from); // obsolete printBuildTrace + clientSettings.buildCores = readInt(conn.from); + clientSettings.useSubstitutes = readInt(conn.from); - if (GET_PROTOCOL_MINOR(clientVersion) >= 12) { - unsigned int n = readInt(from); + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) { + unsigned int n = readInt(conn.from); for (unsigned int i = 0; i < n; i++) { - auto name = readString(from); - auto value = readString(from); + auto name = readString(conn.from); + auto value = readString(conn.from); clientSettings.overrides.emplace(name, value); } } @@ -784,20 +786,20 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QuerySubstitutablePathInfo: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); SubstitutablePathInfos infos; store->querySubstitutablePathInfos({{path, std::nullopt}}, infos); logger->stopWork(); auto i = infos.find(path); if (i == infos.end()) - to << 0; + conn.to << 0; else { - to << 1 + conn.to << 1 << (i->second.deriver ? store->printStorePath(*i->second.deriver) : ""); WorkerProto::write(*store, wconn, i->second.references); - to << i->second.downloadSize - << i->second.narSize; + conn.to << i->second.downloadSize + << i->second.narSize; } break; } @@ -805,7 +807,7 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::QuerySubstitutablePathInfos: { SubstitutablePathInfos infos; StorePathCAMap pathsMap = {}; - if (GET_PROTOCOL_MINOR(clientVersion) < 22) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 22) { auto paths = WorkerProto::Serialise::read(*store, rconn); for (auto & path : paths) pathsMap.emplace(path, std::nullopt); @@ -814,12 +816,12 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); store->querySubstitutablePathInfos(pathsMap, infos); logger->stopWork(); - to << infos.size(); + conn.to << infos.size(); for (auto & i : infos) { - to << store->printStorePath(i.first) - << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); + conn.to << store->printStorePath(i.first) + << (i.second.deriver ? store->printStorePath(*i.second.deriver) : ""); WorkerProto::write(*store, wconn, i.second.references); - to << i.second.downloadSize << i.second.narSize; + conn.to << i.second.downloadSize << i.second.narSize; } break; } @@ -833,22 +835,22 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::QueryPathInfo: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); std::shared_ptr info; logger->startWork(); try { info = store->queryPathInfo(path); } catch (InvalidPath &) { - if (GET_PROTOCOL_MINOR(clientVersion) < 17) throw; + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 17) throw; } logger->stopWork(); if (info) { - if (GET_PROTOCOL_MINOR(clientVersion) >= 17) - to << 1; + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 17) + conn.to << 1; WorkerProto::write(*store, wconn, static_cast(*info)); } else { - assert(GET_PROTOCOL_MINOR(clientVersion) >= 17); - to << 0; + assert(GET_PROTOCOL_MINOR(conn.protoVersion) >= 17); + conn.to << 0; } break; } @@ -857,61 +859,61 @@ static void performOp(TunnelLogger * logger, ref store, logger->startWork(); store->optimiseStore(); logger->stopWork(); - to << 1; + conn.to << 1; break; case WorkerProto::Op::VerifyStore: { bool checkContents, repair; - from >> checkContents >> repair; + conn.from >> checkContents >> repair; logger->startWork(); if (repair && !trusted) throw Error("you are not privileged to repair paths"); bool errors = store->verifyStore(checkContents, (RepairFlag) repair); logger->stopWork(); - to << errors; + conn.to << errors; break; } case WorkerProto::Op::AddSignatures: { - auto path = store->parseStorePath(readString(from)); - StringSet sigs = readStrings(from); + auto path = store->parseStorePath(readString(conn.from)); + StringSet sigs = readStrings(conn.from); logger->startWork(); store->addSignatures(path, sigs); logger->stopWork(); - to << 1; + conn.to << 1; break; } case WorkerProto::Op::NarFromPath: { - auto path = store->parseStorePath(readString(from)); + auto path = store->parseStorePath(readString(conn.from)); logger->startWork(); logger->stopWork(); - dumpPath(store->toRealPath(path), to); + dumpPath(store->toRealPath(path), conn.to); break; } case WorkerProto::Op::AddToStoreNar: { bool repair, dontCheckSigs; - auto path = store->parseStorePath(readString(from)); - auto deriver = readString(from); - auto narHash = Hash::parseAny(readString(from), HashAlgorithm::SHA256); + auto path = store->parseStorePath(readString(conn.from)); + auto deriver = readString(conn.from); + auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256); ValidPathInfo info { path, narHash }; if (deriver != "") info.deriver = store->parseStorePath(deriver); info.references = WorkerProto::Serialise::read(*store, rconn); - from >> info.registrationTime >> info.narSize >> info.ultimate; - info.sigs = readStrings(from); - info.ca = ContentAddress::parseOpt(readString(from)); - from >> repair >> dontCheckSigs; + conn.from >> info.registrationTime >> info.narSize >> info.ultimate; + info.sigs = readStrings(conn.from); + info.ca = ContentAddress::parseOpt(readString(conn.from)); + conn.from >> repair >> dontCheckSigs; if (!trusted && dontCheckSigs) dontCheckSigs = false; if (!trusted) info.ultimate = false; - if (GET_PROTOCOL_MINOR(clientVersion) >= 23) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 23) { logger->startWork(); { - FramedSource source(from); + FramedSource source(conn.from); store->addToStore(info, source, (RepairFlag) repair, dontCheckSigs ? NoCheckSigs : CheckSigs); } @@ -921,10 +923,10 @@ static void performOp(TunnelLogger * logger, ref store, else { std::unique_ptr source; StringSink saved; - if (GET_PROTOCOL_MINOR(clientVersion) >= 21) - source = std::make_unique(from, to); + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 21) + source = std::make_unique(conn.from, conn.to); else { - TeeSource tee { from, saved }; + TeeSource tee { conn.from, saved }; NullFileSystemObjectSink ether; parseDump(ether, tee); source = std::make_unique(saved.s); @@ -952,15 +954,15 @@ static void performOp(TunnelLogger * logger, ref store, WorkerProto::write(*store, wconn, willBuild); WorkerProto::write(*store, wconn, willSubstitute); WorkerProto::write(*store, wconn, unknown); - to << downloadSize << narSize; + conn.to << downloadSize << narSize; break; } case WorkerProto::Op::RegisterDrvOutput: { logger->startWork(); - if (GET_PROTOCOL_MINOR(clientVersion) < 31) { - auto outputId = DrvOutput::parse(readString(from)); - auto outputPath = StorePath(readString(from)); + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { + auto outputId = DrvOutput::parse(readString(conn.from)); + auto outputPath = StorePath(readString(conn.from)); store->registerDrvOutput(Realisation{ .id = outputId, .outPath = outputPath}); } else { @@ -973,10 +975,10 @@ static void performOp(TunnelLogger * logger, ref store, case WorkerProto::Op::QueryRealisation: { logger->startWork(); - auto outputId = DrvOutput::parse(readString(from)); + auto outputId = DrvOutput::parse(readString(conn.from)); auto info = store->queryRealisation(outputId); logger->stopWork(); - if (GET_PROTOCOL_MINOR(clientVersion) < 31) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) < 31) { std::set outPaths; if (info) outPaths.insert(info->outPath); WorkerProto::write(*store, wconn, outPaths); @@ -989,19 +991,19 @@ static void performOp(TunnelLogger * logger, ref store, } case WorkerProto::Op::AddBuildLog: { - StorePath path{readString(from)}; + StorePath path{readString(conn.from)}; logger->startWork(); if (!trusted) throw Error("you are not privileged to add logs"); auto & logStore = require(*store); { - FramedSource source(from); + FramedSource source(conn.from); StringSink sink; source.drainInto(sink); logStore.addBuildLog(path, sink.s); } logger->stopWork(); - to << 1; + conn.to << 1; break; } @@ -1016,8 +1018,8 @@ static void performOp(TunnelLogger * logger, ref store, void processConnection( ref store, - FdSource & from, - FdSink & to, + FdSource && from, + FdSink && to, TrustedFlag trusted, RecursiveFlag recursive) { @@ -1026,16 +1028,20 @@ void processConnection( #endif /* Exchange the greeting. */ - unsigned int magic = readInt(from); - if (magic != WORKER_MAGIC_1) throw Error("protocol mismatch"); - to << WORKER_MAGIC_2 << PROTOCOL_VERSION; - to.flush(); - WorkerProto::Version clientVersion = readInt(from); + auto [protoVersion, features] = + WorkerProto::BasicServerConnection::handshake( + to, from, PROTOCOL_VERSION, WorkerProto::allFeatures); - if (clientVersion < 0x10a) + if (protoVersion < 0x10a) throw Error("the Nix client version is too old"); - auto tunnelLogger = new TunnelLogger(to, clientVersion); + WorkerProto::BasicServerConnection conn; + conn.to = std::move(to); + conn.from = std::move(from); + conn.protoVersion = protoVersion; + conn.features = features; + + auto tunnelLogger = new TunnelLogger(conn.to, protoVersion); auto prevLogger = nix::logger; // FIXME if (!recursive) @@ -1048,29 +1054,14 @@ void processConnection( printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount); }); - if (GET_PROTOCOL_MINOR(clientVersion) >= 14 && readInt(from)) { - // Obsolete CPU affinity. - readInt(from); - } - - if (GET_PROTOCOL_MINOR(clientVersion) >= 11) - readInt(from); // obsolete reserveSpace - - if (GET_PROTOCOL_MINOR(clientVersion) >= 33) - to << nixVersion; - - if (GET_PROTOCOL_MINOR(clientVersion) >= 35) { + conn.postHandshake(*store, { + .daemonNixVersion = nixVersion, // We and the underlying store both need to trust the client for // it to be trusted. - auto temp = trusted + .remoteTrustsUs = trusted ? store->isTrustedClient() - : std::optional { NotTrusted }; - WorkerProto::WriteConn wconn { - .to = to, - .version = clientVersion, - }; - WorkerProto::write(*store, wconn, temp); - } + : std::optional { NotTrusted }, + }); /* Send startup error messages to the client. */ tunnelLogger->startWork(); @@ -1078,13 +1069,13 @@ void processConnection( try { tunnelLogger->stopWork(); - to.flush(); + conn.to.flush(); /* Process client requests. */ while (true) { WorkerProto::Op op; try { - op = (enum WorkerProto::Op) readInt(from); + op = (enum WorkerProto::Op) readInt(conn.from); } catch (Interrupted & e) { break; } catch (EndOfFile & e) { @@ -1098,7 +1089,7 @@ void processConnection( debug("performing daemon worker op: %d", op); try { - performOp(tunnelLogger, store, trusted, recursive, clientVersion, from, to, op); + performOp(tunnelLogger, store, trusted, recursive, conn, op); } catch (Error & e) { /* If we're not in a state where we can send replies, then something went wrong processing the input of the @@ -1114,19 +1105,19 @@ void processConnection( throw; } - to.flush(); + conn.to.flush(); assert(!tunnelLogger->state_.lock()->canSendStderr); }; } catch (Error & e) { tunnelLogger->stopWork(&e); - to.flush(); + conn.to.flush(); return; } catch (std::exception & e) { auto ex = Error(e.what()); tunnelLogger->stopWork(&ex); - to.flush(); + conn.to.flush(); return; } } diff --git a/src/libstore/daemon.hh b/src/libstore/daemon.hh index 1964c0d99..a8ce32d8d 100644 --- a/src/libstore/daemon.hh +++ b/src/libstore/daemon.hh @@ -10,8 +10,8 @@ enum RecursiveFlag : bool { NotRecursive = false, Recursive = true }; void processConnection( ref store, - FdSource & from, - FdSink & to, + FdSource && from, + FdSink && to, TrustedFlag trusted, RecursiveFlag recursive); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index fcf813a37..9b6f67852 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -7,6 +7,9 @@ #include "split.hh" #include "common-protocol.hh" #include "common-protocol-impl.hh" +#include "strings-inline.hh" +#include "json-utils.hh" + #include #include @@ -150,7 +153,7 @@ StorePath writeDerivation(Store & store, }) : ({ StringSource s { contents }; - store.addToStoreFromDump(s, suffix, FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, references, repair); + store.addToStoreFromDump(s, suffix, FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references, repair); }); } @@ -274,7 +277,7 @@ static DerivationOutput parseDerivationOutput( { if (hashAlgoStr != "") { ContentAddressMethod method = ContentAddressMethod::parsePrefix(hashAlgoStr); - if (method == TextIngestionMethod {}) + if (method == ContentAddressMethod::Raw::Text) xpSettings.require(Xp::DynamicDerivations); const auto hashAlgo = parseHashAlgo(hashAlgoStr); if (hashS == "impure") { @@ -930,10 +933,9 @@ DerivationOutputsAndOptPaths BasicDerivation::outputsAndOptPaths(const StoreDirC std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) { + drvPath.requireDerivation(); auto nameWithSuffix = drvPath.name(); - constexpr std::string_view extension = ".drv"; - assert(hasSuffix(nameWithSuffix, extension)); - nameWithSuffix.remove_suffix(extension.size()); + nameWithSuffix.remove_suffix(drvExtension.size()); return nameWithSuffix; } @@ -1216,16 +1218,19 @@ nlohmann::json DerivationOutput::toJSON( }, [&](const DerivationOutput::CAFixed & dof) { res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); - res["hashAlgo"] = dof.ca.printMethodAlgo(); + res["method"] = std::string { dof.ca.method.render() }; + res["hashAlgo"] = printHashAlgo(dof.ca.hash.algo); res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false); // FIXME print refs? }, [&](const DerivationOutput::CAFloating & dof) { - res["hashAlgo"] = std::string { dof.method.renderPrefix() } + printHashAlgo(dof.hashAlgo); + res["method"] = std::string { dof.method.render() }; + res["hashAlgo"] = printHashAlgo(dof.hashAlgo); }, [&](const DerivationOutput::Deferred &) {}, [&](const DerivationOutput::Impure & doi) { - res["hashAlgo"] = std::string { doi.method.renderPrefix() } + printHashAlgo(doi.hashAlgo); + res["method"] = std::string { doi.method.render() }; + res["hashAlgo"] = printHashAlgo(doi.hashAlgo); res["impure"] = true; }, }, raw); @@ -1245,12 +1250,13 @@ DerivationOutput DerivationOutput::fromJSON( keys.insert(key); auto methodAlgo = [&]() -> std::pair { - auto & str = getString(valueAt(json, "hashAlgo")); - std::string_view s = str; - ContentAddressMethod method = ContentAddressMethod::parsePrefix(s); - if (method == TextIngestionMethod {}) + auto & method_ = getString(valueAt(json, "method")); + ContentAddressMethod method = ContentAddressMethod::parse(method_); + if (method == ContentAddressMethod::Raw::Text) xpSettings.require(Xp::DynamicDerivations); - auto hashAlgo = parseHashAlgo(s); + + auto & hashAlgo_ = getString(valueAt(json, "hashAlgo")); + auto hashAlgo = parseHashAlgo(hashAlgo_); return { std::move(method), std::move(hashAlgo) }; }; @@ -1260,7 +1266,7 @@ DerivationOutput DerivationOutput::fromJSON( }; } - else if (keys == (std::set { "path", "hashAlgo", "hash" })) { + else if (keys == (std::set { "path", "method", "hashAlgo", "hash" })) { auto [method, hashAlgo] = methodAlgo(); auto dof = DerivationOutput::CAFixed { .ca = ContentAddress { @@ -1273,7 +1279,7 @@ DerivationOutput DerivationOutput::fromJSON( return dof; } - else if (keys == (std::set { "hashAlgo" })) { + else if (keys == (std::set { "method", "hashAlgo" })) { xpSettings.require(Xp::CaDerivations); auto [method, hashAlgo] = methodAlgo(); return DerivationOutput::CAFloating { @@ -1286,7 +1292,7 @@ DerivationOutput DerivationOutput::fromJSON( return DerivationOutput::Deferred {}; } - else if (keys == (std::set { "hashAlgo", "impure" })) { + else if (keys == (std::set { "method", "hashAlgo", "impure" })) { xpSettings.require(Xp::ImpureDerivations); auto [method, hashAlgo] = methodAlgo(); return DerivationOutput::Impure { diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 522523e45..58e5328a5 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -8,13 +8,11 @@ #include "repair-flag.hh" #include "derived-path-map.hh" #include "sync.hh" -#include "comparator.hh" #include "variant-wrapper.hh" #include #include - namespace nix { struct StoreDirConfig; @@ -33,7 +31,8 @@ struct DerivationOutput { StorePath path; - GENERATE_CMP(InputAddressed, me->path); + bool operator == (const InputAddressed &) const = default; + auto operator <=> (const InputAddressed &) const = default; }; /** @@ -57,7 +56,8 @@ struct DerivationOutput */ StorePath path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; - GENERATE_CMP(CAFixed, me->ca); + bool operator == (const CAFixed &) const = default; + auto operator <=> (const CAFixed &) const = default; }; /** @@ -77,7 +77,8 @@ struct DerivationOutput */ HashAlgorithm hashAlgo; - GENERATE_CMP(CAFloating, me->method, me->hashAlgo); + bool operator == (const CAFloating &) const = default; + auto operator <=> (const CAFloating &) const = default; }; /** @@ -85,7 +86,8 @@ struct DerivationOutput * isn't known yet. */ struct Deferred { - GENERATE_CMP(Deferred); + bool operator == (const Deferred &) const = default; + auto operator <=> (const Deferred &) const = default; }; /** @@ -104,7 +106,8 @@ struct DerivationOutput */ HashAlgorithm hashAlgo; - GENERATE_CMP(Impure, me->method, me->hashAlgo); + bool operator == (const Impure &) const = default; + auto operator <=> (const Impure &) const = default; }; typedef std::variant< @@ -117,7 +120,8 @@ struct DerivationOutput Raw raw; - GENERATE_CMP(DerivationOutput, me->raw); + bool operator == (const DerivationOutput &) const = default; + auto operator <=> (const DerivationOutput &) const = default; MAKE_WRAPPER_CONSTRUCTOR(DerivationOutput); @@ -178,7 +182,8 @@ struct DerivationType { */ bool deferred; - GENERATE_CMP(InputAddressed, me->deferred); + bool operator == (const InputAddressed &) const = default; + auto operator <=> (const InputAddressed &) const = default; }; /** @@ -202,7 +207,8 @@ struct DerivationType { */ bool fixed; - GENERATE_CMP(ContentAddressed, me->sandboxed, me->fixed); + bool operator == (const ContentAddressed &) const = default; + auto operator <=> (const ContentAddressed &) const = default; }; /** @@ -212,7 +218,8 @@ struct DerivationType { * type, but has some restrictions on its usage. */ struct Impure { - GENERATE_CMP(Impure); + bool operator == (const Impure &) const = default; + auto operator <=> (const Impure &) const = default; }; typedef std::variant< @@ -223,7 +230,8 @@ struct DerivationType { Raw raw; - GENERATE_CMP(DerivationType, me->raw); + bool operator == (const DerivationType &) const = default; + auto operator <=> (const DerivationType &) const = default; MAKE_WRAPPER_CONSTRUCTOR(DerivationType); @@ -313,14 +321,9 @@ struct BasicDerivation static std::string_view nameFromPath(const StorePath & storePath); - GENERATE_CMP(BasicDerivation, - me->outputs, - me->inputSrcs, - me->platform, - me->builder, - me->args, - me->env, - me->name); + bool operator == (const BasicDerivation &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const BasicDerivation &) const = default; }; class Store; @@ -378,9 +381,9 @@ struct Derivation : BasicDerivation const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); - GENERATE_CMP(Derivation, - static_cast(*me), - me->inputDrvs); + bool operator == (const Derivation &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + //auto operator <=> (const Derivation &) const = default; }; diff --git a/src/libstore/derived-path-map.cc b/src/libstore/derived-path-map.cc index 4c1ea417a..c97d52773 100644 --- a/src/libstore/derived-path-map.cc +++ b/src/libstore/derived-path-map.cc @@ -54,17 +54,18 @@ typename DerivedPathMap::ChildNode * DerivedPathMap::findSlot(const Single namespace nix { -GENERATE_CMP_EXT( - template<>, - DerivedPathMap>::ChildNode, - me->value, - me->childMap); +template<> +bool DerivedPathMap>::ChildNode::operator == ( + const DerivedPathMap>::ChildNode &) const noexcept = default; -GENERATE_CMP_EXT( - template<>, - DerivedPathMap>, - me->map); +// TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. +#if 0 +template<> +std::strong_ordering DerivedPathMap>::ChildNode::operator <=> ( + const DerivedPathMap>::ChildNode &) const noexcept = default; +#endif +template struct DerivedPathMap>::ChildNode; template struct DerivedPathMap>; }; diff --git a/src/libstore/derived-path-map.hh b/src/libstore/derived-path-map.hh index 393cdedf7..bd60fe887 100644 --- a/src/libstore/derived-path-map.hh +++ b/src/libstore/derived-path-map.hh @@ -47,7 +47,11 @@ struct DerivedPathMap { */ Map childMap; - DECLARE_CMP(ChildNode); + bool operator == (const ChildNode &) const noexcept; + + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + // decltype(std::declval() <=> std::declval()) + // operator <=> (const ChildNode &) const noexcept; }; /** @@ -60,7 +64,10 @@ struct DerivedPathMap { */ Map map; - DECLARE_CMP(DerivedPathMap); + bool operator == (const DerivedPathMap &) const = default; + + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + // auto operator <=> (const DerivedPathMap &) const noexcept; /** * Find the node for `k`, creating it if needed. @@ -83,14 +90,21 @@ struct DerivedPathMap { ChildNode * findSlot(const SingleDerivedPath & k); }; +template<> +bool DerivedPathMap>::ChildNode::operator == ( + const DerivedPathMap>::ChildNode &) const noexcept; -DECLARE_CMP_EXT( - template<>, - DerivedPathMap>::, - DerivedPathMap>); -DECLARE_CMP_EXT( - template<>, - DerivedPathMap>::ChildNode::, - DerivedPathMap>::ChildNode); +// TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. +#if 0 +template<> +std::strong_ordering DerivedPathMap>::ChildNode::operator <=> ( + const DerivedPathMap>::ChildNode &) const noexcept; + +template<> +inline auto DerivedPathMap>::operator <=> (const DerivedPathMap> &) const noexcept = default; +#endif + +extern template struct DerivedPathMap>::ChildNode; +extern template struct DerivedPathMap>; } diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index a7b404321..1eef881de 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -1,6 +1,7 @@ #include "derived-path.hh" #include "derivations.hh" #include "store-api.hh" +#include "comparator.hh" #include @@ -8,26 +9,32 @@ namespace nix { -#define CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, COMPARATOR) \ - bool MY_TYPE ::operator COMPARATOR (const MY_TYPE & other) const \ - { \ - const MY_TYPE* me = this; \ - auto fields1 = std::tie(*me->drvPath, me->FIELD); \ - me = &other; \ - auto fields2 = std::tie(*me->drvPath, me->FIELD); \ - return fields1 COMPARATOR fields2; \ - } -#define CMP(CHILD_TYPE, MY_TYPE, FIELD) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, ==) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, !=) \ - CMP_ONE(CHILD_TYPE, MY_TYPE, FIELD, <) +// Custom implementation to avoid `ref` ptr equality +GENERATE_CMP_EXT( + , + std::strong_ordering, + SingleDerivedPathBuilt, + *me->drvPath, + me->output); -CMP(SingleDerivedPath, SingleDerivedPathBuilt, output) +// Custom implementation to avoid `ref` ptr equality -CMP(SingleDerivedPath, DerivedPathBuilt, outputs) - -#undef CMP -#undef CMP_ONE +// TODO no `GENERATE_CMP_EXT` because no `std::set::operator<=>` on +// Darwin, per header. +GENERATE_EQUAL( + , + DerivedPathBuilt ::, + DerivedPathBuilt, + *me->drvPath, + me->outputs); +GENERATE_ONE_CMP( + , + bool, + DerivedPathBuilt ::, + <, + DerivedPathBuilt, + *me->drvPath, + me->outputs); nlohmann::json DerivedPath::Opaque::toJSON(const StoreDirConfig & store) const { diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh index b238f844e..4ba3fb37d 100644 --- a/src/libstore/derived-path.hh +++ b/src/libstore/derived-path.hh @@ -3,8 +3,8 @@ #include "path.hh" #include "outputs-spec.hh" -#include "comparator.hh" #include "config.hh" +#include "ref.hh" #include @@ -31,7 +31,8 @@ struct DerivedPathOpaque { static DerivedPathOpaque parse(const StoreDirConfig & store, std::string_view); nlohmann::json toJSON(const StoreDirConfig & store) const; - GENERATE_CMP(DerivedPathOpaque, me->path); + bool operator == (const DerivedPathOpaque &) const = default; + auto operator <=> (const DerivedPathOpaque &) const = default; }; struct SingleDerivedPath; @@ -78,7 +79,8 @@ struct SingleDerivedPathBuilt { const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; - DECLARE_CMP(SingleDerivedPathBuilt); + bool operator == (const SingleDerivedPathBuilt &) const noexcept; + std::strong_ordering operator <=> (const SingleDerivedPathBuilt &) const noexcept; }; using _SingleDerivedPathRaw = std::variant< @@ -108,6 +110,9 @@ struct SingleDerivedPath : _SingleDerivedPathRaw { return static_cast(*this); } + bool operator == (const SingleDerivedPath &) const = default; + auto operator <=> (const SingleDerivedPath &) const = default; + /** * Get the store path this is ultimately derived from (by realising * and projecting outputs). @@ -201,7 +206,9 @@ struct DerivedPathBuilt { const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); nlohmann::json toJSON(Store & store) const; - DECLARE_CMP(DerivedPathBuilt); + bool operator == (const DerivedPathBuilt &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + bool operator < (const DerivedPathBuilt &) const noexcept; }; using _DerivedPathRaw = std::variant< @@ -230,6 +237,10 @@ struct DerivedPath : _DerivedPathRaw { return static_cast(*this); } + bool operator == (const DerivedPath &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + //auto operator <=> (const DerivedPath &) const = default; + /** * Get the store path this is ultimately derived from (by realising * and projecting outputs). diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 30f23cff9..c1e871e93 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -6,6 +6,13 @@ namespace nix { struct DummyStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; + DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) + : StoreConfig(params) + { + if (!authority.empty()) + throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); + } + const std::string name() override { return "Dummy Store"; } std::string doc() override @@ -14,18 +21,22 @@ struct DummyStoreConfig : virtual StoreConfig { #include "dummy-store.md" ; } + + static std::set uriSchemes() { + return {"dummy"}; + } }; struct DummyStore : public virtual DummyStoreConfig, public virtual Store { - DummyStore(const std::string scheme, const std::string uri, const Params & params) - : DummyStore(params) + DummyStore(std::string_view scheme, std::string_view authority, const Params & params) + : StoreConfig(params) + , DummyStoreConfig(scheme, authority, params) + , Store(params) { } DummyStore(const Params & params) - : StoreConfig(params) - , DummyStoreConfig(params) - , Store(params) + : DummyStore("dummy", "", params) { } std::string getUri() override @@ -47,10 +58,6 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store return Trusted; } - static std::set uriSchemes() { - return {"dummy"}; - } - std::optional queryPathFromHashPart(const std::string & hashPart) override { unsupported("queryPathFromHashPart"); } @@ -61,8 +68,8 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store virtual StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, - ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 219b60c44..e9e4b2c44 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,5 +1,6 @@ #include "filetransfer.hh" #include "globals.hh" +#include "config-global.hh" #include "store-api.hh" #include "s3.hh" #include "compression.hh" @@ -53,6 +54,8 @@ struct curlFileTransfer : public FileTransfer bool done = false; // whether either the success or failure function has been called Callback callback; CURL * req = 0; + // buffer to accompany the `req` above + char errbuf[CURL_ERROR_SIZE]; bool active = false; // whether the handle has been added to the multi object std::string statusMsg; @@ -70,7 +73,10 @@ struct curlFileTransfer : public FileTransfer curl_off_t writtenToSink = 0; + std::chrono::steady_clock::time_point startTime = std::chrono::steady_clock::now(); + inline static const std::set successfulStatuses {200, 201, 204, 206, 304, 0 /* other protocol */}; + /* Get the HTTP status code, or 0 for other protocols. */ long getHTTPStatus() { @@ -133,7 +139,7 @@ struct curlFileTransfer : public FileTransfer if (!done) fail(FileTransferError(Interrupted, {}, "download of '%s' was interrupted", request.uri)); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -366,16 +372,23 @@ struct curlFileTransfer : public FileTransfer if (writtenToSink) curl_easy_setopt(req, CURLOPT_RESUME_FROM_LARGE, writtenToSink); + curl_easy_setopt(req, CURLOPT_ERRORBUFFER, errbuf); + errbuf[0] = 0; + result.data.clear(); result.bodySize = 0; } void finish(CURLcode code) { + auto finishTime = std::chrono::steady_clock::now(); + auto httpStatus = getHTTPStatus(); - debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes", - request.verb(), request.uri, code, httpStatus, result.bodySize); + debug("finished %s of '%s'; curl status = %d, HTTP status = %d, body = %d bytes, duration = %.2f s", + request.verb(), request.uri, code, httpStatus, result.bodySize, + std::chrono::duration_cast(finishTime - startTime).count() / 1000.0f + ); appendCurrentUrl(); @@ -476,8 +489,8 @@ struct curlFileTransfer : public FileTransfer code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code))) : FileTransferError(err, std::move(response), - "unable to %s '%s': %s (%d)", - request.verb(), request.uri, curl_easy_strerror(code), code); + "unable to %s '%s': %s (%d) %s", + request.verb(), request.uri, curl_easy_strerror(code), code, errbuf); /* If this is a transient error, then maybe retry the download after a while. If we're writing to a @@ -580,7 +593,12 @@ struct curlFileTransfer : public FileTransfer #endif #if __linux__ - unshareFilesystem(); + try { + tryUnshareFilesystem(); + } catch (nix::Error & e) { + e.addTrace({}, "in download thread"); + throw; + } #endif std::map> items; @@ -741,12 +759,17 @@ struct curlFileTransfer : public FileTransfer S3Helper s3Helper(profile, region, scheme, endpoint); + Activity act(*logger, lvlTalkative, actFileTransfer, + fmt("downloading '%s'", request.uri), + {request.uri}, request.parentAct); + // FIXME: implement ETag auto s3Res = s3Helper.getObject(bucketName, key); FileTransferResult res; if (!s3Res.data) throw FileTransferError(NotFound, "S3 object '%s' does not exist", request.uri); res.data = std::move(*s3Res.data); + res.urls.push_back(request.uri); callback(std::move(res)); #else throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri); @@ -845,8 +868,10 @@ void FileTransfer::download( buffer). We don't wait forever to prevent stalling the download thread. (Hopefully sleeping will throttle the sender.) */ - if (state->data.size() > 1024 * 1024) { + if (state->data.size() > fileTransferSettings.downloadBufferSize) { debug("download buffer is full; going to sleep"); + static bool haveWarned = false; + warnOnce(haveWarned, "download buffer is full; consider increasing the 'download-buffer-size' setting"); state.wait_for(state->request, std::chrono::seconds(10)); } diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index 1c271cbec..d836ab2c4 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -1,13 +1,15 @@ #pragma once ///@file -#include "types.hh" -#include "hash.hh" -#include "config.hh" - #include #include +#include "logging.hh" +#include "types.hh" +#include "ref.hh" +#include "config.hh" +#include "serialise.hh" + namespace nix { struct FileTransferSettings : Config @@ -45,6 +47,12 @@ struct FileTransferSettings : Config Setting tries{this, 5, "download-attempts", "How often Nix will attempt to download a file before giving up."}; + + Setting downloadBufferSize{this, 64 * 1024 * 1024, "download-buffer-size", + R"( + The size of Nix's internal download buffer during `curl` transfers. If data is + not processed quickly enough to exceed the size of this buffer, downloads may stall. + )"}; }; extern FileTransferSettings fileTransferSettings; diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh index ab1059fb1..020f770b0 100644 --- a/src/libstore/gc-store.hh +++ b/src/libstore/gc-store.hh @@ -1,8 +1,9 @@ #pragma once ///@file -#include "store-api.hh" +#include +#include "store-api.hh" namespace nix { diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 8286dff27..73195794a 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -162,6 +162,7 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) /* Read the `temproots' directory for per-process temporary root files. */ for (auto & i : std::filesystem::directory_iterator{tempRootsDir}) { + checkInterrupt(); auto name = i.path().filename().string(); if (name[0] == '.') { // Ignore hidden files. Some package managers (notably portage) create @@ -228,8 +229,10 @@ void LocalStore::findRoots(const Path & path, std::filesystem::file_type type, R type = std::filesystem::symlink_status(path).type(); if (type == std::filesystem::file_type::directory) { - for (auto & i : std::filesystem::directory_iterator{path}) + for (auto & i : std::filesystem::directory_iterator{path}) { + checkInterrupt(); findRoots(i.path().string(), i.symlink_status().type(), roots); + } } else if (type == std::filesystem::file_type::symlink) { @@ -330,7 +333,7 @@ static std::string quoteRegexChars(const std::string & raw) } #if __linux__ -static void readFileRoots(const char * path, UncheckedRoots & roots) +static void readFileRoots(const std::filesystem::path & path, UncheckedRoots & roots) { try { roots[readFile(path)].emplace(path); @@ -556,7 +559,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) non-blocking flag from the server socket, so explicitly make it blocking. */ if (fcntl(fdClient.get(), F_SETFL, fcntl(fdClient.get(), F_GETFL) & ~O_NONBLOCK) == -1) - abort(); + panic("Could not set non-blocking flag on client socket"); while (true) { try { @@ -781,7 +784,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) throw Error( "Cannot delete path '%1%' since it is still alive. " "To find out why, use: " - "nix-store --query --roots", + "nix-store --query --roots and nix-store --query --referrers", printStorePath(i)); } @@ -888,7 +891,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) void LocalStore::autoGC(bool sync) { -#ifdef HAVE_STATVFS +#if HAVE_STATVFS static auto fakeFreeSpaceFile = getEnv("_NIX_TEST_FREE_SPACE_FILE"); auto getAvail = [this]() -> uint64_t { @@ -955,8 +958,8 @@ void LocalStore::autoGC(bool sync) } catch (...) { // FIXME: we could propagate the exception to the - // future, but we don't really care. - ignoreException(); + // future, but we don't really care. (what??) + ignoreExceptionInDestructor(); } }).detach(); diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index d9cab2fb8..b64e73c26 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -1,19 +1,21 @@ #include "globals.hh" +#include "config-global.hh" #include "current-process.hh" #include "archive.hh" #include "args.hh" #include "abstract-setting-to-json.hh" #include "compute-levels.hh" +#include "signals.hh" #include #include #include #include +#include #include #ifndef _WIN32 -# include # include #endif @@ -33,6 +35,8 @@ #include #endif +#include "strings.hh" + namespace nix { @@ -61,7 +65,6 @@ Settings::Settings() , nixStateDir(canonPath(getEnvNonEmpty("NIX_STATE_DIR").value_or(NIX_STATE_DIR))) , nixConfDir(canonPath(getEnvNonEmpty("NIX_CONF_DIR").value_or(NIX_CONF_DIR))) , nixUserConfFiles(getUserConfigFiles()) - , nixBinDir(canonPath(getEnvNonEmpty("NIX_BIN_DIR").value_or(NIX_BIN_DIR))) , nixManDir(canonPath(NIX_MAN_DIR)) , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { @@ -80,7 +83,7 @@ Settings::Settings() Strings ss; for (auto & p : tokenizeString(*s, ":")) ss.push_back("@" + p); - builders = concatStringsSep(" ", ss); + builders = concatStringsSep("\n", ss); } #if defined(__linux__) && defined(SANDBOX_SHELL) @@ -92,42 +95,14 @@ Settings::Settings() sandboxPaths = tokenizeString("/System/Library/Frameworks /System/Library/PrivateFrameworks /bin/sh /bin/bash /private/tmp /private/var/tmp /usr/lib"); allowedImpureHostPrefixes = tokenizeString("/System/Library /usr/lib /dev /bin/sh"); #endif - - /* Set the build hook location - - For builds we perform a self-invocation, so Nix has to be self-aware. - That is, it has to know where it is installed. We don't think it's sentient. - - Normally, nix is installed according to `nixBinDir`, which is set at compile time, - but can be overridden. This makes for a great default that works even if this - code is linked as a library into some other program whose main is not aware - that it might need to be a build remote hook. - - However, it may not have been installed at all. For example, if it's a static build, - there's a good chance that it has been moved out of its installation directory. - That makes `nixBinDir` useless. Instead, we'll query the OS for the path to the - current executable, using `getSelfExe()`. - - As a last resort, we resort to `PATH`. Hopefully we find a `nix` there that's compatible. - If you're porting Nix to a new platform, that might be good enough for a while, but - you'll want to improve `getSelfExe()` to work on your platform. - */ - std::string nixExePath = nixBinDir + "/nix"; - if (!pathExists(nixExePath)) { - nixExePath = getSelfExe().value_or("nix"); - } - buildHook = { - nixExePath, - "__build-remote", - }; } -void loadConfFile() +void loadConfFile(AbstractConfig & config) { auto applyConfigFile = [&](const Path & path) { try { std::string contents = readFile(path); - globalConfig.applyConfig(contents, path); + config.applyConfig(contents, path); } catch (SystemError &) { } }; @@ -135,7 +110,7 @@ void loadConfFile() /* We only want to send overrides to the daemon, i.e. stuff from ~/.nix/nix.conf or the command line. */ - globalConfig.resetOverridden(); + config.resetOverridden(); auto files = settings.nixUserConfFiles; for (auto file = files.rbegin(); file != files.rend(); file++) { @@ -144,7 +119,7 @@ void loadConfFile() auto nixConfEnv = getEnv("NIX_CONFIG"); if (nixConfEnv.has_value()) { - globalConfig.applyConfig(nixConfEnv.value(), "NIX_CONFIG"); + config.applyConfig(nixConfEnv.value(), "NIX_CONFIG"); } } @@ -161,7 +136,7 @@ std::vector getUserConfigFiles() std::vector files; auto dirs = getConfigDirs(); for (auto & dir : dirs) { - files.insert(files.end(), dir + "/nix/nix.conf"); + files.insert(files.end(), dir + "/nix.conf"); } return files; } @@ -294,25 +269,28 @@ template<> std::string BaseSetting::to_string() const if (value == smEnabled) return "true"; else if (value == smRelaxed) return "relaxed"; else if (value == smDisabled) return "false"; - else abort(); + else unreachable(); } template<> void BaseSetting::convertToArg(Args & args, const std::string & category) { args.addFlag({ .longName = name, + .aliases = aliases, .description = "Enable sandboxing.", .category = category, .handler = {[this]() { override(smEnabled); }} }); args.addFlag({ .longName = "no-" + name, + .aliases = aliases, .description = "Disable sandboxing.", .category = category, .handler = {[this]() { override(smDisabled); }} }); args.addFlag({ .longName = "relaxed-" + name, + .aliases = aliases, .description = "Enable sandboxing, but allow builds to disable it.", .category = category, .handler = {[this]() { override(smRelaxed); }} @@ -331,57 +309,6 @@ unsigned int MaxBuildJobsSetting::parse(const std::string & str) const } -Paths PluginFilesSetting::parse(const std::string & str) const -{ - if (pluginsLoaded) - throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand"); - return BaseSetting::parse(str); -} - - -void initPlugins() -{ - assert(!settings.pluginFiles.pluginsLoaded); - for (const auto & pluginFile : settings.pluginFiles.get()) { - std::vector pluginFiles; - try { - auto ents = std::filesystem::directory_iterator{pluginFile}; - for (const auto & ent : ents) - pluginFiles.emplace_back(ent.path()); - } catch (std::filesystem::filesystem_error & e) { - if (e.code() != std::errc::not_a_directory) - throw; - pluginFiles.emplace_back(pluginFile); - } - for (const auto & file : pluginFiles) { - /* handle is purposefully leaked as there may be state in the - DSO needed by the action of the plugin. */ -#ifndef _WIN32 // TODO implement via DLL loading on Windows - void *handle = - dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); - if (!handle) - throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); - - /* Older plugins use a statically initialized object to run their code. - Newer plugins can also export nix_plugin_entry() */ - void (*nix_plugin_entry)() = (void (*)())dlsym(handle, "nix_plugin_entry"); - if (nix_plugin_entry) - nix_plugin_entry(); -#else - throw Error("could not dynamically open plugin file '%s'", file); -#endif - } - } - - /* Since plugins can add settings, try to re-apply previously - unknown settings. */ - globalConfig.reapplyUnknownSettings(); - globalConfig.warnUnknownSettings(); - - /* Tell the user if they try to set plugin-files after we've already loaded */ - settings.pluginFiles.pluginsLoaded = true; -} - static void preloadNSS() { /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of @@ -433,14 +360,25 @@ void initLibStore(bool loadConfig) { initLibUtil(); if (loadConfig) - loadConfFile(); + loadConfFile(globalConfig); preloadNSS(); + /* Because of an objc quirk[1], calling curl_global_init for the first time + after fork() will always result in a crash. + Up until now the solution has been to set OBJC_DISABLE_INITIALIZE_FORK_SAFETY + for every nix process to ignore that error. + Instead of working around that error we address it at the core - + by calling curl_global_init here, which should mean curl will already + have been initialized by the time we try to do so in a forked process. + + [1] https://github.com/apple-oss-distributions/objc4/blob/01edf1705fbc3ff78a423cd21e03dfc21eb4d780/runtime/objc-initialize.mm#L614-L636 + */ + curl_global_init(CURL_GLOBAL_ALL); +#if __APPLE__ /* On macOS, don't use the per-session TMPDIR (as set e.g. by sshd). This breaks build users because they don't have access to the TMPDIR, in particular in ‘nix-store --serve’. */ -#if __APPLE__ if (hasPrefix(defaultTempDir(), "/var/folders/")) unsetenv("TMPDIR"); #endif diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index dc18a11fc..be922c9f7 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -31,23 +31,6 @@ struct MaxBuildJobsSetting : public BaseSetting unsigned int parse(const std::string & str) const override; }; -struct PluginFilesSetting : public BaseSetting -{ - bool pluginsLoaded = false; - - PluginFilesSetting(Config * options, - const Paths & def, - const std::string & name, - const std::string & description, - const std::set & aliases = {}) - : BaseSetting(def, true, name, description, aliases) - { - options->addSetting(this); - } - - Paths parse(const std::string & str) const override; -}; - const uint32_t maxIdsPerBuild = #if __linux__ 1 << 16 @@ -101,11 +84,6 @@ public: */ std::vector nixUserConfFiles; - /** - * The directory where the main programs are stored. - */ - Path nixBinDir; - /** * The directory where the man pages are stored. */ @@ -228,7 +206,7 @@ public: While you can force Nix to run a Darwin-specific `builder` executable on a Linux machine, the result would obviously be wrong. This value is available in the Nix language as - [`builtins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) + [`builtins.currentSystem`](@docroot@/language/builtins.md#builtins-currentSystem) if the [`eval-system`](#conf-eval-system) configuration option is set as the empty string. @@ -263,7 +241,7 @@ public: )", {"build-timeout"}}; - Setting buildHook{this, {}, "build-hook", + Setting buildHook{this, {"nix", "__build-remote"}, "build-hook", R"( The path to the helper program that executes remote builds. @@ -303,7 +281,7 @@ public: For backward compatibility, `ssh://` may be omitted. The hostname may be an alias defined in `~/.ssh/config`. - 2. A comma-separated list of [Nix system types](@docroot@/contributing/hacking.md#system-type). + 2. A comma-separated list of [Nix system types](@docroot@/development/building.md#system-type). If omitted, this defaults to the local platform type. > **Example** @@ -421,10 +399,18 @@ public: default is `true`. )"}; + Setting fsyncStorePaths{this, false, "fsync-store-paths", + R"( + Whether to call `fsync()` on store paths before registering them, to + flush them to disk. This improves robustness in case of system crashes, + but reduces performance. The default is `false`. + )"}; + Setting useSQLiteWAL{this, !isWSL1(), "use-sqlite-wal", "Whether SQLite should use WAL mode."}; #ifndef _WIN32 + // FIXME: remove this option, `fsync-store-paths` is faster. Setting syncBeforeRegistering{this, false, "sync-before-registering", "Whether to call `sync()` before registering a path as valid."}; #endif @@ -883,13 +869,13 @@ public: - `ca-derivations` - Included by default if the [`ca-derivations` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-ca-derivations) is enabled. + Included by default if the [`ca-derivations` experimental feature](@docroot@/development/experimental-features.md#xp-feature-ca-derivations) is enabled. This system feature is implicitly required by derivations with the [`__contentAddressed` attribute](@docroot@/language/advanced-attributes.md#adv-attr-__contentAddressed). - `recursive-nix` - Included by default if the [`recursive-nix` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-recursive-nix) is enabled. + Included by default if the [`recursive-nix` experimental feature](@docroot@/development/experimental-features.md#xp-feature-recursive-nix) is enabled. - `uid-range` @@ -1148,7 +1134,10 @@ public: )"}; Setting maxFree{ - this, std::numeric_limits::max(), "max-free", + // n.b. this is deliberately int64 max rather than uint64 max because + // this goes through the Nix language JSON parser and thus needs to be + // representable in Nix language integers. + this, std::numeric_limits::max(), "max-free", R"( When a garbage collection is triggered by the `min-free` option, it stops as soon as `max-free` bytes are available. The default is @@ -1158,33 +1147,6 @@ public: Setting minFreeCheckInterval{this, 5, "min-free-check-interval", "Number of seconds between checking free disk space."}; - PluginFilesSetting pluginFiles{ - this, {}, "plugin-files", - R"( - A list of plugin files to be loaded by Nix. Each of these files will - be dlopened by Nix. If they contain the symbol `nix_plugin_entry()`, - this symbol will be called. Alternatively, they can affect execution - through static initialization. In particular, these plugins may construct - static instances of RegisterPrimOp to add new primops or constants to the - expression language, RegisterStoreImplementation to add new store - implementations, RegisterCommand to add new subcommands to the `nix` - command, and RegisterSetting to add new nix config settings. See the - constructors for those types for more details. - - Warning! These APIs are inherently unstable and may change from - release to release. - - Since these files are loaded into the same address space as Nix - itself, they must be DSOs compatible with the instance of Nix - running at the time (i.e. compiled against the same headers, not - linked to any incompatible libraries). They should not be linked to - any Nix libs directly, as those will be available already at load - time. - - If an entry in the list is a directory, all files in the directory - are loaded as plugins (non-recursively). - )"}; - Setting narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size", "Maximum size of NARs before spilling them to disk."}; @@ -1242,7 +1204,7 @@ public: If the user is trusted (see `trusted-users` option), when building a fixed-output derivation, environment variables set in this option - will be passed to the builder if they are listed in [`impureEnvVars`](@docroot@/language/advanced-attributes.md##adv-attr-impureEnvVars). + will be passed to the builder if they are listed in [`impureEnvVars`](@docroot@/language/advanced-attributes.md#adv-attr-impureEnvVars). This option is useful for, e.g., setting `https_proxy` for fixed-output derivations and in a multi-user Nix installation, or @@ -1262,6 +1224,19 @@ public: store paths of the latest Nix release. )" }; + + Setting warnLargePathThreshold{ + this, + // n.b. this is deliberately int64 max rather than uint64 max because + // this goes through the Nix language JSON parser and thus needs to be + // representable in Nix language integers. + std::numeric_limits::max(), + "warn-large-path-threshold", + R"( + Warn when copying a path larger than this number of bytes to the Nix store + (as determined by its NAR serialisation). + )" + }; }; @@ -1269,12 +1244,12 @@ public: extern Settings settings; /** - * This should be called after settings are initialized, but before - * anything else + * Load the configuration (from `nix.conf`, `NIX_CONFIG`, etc.) into the + * given configuration object. + * + * Usually called with `globalConfig`. */ -void initPlugins(); - -void loadConfFile(); +void loadConfFile(AbstractConfig & config); // Used by the Settings constructor std::vector getUserConfigFiles(); diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 5da87e935..fc7ac2dea 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -1,4 +1,4 @@ -#include "binary-cache-store.hh" +#include "http-binary-cache-store.hh" #include "filetransfer.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" @@ -8,26 +8,37 @@ namespace nix { MakeError(UploadToHTTP, Error); -struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig + +HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( + std::string_view scheme, + std::string_view _cacheUri, + const Params & params) + : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , cacheUri( + std::string { scheme } + + "://" + + (!_cacheUri.empty() + ? _cacheUri + : throw UsageError("`%s` Store requires a non-empty authority in Store URL", scheme))) { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + while (!cacheUri.empty() && cacheUri.back() == '/') + cacheUri.pop_back(); +} - const std::string name() override { return "HTTP Binary Cache Store"; } - std::string doc() override - { - return - #include "http-binary-cache-store.md" - ; - } -}; +std::string HttpBinaryCacheStoreConfig::doc() +{ + return + #include "http-binary-cache-store.md" + ; +} + class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore { private: - Path cacheUri; - struct State { bool enabled = true; @@ -39,19 +50,15 @@ private: public: HttpBinaryCacheStore( - const std::string & scheme, - const Path & _cacheUri, + std::string_view scheme, + PathView cacheUri, const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) - , HttpBinaryCacheStoreConfig(params) + , HttpBinaryCacheStoreConfig(scheme, cacheUri, params) , Store(params) , BinaryCacheStore(params) - , cacheUri(scheme + "://" + _cacheUri) { - while (!cacheUri.empty() && cacheUri.back() == '/') - cacheUri.pop_back(); - diskCache = getNarInfoDiskCache(); } @@ -76,14 +83,6 @@ public: } } - static std::set uriSchemes() - { - static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; - auto ret = std::set({"http", "https"}); - if (forceHttp) ret.insert("file"); - return ret; - } - protected: void maybeDisable() @@ -170,28 +169,29 @@ protected: { try { checkEnabled(); + + auto request(makeRequest(path)); + + auto callbackPtr = std::make_shared(std::move(callback)); + + getFileTransfer()->enqueueFileTransfer(request, + {[callbackPtr, this](std::future result) { + try { + (*callbackPtr)(std::move(result.get().data)); + } catch (FileTransferError & e) { + if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) + return (*callbackPtr)({}); + maybeDisable(); + callbackPtr->rethrow(); + } catch (...) { + callbackPtr->rethrow(); + } + }}); + } catch (...) { callback.rethrow(); return; } - - auto request(makeRequest(path)); - - auto callbackPtr = std::make_shared(std::move(callback)); - - getFileTransfer()->enqueueFileTransfer(request, - {[callbackPtr, this](std::future result) { - try { - (*callbackPtr)(std::move(result.get().data)); - } catch (FileTransferError & e) { - if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden) - return (*callbackPtr)({}); - maybeDisable(); - callbackPtr->rethrow(); - } catch (...) { - callbackPtr->rethrow(); - } - }}); } /** diff --git a/src/libstore/http-binary-cache-store.hh b/src/libstore/http-binary-cache-store.hh new file mode 100644 index 000000000..d2fc43210 --- /dev/null +++ b/src/libstore/http-binary-cache-store.hh @@ -0,0 +1,30 @@ +#include "binary-cache-store.hh" + +namespace nix { + +struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + HttpBinaryCacheStoreConfig(std::string_view scheme, std::string_view _cacheUri, const Params & params); + + Path cacheUri; + + const std::string name() override + { + return "HTTP Binary Cache Store"; + } + + static std::set uriSchemes() + { + static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; + auto ret = std::set({"http", "https"}); + if (forceHttp) + ret.insert("file"); + return ret; + } + + std::string doc() override; +}; + +} diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index e422adeec..eac360a4f 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -1,9 +1,10 @@ #include "legacy-ssh-store.hh" -#include "ssh-store-config.hh" +#include "common-ssh-store-config.hh" #include "archive.hh" #include "pool.hh" #include "remote-store.hh" #include "serve-protocol.hh" +#include "serve-protocol-connection.hh" #include "serve-protocol-impl.hh" #include "build-result.hh" #include "store-api.hh" @@ -14,6 +15,15 @@ namespace nix { +LegacySSHStoreConfig::LegacySSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , CommonSSHStoreConfig(scheme, authority, params) +{ +} + std::string LegacySSHStoreConfig::doc() { return @@ -28,26 +38,23 @@ struct LegacySSHStore::Connection : public ServeProto::BasicClientConnection bool good = true; }; - -LegacySSHStore::LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params) +LegacySSHStore::LegacySSHStore( + std::string_view scheme, + std::string_view host, + const Params & params) : StoreConfig(params) - , CommonSSHStoreConfig(params) - , LegacySSHStoreConfig(params) + , CommonSSHStoreConfig(scheme, host, params) + , LegacySSHStoreConfig(scheme, host, params) , Store(params) - , host(host) , connections(make_ref>( std::max(1, (int) maxConnections), [this]() { return openConnection(); }, [](const ref & r) { return r->good; } )) - , master( - host, - sshKey, - sshPublicHostKey, + , master(createSSHMaster( // Use SSH master only if using more than 1 connection. connections->capacity() > 1, - compress, - logFD) + logFD)) { } @@ -76,7 +83,7 @@ ref LegacySSHStore::openConnection() conn->sshConn->in.close(); { NullSink nullSink; - conn->from.drainInto(nullSink); + tee.drainInto(nullSink); } throw Error("'nix-store --serve' protocol mismatch from '%s', got '%s'", host, chomp(saved.s)); @@ -105,24 +112,26 @@ void LegacySSHStore::queryPathInfoUncached(const StorePath & path, debug("querying remote host '%s' for info on '%s'", host, printStorePath(path)); - conn->to << ServeProto::Command::QueryPathInfos << PathSet{printStorePath(path)}; - conn->to.flush(); + auto infos = conn->queryPathInfos(*this, {path}); - auto p = readString(conn->from); - if (p.empty()) return callback(nullptr); - auto path2 = parseStorePath(p); - assert(path == path2); - auto info = std::make_shared( - path, - ServeProto::Serialise::read(*this, *conn)); + switch (infos.size()) { + case 0: + return callback(nullptr); + case 1: { + auto & [path2, info] = *infos.begin(); - if (info->narHash == Hash::dummy) - throw Error("NAR hash is now mandatory"); + if (info.narHash == Hash::dummy) + throw Error("NAR hash is now mandatory"); - auto s = readString(conn->from); - assert(s == ""); - - callback(std::move(info)); + assert(path == path2); + return callback(std::make_shared( + std::move(path), + std::move(info) + )); + } + default: + throw Error("More path infos returned than queried"); + } } catch (...) { callback.rethrow(); } } @@ -156,41 +165,38 @@ void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, } conn->to.flush(); + if (readInt(conn->from) != 1) + throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); + } else { - conn->to - << ServeProto::Command::ImportPaths - << 1; - try { - copyNAR(source, conn->to); - } catch (...) { - conn->good = false; - throw; - } - conn->to - << exportMagic - << printStorePath(info.path); - ServeProto::write(*this, *conn, info.references); - conn->to - << (info.deriver ? printStorePath(*info.deriver) : "") - << 0 - << 0; - conn->to.flush(); + conn->importPaths(*this, [&](Sink & sink) { + try { + copyNAR(source, sink); + } catch (...) { + conn->good = false; + throw; + } + sink + << exportMagic + << printStorePath(info.path); + ServeProto::write(*this, *conn, info.references); + sink + << (info.deriver ? printStorePath(*info.deriver) : "") + << 0 + << 0; + }); } - - if (readInt(conn->from) != 1) - throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), host); } void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink) { auto conn(connections->get()); - - conn->to << ServeProto::Command::DumpStorePath << printStorePath(path); - conn->to.flush(); - copyNAR(conn->from, sink); + conn->narFromPath(*this, path, [&](auto & source) { + copyNAR(source, sink); + }); } @@ -214,7 +220,7 @@ BuildResult LegacySSHStore::buildDerivation(const StorePath & drvPath, const Bas conn->putBuildDerivationRequest(*this, drvPath, drv, buildSettings()); - return ServeProto::Serialise::read(*this, *conn); + return conn->getBuildDerivationResponse(*this); } diff --git a/src/libstore/legacy-ssh-store.hh b/src/libstore/legacy-ssh-store.hh index 343823693..b541455b4 100644 --- a/src/libstore/legacy-ssh-store.hh +++ b/src/libstore/legacy-ssh-store.hh @@ -1,7 +1,7 @@ #pragma once ///@file -#include "ssh-store-config.hh" +#include "common-ssh-store-config.hh" #include "store-api.hh" #include "ssh.hh" #include "callback.hh" @@ -13,6 +13,11 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig { using CommonSSHStoreConfig::CommonSSHStoreConfig; + LegacySSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params); + const Setting remoteProgram{this, {"nix-store"}, "remote-program", "Path to the `nix-store` executable on the remote machine."}; @@ -21,27 +26,32 @@ struct LegacySSHStoreConfig : virtual CommonSSHStoreConfig const std::string name() override { return "SSH Store"; } + static std::set uriSchemes() { return {"ssh"}; } + std::string doc() override; }; struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store { +#ifndef _WIN32 // Hack for getting remote build log output. // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in // the documentation - const Setting logFD{this, -1, "log-fd", "file descriptor to which SSH's stderr is connected"}; + const Setting logFD{this, INVALID_DESCRIPTOR, "log-fd", "file descriptor to which SSH's stderr is connected"}; +#else + Descriptor logFD = INVALID_DESCRIPTOR; +#endif struct Connection; - std::string host; - ref> connections; SSHMaster master; - static std::set uriSchemes() { return {"ssh"}; } - - LegacySSHStore(const std::string & scheme, const std::string & host, const Params & params); + LegacySSHStore( + std::string_view scheme, + std::string_view host, + const Params & params); ref openConnection(); @@ -71,8 +81,8 @@ struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Stor virtual StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, - ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override diff --git a/src/libstore/linux/meson.build b/src/libstore/linux/meson.build new file mode 100644 index 000000000..0c494b5d6 --- /dev/null +++ b/src/libstore/linux/meson.build @@ -0,0 +1,10 @@ +sources += files( + 'personality.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'fchmodat2-compat.hh', + 'personality.hh', +) diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index 87a6026f1..dcc6affe4 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -1,43 +1,46 @@ -#include "binary-cache-store.hh" +#include "local-binary-cache-store.hh" #include "globals.hh" #include "nar-info-disk-cache.hh" +#include "signals.hh" #include namespace nix { -struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +LocalBinaryCacheStoreConfig::LocalBinaryCacheStoreConfig( + std::string_view scheme, + PathView binaryCacheDir, + const Params & params) + : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , binaryCacheDir(binaryCacheDir) { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; +} - const std::string name() override { return "Local Binary Cache Store"; } - std::string doc() override - { - return - #include "local-binary-cache-store.md" - ; - } -}; - -class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore +std::string LocalBinaryCacheStoreConfig::doc() { -private: + return + #include "local-binary-cache-store.md" + ; +} - Path binaryCacheDir; - -public: +struct LocalBinaryCacheStore : virtual LocalBinaryCacheStoreConfig, virtual BinaryCacheStore +{ + /** + * @param binaryCacheDir `file://` is a short-hand for `file:///` + * for now. + */ LocalBinaryCacheStore( - const std::string scheme, - const Path & binaryCacheDir, + std::string_view scheme, + PathView binaryCacheDir, const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) - , LocalBinaryCacheStoreConfig(params) + , LocalBinaryCacheStoreConfig(scheme, binaryCacheDir, params) , Store(params) , BinaryCacheStore(params) - , binaryCacheDir(binaryCacheDir) { } @@ -48,8 +51,6 @@ public: return "file://" + binaryCacheDir; } - static std::set uriSchemes(); - protected: bool fileExists(const std::string & path) override; @@ -84,6 +85,7 @@ protected: StorePathSet paths; for (auto & entry : std::filesystem::directory_iterator{binaryCacheDir}) { + checkInterrupt(); auto name = entry.path().filename().string(); if (name.size() != 40 || !hasSuffix(name, ".narinfo")) @@ -117,7 +119,7 @@ bool LocalBinaryCacheStore::fileExists(const std::string & path) return pathExists(binaryCacheDir + "/" + path); } -std::set LocalBinaryCacheStore::uriSchemes() +std::set LocalBinaryCacheStoreConfig::uriSchemes() { if (getEnv("_NIX_FORCE_HTTP") == "1") return {}; diff --git a/src/libstore/local-binary-cache-store.hh b/src/libstore/local-binary-cache-store.hh new file mode 100644 index 000000000..997e8ecbb --- /dev/null +++ b/src/libstore/local-binary-cache-store.hh @@ -0,0 +1,23 @@ +#include "binary-cache-store.hh" + +namespace nix { + +struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + LocalBinaryCacheStoreConfig(std::string_view scheme, PathView binaryCacheDir, const Params & params); + + Path binaryCacheDir; + + const std::string name() override + { + return "Local Binary Cache Store"; + } + + static std::set uriSchemes(); + + std::string doc() override; +}; + +} diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 843c0d288..5449b20eb 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -8,6 +8,20 @@ namespace nix { +LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params) + : StoreConfig(params) + // Default `?root` from `rootDir` if non set + // FIXME don't duplicate description once we don't have root setting + , rootDir{ + this, + !rootDir.empty() && params.count("root") == 0 + ? (std::optional{rootDir}) + : std::nullopt, + "root", + "Directory prefixed to all other paths."} +{ +} + LocalFSStore::LocalFSStore(const Params & params) : Store(params) { diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 8fb081200..9bb569f0b 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -11,6 +11,15 @@ struct LocalFSStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; + /** + * Used to override the `root` settings. Can't be done via modifying + * `params` reliably because this parameter is unused except for + * passing to base class constructors. + * + * @todo Make this less error-prone with new store settings system. + */ + LocalFSStoreConfig(PathView path, const Params & params); + const OptionalPathSetting rootDir{this, std::nullopt, "root", "Directory prefixed to all other paths."}; diff --git a/src/libstore/unix/local-overlay-store.cc b/src/libstore/local-overlay-store.cc similarity index 97% rename from src/libstore/unix/local-overlay-store.cc rename to src/libstore/local-overlay-store.cc index 598415db8..b86beba2c 100644 --- a/src/libstore/unix/local-overlay-store.cc +++ b/src/libstore/local-overlay-store.cc @@ -18,11 +18,11 @@ Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { return upperLayer + "/" + path.to_string(); } -LocalOverlayStore::LocalOverlayStore(const Params & params) +LocalOverlayStore::LocalOverlayStore(std::string_view scheme, PathView path, const Params & params) : StoreConfig(params) - , LocalFSStoreConfig(params) + , LocalFSStoreConfig(path, params) , LocalStoreConfig(params) - , LocalOverlayStoreConfig(params) + , LocalOverlayStoreConfig(scheme, path, params) , Store(params) , LocalFSStore(params) , LocalStore(params) @@ -31,7 +31,7 @@ LocalOverlayStore::LocalOverlayStore(const Params & params) if (checkMount.get()) { std::smatch match; std::string mountInfo; - auto mounts = readFile("/proc/self/mounts"); + auto mounts = readFile(std::filesystem::path{"/proc/self/mounts"}); auto regex = std::regex(R"((^|\n)overlay )" + realStoreDir.get() + R"( .*(\n|$))"); // Mount points can be stacked, so there might be multiple matching entries. diff --git a/src/libstore/unix/local-overlay-store.hh b/src/libstore/local-overlay-store.hh similarity index 94% rename from src/libstore/unix/local-overlay-store.hh rename to src/libstore/local-overlay-store.hh index 2c24285dd..63628abed 100644 --- a/src/libstore/unix/local-overlay-store.hh +++ b/src/libstore/local-overlay-store.hh @@ -8,11 +8,16 @@ namespace nix { struct LocalOverlayStoreConfig : virtual LocalStoreConfig { LocalOverlayStoreConfig(const StringMap & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , LocalStoreConfig(params) + : LocalOverlayStoreConfig("local-overlay", "", params) { } + LocalOverlayStoreConfig(std::string_view scheme, PathView path, const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(path, params) + , LocalStoreConfig(scheme, path, params) + { + } + const Setting lowerStoreUri{(StoreConfig*) this, "", "lower-store", R"( [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) @@ -58,6 +63,11 @@ struct LocalOverlayStoreConfig : virtual LocalStoreConfig return ExperimentalFeature::LocalOverlayStore; } + static std::set uriSchemes() + { + return { "local-overlay" }; + } + std::string doc() override; protected: @@ -90,19 +100,12 @@ class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual ref lowerStore; public: - LocalOverlayStore(const Params & params); - - LocalOverlayStore(std::string scheme, std::string path, const Params & params) - : LocalOverlayStore(params) + LocalOverlayStore(const Params & params) + : LocalOverlayStore("local-overlay", "", params) { - if (!path.empty()) - throw UsageError("local-overlay:// store url doesn't support path part, only scheme and query params"); } - static std::set uriSchemes() - { - return { "local-overlay" }; - } + LocalOverlayStore(std::string_view scheme, PathView path, const Params & params); std::string getUri() override { diff --git a/src/libstore/unix/local-overlay-store.md b/src/libstore/local-overlay-store.md similarity index 95% rename from src/libstore/unix/local-overlay-store.md rename to src/libstore/local-overlay-store.md index 1e1a3d26c..9434ebfb9 100644 --- a/src/libstore/unix/local-overlay-store.md +++ b/src/libstore/local-overlay-store.md @@ -4,7 +4,7 @@ R"( This store type is a variation of the [local store] designed to leverage Linux's [Overlay Filesystem](https://docs.kernel.org/filesystems/overlayfs.html) (OverlayFS for short). Just as OverlayFS combines a lower and upper filesystem by treating the upper one as a patch against the lower, the local overlay store combines a lower store with an upper almost-[local store]. -("almost" because while the upper fileystems for OverlayFS is valid on its own, the upper almost-store is not a valid local store on its own because some references will dangle.) +("almost" because while the upper filesystems for OverlayFS is valid on its own, the upper almost-store is not a valid local store on its own because some references will dangle.) To use this store, you will first need to configure an OverlayFS mountpoint [appropriately](#example-filesystem-layout) as Nix will not do this for you (though it will verify the mountpoint is configured correctly). ### Conceptual parts of a local overlay store @@ -77,13 +77,13 @@ The parts of a local overlay store are as follows: The lower store directory and upper layer directory are combined via OverlayFS to create this directory. Nix doesn't do this itself, because it typically wouldn't have the permissions to do so, so it is the responsibility of the user to set this up first. - Nix can, however, optionally check that that the OverlayFS mount settings appear as expected, matching Nix's own settings. + Nix can, however, optionally check that the OverlayFS mount settings appear as expected, matching Nix's own settings. - **Upper SQLite database**: > Not directly specified. > The location of the database instead depends on the [`state`](#store-experimental-local-overlay-store-state) setting. - > It is is always `${state}/db`. + > It is always `${state}/db`. This contains the metadata of all of the upper layer [store objects][store object] (everything beyond their file system objects), and also duplicate copies of some lower layer store object's metadta. The duplication is so the metadata for the [closure](@docroot@/glossary.md#gloss-closure) of upper layer [store objects][store object] can be found entirely within the upper layer. diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index dd06e5b65..eafdac0cd 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -42,7 +42,6 @@ # include # include # include -# include #endif #ifdef __CYGWIN__ @@ -51,9 +50,22 @@ #include +#include + +#include "strings.hh" + namespace nix { +LocalStoreConfig::LocalStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(authority, params) +{ +} + std::string LocalStoreConfig::doc() { return @@ -83,7 +95,7 @@ struct LocalStore::State::Stmts { SQLiteStmt AddRealisationReference; }; -int getSchema(Path schemaPath) +static int getSchema(Path schemaPath) { int curSchema = 0; if (pathExists(schemaPath)) { @@ -120,71 +132,21 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) curCASchema = nixCASchemaVersion; } - if (curCASchema < 2) { - SQLiteTxn txn(db); - // Ugly little sql dance to add a new `id` column and make it the primary key - db.exec(R"( - create table Realisations2 ( - id integer primary key autoincrement not null, - drvPath text not null, - outputName text not null, -- symbolic output id, usually "out" - outputPath integer not null, - signatures text, -- space-separated list - foreign key (outputPath) references ValidPaths(id) on delete cascade - ); - insert into Realisations2 (drvPath, outputName, outputPath, signatures) - select drvPath, outputName, outputPath, signatures from Realisations; - drop table Realisations; - alter table Realisations2 rename to Realisations; - )"); - db.exec(R"( - create index if not exists IndexRealisations on Realisations(drvPath, outputName); - - create table if not exists RealisationsRefs ( - referrer integer not null, - realisationReference integer, - foreign key (referrer) references Realisations(id) on delete cascade, - foreign key (realisationReference) references Realisations(id) on delete restrict - ); - )"); - txn.commit(); - } - - if (curCASchema < 3) { - SQLiteTxn txn(db); - // Apply new indices added in this schema update. - db.exec(R"( - -- used by QueryRealisationReferences - create index if not exists IndexRealisationsRefs on RealisationsRefs(referrer); - -- used by cascade deletion when ValidPaths is deleted - create index if not exists IndexRealisationsRefsOnOutputPath on Realisations(outputPath); - )"); - txn.commit(); - } - if (curCASchema < 4) { - SQLiteTxn txn(db); - db.exec(R"( - create trigger if not exists DeleteSelfRefsViaRealisations before delete on ValidPaths - begin - delete from RealisationsRefs where realisationReference in ( - select id from Realisations where outputPath = old.id - ); - end; - -- used by deletion trigger - create index if not exists IndexRealisationsRefsRealisationReference on RealisationsRefs(realisationReference); - )"); - txn.commit(); - } + if (curCASchema < 4) + throw Error("experimental CA schema version %d is no longer supported", curCASchema); writeFile(schemaPath, fmt("%d", nixCASchemaVersion), 0666, true); lockFile(lockFd.get(), ltRead, true); } } -LocalStore::LocalStore(const Params & params) +LocalStore::LocalStore( + std::string_view scheme, + PathView path, + const Params & params) : StoreConfig(params) - , LocalFSStoreConfig(params) - , LocalStoreConfig(params) + , LocalFSStoreConfig(path, params) + , LocalStoreConfig(scheme, path, params) , Store(params) , LocalFSStore(params) , dbDir(stateDir + "/db") @@ -233,7 +195,7 @@ LocalStore::LocalStore(const Params & params) struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); if (!gr) printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup); - else { + else if (!readOnly) { struct stat st; if (stat(realStoreDir.get().c_str(), &st)) throw SysError("getting attributes of path '%1%'", realStoreDir); @@ -352,8 +314,6 @@ LocalStore::LocalStore(const Params & params) have performed the upgrade already. */ curSchema = getSchema(); - if (curSchema < 7) { upgradeStore7(); } - openDB(*state, false); if (curSchema < 8) { @@ -463,10 +423,9 @@ LocalStore::LocalStore(const Params & params) } -LocalStore::LocalStore(std::string scheme, std::string path, const Params & params) - : LocalStore(params) +LocalStore::LocalStore(const Params & params) + : LocalStore("local", "", params) { - throw UnimplementedError("LocalStore"); } @@ -512,7 +471,7 @@ LocalStore::~LocalStore() unlink(fnTempRoots.c_str()); } } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -1086,103 +1045,114 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, if (checkSigs && pathInfoIsUntrusted(info)) throw Error("cannot add path '%s' because it lacks a signature by a trusted key", printStorePath(info.path)); - /* In case we are not interested in reading the NAR: discard it. */ - bool narRead = false; - Finally cleanup = [&]() { - if (!narRead) { - NullFileSystemObjectSink sink; - try { - parseDump(sink, source); - } catch (...) { - ignoreException(); + { + /* In case we are not interested in reading the NAR: discard it. */ + bool narRead = false; + Finally cleanup = [&]() { + if (!narRead) { + NullFileSystemObjectSink sink; + try { + parseDump(sink, source); + } catch (...) { + // TODO: should Interrupted be handled here? + ignoreExceptionInDestructor(); + } } - } - }; + }; - addTempRoot(info.path); - - if (repair || !isValidPath(info.path)) { - - PathLocks outputLock; - - auto realPath = Store::toRealPath(info.path); - - /* Lock the output path. But don't lock if we're being called - from a build hook (whose parent process already acquired a - lock on this path). */ - if (!locksHeld.count(printStorePath(info.path))) - outputLock.lockPaths({realPath}); + addTempRoot(info.path); if (repair || !isValidPath(info.path)) { - deletePath(realPath); + PathLocks outputLock; - /* While restoring the path from the NAR, compute the hash - of the NAR. */ - HashSink hashSink(HashAlgorithm::SHA256); + auto realPath = Store::toRealPath(info.path); - TeeSource wrapperSource { source, hashSink }; + /* Lock the output path. But don't lock if we're being called + from a build hook (whose parent process already acquired a + lock on this path). */ + if (!locksHeld.count(printStorePath(info.path))) + outputLock.lockPaths({realPath}); - narRead = true; - restorePath(realPath, wrapperSource); + if (repair || !isValidPath(info.path)) { - auto hashResult = hashSink.finish(); + deletePath(realPath); - if (hashResult.first != info.narHash) - throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), info.narHash.to_string(HashFormat::Nix32, true), hashResult.first.to_string(HashFormat::Nix32, true)); + /* While restoring the path from the NAR, compute the hash + of the NAR. */ + HashSink hashSink(HashAlgorithm::SHA256); - if (hashResult.second != info.narSize) - throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), info.narSize, hashResult.second); + TeeSource wrapperSource { source, hashSink }; - if (info.ca) { - auto & specified = *info.ca; - auto actualHash = ({ - auto accessor = getFSAccessor(false); - CanonPath path { printStorePath(info.path) }; - Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++ - auto fim = specified.method.getFileIngestionMethod(); - switch (fim) { - case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: - { - HashModuloSink caSink { - specified.hash.algo, - std::string { info.path.hashPart() }, + narRead = true; + restorePath(realPath, wrapperSource, settings.fsyncStorePaths); + + auto hashResult = hashSink.finish(); + + if (hashResult.first != info.narHash) + throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s", + printStorePath(info.path), info.narHash.to_string(HashFormat::Nix32, true), hashResult.first.to_string(HashFormat::Nix32, true)); + + if (hashResult.second != info.narSize) + throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s", + printStorePath(info.path), info.narSize, hashResult.second); + + if (info.ca) { + auto & specified = *info.ca; + auto actualHash = ({ + auto accessor = getFSAccessor(false); + CanonPath path { printStorePath(info.path) }; + Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++ + auto fim = specified.method.getFileIngestionMethod(); + switch (fim) { + case FileIngestionMethod::Flat: + case FileIngestionMethod::NixArchive: + { + HashModuloSink caSink { + specified.hash.algo, + std::string { info.path.hashPart() }, + }; + dumpPath({accessor, path}, caSink, (FileSerialisationMethod) fim); + h = caSink.finish().first; + break; + } + case FileIngestionMethod::Git: + h = git::dumpHash(specified.hash.algo, {accessor, path}).hash; + break; + } + ContentAddress { + .method = specified.method, + .hash = std::move(h), }; - dumpPath({accessor, path}, caSink, (FileSerialisationMethod) fim); - h = caSink.finish().first; - break; + }); + if (specified.hash != actualHash.hash) { + throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", + printStorePath(info.path), + specified.hash.to_string(HashFormat::Nix32, true), + actualHash.hash.to_string(HashFormat::Nix32, true)); } - case FileIngestionMethod::Git: - h = git::dumpHash(specified.hash.algo, {accessor, path}).hash; - break; - } - ContentAddress { - .method = specified.method, - .hash = std::move(h), - }; - }); - if (specified.hash != actualHash.hash) { - throw Error("ca hash mismatch importing path '%s';\n specified: %s\n got: %s", - printStorePath(info.path), - specified.hash.to_string(HashFormat::Nix32, true), - actualHash.hash.to_string(HashFormat::Nix32, true)); } + + autoGC(); + + canonicalisePathMetaData(realPath); + + optimisePath(realPath, repair); // FIXME: combine with hashPath() + + if (settings.fsyncStorePaths) { + recursiveSync(realPath); + syncParent(realPath); + } + + registerValidPath(info); } - autoGC(); - - canonicalisePathMetaData(realPath); - - optimisePath(realPath, repair); // FIXME: combine with hashPath() - - registerValidPath(info); + outputLock.setDeletion(true); } - - outputLock.setDeletion(true); } + + // In case `cleanup` ignored an `Interrupted` exception + checkInterrupt(); } @@ -1243,7 +1213,7 @@ StorePath LocalStore::addToStoreFromDump( std::filesystem::path tempDir; AutoCloseFD tempDirFd; - bool methodsMatch = ContentAddressMethod(FileIngestionMethod(dumpMethod)) == hashMethod; + bool methodsMatch = static_cast(dumpMethod) == hashMethod.getFileIngestionMethod(); /* If the methods don't match, our streaming hash of the dump is the wrong sort, and we need to rehash. */ @@ -1258,7 +1228,7 @@ StorePath LocalStore::addToStoreFromDump( delTempDir = std::make_unique(tempDir); tempPath = tempDir / "x"; - restorePath(tempPath.string(), bothSource, dumpMethod); + restorePath(tempPath.string(), bothSource, dumpMethod, settings.fsyncStorePaths); dumpBuffer.reset(); dump = {}; @@ -1272,7 +1242,7 @@ StorePath LocalStore::addToStoreFromDump( ? dumpHash : hashPath( PosixSourceAccessor::createAtRoot(tempPath), - hashMethod.getFileIngestionMethod(), hashAlgo), + hashMethod.getFileIngestionMethod(), hashAlgo).first, { .others = references, // caller is not capable of creating a self-reference, because this is content-addressed without modulus @@ -1304,8 +1274,8 @@ StorePath LocalStore::addToStoreFromDump( auto fim = hashMethod.getFileIngestionMethod(); switch (fim) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: - restorePath(realPath, dumpSource, (FileSerialisationMethod) fim); + case FileIngestionMethod::NixArchive: + restorePath(realPath, dumpSource, (FileSerialisationMethod) fim, settings.fsyncStorePaths); break; case FileIngestionMethod::Git: // doesn't correspond to serialization method, so @@ -1320,7 +1290,7 @@ StorePath LocalStore::addToStoreFromDump( /* For computing the nar hash. In recursive SHA-256 mode, this is the same as the store hash, so no need to do it again. */ auto narHash = std::pair { dumpHash, size }; - if (dumpMethod != FileSerialisationMethod::Recursive || hashAlgo != HashAlgorithm::SHA256) { + if (dumpMethod != FileSerialisationMethod::NixArchive || hashAlgo != HashAlgorithm::SHA256) { HashSink narSink { HashAlgorithm::SHA256 }; dumpPath(realPath, narSink); narHash = narSink.finish(); @@ -1330,6 +1300,11 @@ StorePath LocalStore::addToStoreFromDump( optimisePath(realPath, repair); + if (settings.fsyncStorePaths) { + recursiveSync(realPath); + syncParent(realPath); + } + ValidPathInfo info { *this, name, @@ -1407,12 +1382,13 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) printInfo("checking link hashes..."); for (auto & link : std::filesystem::directory_iterator{linksDir}) { + checkInterrupt(); auto name = link.path().filename(); printMsg(lvlTalkative, "checking contents of '%s'", name); PosixSourceAccessor accessor; std::string hash = hashPath( PosixSourceAccessor::createAtRoot(link.path()), - FileIngestionMethod::Recursive, HashAlgorithm::SHA256).to_string(HashFormat::Nix32, false); + FileIngestionMethod::NixArchive, HashAlgorithm::SHA256).first.to_string(HashFormat::Nix32, false); if (hash != name.string()) { printError("link '%s' was modified! expected hash '%s', got '%s'", link.path(), name, hash); @@ -1499,6 +1475,7 @@ LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair invalid states. */ for (auto & i : std::filesystem::directory_iterator{realStoreDir.to_string()}) { + checkInterrupt(); try { storePathsInStoreDir.insert({i.path().filename().string()}); } catch (BadStorePath &) { } @@ -1581,62 +1558,6 @@ std::optional LocalStore::isTrustedClient() } -#if defined(FS_IOC_SETFLAGS) && defined(FS_IOC_GETFLAGS) && defined(FS_IMMUTABLE_FL) - -static void makeMutable(const Path & path) -{ - checkInterrupt(); - - auto st = lstat(path); - - if (!S_ISDIR(st.st_mode) && !S_ISREG(st.st_mode)) return; - - if (S_ISDIR(st.st_mode)) { - for (auto & i : readDirectory(path)) - makeMutable(path + "/" + i.name); - } - - /* The O_NOFOLLOW is important to prevent us from changing the - mutable bit on the target of a symlink (which would be a - security hole). */ - AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_NOFOLLOW -#ifndef _WIN32 - | O_CLOEXEC -#endif - ); - if (fd == INVALID_DESCRIPTOR) { - if (errno == ELOOP) return; // it's a symlink - throw SysError("opening file '%1%'", path); - } - - unsigned int flags = 0, old; - - /* Silently ignore errors getting/setting the immutable flag so - that we work correctly on filesystems that don't support it. */ - if (ioctl(fd, FS_IOC_GETFLAGS, &flags)) return; - old = flags; - flags &= ~FS_IMMUTABLE_FL; - if (old == flags) return; - if (ioctl(fd, FS_IOC_SETFLAGS, &flags)) return; -} - -/* Upgrade from schema 6 (Nix 0.15) to schema 7 (Nix >= 1.3). */ -void LocalStore::upgradeStore7() -{ - if (!isRootUser()) return; - printInfo("removing immutable bits from the Nix store (this may take a while)..."); - makeMutable(realStoreDir); -} - -#else - -void LocalStore::upgradeStore7() -{ -} - -#endif - - void LocalStore::vacuumDB() { auto state(_state.lock()); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index b3d7bd6d0..21848cc4d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -38,6 +38,11 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; + LocalStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params); + Setting requireSigs{this, settings.requireSigs, "require-sigs", @@ -62,6 +67,9 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig const std::string name() override { return "Local Store"; } + static std::set uriSchemes() + { return {"local"}; } + std::string doc() override; }; @@ -137,13 +145,13 @@ public: * necessary. */ LocalStore(const Params & params); - LocalStore(std::string scheme, std::string path, const Params & params); + LocalStore( + std::string_view scheme, + PathView path, + const Params & params); ~LocalStore(); - static std::set uriSchemes() - { return {}; } - /** * Implementations of abstract store API methods. */ @@ -365,8 +373,6 @@ private: void updatePathInfo(State & state, const ValidPathInfo & info); - void upgradeStore6(); - void upgradeStore7(); PathSet queryValidPathsOld(); ValidPathInfo queryPathInfoOld(const Path & path); diff --git a/src/libstore/local.mk b/src/libstore/local.mk index cc67da786..43d8993ba 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -4,9 +4,9 @@ libstore_NAME = libnixstore libstore_DIR := $(d) -libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc) +libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc) ifdef HOST_UNIX - libstore_SOURCES += $(wildcard $(d)/unix/*.cc $(d)/unix/builtins/*.cc $(d)/unix/build/*.cc) + libstore_SOURCES += $(wildcard $(d)/unix/*.cc $(d)/unix/build/*.cc) endif ifdef HOST_LINUX libstore_SOURCES += $(wildcard $(d)/linux/*.cc) @@ -43,7 +43,7 @@ endif INCLUDE_libstore := -I $(d) -I $(d)/build ifdef HOST_UNIX - INCLUDE_libstore += -I $(d)/unix + INCLUDE_libstore += -I $(d)/unix -I $(d)/unix/build endif ifdef HOST_LINUX INCLUDE_libstore += -I $(d)/linux @@ -59,7 +59,7 @@ NIX_ROOT = endif # Prefix all but `NIX_STORE_DIR`, since we aren't doing a local store -# yet so a "logical" store dir that is the same as unix is prefered. +# yet so a "logical" store dir that is the same as unix is preferred. # # Also, it keeps the unit tests working. @@ -71,7 +71,6 @@ libstore_CXXFLAGS += \ -DNIX_STATE_DIR=\"$(NIX_ROOT)$(localstatedir)/nix\" \ -DNIX_LOG_DIR=\"$(NIX_ROOT)$(localstatedir)/log/nix\" \ -DNIX_CONF_DIR=\"$(NIX_ROOT)$(sysconfdir)/nix\" \ - -DNIX_BIN_DIR=\"$(NIX_ROOT)$(bindir)\" \ -DNIX_MAN_DIR=\"$(NIX_ROOT)$(mandir)\" \ -DLSOF=\"$(NIX_ROOT)$(lsof)\" diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index 2d461c63a..5e038fb28 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -6,7 +6,8 @@ namespace nix { -Machine::Machine(decltype(storeUri) storeUri, +Machine::Machine( + const std::string & storeUri, decltype(systemTypes) systemTypes, decltype(sshKey) sshKey, decltype(maxJobs) maxJobs, @@ -14,7 +15,7 @@ Machine::Machine(decltype(storeUri) storeUri, decltype(supportedFeatures) supportedFeatures, decltype(mandatoryFeatures) mandatoryFeatures, decltype(sshPublicHostKey) sshPublicHostKey) : - storeUri( + storeUri(StoreReference::parse( // Backwards compatibility: if the URI is schemeless, is not a path, // and is not one of the special store connection words, prepend // ssh://. @@ -28,7 +29,7 @@ Machine::Machine(decltype(storeUri) storeUri, || hasPrefix(storeUri, "local?") || hasPrefix(storeUri, "?") ? storeUri - : "ssh://" + storeUri), + : "ssh://" + storeUri)), systemTypes(systemTypes), sshKey(sshKey), maxJobs(maxJobs), @@ -63,23 +64,26 @@ bool Machine::mandatoryMet(const std::set & features) const }); } -ref Machine::openStore() const +StoreReference Machine::completeStoreReference() const { - Store::Params storeParams; - if (hasPrefix(storeUri, "ssh://")) { - storeParams["max-connections"] = "1"; - storeParams["log-fd"] = "4"; + auto storeUri = this->storeUri; + + auto * generic = std::get_if(&storeUri.variant); + + if (generic && generic->scheme == "ssh") { + storeUri.params["max-connections"] = "1"; + storeUri.params["log-fd"] = "4"; } - if (hasPrefix(storeUri, "ssh://") || hasPrefix(storeUri, "ssh-ng://")) { + if (generic && (generic->scheme == "ssh" || generic->scheme == "ssh-ng")) { if (sshKey != "") - storeParams["ssh-key"] = sshKey; + storeUri.params["ssh-key"] = sshKey; if (sshPublicHostKey != "") - storeParams["base64-ssh-public-host-key"] = sshPublicHostKey; + storeUri.params["base64-ssh-public-host-key"] = sshPublicHostKey; } { - auto & fs = storeParams["system-features"]; + auto & fs = storeUri.params["system-features"]; auto append = [&](auto feats) { for (auto & f : feats) { if (fs.size() > 0) fs += ' '; @@ -90,7 +94,12 @@ ref Machine::openStore() const append(mandatoryFeatures); } - return nix::openStore(storeUri, storeParams); + return storeUri; +} + +ref Machine::openStore() const +{ + return nix::openStore(completeStoreReference()); } static std::vector expandBuilderLines(const std::string & builders) @@ -122,7 +131,7 @@ static std::vector expandBuilderLines(const std::string & builders) return result; } -static Machine parseBuilderLine(const std::string & line) +static Machine parseBuilderLine(const std::set & defaultSystems, const std::string & line) { const auto tokens = tokenizeString>(line); @@ -139,7 +148,7 @@ static Machine parseBuilderLine(const std::string & line) }; auto parseFloatField = [&](size_t fieldIndex) { - const auto result = string2Int(tokens[fieldIndex]); + const auto result = string2Float(tokens[fieldIndex]); if (!result) { throw FormatError("bad machine specification: failed to convert column #%lu in a row: '%s' to 'float'", fieldIndex, line); } @@ -150,8 +159,9 @@ static Machine parseBuilderLine(const std::string & line) const auto & str = tokens[fieldIndex]; try { base64Decode(str); - } catch (const Error & e) { - throw FormatError("bad machine specification: a column #%lu in a row: '%s' is not valid base64 string: %s", fieldIndex, line, e.what()); + } catch (FormatError & e) { + e.addTrace({}, "while parsing machine specification at a column #%lu in a row: '%s'", fieldIndex, line); + throw; } return str; }; @@ -159,29 +169,46 @@ static Machine parseBuilderLine(const std::string & line) if (!isSet(0)) throw FormatError("bad machine specification: store URL was not found at the first column of a row: '%s'", line); + // TODO use designated initializers, once C++ supports those with + // custom constructors. return { + // `storeUri` tokens[0], - isSet(1) ? tokenizeString>(tokens[1], ",") : std::set{settings.thisSystem}, + // `systemTypes` + isSet(1) ? tokenizeString>(tokens[1], ",") : defaultSystems, + // `sshKey` isSet(2) ? tokens[2] : "", + // `maxJobs` isSet(3) ? parseUnsignedIntField(3) : 1U, + // `speedFactor` isSet(4) ? parseFloatField(4) : 1.0f, + // `supportedFeatures` isSet(5) ? tokenizeString>(tokens[5], ",") : std::set{}, + // `mandatoryFeatures` isSet(6) ? tokenizeString>(tokens[6], ",") : std::set{}, + // `sshPublicHostKey` isSet(7) ? ensureBase64(7) : "" }; } -static Machines parseBuilderLines(const std::vector & builders) +static Machines parseBuilderLines(const std::set & defaultSystems, const std::vector & builders) { Machines result; - std::transform(builders.begin(), builders.end(), std::back_inserter(result), parseBuilderLine); + std::transform( + builders.begin(), builders.end(), std::back_inserter(result), + [&](auto && line) { return parseBuilderLine(defaultSystems, line); }); return result; } +Machines Machine::parseConfig(const std::set & defaultSystems, const std::string & s) +{ + const auto builderLines = expandBuilderLines(s); + return parseBuilderLines(defaultSystems, builderLines); +} + Machines getMachines() { - const auto builderLines = expandBuilderLines(settings.builders); - return parseBuilderLines(builderLines); + return Machine::parseConfig({settings.thisSystem}, settings.builders); } } diff --git a/src/libstore/machines.hh b/src/libstore/machines.hh index 8516409d4..983652d5f 100644 --- a/src/libstore/machines.hh +++ b/src/libstore/machines.hh @@ -1,15 +1,20 @@ #pragma once ///@file -#include "types.hh" +#include "ref.hh" +#include "store-reference.hh" namespace nix { class Store; +struct Machine; + +typedef std::vector Machines; + struct Machine { - const std::string storeUri; + const StoreReference storeUri; const std::set systemTypes; const std::string sshKey; const unsigned int maxJobs; @@ -36,7 +41,8 @@ struct Machine { */ bool mandatoryMet(const std::set & features) const; - Machine(decltype(storeUri) storeUri, + Machine( + const std::string & storeUri, decltype(systemTypes) systemTypes, decltype(sshKey) sshKey, decltype(maxJobs) maxJobs, @@ -45,13 +51,38 @@ struct Machine { decltype(mandatoryFeatures) mandatoryFeatures, decltype(sshPublicHostKey) sshPublicHostKey); + /** + * Elaborate `storeUri` into a complete store reference, + * incorporating information from the other fields of the `Machine` + * as applicable. + */ + StoreReference completeStoreReference() const; + + /** + * Open a `Store` for this machine. + * + * Just a simple function composition: + * ```c++ + * nix::openStore(completeStoreReference()) + * ``` + */ ref openStore() const; + + /** + * Parse a machine configuration. + * + * Every machine is specified on its own line, and lines beginning + * with `@` are interpreted as paths to other configuration files in + * the same format. + */ + static Machines parseConfig(const std::set & defaultSystems, const std::string & config); }; -typedef std::vector Machines; - -void parseMachines(const std::string & s, Machines & machines); - +/** + * Parse machines from the global config + * + * @todo Remove, globals are bad. + */ Machines getMachines(); } diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index 170fe67b9..a3130d7cc 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -52,7 +52,7 @@ std::map makeContentAddressed( dstStore, path.name(), FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = narModuloHash, .references = std::move(refs), }, diff --git a/src/libstore/meson.build b/src/libstore/meson.build new file mode 100644 index 000000000..6a6aabf97 --- /dev/null +++ b/src/libstore/meson.build @@ -0,0 +1,436 @@ +project('nix-store', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + 'localstatedir=/nix/var', + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +configdata.set_quoted('SYSTEM', host_machine.cpu_family() + '-' + host_machine.system()) + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), +] +subdir('build-utils-meson/subprojects') + +run_command('ln', '-s', + meson.project_build_root() / '__nothing_link_target', + meson.project_build_root() / '__nothing_symlink', + check : true, +) +can_link_symlink = run_command('ln', + meson.project_build_root() / '__nothing_symlink', + meson.project_build_root() / '__nothing_hardlink', + check : false, +).returncode() == 0 +run_command('rm', '-f', + meson.project_build_root() / '__nothing_symlink', + meson.project_build_root() / '__nothing_hardlink', + check : true, +) +summary('can hardlink to symlink', can_link_symlink, bool_yn : true) +configdata.set('CAN_LINK_SYMLINK', can_link_symlink.to_int()) + +# Check for each of these functions, and create a define like `#define HAVE_LCHOWN 1`. +# +# Only need to do functions that deps (like `libnixutil`) didn't already +# check for. +check_funcs = [ + # Optionally used for canonicalising files from the build + 'lchown', + 'statvfs', +] +foreach funcspec : check_funcs + define_name = 'HAVE_' + funcspec.underscorify().to_upper() + define_value = cxx.has_function(funcspec).to_int() + configdata.set(define_name, define_value) +endforeach + +has_acl_support = cxx.has_header('sys/xattr.h') \ + and cxx.has_function('llistxattr') \ + and cxx.has_function('lremovexattr') +configdata.set('HAVE_ACL_SUPPORT', has_acl_support.to_int()) + +subdir('build-utils-meson/threads') + +boost = dependency( + 'boost', + modules : ['container'], + include_type: 'system', +) +# boost is a public dependency, but not a pkg-config dependency unfortunately, so we +# put in `deps_other`. +deps_other += boost + +curl = dependency('libcurl', 'curl') +deps_private += curl + +# seccomp only makes sense on Linux +is_linux = host_machine.system() == 'linux' +seccomp_required = get_option('seccomp-sandboxing') +if not is_linux and seccomp_required.enabled() + warning('Force-enabling seccomp on non-Linux does not make sense') +endif +seccomp = dependency('libseccomp', 'seccomp', required : seccomp_required, version : '>=2.5.5') +if is_linux and not seccomp.found() + warning('Sandbox security is reduced because libseccomp has not been found! Please provide libseccomp if it supports your CPU architecture.') +endif +configdata.set('HAVE_SECCOMP', seccomp.found().to_int()) +deps_private += seccomp + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') +deps_private += sqlite + +# AWS C++ SDK has bad pkg-config +aws_s3 = dependency('aws-cpp-sdk-s3', required : false) +configdata.set('ENABLE_S3', aws_s3.found().to_int()) +if aws_s3.found() + aws_s3 = declare_dependency( + include_directories: include_directories(aws_s3.get_variable('includedir')), + link_args: [ + '-L' + aws_s3.get_variable('libdir'), + '-laws-cpp-sdk-transfer', + '-laws-cpp-sdk-s3', + '-laws-cpp-sdk-core', + '-laws-crt-cpp', + ], + ).as_system('system') +endif +deps_other += aws_s3 + +subdir('build-utils-meson/generate-header') + +generated_headers = [] +foreach header : [ + 'schema.sql', + 'ca-specific-schema.sql', +] + generated_headers += gen_header.process(header) +endforeach + +busybox = find_program(get_option('sandbox-shell'), required : false) + +if get_option('embedded-sandbox-shell') + # This one goes in config.h + # The path to busybox is passed as a -D flag when compiling this_library. + # This solution is inherited from the old make buildsystem + # TODO: do this differently? + configdata.set('HAVE_EMBEDDED_SANDBOX_SHELL', 1) + hexdump = find_program('hexdump', native : true) + embedded_sandbox_shell_gen = custom_target( + 'embedded-sandbox-shell.gen.hh', + command : [ + hexdump, + '-v', + '-e', + '1/1 "0x%x," "\n"' + ], + input : busybox.full_path(), + output : 'embedded-sandbox-shell.gen.hh', + capture : true, + feed : true, + ) + generated_headers += embedded_sandbox_shell_gen +endif + +config_h = configure_file( + configuration : configdata, + output : 'config-store.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'binary-cache-store.cc', + 'build-result.cc', + 'build/derivation-goal.cc', + 'build/drv-output-substitution-goal.cc', + 'build/entry-points.cc', + 'build/goal.cc', + 'build/substitution-goal.cc', + 'build/worker.cc', + 'builtins/buildenv.cc', + 'builtins/fetchurl.cc', + 'builtins/unpack-channel.cc', + 'common-protocol.cc', + 'common-ssh-store-config.cc', + 'content-address.cc', + 'daemon.cc', + 'derivations.cc', + 'derived-path-map.cc', + 'derived-path.cc', + 'downstream-placeholder.cc', + 'dummy-store.cc', + 'export-import.cc', + 'filetransfer.cc', + 'gc.cc', + 'globals.cc', + 'http-binary-cache-store.cc', + 'indirect-root-store.cc', + 'keys.cc', + 'legacy-ssh-store.cc', + 'local-binary-cache-store.cc', + 'local-fs-store.cc', + 'local-overlay-store.cc', + 'local-store.cc', + 'log-store.cc', + 'machines.cc', + 'make-content-addressed.cc', + 'misc.cc', + 'names.cc', + 'nar-accessor.cc', + 'nar-info-disk-cache.cc', + 'nar-info.cc', + 'optimise-store.cc', + 'outputs-spec.cc', + 'parsed-derivations.cc', + 'path-info.cc', + 'path-references.cc', + 'path-with-outputs.cc', + 'path.cc', + 'pathlocks.cc', + 'posix-fs-canonicalise.cc', + 'profiles.cc', + 'realisation.cc', + 'remote-fs-accessor.cc', + 'remote-store.cc', + 's3-binary-cache-store.cc', + 'serve-protocol-connection.cc', + 'serve-protocol.cc', + 'sqlite.cc', + 'ssh-store.cc', + 'ssh.cc', + 'store-api.cc', + 'store-reference.cc', + 'uds-remote-store.cc', + 'worker-protocol-connection.cc', + 'worker-protocol.cc', +) + +include_dirs = [ + include_directories('.'), + include_directories('build'), +] + +headers = [config_h] + files( + 'binary-cache-store.hh', + 'build-result.hh', + 'build/derivation-goal.hh', + 'build/drv-output-substitution-goal.hh', + 'build/goal.hh', + 'build/substitution-goal.hh', + 'build/worker.hh', + 'builtins.hh', + 'builtins/buildenv.hh', + 'common-protocol-impl.hh', + 'common-protocol.hh', + 'common-ssh-store-config.hh', + 'content-address.hh', + 'daemon.hh', + 'derivations.hh', + 'derived-path-map.hh', + 'derived-path.hh', + 'downstream-placeholder.hh', + 'filetransfer.hh', + 'gc-store.hh', + 'globals.hh', + 'http-binary-cache-store.hh', + 'indirect-root-store.hh', + 'keys.hh', + 'legacy-ssh-store.hh', + 'length-prefixed-protocol-helper.hh', + 'local-binary-cache-store.hh', + 'local-fs-store.hh', + 'local-overlay-store.hh', + 'local-store.hh', + 'log-store.hh', + 'machines.hh', + 'make-content-addressed.hh', + 'names.hh', + 'nar-accessor.hh', + 'nar-info-disk-cache.hh', + 'nar-info.hh', + 'outputs-spec.hh', + 'parsed-derivations.hh', + 'path-info.hh', + 'path-references.hh', + 'path-regex.hh', + 'path-with-outputs.hh', + 'path.hh', + 'pathlocks.hh', + 'posix-fs-canonicalise.hh', + 'profiles.hh', + 'realisation.hh', + 'remote-fs-accessor.hh', + 'remote-store-connection.hh', + 'remote-store.hh', + 's3-binary-cache-store.hh', + 's3.hh', + 'ssh-store.hh', + 'serve-protocol-connection.hh', + 'serve-protocol-impl.hh', + 'serve-protocol.hh', + 'sqlite.hh', + 'ssh.hh', + 'store-api.hh', + 'store-cast.hh', + 'store-dir-config.hh', + 'store-reference.hh', + 'uds-remote-store.hh', + 'worker-protocol-connection.hh', + 'worker-protocol-impl.hh', + 'worker-protocol.hh', +) + +if host_machine.system() == 'linux' + subdir('linux') +endif + +if host_machine.system() == 'windows' + subdir('windows') +else + subdir('unix') +endif + +fs = import('fs') + +prefix = get_option('prefix') +# For each of these paths, assume that it is relative to the prefix unless +# it is already an absolute path (which is the default for store-dir, localstatedir, and log-dir). +path_opts = [ + # Meson built-ins. + 'datadir', + 'mandir', + 'libdir', + 'includedir', + 'libexecdir', + # Homecooked Nix directories. + 'store-dir', + 'localstatedir', + 'log-dir', +] +# For your grepping pleasure, this loop sets the following variables that aren't mentioned +# literally above: +# store_dir +# localstatedir +# log_dir +# profile_dir +foreach optname : path_opts + varname = optname.replace('-', '_') + path = get_option(optname) + if fs.is_absolute(path) + set_variable(varname, path) + else + set_variable(varname, prefix / path) + endif +endforeach + +# sysconfdir doesn't get anything installed to directly, and is only used to +# tell Nix where to look for nix.conf, so it doesn't get appended to prefix. +sysconfdir = get_option('sysconfdir') +if not fs.is_absolute(sysconfdir) + sysconfdir = '/' / sysconfdir +endif + +lsof = find_program('lsof', required : false) + +# Aside from prefix itself, each of these was made into an absolute path +# by joining it with prefix, unless it was already an absolute path +# (which is the default for store-dir, localstatedir, and log-dir). +cpp_str_defines = { + 'NIX_PREFIX': prefix, + 'NIX_STORE_DIR': store_dir, + 'NIX_DATA_DIR': datadir, + 'NIX_STATE_DIR': localstatedir / 'nix', + 'NIX_LOG_DIR': log_dir, + 'NIX_CONF_DIR': sysconfdir / 'nix', + 'NIX_MAN_DIR': mandir, +} + +if lsof.found() + lsof_path = lsof.full_path() +else + # Just look up on the PATH + lsof_path = 'lsof' +endif +cpp_str_defines += { + 'LSOF': lsof_path +} + +if get_option('embedded-sandbox-shell') + cpp_str_defines += { + 'SANDBOX_SHELL': '__embedded_sandbox_shell__' + } +elif busybox.found() + cpp_str_defines += { + 'SANDBOX_SHELL': busybox.full_path() + } +endif + +cpp_args = [] + +foreach name, value : cpp_str_defines + cpp_args += [ + '-D' + name + '=' + '"' + value + '"' + ] +endforeach + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixstore', + generated_headers, + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + cpp_args : cpp_args, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +extra_pkg_config_variables = { + 'storedir' : get_option('store-dir'), +} + +# Working around https://github.com/mesonbuild/meson/issues/13584 +if host_machine.system() != 'darwin' + extra_pkg_config_variables += { + 'localstatedir' : get_option('localstatedir'), + } +endif + +subdir('build-utils-meson/export') diff --git a/src/libstore/meson.options b/src/libstore/meson.options new file mode 100644 index 000000000..ebad24dc4 --- /dev/null +++ b/src/libstore/meson.options @@ -0,0 +1,21 @@ +# vim: filetype=meson + +option('embedded-sandbox-shell', type : 'boolean', value : false, + description : 'include the sandbox shell in the Nix binary', +) + +option('seccomp-sandboxing', type : 'feature', + description : 'build support for seccomp sandboxing (recommended unless your arch doesn\'t support libseccomp, only relevant on Linux)', +) + +option('sandbox-shell', type : 'string', value : 'busybox', + description : 'path to a statically-linked shell to use as /bin/sh in sandboxes (usually busybox)', +) + +option('store-dir', type : 'string', value : '/nix/store', + description : 'path of the Nix store', +) + +option('log-dir', type : 'string', value : '/nix/var/log/nix', + description : 'path to store logs in for Nix', +) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index cc3f4884f..bcc02206b 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,3 +1,5 @@ +#include + #include "derivations.hh" #include "parsed-derivations.hh" #include "globals.hh" @@ -8,6 +10,7 @@ #include "callback.hh" #include "closure.hh" #include "filetransfer.hh" +#include "strings.hh" namespace nix { diff --git a/src/libstore/names.cc b/src/libstore/names.cc index 277aabf0f..c0e1b1022 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -94,7 +94,7 @@ static bool componentsLT(const std::string_view c1, const std::string_view c2) } -int compareVersions(const std::string_view v1, const std::string_view v2) +std::strong_ordering compareVersions(const std::string_view v1, const std::string_view v2) { auto p1 = v1.begin(); auto p2 = v2.begin(); @@ -102,11 +102,11 @@ int compareVersions(const std::string_view v1, const std::string_view v2) while (p1 != v1.end() || p2 != v2.end()) { auto c1 = nextComponent(p1, v1.end()); auto c2 = nextComponent(p2, v2.end()); - if (componentsLT(c1, c2)) return -1; - else if (componentsLT(c2, c1)) return 1; + if (componentsLT(c1, c2)) return std::strong_ordering::less; + else if (componentsLT(c2, c1)) return std::strong_ordering::greater; } - return 0; + return std::strong_ordering::equal; } diff --git a/src/libstore/names.hh b/src/libstore/names.hh index d82b99bb4..a6909d545 100644 --- a/src/libstore/names.hh +++ b/src/libstore/names.hh @@ -30,7 +30,7 @@ typedef std::list DrvNames; std::string_view nextComponent(std::string_view::const_iterator & p, const std::string_view::const_iterator end); -int compareVersions(const std::string_view v1, const std::string_view v2); +std::strong_ordering compareVersions(const std::string_view v1, const std::string_view v2); DrvNames drvNamesFromArgs(const Strings & opArgs); } diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index cecf8148f..9a541bb77 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -3,7 +3,6 @@ #include #include -#include #include @@ -71,9 +70,14 @@ struct NarAccessor : public SourceAccessor : acc(acc), source(source) { } - NarMember & createMember(const Path & path, NarMember member) + NarMember & createMember(const CanonPath & path, NarMember member) { - size_t level = std::count(path.begin(), path.end(), '/'); + size_t level = 0; + for (auto _ : path) { + (void)_; + ++level; + } + while (parents.size() > level) parents.pop(); if (parents.empty()) { @@ -83,14 +87,14 @@ struct NarAccessor : public SourceAccessor } else { if (parents.top()->stat.type != Type::tDirectory) throw Error("NAR file missing parent directory of path '%s'", path); - auto result = parents.top()->children.emplace(baseNameOf(path), std::move(member)); + auto result = parents.top()->children.emplace(*path.baseName(), std::move(member)); auto & ref = result.first->second; parents.push(&ref); return ref; } } - void createDirectory(const Path & path) override + void createDirectory(const CanonPath & path) override { createMember(path, NarMember{ .stat = { .type = Type::tDirectory, @@ -100,7 +104,7 @@ struct NarAccessor : public SourceAccessor } }); } - void createRegularFile(const Path & path, std::function func) override + void createRegularFile(const CanonPath & path, std::function func) override { auto & nm = createMember(path, NarMember{ .stat = { .type = Type::tRegular, @@ -112,7 +116,7 @@ struct NarAccessor : public SourceAccessor func(nmc); } - void createSymlink(const Path & path, const std::string & target) override + void createSymlink(const CanonPath & path, const std::string & target) override { createMember(path, NarMember{ diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 07beb8acb..80e8d3414 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -7,6 +7,8 @@ #include #include +#include "strings.hh" + namespace nix { static const char * schema = R"sql( @@ -85,7 +87,7 @@ public: Sync _state; - NarInfoDiskCacheImpl(Path dbPath = getCacheDir() + "/nix/binary-cache-v6.sqlite") + NarInfoDiskCacheImpl(Path dbPath = getCacheDir() + "/binary-cache-v6.sqlite") { auto state(_state.lock()); @@ -162,7 +164,7 @@ public: Cache & getCache(State & state, const std::string & uri) { auto i = state.caches.find(uri); - if (i == state.caches.end()) abort(); + if (i == state.caches.end()) unreachable(); return i->second; } @@ -209,7 +211,7 @@ public: { auto r(state->insertCache.use()(uri)(time(0))(storeDir)(wantMassQuery)(priority)); - if (!r.next()) { abort(); } + if (!r.next()) { unreachable(); } ret.id = (int) r.getInt(0); } diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 0d219a489..8b2557060 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -1,18 +1,11 @@ #include "globals.hh" #include "nar-info.hh" #include "store-api.hh" +#include "strings.hh" +#include "json-utils.hh" namespace nix { -GENERATE_CMP_EXT( - , - NarInfo, - me->url, - me->compression, - me->fileHash, - me->fileSize, - static_cast(*me)); - NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & whence) : ValidPathInfo(StorePath(StorePath::dummy), Hash(Hash::dummy)) // FIXME: hack { diff --git a/src/libstore/nar-info.hh b/src/libstore/nar-info.hh index fd538a7cd..561c9a863 100644 --- a/src/libstore/nar-info.hh +++ b/src/libstore/nar-info.hh @@ -24,7 +24,9 @@ struct NarInfo : ValidPathInfo NarInfo(const ValidPathInfo & info) : ValidPathInfo(info) { } NarInfo(const Store & store, const std::string & s, const std::string & whence); - DECLARE_CMP(NarInfo); + bool operator ==(const NarInfo &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::optional::operator <=>`, can't do yet + //auto operator <=>(const NarInfo &) const = default; std::string to_string(const Store & store) const; diff --git a/src/libstore/nix-store.pc.in b/src/libstore/nix-store.pc.in index dc42d0bca..cd3f2b8da 100644 --- a/src/libstore/nix-store.pc.in +++ b/src/libstore/nix-store.pc.in @@ -5,5 +5,6 @@ includedir=@includedir@ Name: Nix Description: Nix Package Manager Version: @PACKAGE_VERSION@ -Libs: -L${libdir} -lnixstore -lnixutil +Requires: nix-util +Libs: -L${libdir} -lnixstore Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 2477cf0c0..aeff24c64 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -35,7 +35,7 @@ struct MakeReadOnly /* This will make the path read-only. */ if (path != "") canonicaliseTimestampAndPermissions(path); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } }; @@ -98,9 +98,10 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, #if __APPLE__ /* HFS/macOS has some undocumented security feature disabling hardlinking for - special files within .app dirs. *.app/Contents/PkgInfo and - *.app/Contents/Resources/\*.lproj seem to be the only paths affected. See - https://github.com/NixOS/nix/issues/1443 for more discussion. */ + special files within .app dirs. Known affected paths include + *.app/Contents/{PkgInfo,Resources/\*.lproj,_CodeSignature} and .DS_Store. + See https://github.com/NixOS/nix/issues/1443 and + https://github.com/NixOS/nix/pull/2230 for more discussion. */ if (std::regex_search(path, std::regex("\\.app/Contents/.+$"))) { @@ -150,7 +151,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, Hash hash = ({ hashPath( {make_ref(), CanonPath(path)}, - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first; + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256).first; }); debug("'%1%' has hash '%2%'", path, hash.to_string(HashFormat::Nix32, true)); @@ -164,7 +165,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats, || (repair && hash != ({ hashPath( PosixSourceAccessor::createAtRoot(linkPath), - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256).first; + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256).first; }))) { // XXX: Consider overwriting linkPath with our valid version. diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index 21c069223..f5ecbd74b 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -5,6 +5,7 @@ #include "regex-combinators.hh" #include "outputs-spec.hh" #include "path-regex.hh" +#include "strings-inline.hh" namespace nix { @@ -29,16 +30,15 @@ std::optional OutputsSpec::parseOpt(std::string_view s) { static std::regex regex(std::string { outputSpecRegexStr }); - std::smatch match; - std::string s2 { s }; // until some improves std::regex - if (!std::regex_match(s2, match, regex)) + std::cmatch match; + if (!std::regex_match(s.cbegin(), s.cend(), match, regex)) return std::nullopt; if (match[1].matched) return { OutputsSpec::All {} }; if (match[2].matched) - return OutputsSpec::Names { tokenizeString(match[2].str(), ",") }; + return OutputsSpec::Names { tokenizeString({match[2].first, match[2].second}, ",") }; assert(false); } diff --git a/src/libstore/outputs-spec.hh b/src/libstore/outputs-spec.hh index 1ef99a5fc..30d15311d 100644 --- a/src/libstore/outputs-spec.hh +++ b/src/libstore/outputs-spec.hh @@ -6,9 +6,7 @@ #include #include -#include "comparator.hh" #include "json-impls.hh" -#include "comparator.hh" #include "variant-wrapper.hh" namespace nix { @@ -60,7 +58,11 @@ struct OutputsSpec { Raw raw; - GENERATE_CMP(OutputsSpec, me->raw); + bool operator == (const OutputsSpec &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + bool operator < (const OutputsSpec & other) const { + return raw < other.raw; + } MAKE_WRAPPER_CONSTRUCTOR(OutputsSpec); @@ -99,7 +101,9 @@ struct ExtendedOutputsSpec { Raw raw; - GENERATE_CMP(ExtendedOutputsSpec, me->raw); + bool operator == (const ExtendedOutputsSpec &) const = default; + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + bool operator < (const ExtendedOutputsSpec &) const; MAKE_WRAPPER_CONSTRUCTOR(ExtendedOutputsSpec); diff --git a/src/libstore/package.nix b/src/libstore/package.nix new file mode 100644 index 000000000..4582ba0d2 --- /dev/null +++ b/src/libstore/package.nix @@ -0,0 +1,109 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config +, unixtools + +, nix-util +, boost +, curl +, aws-sdk-cpp +, libseccomp +, nlohmann_json +, sqlite + +, busybox-sandbox-shell ? null + +# Configuration Options + +, version + +, embeddedSandboxShell ? stdenv.hostPlatform.isStatic +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-store"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + ./linux/meson.build + ./unix/meson.build + ./windows/meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "sb") ./.) + (fileset.fileFilter (file: file.hasExt "md") ./.) + (fileset.fileFilter (file: file.hasExt "sql") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ] ++ lib.optional embeddedSandboxShell unixtools.hexdump; + + buildInputs = [ + boost + curl + sqlite + ] ++ lib.optional stdenv.hostPlatform.isLinux libseccomp + # There have been issues building these dependencies + ++ lib.optional (stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin)) + aws-sdk-cpp + ; + + propagatedBuildInputs = [ + nix-util + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "seccomp-sandboxing" stdenv.hostPlatform.isLinux) + (lib.mesonBool "embedded-sandbox-shell" embeddedSandboxShell) + ] ++ lib.optionals stdenv.hostPlatform.isLinux [ + (lib.mesonOption "sandbox-shell" "${busybox-sandbox-shell}/bin/busybox") + ]; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index a29281953..d8459d4d7 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -135,18 +135,37 @@ static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*"); /** * Write a JSON representation of store object metadata, such as the * hash and the references. + * + * @note Do *not* use `ValidPathInfo::toJSON` because this function is + * subject to stronger stability requirements since it is used to + * prepare build environments. Perhaps someday we'll have a versionining + * mechanism to allow this to evolve again and get back in sync, but for + * now we must not change - not even extend - the behavior. */ static nlohmann::json pathInfoToJSON( Store & store, const StorePathSet & storePaths) { - nlohmann::json::array_t jsonList = nlohmann::json::array(); + using nlohmann::json; + + nlohmann::json::array_t jsonList = json::array(); for (auto & storePath : storePaths) { auto info = store.queryPathInfo(storePath); - auto & jsonPath = jsonList.emplace_back( - info->toJSON(store, false, HashFormat::Nix32)); + auto & jsonPath = jsonList.emplace_back(json::object()); + + jsonPath["narHash"] = info->narHash.to_string(HashFormat::Nix32, true); + jsonPath["narSize"] = info->narSize; + + { + auto & jsonRefs = jsonPath["references"] = json::array(); + for (auto & ref : info->references) + jsonRefs.emplace_back(store.printStorePath(ref)); + } + + if (info->ca) + jsonPath["ca"] = renderContentAddress(info->ca); // Add the path to the object whose metadata we are including. jsonPath["path"] = store.printStorePath(storePath); diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index 6523cb425..6e87e60f4 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -3,11 +3,14 @@ #include "path-info.hh" #include "store-api.hh" #include "json-utils.hh" +#include "comparator.hh" +#include "strings.hh" namespace nix { GENERATE_CMP_EXT( , + std::weak_ordering, UnkeyedValidPathInfo, me->deriver, me->narHash, @@ -19,12 +22,6 @@ GENERATE_CMP_EXT( me->sigs, me->ca); -GENERATE_CMP_EXT( - , - ValidPathInfo, - me->path, - static_cast(*me)); - std::string ValidPathInfo::fingerprint(const Store & store) const { if (narSize == 0) @@ -48,15 +45,21 @@ std::optional ValidPathInfo::contentAddressWithRef if (! ca) return std::nullopt; - return std::visit(overloaded { - [&](const TextIngestionMethod &) -> ContentAddressWithReferences { + switch (ca->method.raw) { + case ContentAddressMethod::Raw::Text: + { assert(references.count(path) == 0); return TextInfo { .hash = ca->hash, .references = references, }; - }, - [&](const FileIngestionMethod & m2) -> ContentAddressWithReferences { + } + + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + default: + { auto refs = references; bool hasSelfReference = false; if (refs.count(path)) { @@ -64,15 +67,15 @@ std::optional ValidPathInfo::contentAddressWithRef refs.erase(path); } return FixedOutputInfo { - .method = m2, + .method = ca->method.getFileIngestionMethod(), .hash = ca->hash, .references = { .others = std::move(refs), .self = hasSelfReference, }, }; - }, - }, ca->method.raw); + } + } } bool ValidPathInfo::isContentAddressed(const Store & store) const @@ -127,22 +130,18 @@ ValidPathInfo::ValidPathInfo( : UnkeyedValidPathInfo(narHash) , path(store.makeFixedOutputPathFromCA(name, ca)) { + this->ca = ContentAddress { + .method = ca.getMethod(), + .hash = ca.getHash(), + }; std::visit(overloaded { [this](TextInfo && ti) { this->references = std::move(ti.references); - this->ca = ContentAddress { - .method = TextIngestionMethod {}, - .hash = std::move(ti.hash), - }; }, [this](FixedOutputInfo && foi) { this->references = std::move(foi.references.others); if (foi.references.self) this->references.insert(path); - this->ca = ContentAddress { - .method = std::move(foi.method), - .hash = std::move(foi.hash), - }; }, }, std::move(ca).raw); } @@ -161,28 +160,23 @@ nlohmann::json UnkeyedValidPathInfo::toJSON( jsonObject["narSize"] = narSize; { - auto& jsonRefs = (jsonObject["references"] = json::array()); + auto & jsonRefs = jsonObject["references"] = json::array(); for (auto & ref : references) jsonRefs.emplace_back(store.printStorePath(ref)); } - if (ca) - jsonObject["ca"] = renderContentAddress(ca); + jsonObject["ca"] = ca ? (std::optional { renderContentAddress(*ca) }) : std::nullopt; if (includeImpureInfo) { - if (deriver) - jsonObject["deriver"] = store.printStorePath(*deriver); + jsonObject["deriver"] = deriver ? (std::optional { store.printStorePath(*deriver) }) : std::nullopt; - if (registrationTime) - jsonObject["registrationTime"] = registrationTime; + jsonObject["registrationTime"] = registrationTime ? (std::optional { registrationTime }) : std::nullopt; - if (ultimate) - jsonObject["ultimate"] = ultimate; + jsonObject["ultimate"] = ultimate; - if (!sigs.empty()) { - for (auto & sig : sigs) - jsonObject["signatures"].push_back(sig); - } + auto & sigsObj = jsonObject["signatures"] = json::array(); + for (auto & sig : sigs) + sigsObj.push_back(sig); } return jsonObject; @@ -210,20 +204,25 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON( throw; } + // New format as this as nullable but mandatory field; handling + // missing is for back-compat. if (json.contains("ca")) - res.ca = ContentAddress::parse(getString(valueAt(json, "ca"))); + if (auto * rawCa = getNullable(valueAt(json, "ca"))) + res.ca = ContentAddress::parse(getString(*rawCa)); if (json.contains("deriver")) - res.deriver = store.parseStorePath(getString(valueAt(json, "deriver"))); + if (auto * rawDeriver = getNullable(valueAt(json, "deriver"))) + res.deriver = store.parseStorePath(getString(*rawDeriver)); if (json.contains("registrationTime")) - res.registrationTime = getInteger(valueAt(json, "registrationTime")); + if (auto * rawRegistrationTime = getNullable(valueAt(json, "registrationTime"))) + res.registrationTime = getInteger(*rawRegistrationTime); if (json.contains("ultimate")) res.ultimate = getBoolean(valueAt(json, "ultimate")); if (json.contains("signatures")) - res.sigs = valueAt(json, "signatures"); + res.sigs = getStringSet(valueAt(json, "signatures")); return res; } diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh index b6dc0855d..71f1476a6 100644 --- a/src/libstore/path-info.hh +++ b/src/libstore/path-info.hh @@ -32,17 +32,47 @@ struct SubstitutablePathInfo using SubstitutablePathInfos = std::map; +/** + * Information about a store object. + * + * See `store/store-object` and `protocols/json/store-object-info` in + * the Nix manual + */ struct UnkeyedValidPathInfo { + /** + * Path to derivation that produced this store object, if known. + */ std::optional deriver; + /** * \todo document this */ Hash narHash; + + /** + * Other store objects this store object referes to. + */ StorePathSet references; + + /** + * When this store object was registered in the store that contains + * it, if known. + */ time_t registrationTime = 0; - uint64_t narSize = 0; // 0 = unknown - uint64_t id = 0; // internal use only + + /** + * 0 = unknown + */ + uint64_t narSize = 0; + + /** + * internal use only: SQL primary key for on-disk store objects with + * `LocalStore`. + * + * @todo Remove, layer violation + */ + uint64_t id = 0; /** * Whether the path is ultimately trusted, that is, it's a @@ -75,7 +105,12 @@ struct UnkeyedValidPathInfo UnkeyedValidPathInfo(Hash narHash) : narHash(narHash) { }; - DECLARE_CMP(UnkeyedValidPathInfo); + bool operator == (const UnkeyedValidPathInfo &) const noexcept; + + /** + * @todo return `std::strong_ordering` once `id` is removed + */ + std::weak_ordering operator <=> (const UnkeyedValidPathInfo &) const noexcept; virtual ~UnkeyedValidPathInfo() { } @@ -95,7 +130,8 @@ struct UnkeyedValidPathInfo struct ValidPathInfo : UnkeyedValidPathInfo { StorePath path; - DECLARE_CMP(ValidPathInfo); + bool operator == (const ValidPathInfo &) const = default; + auto operator <=> (const ValidPathInfo &) const = default; /** * Return a fingerprint of the store path to be used in binary @@ -135,6 +171,9 @@ struct ValidPathInfo : UnkeyedValidPathInfo { */ bool checkSignature(const Store & store, const PublicKeys & publicKeys, const std::string & sig) const; + /** + * References as store path basenames, including a self reference if it has one. + */ Strings shortRefs() const; ValidPathInfo(const ValidPathInfo & other) = default; diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc index 026e37647..161d023d1 100644 --- a/src/libstore/path-with-outputs.cc +++ b/src/libstore/path-with-outputs.cc @@ -1,7 +1,9 @@ +#include + #include "path-with-outputs.hh" #include "store-api.hh" +#include "strings.hh" -#include namespace nix { diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 4b806e408..3e9d05477 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -2,25 +2,24 @@ namespace nix { -static void checkName(std::string_view path, std::string_view name) +void checkName(std::string_view name) { if (name.empty()) - throw BadStorePath("store path '%s' has an empty name", path); + throw BadStorePathName("name must not be empty"); if (name.size() > StorePath::MaxPathLen) - throw BadStorePath("store path '%s' has a name longer than %d characters", - path, StorePath::MaxPathLen); + throw BadStorePathName("name '%s' must be no longer than %d characters", name, StorePath::MaxPathLen); // See nameRegexStr for the definition if (name[0] == '.') { // check against "." and "..", followed by end or dash if (name.size() == 1) - throw BadStorePath("store path '%s' has invalid name '%s'", path, name); + throw BadStorePathName("name '%s' is not valid", name); if (name[1] == '-') - throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, "."); + throw BadStorePathName("name '%s' is not valid: first dash-separated component must not be '%s'", name, "."); if (name[1] == '.') { if (name.size() == 2) - throw BadStorePath("store path '%s' has invalid name '%s'", path, name); + throw BadStorePathName("name '%s' is not valid", name); if (name[2] == '-') - throw BadStorePath("store path '%s' has invalid name '%s': first dash-separated component must not be '%s'", path, name, ".."); + throw BadStorePathName("name '%s' is not valid: first dash-separated component must not be '%s'", name, ".."); } } for (auto c : name) @@ -28,7 +27,16 @@ static void checkName(std::string_view path, std::string_view name) || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '+' || c == '-' || c == '.' || c == '_' || c == '?' || c == '=')) - throw BadStorePath("store path '%s' contains illegal character '%s'", path, c); + throw BadStorePathName("name '%s' contains illegal character '%s'", name, c); +} + +static void checkPathName(std::string_view path, std::string_view name) +{ + try { + checkName(name); + } catch (BadStorePathName & e) { + throw BadStorePath("path '%s' is not a valid store path: %s", path, Uncolored(e.message())); + } } StorePath::StorePath(std::string_view _baseName) @@ -40,20 +48,26 @@ StorePath::StorePath(std::string_view _baseName) if (c == 'e' || c == 'o' || c == 'u' || c == 't' || !((c >= '0' && c <= '9') || (c >= 'a' && c <= 'z'))) throw BadStorePath("store path '%s' contains illegal base-32 character '%s'", baseName, c); - checkName(baseName, name()); + checkPathName(baseName, name()); } StorePath::StorePath(const Hash & hash, std::string_view _name) : baseName((hash.to_string(HashFormat::Nix32, false) + "-").append(std::string(_name))) { - checkName(baseName, name()); + checkPathName(baseName, name()); } -bool StorePath::isDerivation() const +bool StorePath::isDerivation() const noexcept { return hasSuffix(name(), drvExtension); } +void StorePath::requireDerivation() const +{ + if (!isDerivation()) + throw FormatError("store path '%s' is not a valid derivation path", to_string()); +} + StorePath StorePath::dummy("ffffffffffffffffffffffffffffffff-x"); StorePath StorePath::random(std::string_view name) diff --git a/src/libstore/path.hh b/src/libstore/path.hh index 3c26fc515..2380dc6a2 100644 --- a/src/libstore/path.hh +++ b/src/libstore/path.hh @@ -9,6 +9,13 @@ namespace nix { struct Hash; +/** + * Check whether a name is a valid store path name. + * + * @throws BadStorePathName if the name is invalid. The message is of the format "name %s is not valid, for this specific reason". + */ +void checkName(std::string_view name); + /** * \ref StorePath "Store path" is the fundamental reference type of Nix. * A store paths refers to a Store object. @@ -31,34 +38,29 @@ public: StorePath() = delete; + /** @throws BadStorePath */ StorePath(std::string_view baseName); + /** @throws BadStorePath */ StorePath(const Hash & hash, std::string_view name); - std::string_view to_string() const + std::string_view to_string() const noexcept { return baseName; } - bool operator < (const StorePath & other) const - { - return baseName < other.baseName; - } - - bool operator == (const StorePath & other) const - { - return baseName == other.baseName; - } - - bool operator != (const StorePath & other) const - { - return baseName != other.baseName; - } + bool operator == (const StorePath & other) const noexcept = default; + auto operator <=> (const StorePath & other) const noexcept = default; /** * Check whether a file name ends with the extension for derivations. */ - bool isDerivation() const; + bool isDerivation() const noexcept; + + /** + * Throw an exception if `isDerivation` is false. + */ + void requireDerivation() const; std::string_view name() const { @@ -82,7 +84,7 @@ typedef std::vector StorePaths; * The file extension of \ref Derivation derivations when serialized * into store objects. */ -const std::string drvExtension = ".drv"; +constexpr std::string_view drvExtension = ".drv"; } diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc index 37793db5b..c855e797f 100644 --- a/src/libstore/pathlocks.cc +++ b/src/libstore/pathlocks.cc @@ -27,7 +27,7 @@ PathLocks::~PathLocks() try { unlock(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc index d8bae13f5..46a78cc86 100644 --- a/src/libstore/posix-fs-canonicalise.cc +++ b/src/libstore/posix-fs-canonicalise.cc @@ -33,19 +33,9 @@ static void canonicaliseTimestampAndPermissions(const Path & path, const struct #ifndef _WIN32 // TODO implement if (st.st_mtime != mtimeStore) { - struct timeval times[2]; - times[0].tv_sec = st.st_atime; - times[0].tv_usec = 0; - times[1].tv_sec = mtimeStore; - times[1].tv_usec = 0; -#if HAVE_LUTIMES - if (lutimes(path.c_str(), times) == -1) - if (errno != ENOSYS || - (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1)) -#else - if (!S_ISLNK(st.st_mode) && utimes(path.c_str(), times) == -1) -#endif - throw SysError("changing modification time of '%1%'", path); + struct stat st2 = st; + st2.st_mtime = mtimeStore, + setWriteTime(path, st2); } #endif } @@ -144,13 +134,15 @@ static void canonicalisePathMetaData_( #endif if (S_ISDIR(st.st_mode)) { - for (auto & i : std::filesystem::directory_iterator{path}) + for (auto & i : std::filesystem::directory_iterator{path}) { + checkInterrupt(); canonicalisePathMetaData_( i.path().string(), #ifndef _WIN32 uidRange, #endif inodesSeen); + } } } diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index d0da96262..46efedfe3 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -1,4 +1,5 @@ #include "profiles.hh" +#include "signals.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "users.hh" @@ -38,6 +39,7 @@ std::pair> findGenerations(Path pro auto profileName = std::string(baseNameOf(profile)); for (auto & i : std::filesystem::directory_iterator{profileDir}) { + checkInterrupt(); if (auto n = parseName(profileName, i.path().filename().string())) { auto path = i.path().string(); gens.push_back({ diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 20f1d826c..7e360b5fe 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -30,7 +30,7 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std: /* FIXME: do this asynchronously. */ writeFile(makeCacheFile(hashPart, "nar"), nar); } catch (...) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } } @@ -42,7 +42,7 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std: nlohmann::json j = listNar(narAccessor, CanonPath::root, true); writeFile(makeCacheFile(hashPart, "ls"), j.dump()); } catch (...) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } } diff --git a/src/libstore/remote-store-connection.hh b/src/libstore/remote-store-connection.hh index 44328b06b..513bd6838 100644 --- a/src/libstore/remote-store-connection.hh +++ b/src/libstore/remote-store-connection.hh @@ -3,6 +3,7 @@ #include "remote-store.hh" #include "worker-protocol.hh" +#include "worker-protocol-connection.hh" #include "pool.hh" namespace nix { @@ -14,90 +15,13 @@ namespace nix { * Contains `Source` and `Sink` for actual communication, along with * other information learned when negotiating the connection. */ -struct RemoteStore::Connection +struct RemoteStore::Connection : WorkerProto::BasicClientConnection, + WorkerProto::ClientHandshakeInfo { - /** - * Send with this. - */ - FdSink to; - - /** - * Receive with this. - */ - FdSource from; - - /** - * Worker protocol version used for the connection. - * - * Despite its name, I think it is actually the maximum version both - * sides support. (If the maximum doesn't exist, we would fail to - * establish a connection and produce a value of this type.) - */ - WorkerProto::Version daemonVersion; - - /** - * Whether the remote side trusts us or not. - * - * 3 values: "yes", "no", or `std::nullopt` for "unknown". - * - * Note that the "remote side" might not be just the end daemon, but - * also an intermediary forwarder that can make its own trusting - * decisions. This would be the intersection of all their trust - * decisions, since it takes only one link in the chain to start - * denying operations. - */ - std::optional remoteTrustsUs; - - /** - * The version of the Nix daemon that is processing our requests. - * - * Do note, it may or may not communicating with another daemon, - * rather than being an "end" `LocalStore` or similar. - */ - std::optional daemonNixVersion; - /** * Time this connection was established. */ std::chrono::time_point startTime; - - /** - * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the - * factored out worker protocol searlizers with a - * `RemoteStore::Connection`. - * - * The worker protocol connection types are unidirectional, unlike - * this type. - */ - operator WorkerProto::ReadConn () - { - return WorkerProto::ReadConn { - .from = from, - .version = daemonVersion, - }; - } - - /** - * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the - * factored out worker protocol searlizers with a - * `RemoteStore::Connection`. - * - * The worker protocol connection types are unidirectional, unlike - * this type. - */ - operator WorkerProto::WriteConn () - { - return WorkerProto::WriteConn { - .to = to, - .version = daemonVersion, - }; - } - - virtual ~Connection(); - - virtual void closeWrite() = 0; - - std::exception_ptr processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); }; /** @@ -125,7 +49,7 @@ struct RemoteStore::ConnectionHandle RemoteStore::Connection & operator * () { return *handle; } RemoteStore::Connection * operator -> () { return &*handle; } - void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true); + void processStderr(Sink * sink = 0, Source * source = 0, bool flush = true, bool block = true); void withFramedSink(std::function fun); }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 09196481b..69bbc64fc 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -69,50 +69,32 @@ void RemoteStore::initConnection(Connection & conn) /* Send the magic greeting, check for the reply. */ try { conn.from.endOfFileError = "Nix daemon disconnected unexpectedly (maybe it crashed?)"; - conn.to << WORKER_MAGIC_1; - conn.to.flush(); + StringSink saved; + TeeSource tee(conn.from, saved); try { - TeeSource tee(conn.from, saved); - unsigned int magic = readInt(tee); - if (magic != WORKER_MAGIC_2) - throw Error("protocol mismatch"); + auto [protoVersion, features] = WorkerProto::BasicClientConnection::handshake( + conn.to, tee, PROTOCOL_VERSION, + WorkerProto::allFeatures); + conn.protoVersion = protoVersion; + conn.features = features; } catch (SerialisationError & e) { /* In case the other side is waiting for our input, close it. */ conn.closeWrite(); - auto msg = conn.from.drain(); - throw Error("protocol mismatch, got '%s'", chomp(saved.s + msg)); + { + NullSink nullSink; + tee.drainInto(nullSink); + } + throw Error("protocol mismatch, got '%s'", chomp(saved.s)); } - conn.from >> conn.daemonVersion; - if (GET_PROTOCOL_MAJOR(conn.daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) - throw Error("Nix daemon protocol version not supported"); - if (GET_PROTOCOL_MINOR(conn.daemonVersion) < 10) - throw Error("the Nix daemon version is too old"); - conn.to << PROTOCOL_VERSION; + static_cast(conn) = conn.postHandshake(*this); - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 14) { - // Obsolete CPU affinity. - conn.to << 0; - } + for (auto & feature : conn.features) + debug("negotiated feature '%s'", feature); - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 11) - conn.to << false; // obsolete reserveSpace - - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 33) { - conn.to.flush(); - conn.daemonNixVersion = readString(conn.from); - } - - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 35) { - conn.remoteTrustsUs = WorkerProto::Serialise>::read(*this, conn); - } else { - // We don't know the answer; protocol to old. - conn.remoteTrustsUs = std::nullopt; - } - - auto ex = conn.processStderr(); + auto ex = conn.processStderrReturn(); if (ex) std::rethrow_exception(ex); } catch (Error & e) { @@ -139,7 +121,7 @@ void RemoteStore::setOptions(Connection & conn) << settings.buildCores << settings.useSubstitutes; - if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) { + if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) { std::map overrides; settings.getSettings(overrides, true); // libstore settings fileTransferSettings.getSettings(overrides, true); @@ -152,13 +134,13 @@ void RemoteStore::setOptions(Connection & conn) overrides.erase(settings.useSubstitutes.name); overrides.erase(loggerSettings.showTrace.name); overrides.erase(experimentalFeatureSettings.experimentalFeatures.name); - overrides.erase(settings.pluginFiles.name); + overrides.erase("plugin-files"); conn.to << overrides.size(); for (auto & i : overrides) conn.to << i.first << i.second.value; } - auto ex = conn.processStderr(); + auto ex = conn.processStderrReturn(); if (ex) std::rethrow_exception(ex); } @@ -171,30 +153,9 @@ RemoteStore::ConnectionHandle::~ConnectionHandle() } } -void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, bool flush) +void RemoteStore::ConnectionHandle::processStderr(Sink * sink, Source * source, bool flush, bool block) { - auto ex = handle->processStderr(sink, source, flush); - if (ex) { - daemonException = true; - try { - std::rethrow_exception(ex); - } catch (const Error & e) { - // Nix versions before #4628 did not have an adequate behavior for reporting that the derivation format was upgraded. - // To avoid having to add compatibility logic in many places, we expect to catch almost all occurrences of the - // old incomprehensible error here, so that we can explain to users what's going on when their daemon is - // older than #4628 (2023). - if (experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations) && - GET_PROTOCOL_MINOR(handle->daemonVersion) <= 35) - { - auto m = e.msg(); - if (m.find("parsing derivation") != std::string::npos && - m.find("expected string") != std::string::npos && - m.find("Derive([") != std::string::npos) - throw Error("%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw derivation is in the form '%s'", std::move(m), "DrvWithVersion(..)"); - } - throw; - } - } + handle->processStderr(&daemonException, sink, source, flush, block); } @@ -220,19 +181,13 @@ bool RemoteStore::isValidPathUncached(const StorePath & path) StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { StorePathSet res; for (auto & i : paths) if (isValidPath(i)) res.insert(i); return res; } else { - conn->to << WorkerProto::Op::QueryValidPaths; - WorkerProto::write(*this, *conn, paths); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) { - conn->to << maybeSubstitute; - } - conn.processStderr(); - return WorkerProto::Serialise::read(*this, *conn); + return conn->queryValidPaths(*this, &conn.daemonException, paths, maybeSubstitute); } } @@ -249,7 +204,7 @@ StorePathSet RemoteStore::queryAllValidPaths() StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { StorePathSet res; for (auto & i : paths) { conn->to << WorkerProto::Op::HasSubstitutes << printStorePath(i); @@ -272,7 +227,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { for (auto & i : pathsMap) { SubstitutablePathInfo info; @@ -292,7 +247,7 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S } else { conn->to << WorkerProto::Op::QuerySubstitutablePathInfos; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 22) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 22) { StorePathSet paths; for (auto & path : pathsMap) paths.insert(path.first); @@ -322,22 +277,10 @@ void RemoteStore::queryPathInfoUncached(const StorePath & path, std::shared_ptr info; { auto conn(getConnection()); - conn->to << WorkerProto::Op::QueryPathInfo << printStorePath(path); - try { - conn.processStderr(); - } catch (Error & e) { - // Ugly backwards compatibility hack. - if (e.msg().find("is not valid") != std::string::npos) - throw InvalidPath(std::move(e.info())); - throw; - } - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 17) { - bool valid; conn->from >> valid; - if (!valid) throw InvalidPath("path '%s' is not valid", printStorePath(path)); - } info = std::make_shared( StorePath{path}, - WorkerProto::Serialise::read(*this, *conn)); + conn->queryPathInfo(*this, &conn.daemonException, path)); + } callback(std::move(info)); } catch (...) { callback.rethrow(); } @@ -431,7 +374,7 @@ ref RemoteStore::addCAToStore( std::optional conn_(getConnection()); auto & conn = *conn_; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 25) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 25) { conn->to << WorkerProto::Op::AddToStore @@ -455,8 +398,9 @@ ref RemoteStore::addCAToStore( else { if (repair) throw Error("repairing is not supported when building through the Nix daemon protocol < 1.25"); - std::visit(overloaded { - [&](const TextIngestionMethod & thm) -> void { + switch (caMethod.raw) { + case ContentAddressMethod::Raw::Text: + { if (hashAlgo != HashAlgorithm::SHA256) throw UnimplementedError("When adding text-hashed data called '%s', only SHA-256 is supported but '%s' was given", name, printHashAlgo(hashAlgo)); @@ -464,13 +408,19 @@ ref RemoteStore::addCAToStore( conn->to << WorkerProto::Op::AddTextToStore << name << s; WorkerProto::write(*this, *conn, references); conn.processStderr(); - }, - [&](const FileIngestionMethod & fim) -> void { + break; + } + case ContentAddressMethod::Raw::Flat: + case ContentAddressMethod::Raw::NixArchive: + case ContentAddressMethod::Raw::Git: + default: + { + auto fim = caMethod.getFileIngestionMethod(); conn->to << WorkerProto::Op::AddToStore << name - << ((hashAlgo == HashAlgorithm::SHA256 && fim == FileIngestionMethod::Recursive) ? 0 : 1) /* backwards compatibility hack */ - << (fim == FileIngestionMethod::Recursive ? 1 : 0) + << ((hashAlgo == HashAlgorithm::SHA256 && fim == FileIngestionMethod::NixArchive) ? 0 : 1) /* backwards compatibility hack */ + << (fim == FileIngestionMethod::NixArchive ? 1 : 0) << printHashAlgo(hashAlgo); try { @@ -478,7 +428,7 @@ ref RemoteStore::addCAToStore( connections->incCapacity(); { Finally cleanup([&]() { connections->decCapacity(); }); - if (fim == FileIngestionMethod::Recursive) { + if (fim == FileIngestionMethod::NixArchive) { dump.drainInto(conn->to); } else { std::string contents = dump.drain(); @@ -495,9 +445,9 @@ ref RemoteStore::addCAToStore( } catch (EndOfFile & e) { } throw; } - + break; } - }, caMethod.raw); + } auto path = parseStorePath(readString(conn->from)); // Release our connection to prevent a deadlock in queryPathInfo(). conn_.reset(); @@ -520,12 +470,12 @@ StorePath RemoteStore::addToStoreFromDump( case FileIngestionMethod::Flat: fsm = FileSerialisationMethod::Flat; break; - case FileIngestionMethod::Recursive: - fsm = FileSerialisationMethod::Recursive; + case FileIngestionMethod::NixArchive: + fsm = FileSerialisationMethod::NixArchive; break; case FileIngestionMethod::Git: // Use NAR; Git is not a serialization method - fsm = FileSerialisationMethod::Recursive; + fsm = FileSerialisationMethod::NixArchive; break; default: assert(false); @@ -541,9 +491,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 18) { - conn->to << WorkerProto::Op::ImportPaths; - + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 18) { auto source2 = sinkToSource([&](Sink & sink) { sink << 1 // == path follows ; @@ -558,11 +506,7 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << 0 // == no path follows ; }); - - conn.processStderr(0, source2.get()); - - auto importedPaths = WorkerProto::Serialise::read(*this, *conn); - assert(importedPaths.size() <= 1); + conn->importPaths(*this, &conn.daemonException, *source2); } else { @@ -575,11 +519,11 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, << info.ultimate << info.sigs << renderContentAddress(info.ca) << repair << !checkSigs; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 23) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 23) { conn.withFramedSink([&](Sink & sink) { copyNAR(source, sink); }); - } else if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 21) { + } else if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 21) { conn.processStderr(0, &source); } else { copyNAR(source, conn->to); @@ -616,7 +560,7 @@ void RemoteStore::addMultipleToStore( RepairFlag repair, CheckSigsFlag checkSigs) { - if (GET_PROTOCOL_MINOR(getConnection()->daemonVersion) >= 32) { + if (GET_PROTOCOL_MINOR(getConnection()->protoVersion) >= 32) { auto conn(getConnection()); conn->to << WorkerProto::Op::AddMultipleToStore @@ -634,7 +578,7 @@ void RemoteStore::registerDrvOutput(const Realisation & info) { auto conn(getConnection()); conn->to << WorkerProto::Op::RegisterDrvOutput; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) { conn->to << info.id.to_string(); conn->to << std::string(info.outPath.to_string()); } else { @@ -649,7 +593,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, try { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 27) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 27) { warn("the daemon is too old to support content-addressed derivations, please upgrade it to 2.4"); return callback(nullptr); } @@ -659,7 +603,7 @@ void RemoteStore::queryRealisationUncached(const DrvOutput & id, conn.processStderr(); auto real = [&]() -> std::shared_ptr { - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 31) { auto outPaths = WorkerProto::Serialise>::read( *this, *conn); if (outPaths.empty()) @@ -706,9 +650,9 @@ void RemoteStore::buildPaths(const std::vector & drvPaths, BuildMod auto conn(getConnection()); conn->to << WorkerProto::Op::BuildPaths; - assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13); + assert(GET_PROTOCOL_MINOR(conn->protoVersion) >= 13); WorkerProto::write(*this, *conn, drvPaths); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15) + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 15) conn->to << buildMode; else /* Old daemons did not take a 'buildMode' parameter, so we @@ -729,7 +673,7 @@ std::vector RemoteStore::buildPathsWithResults( std::optional conn_(getConnection()); auto & conn = *conn_; - if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 34) { + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 34) { conn->to << WorkerProto::Op::BuildPathsWithResults; WorkerProto::write(*this, *conn, paths); conn->to << buildMode; @@ -807,9 +751,7 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD BuildMode buildMode) { auto conn(getConnection()); - conn->to << WorkerProto::Op::BuildDerivation << printStorePath(drvPath); - writeDerivation(conn->to, *this, drv); - conn->to << buildMode; + conn->putBuildDerivationRequest(*this, &conn.daemonException, drvPath, drv, buildMode); conn.processStderr(); return WorkerProto::Serialise::read(*this, *conn); } @@ -827,9 +769,7 @@ void RemoteStore::ensurePath(const StorePath & path) void RemoteStore::addTempRoot(const StorePath & path) { auto conn(getConnection()); - conn->to << WorkerProto::Op::AddTempRoot << printStorePath(path); - conn.processStderr(); - readInt(conn->from); + conn->addTempRoot(*this, &conn.daemonException, path); } @@ -907,7 +847,7 @@ void RemoteStore::queryMissing(const std::vector & targets, { { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 19) + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 19) // Don't hold the connection handle in the fallback case // to prevent a deadlock. goto fallback; @@ -955,7 +895,7 @@ void RemoteStore::connect() unsigned int RemoteStore::getProtocol() { auto conn(connections->get()); - return conn->daemonVersion; + return conn->protoVersion; } std::optional RemoteStore::isTrustedClient() @@ -969,22 +909,12 @@ void RemoteStore::flushBadConnections() connections->flushBad(); } - -RemoteStore::Connection::~Connection() -{ - try { - to.flush(); - } catch (...) { - ignoreException(); - } -} - void RemoteStore::narFromPath(const StorePath & path, Sink & sink) { - auto conn(connections->get()); - conn->to << WorkerProto::Op::NarFromPath << printStorePath(path); - conn->processStderr(); - copyNAR(conn->from, sink); + auto conn(getConnection()); + conn->narFromPath(*this, &conn.daemonException, path, [&](Source & source) { + copyNAR(conn->from, sink); + }); } ref RemoteStore::getFSAccessor(bool requireValidPath) @@ -992,132 +922,21 @@ ref RemoteStore::getFSAccessor(bool requireValidPath) return make_ref(ref(shared_from_this())); } -static Logger::Fields readFields(Source & from) -{ - Logger::Fields fields; - size_t size = readInt(from); - for (size_t n = 0; n < size; n++) { - auto type = (decltype(Logger::Field::type)) readInt(from); - if (type == Logger::Field::tInt) - fields.push_back(readNum(from)); - else if (type == Logger::Field::tString) - fields.push_back(readString(from)); - else - throw Error("got unsupported field type %x from Nix daemon", (int) type); - } - return fields; -} - - -std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source * source, bool flush) -{ - if (flush) - to.flush(); - - while (true) { - - auto msg = readNum(from); - - if (msg == STDERR_WRITE) { - auto s = readString(from); - if (!sink) throw Error("no sink"); - (*sink)(s); - } - - else if (msg == STDERR_READ) { - if (!source) throw Error("no source"); - size_t len = readNum(from); - auto buf = std::make_unique(len); - writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to); - to.flush(); - } - - else if (msg == STDERR_ERROR) { - if (GET_PROTOCOL_MINOR(daemonVersion) >= 26) { - return std::make_exception_ptr(readError(from)); - } else { - auto error = readString(from); - unsigned int status = readInt(from); - return std::make_exception_ptr(Error(status, error)); - } - } - - else if (msg == STDERR_NEXT) - printError(chomp(readString(from))); - - else if (msg == STDERR_START_ACTIVITY) { - auto act = readNum(from); - auto lvl = (Verbosity) readInt(from); - auto type = (ActivityType) readInt(from); - auto s = readString(from); - auto fields = readFields(from); - auto parent = readNum(from); - logger->startActivity(act, lvl, type, s, fields, parent); - } - - else if (msg == STDERR_STOP_ACTIVITY) { - auto act = readNum(from); - logger->stopActivity(act); - } - - else if (msg == STDERR_RESULT) { - auto act = readNum(from); - auto type = (ResultType) readInt(from); - auto fields = readFields(from); - logger->result(act, type, fields); - } - - else if (msg == STDERR_LAST) - break; - - else - throw Error("got unknown message type %x from Nix daemon", msg); - } - - return nullptr; -} - void RemoteStore::ConnectionHandle::withFramedSink(std::function fun) { (*this)->to.flush(); - std::exception_ptr ex; - - /* Handle log messages / exceptions from the remote on a separate - thread. */ - std::thread stderrThread([&]() { - try { - ReceiveInterrupts receiveInterrupts; - processStderr(nullptr, nullptr, false); - } catch (...) { - ex = std::current_exception(); - } - }); - - Finally joinStderrThread([&]() - { - if (stderrThread.joinable()) { - stderrThread.join(); - if (ex) { - try { - std::rethrow_exception(ex); - } catch (...) { - ignoreException(); - } - } - } - }); - - { - FramedSink sink((*this)->to, ex); + FramedSink sink((*this)->to, [&]() { + /* Periodically process stderr messages and exceptions + from the daemon. */ + processStderr(nullptr, nullptr, false, false); + }); fun(sink); sink.flush(); } - stderrThread.join(); - if (ex) - std::rethrow_exception(ex); + processStderr(nullptr, nullptr, false); } } diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index d630adc08..4e1896268 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -87,8 +87,8 @@ public: StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, - ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override; diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 1a62d92d4..bcbf0b55e 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -1,5 +1,7 @@ #if ENABLE_S3 +#include + #include "s3.hh" #include "s3-binary-cache-store.hh" #include "nar-info.hh" @@ -7,6 +9,7 @@ #include "globals.hh" #include "compression.hh" #include "filetransfer.hh" +#include "signals.hh" #include #include @@ -58,7 +61,7 @@ class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem debug("AWS: %s", chomp(statement)); } -#if !(AWS_VERSION_MAJOR <= 1 && AWS_VERSION_MINOR <= 7 && AWS_VERSION_PATCH <= 115) +#if !(AWS_SDK_VERSION_MAJOR <= 1 && AWS_SDK_VERSION_MINOR <= 7 && AWS_SDK_VERSION_PATCH <= 115) void Flush() override {} #endif }; @@ -101,7 +104,7 @@ S3Helper::S3Helper( std::make_shared(profile.c_str())), *config, // FIXME: https://github.com/aws/aws-sdk-cpp/issues/759 -#if AWS_VERSION_MAJOR == 1 && AWS_VERSION_MINOR < 3 +#if AWS_SDK_VERSION_MAJOR == 1 && AWS_SDK_VERSION_MINOR < 3 false, #else Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, @@ -115,6 +118,7 @@ class RetryStrategy : public Aws::Client::DefaultRetryStrategy { bool ShouldRetry(const Aws::Client::AWSError& error, long attemptedRetries) const override { + checkInterrupt(); auto retry = Aws::Client::DefaultRetryStrategy::ShouldRetry(error, attemptedRetries); if (retry) printError("AWS error '%s' (%s), will retry in %d ms", @@ -132,6 +136,7 @@ ref S3Helper::makeConfig( { initAWS(); auto res = make_ref(); + res->allowSystemProxy = true; res->region = region; if (!scheme.empty()) { res->scheme = Aws::Http::SchemeMapper::FromString(scheme.c_str()); @@ -189,96 +194,48 @@ S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) , BinaryCacheStore(params) { } -struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig + +S3BinaryCacheStoreConfig::S3BinaryCacheStoreConfig( + std::string_view uriScheme, + std::string_view bucketName, + const Params & params) + : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , bucketName(bucketName) { - using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + // Don't want to use use AWS SDK in header, so we check the default + // here. TODO do this better after we overhaul the store settings + // system. + assert(std::string{defaultRegion} == std::string{Aws::Region::US_EAST_1}); - const Setting profile{this, "", "profile", - R"( - The name of the AWS configuration profile to use. By default - Nix will use the `default` profile. - )"}; + if (bucketName.empty()) + throw UsageError("`%s` store requires a bucket name in its Store URI", uriScheme); +} - const Setting region{this, Aws::Region::US_EAST_1, "region", - R"( - The region of the S3 bucket. If your bucket is not in - `us–east-1`, you should always explicitly specify the region - parameter. - )"}; +std::string S3BinaryCacheStoreConfig::doc() +{ + return + #include "s3-binary-cache-store.md" + ; +} - const Setting scheme{this, "", "scheme", - R"( - The scheme used for S3 requests, `https` (default) or `http`. This - option allows you to disable HTTPS for binary caches which don't - support it. - - > **Note** - > - > HTTPS should be used if the cache might contain sensitive - > information. - )"}; - - const Setting endpoint{this, "", "endpoint", - R"( - The URL of the endpoint of an S3-compatible service such as MinIO. - Do not specify this setting if you're using Amazon S3. - - > **Note** - > - > This endpoint must support HTTPS and will use path-based - > addressing instead of virtual host based addressing. - )"}; - - const Setting narinfoCompression{this, "", "narinfo-compression", - "Compression method for `.narinfo` files."}; - - const Setting lsCompression{this, "", "ls-compression", - "Compression method for `.ls` files."}; - - const Setting logCompression{this, "", "log-compression", - R"( - Compression method for `log/*` files. It is recommended to - use a compression method supported by most web browsers - (e.g. `brotli`). - )"}; - - const Setting multipartUpload{ - this, false, "multipart-upload", - "Whether to use multi-part uploads."}; - - const Setting bufferSize{ - this, 5 * 1024 * 1024, "buffer-size", - "Size (in bytes) of each part in multi-part uploads."}; - - const std::string name() override { return "S3 Binary Cache Store"; } - - std::string doc() override - { - return - #include "s3-binary-cache-store.md" - ; - } -}; struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore { - std::string bucketName; - Stats stats; S3Helper s3Helper; S3BinaryCacheStoreImpl( - const std::string & uriScheme, - const std::string & bucketName, + std::string_view uriScheme, + std::string_view bucketName, const Params & params) : StoreConfig(params) , BinaryCacheStoreConfig(params) - , S3BinaryCacheStoreConfig(params) + , S3BinaryCacheStoreConfig(uriScheme, bucketName, params) , Store(params) , BinaryCacheStore(params) , S3BinaryCacheStore(params) - , bucketName(bucketName) , s3Helper(profile, region, scheme, endpoint) { diskCache = getNarInfoDiskCache(); @@ -518,9 +475,6 @@ struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual { return std::nullopt; } - - static std::set uriSchemes() { return {"s3"}; } - }; static RegisterStoreImplementation regS3BinaryCacheStore; diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index c62ea5147..7d303a115 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -7,6 +7,101 @@ namespace nix { +struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig +{ + std::string bucketName; + + using BinaryCacheStoreConfig::BinaryCacheStoreConfig; + + S3BinaryCacheStoreConfig(std::string_view uriScheme, std::string_view bucketName, const Params & params); + + const Setting profile{ + this, + "", + "profile", + R"( + The name of the AWS configuration profile to use. By default + Nix will use the `default` profile. + )"}; + +protected: + + constexpr static const char * defaultRegion = "us-east-1"; + +public: + + const Setting region{ + this, + defaultRegion, + "region", + R"( + The region of the S3 bucket. If your bucket is not in + `us–east-1`, you should always explicitly specify the region + parameter. + )"}; + + const Setting scheme{ + this, + "", + "scheme", + R"( + The scheme used for S3 requests, `https` (default) or `http`. This + option allows you to disable HTTPS for binary caches which don't + support it. + + > **Note** + > + > HTTPS should be used if the cache might contain sensitive + > information. + )"}; + + const Setting endpoint{ + this, + "", + "endpoint", + R"( + The URL of the endpoint of an S3-compatible service such as MinIO. + Do not specify this setting if you're using Amazon S3. + + > **Note** + > + > This endpoint must support HTTPS and will use path-based + > addressing instead of virtual host based addressing. + )"}; + + const Setting narinfoCompression{ + this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; + + const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; + + const Setting logCompression{ + this, + "", + "log-compression", + R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )"}; + + const Setting multipartUpload{this, false, "multipart-upload", "Whether to use multi-part uploads."}; + + const Setting bufferSize{ + this, 5 * 1024 * 1024, "buffer-size", "Size (in bytes) of each part in multi-part uploads."}; + + const std::string name() override + { + return "S3 Binary Cache Store"; + } + + static std::set uriSchemes() + { + return {"s3"}; + } + + std::string doc() override; +}; + class S3BinaryCacheStore : public virtual BinaryCacheStore { protected: diff --git a/src/libstore/s3-binary-cache-store.md b/src/libstore/s3-binary-cache-store.md index 675470261..daa41defd 100644 --- a/src/libstore/s3-binary-cache-store.md +++ b/src/libstore/s3-binary-cache-store.md @@ -3,7 +3,7 @@ R"( **Store URL format**: `s3://`*bucket-name* This store allows reading and writing a binary cache stored in an AWS S3 (or S3-compatible service) bucket. -This store shares many idioms with the [HTTP Binary Cache Store](#http-binary-cache-store). +This store shares many idioms with the [HTTP Binary Cache Store](@docroot@/store/types/http-binary-cache-store.md). For AWS S3, the binary cache URL for a bucket named `example-nix-cache` will be exactly . For S3 compatible binary caches, consult that cache's documentation. diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh index f0aeb3bed..18de115ae 100644 --- a/src/libstore/s3.hh +++ b/src/libstore/s3.hh @@ -8,7 +8,7 @@ #include #include -namespace Aws { namespace Client { class ClientConfiguration; } } +namespace Aws { namespace Client { struct ClientConfiguration; } } namespace Aws { namespace S3 { class S3Client; } } namespace nix { diff --git a/src/libstore/serve-protocol-connection.cc b/src/libstore/serve-protocol-connection.cc new file mode 100644 index 000000000..07379999b --- /dev/null +++ b/src/libstore/serve-protocol-connection.cc @@ -0,0 +1,106 @@ +#include "serve-protocol-connection.hh" +#include "serve-protocol-impl.hh" +#include "build-result.hh" +#include "derivations.hh" + +namespace nix { + +ServeProto::Version ServeProto::BasicClientConnection::handshake( + BufferedSink & to, Source & from, ServeProto::Version localVersion, std::string_view host) +{ + to << SERVE_MAGIC_1 << localVersion; + to.flush(); + + unsigned int magic = readInt(from); + if (magic != SERVE_MAGIC_2) + throw Error("'nix-store --serve' protocol mismatch from '%s'", host); + auto remoteVersion = readInt(from); + if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200) + throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); + return std::min(remoteVersion, localVersion); +} + +ServeProto::Version +ServeProto::BasicServerConnection::handshake(BufferedSink & to, Source & from, ServeProto::Version localVersion) +{ + unsigned int magic = readInt(from); + if (magic != SERVE_MAGIC_1) + throw Error("protocol mismatch"); + to << SERVE_MAGIC_2 << localVersion; + to.flush(); + auto remoteVersion = readInt(from); + return std::min(remoteVersion, localVersion); +} + +StorePathSet ServeProto::BasicClientConnection::queryValidPaths( + const StoreDirConfig & store, bool lock, const StorePathSet & paths, SubstituteFlag maybeSubstitute) +{ + to << ServeProto::Command::QueryValidPaths << lock << maybeSubstitute; + write(store, *this, paths); + to.flush(); + + return Serialise::read(store, *this); +} + +std::map +ServeProto::BasicClientConnection::queryPathInfos(const StoreDirConfig & store, const StorePathSet & paths) +{ + std::map infos; + + to << ServeProto::Command::QueryPathInfos; + ServeProto::write(store, *this, paths); + to.flush(); + + while (true) { + auto storePathS = readString(from); + if (storePathS == "") + break; + + auto storePath = store.parseStorePath(storePathS); + assert(paths.count(storePath) == 1); + auto info = ServeProto::Serialise::read(store, *this); + infos.insert_or_assign(std::move(storePath), std::move(info)); + } + + return infos; +} + +void ServeProto::BasicClientConnection::putBuildDerivationRequest( + const StoreDirConfig & store, + const StorePath & drvPath, + const BasicDerivation & drv, + const ServeProto::BuildOptions & options) +{ + to << ServeProto::Command::BuildDerivation << store.printStorePath(drvPath); + writeDerivation(to, store, drv); + + ServeProto::write(store, *this, options); + + to.flush(); +} + +BuildResult ServeProto::BasicClientConnection::getBuildDerivationResponse(const StoreDirConfig & store) +{ + return ServeProto::Serialise::read(store, *this); +} + +void ServeProto::BasicClientConnection::narFromPath( + const StoreDirConfig & store, const StorePath & path, std::function fun) +{ + to << ServeProto::Command::DumpStorePath << store.printStorePath(path); + to.flush(); + + fun(from); +} + +void ServeProto::BasicClientConnection::importPaths(const StoreDirConfig & store, std::function fun) +{ + to << ServeProto::Command::ImportPaths; + fun(to); + to.flush(); + + if (readInt(from) != 1) + throw Error("remote machine failed to import closure"); +} + +} diff --git a/src/libstore/serve-protocol-connection.hh b/src/libstore/serve-protocol-connection.hh new file mode 100644 index 000000000..73bf71443 --- /dev/null +++ b/src/libstore/serve-protocol-connection.hh @@ -0,0 +1,108 @@ +#pragma once +///@file + +#include "serve-protocol.hh" +#include "store-api.hh" + +namespace nix { + +struct ServeProto::BasicClientConnection +{ + FdSink to; + FdSource from; + ServeProto::Version remoteVersion; + + /** + * Establishes connection, negotiating version. + * + * @return the version provided by the other side of the + * connection. + * + * @param to Taken by reference to allow for various error handling + * mechanisms. + * + * @param from Taken by reference to allow for various error + * handling mechanisms. + * + * @param localVersion Our version which is sent over + * + * @param host Just used to add context to thrown exceptions. + */ + static ServeProto::Version + handshake(BufferedSink & to, Source & from, ServeProto::Version localVersion, std::string_view host); + + /** + * Coercion to `ServeProto::ReadConn`. This makes it easy to use the + * factored out serve protocol serializers with a + * `LegacySSHStore::Connection`. + * + * The serve protocol connection types are unidirectional, unlike + * this type. + */ + operator ServeProto::ReadConn() + { + return ServeProto::ReadConn{ + .from = from, + .version = remoteVersion, + }; + } + + /** + * Coercion to `ServeProto::WriteConn`. This makes it easy to use the + * factored out serve protocol serializers with a + * `LegacySSHStore::Connection`. + * + * The serve protocol connection types are unidirectional, unlike + * this type. + */ + operator ServeProto::WriteConn() + { + return ServeProto::WriteConn{ + .to = to, + .version = remoteVersion, + }; + } + + StorePathSet queryValidPaths( + const StoreDirConfig & remoteStore, bool lock, const StorePathSet & paths, SubstituteFlag maybeSubstitute); + + std::map queryPathInfos(const StoreDirConfig & store, const StorePathSet & paths); + ; + + void putBuildDerivationRequest( + const StoreDirConfig & store, + const StorePath & drvPath, + const BasicDerivation & drv, + const ServeProto::BuildOptions & options); + + /** + * Get the response, must be paired with + * `putBuildDerivationRequest`. + */ + BuildResult getBuildDerivationResponse(const StoreDirConfig & store); + + void narFromPath(const StoreDirConfig & store, const StorePath & path, std::function fun); + + void importPaths(const StoreDirConfig & store, std::function fun); +}; + +struct ServeProto::BasicServerConnection +{ + /** + * Establishes connection, negotiating version. + * + * @return the version provided by the other side of the + * connection. + * + * @param to Taken by reference to allow for various error handling + * mechanisms. + * + * @param from Taken by reference to allow for various error + * handling mechanisms. + * + * @param localVersion Our version which is sent over + */ + static ServeProto::Version handshake(BufferedSink & to, Source & from, ServeProto::Version localVersion); +}; + +} diff --git a/src/libstore/serve-protocol-impl.cc b/src/libstore/serve-protocol-impl.cc deleted file mode 100644 index b39212884..000000000 --- a/src/libstore/serve-protocol-impl.cc +++ /dev/null @@ -1,69 +0,0 @@ -#include "serve-protocol-impl.hh" -#include "build-result.hh" -#include "derivations.hh" - -namespace nix { - -ServeProto::Version ServeProto::BasicClientConnection::handshake( - BufferedSink & to, - Source & from, - ServeProto::Version localVersion, - std::string_view host) -{ - to << SERVE_MAGIC_1 << localVersion; - to.flush(); - - unsigned int magic = readInt(from); - if (magic != SERVE_MAGIC_2) - throw Error("'nix-store --serve' protocol mismatch from '%s'", host); - auto remoteVersion = readInt(from); - if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200) - throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); - return remoteVersion; -} - -ServeProto::Version ServeProto::BasicServerConnection::handshake( - BufferedSink & to, - Source & from, - ServeProto::Version localVersion) -{ - unsigned int magic = readInt(from); - if (magic != SERVE_MAGIC_1) throw Error("protocol mismatch"); - to << SERVE_MAGIC_2 << localVersion; - to.flush(); - return readInt(from); -} - - -StorePathSet ServeProto::BasicClientConnection::queryValidPaths( - const Store & store, - bool lock, const StorePathSet & paths, - SubstituteFlag maybeSubstitute) -{ - to - << ServeProto::Command::QueryValidPaths - << lock - << maybeSubstitute; - write(store, *this, paths); - to.flush(); - - return Serialise::read(store, *this); -} - - -void ServeProto::BasicClientConnection::putBuildDerivationRequest( - const Store & store, - const StorePath & drvPath, const BasicDerivation & drv, - const ServeProto::BuildOptions & options) -{ - to - << ServeProto::Command::BuildDerivation - << store.printStorePath(drvPath); - writeDerivation(to, store, drv); - - ServeProto::write(store, *this, options); - - to.flush(); -} - -} diff --git a/src/libstore/serve-protocol-impl.hh b/src/libstore/serve-protocol-impl.hh index fd8d94697..6f3b177ac 100644 --- a/src/libstore/serve-protocol-impl.hh +++ b/src/libstore/serve-protocol-impl.hh @@ -10,7 +10,6 @@ #include "serve-protocol.hh" #include "length-prefixed-protocol-helper.hh" -#include "store-api.hh" namespace nix { @@ -57,101 +56,4 @@ struct ServeProto::Serialise /* protocol-specific templates */ -struct ServeProto::BasicClientConnection -{ - FdSink to; - FdSource from; - ServeProto::Version remoteVersion; - - /** - * Establishes connection, negotiating version. - * - * @return the version provided by the other side of the - * connection. - * - * @param to Taken by reference to allow for various error handling - * mechanisms. - * - * @param from Taken by reference to allow for various error - * handling mechanisms. - * - * @param localVersion Our version which is sent over - * - * @param host Just used to add context to thrown exceptions. - */ - static ServeProto::Version handshake( - BufferedSink & to, - Source & from, - ServeProto::Version localVersion, - std::string_view host); - - /** - * Coercion to `ServeProto::ReadConn`. This makes it easy to use the - * factored out serve protocol serializers with a - * `LegacySSHStore::Connection`. - * - * The serve protocol connection types are unidirectional, unlike - * this type. - */ - operator ServeProto::ReadConn () - { - return ServeProto::ReadConn { - .from = from, - .version = remoteVersion, - }; - } - - /** - * Coercion to `ServeProto::WriteConn`. This makes it easy to use the - * factored out serve protocol serializers with a - * `LegacySSHStore::Connection`. - * - * The serve protocol connection types are unidirectional, unlike - * this type. - */ - operator ServeProto::WriteConn () - { - return ServeProto::WriteConn { - .to = to, - .version = remoteVersion, - }; - } - - StorePathSet queryValidPaths( - const Store & remoteStore, - bool lock, const StorePathSet & paths, - SubstituteFlag maybeSubstitute); - - /** - * Just the request half, because Hydra may do other things between - * issuing the request and reading the `BuildResult` response. - */ - void putBuildDerivationRequest( - const Store & store, - const StorePath & drvPath, const BasicDerivation & drv, - const ServeProto::BuildOptions & options); -}; - -struct ServeProto::BasicServerConnection -{ - /** - * Establishes connection, negotiating version. - * - * @return the version provided by the other side of the - * connection. - * - * @param to Taken by reference to allow for various error handling - * mechanisms. - * - * @param from Taken by reference to allow for various error - * handling mechanisms. - * - * @param localVersion Our version which is sent over - */ - static ServeProto::Version handshake( - BufferedSink & to, - Source & from, - ServeProto::Version localVersion); -}; - } diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 3175c1978..f02e472fd 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -86,7 +86,7 @@ SQLite::~SQLite() if (db && sqlite3_close(db) != SQLITE_OK) SQLiteError::throw_(db, "closing database"); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -125,7 +125,7 @@ SQLiteStmt::~SQLiteStmt() if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) SQLiteError::throw_(db, "finalizing statement '%s'", sql); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -240,7 +240,7 @@ SQLiteTxn::~SQLiteTxn() if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) SQLiteError::throw_(db, "aborting transaction"); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libstore/ssh-store-config.hh b/src/libstore/ssh-store-config.hh deleted file mode 100644 index 4ce4ffc4c..000000000 --- a/src/libstore/ssh-store-config.hh +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once -///@file - -#include "store-api.hh" - -namespace nix { - -struct CommonSSHStoreConfig : virtual StoreConfig -{ - using StoreConfig::StoreConfig; - - const Setting sshKey{this, "", "ssh-key", - "Path to the SSH private key used to authenticate to the remote machine."}; - - const Setting sshPublicHostKey{this, "", "base64-ssh-public-host-key", - "The public host key of the remote machine."}; - - const Setting compress{this, false, "compress", - "Whether to enable SSH compression."}; - - const Setting remoteStore{this, "", "remote-store", - R"( - [Store URL](@docroot@/store/types/index.md#store-url-format) - to be used on the remote machine. The default is `auto` - (i.e. use the Nix daemon or `/nix/store` directly). - )"}; -}; - -} diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 220d5d31b..954a97467 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -1,7 +1,5 @@ -#include "ssh-store-config.hh" -#include "store-api.hh" +#include "ssh-store.hh" #include "local-fs-store.hh" -#include "remote-store.hh" #include "remote-store-connection.hh" #include "source-accessor.hh" #include "archive.hh" @@ -12,48 +10,43 @@ namespace nix { -struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig +SSHStoreConfig::SSHStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(scheme, authority, params) { - using RemoteStoreConfig::RemoteStoreConfig; - using CommonSSHStoreConfig::CommonSSHStoreConfig; +} - const Setting remoteProgram{this, {"nix-daemon"}, "remote-program", - "Path to the `nix-daemon` executable on the remote machine."}; - - const std::string name() override { return "Experimental SSH Store"; } - - std::string doc() override - { - return - #include "ssh-store.md" - ; - } -}; +std::string SSHStoreConfig::doc() +{ + return + #include "ssh-store.md" + ; +} class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore { public: - SSHStore(const std::string & scheme, const std::string & host, const Params & params) + SSHStore( + std::string_view scheme, + std::string_view host, + const Params & params) : StoreConfig(params) , RemoteStoreConfig(params) - , CommonSSHStoreConfig(params) - , SSHStoreConfig(params) + , CommonSSHStoreConfig(scheme, host, params) + , SSHStoreConfig(scheme, host, params) , Store(params) , RemoteStore(params) - , host(host) - , master( - host, - sshKey, - sshPublicHostKey, + , master(createSSHMaster( // Use SSH master only if using more than 1 connection. - connections->capacity() > 1, - compress) + connections->capacity() > 1)) { } - static std::set uriSchemes() { return {"ssh-ng"}; } - std::string getUri() override { return *uriSchemes().begin() + "://" + host; @@ -94,34 +87,32 @@ protected: }; }; -struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig + +MountedSSHStoreConfig::MountedSSHStoreConfig(StringMap params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) { - using SSHStoreConfig::SSHStoreConfig; - using LocalFSStoreConfig::LocalFSStoreConfig; +} - MountedSSHStoreConfig(StringMap params) - : StoreConfig(params) - , RemoteStoreConfig(params) - , CommonSSHStoreConfig(params) - , SSHStoreConfig(params) - , LocalFSStoreConfig(params) - { - } +MountedSSHStoreConfig::MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params) + : StoreConfig(params) + , RemoteStoreConfig(params) + , CommonSSHStoreConfig(scheme, host, params) + , SSHStoreConfig(params) + , LocalFSStoreConfig(params) +{ +} - const std::string name() override { return "Experimental SSH Store with filesystem mounted"; } +std::string MountedSSHStoreConfig::doc() +{ + return + #include "mounted-ssh-store.md" + ; +} - std::string doc() override - { - return - #include "mounted-ssh-store.md" - ; - } - - std::optional experimentalFeature() const override - { - return ExperimentalFeature::MountedSSHStore; - } -}; /** * The mounted ssh store assumes that filesystems on the remote host are @@ -141,10 +132,13 @@ class MountedSSHStore : public virtual MountedSSHStoreConfig, public virtual SSH { public: - MountedSSHStore(const std::string & scheme, const std::string & host, const Params & params) + MountedSSHStore( + std::string_view scheme, + std::string_view host, + const Params & params) : StoreConfig(params) , RemoteStoreConfig(params) - , CommonSSHStoreConfig(params) + , CommonSSHStoreConfig(scheme, host, params) , SSHStoreConfig(params) , LocalFSStoreConfig(params) , MountedSSHStoreConfig(params) @@ -158,11 +152,6 @@ public: }; } - static std::set uriSchemes() - { - return {"mounted-ssh-ng"}; - } - std::string getUri() override { return *uriSchemes().begin() + "://" + host; diff --git a/src/libstore/ssh-store.hh b/src/libstore/ssh-store.hh new file mode 100644 index 000000000..29a2a8b2c --- /dev/null +++ b/src/libstore/ssh-store.hh @@ -0,0 +1,61 @@ +#pragma once +///@file + +#include "common-ssh-store-config.hh" +#include "store-api.hh" +#include "local-fs-store.hh" +#include "remote-store.hh" + +namespace nix { + +struct SSHStoreConfig : virtual RemoteStoreConfig, virtual CommonSSHStoreConfig +{ + using CommonSSHStoreConfig::CommonSSHStoreConfig; + using RemoteStoreConfig::RemoteStoreConfig; + + SSHStoreConfig(std::string_view scheme, std::string_view authority, const Params & params); + + const Setting remoteProgram{ + this, {"nix-daemon"}, "remote-program", "Path to the `nix-daemon` executable on the remote machine."}; + + const std::string name() override + { + return "Experimental SSH Store"; + } + + static std::set uriSchemes() + { + return {"ssh-ng"}; + } + + std::string doc() override; +}; + +struct MountedSSHStoreConfig : virtual SSHStoreConfig, virtual LocalFSStoreConfig +{ + using LocalFSStoreConfig::LocalFSStoreConfig; + using SSHStoreConfig::SSHStoreConfig; + + MountedSSHStoreConfig(StringMap params); + + MountedSSHStoreConfig(std::string_view scheme, std::string_view host, StringMap params); + + const std::string name() override + { + return "Experimental SSH Store with filesystem mounted"; + } + + static std::set uriSchemes() + { + return {"mounted-ssh-ng"}; + } + + std::string doc() override; + + std::optional experimentalFeature() const override + { + return ExperimentalFeature::MountedSSHStore; + } +}; + +} diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 7e730299a..dec733fd5 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -3,14 +3,29 @@ #include "current-process.hh" #include "environment-variables.hh" #include "util.hh" +#include "exec.hh" namespace nix { -SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD) +static std::string parsePublicHostKey(std::string_view host, std::string_view sshPublicHostKey) +{ + try { + return base64Decode(sshPublicHostKey); + } catch (Error & e) { + e.addTrace({}, "while decoding ssh public host key for host '%s'", host); + throw; + } +} + +SSHMaster::SSHMaster( + std::string_view host, + std::string_view keyFile, + std::string_view sshPublicHostKey, + bool useMaster, bool compress, Descriptor logFD) : host(host) , fakeSSH(host == "localhost") , keyFile(keyFile) - , sshPublicHostKey(sshPublicHostKey) + , sshPublicHostKey(parsePublicHostKey(host, sshPublicHostKey)) , useMaster(useMaster && !fakeSSH) , compress(compress) , logFD(logFD) @@ -34,12 +49,16 @@ void SSHMaster::addCommonSSHOpts(Strings & args) std::filesystem::path fileName = state->tmpDir->path() / "host-key"; auto p = host.rfind("@"); std::string thost = p != std::string::npos ? std::string(host, p + 1) : host; - writeFile(fileName.string(), thost + " " + base64Decode(sshPublicHostKey) + "\n"); + writeFile(fileName.string(), thost + " " + sshPublicHostKey + "\n"); args.insert(args.end(), {"-oUserKnownHostsFile=" + fileName.string()}); } if (compress) args.push_back("-C"); + // We use this to make ssh signal back to us that the connection is established. + // It really does run locally; see createSSHEnv which sets up SHELL to make + // it launch more reliably. The local command runs synchronously, so presumably + // the remote session won't be garbled if the local command is slow. args.push_back("-oPermitLocalCommand=yes"); args.push_back("-oLocalCommand=echo started"); } @@ -52,6 +71,27 @@ bool SSHMaster::isMasterRunning() { return res.first == 0; } +Strings createSSHEnv() +{ + // Copy the environment and set SHELL=/bin/sh + std::map env = getEnv(); + + // SSH will invoke the "user" shell for -oLocalCommand, but that means + // $SHELL. To keep things simple and avoid potential issues with other + // shells, we set it to /bin/sh. + // Technically, we don't need that, and we could reinvoke ourselves to print + // "started". Self-reinvocation is tricky with library consumers, but mostly + // solved; refer to the development history of nixExePath in libstore/globals.cc. + env.insert_or_assign("SHELL", "/bin/sh"); + + Strings r; + for (auto & [k, v] : env) { + r.push_back(k + "=" + v); + } + + return r; +} + std::unique_ptr SSHMaster::startCommand( Strings && command, Strings && extraSshArgs) { @@ -100,8 +140,8 @@ std::unique_ptr SSHMaster::startCommand( } args.splice(args.end(), std::move(command)); - - execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); + auto env = createSSHEnv(); + nix::execvpe(args.begin()->c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(env).data()); // could not exec ssh/bash throw SysError("unable to execute '%s'", args.front()); @@ -168,7 +208,8 @@ Path SSHMaster::startMaster() if (verbosity >= lvlChatty) args.push_back("-v"); addCommonSSHOpts(args); - execvp(args.begin()->c_str(), stringsToCharPtrs(args).data()); + auto env = createSSHEnv(); + nix::execvpe(args.begin()->c_str(), stringsToCharPtrs(args).data(), stringsToCharPtrs(env).data()); throw SysError("unable to execute '%s'", args.front()); }, options); diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh index 3b1a0827a..4097134d0 100644 --- a/src/libstore/ssh.hh +++ b/src/libstore/ssh.hh @@ -14,10 +14,13 @@ private: const std::string host; bool fakeSSH; const std::string keyFile; + /** + * Raw bytes, not Base64 encoding. + */ const std::string sshPublicHostKey; const bool useMaster; const bool compress; - const int logFD; + const Descriptor logFD; struct State { @@ -39,7 +42,11 @@ private: public: - SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD = -1); + SSHMaster( + std::string_view host, + std::string_view keyFile, + std::string_view sshPublicHostKey, + bool useMaster, bool compress, Descriptor logFD = INVALID_DESCRIPTOR); struct Connection { diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 419c55e92..8109ea322 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -8,7 +8,6 @@ #include "util.hh" #include "nar-info-disk-cache.hh" #include "thread-pool.hh" -#include "url.hh" #include "references.hh" #include "archive.hh" #include "callback.hh" @@ -20,8 +19,10 @@ #include "signals.hh" #include "users.hh" +#include #include -#include + +#include "strings.hh" using json = nlohmann::json; @@ -123,7 +124,7 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed if (info.method == FileIngestionMethod::Git && info.hash.algo != HashAlgorithm::SHA1) throw Error("Git file ingestion must use SHA-1 hash"); - if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::Recursive) { + if (info.hash.algo == HashAlgorithm::SHA256 && info.method == FileIngestionMethod::NixArchive) { return makeStorePath(makeType(*this, "source", info.references), info.hash, name); } else { if (!info.references.empty()) { @@ -169,7 +170,9 @@ std::pair StoreDirConfig::computeStorePath( const StorePathSet & references, PathFilter & filter) const { - auto h = hashPath(path, method.getFileIngestionMethod(), hashAlgo, filter); + auto [h, size] = hashPath(path, method.getFileIngestionMethod(), hashAlgo, filter); + if (size && *size >= settings.warnLargePathThreshold) + warn("hashed large path '%s' (%s)", path, renderSize(*size)); return { makeFixedOutputPathFromCA( name, @@ -199,18 +202,24 @@ StorePath Store::addToStore( case FileIngestionMethod::Flat: fsm = FileSerialisationMethod::Flat; break; - case FileIngestionMethod::Recursive: - fsm = FileSerialisationMethod::Recursive; + case FileIngestionMethod::NixArchive: + fsm = FileSerialisationMethod::NixArchive; break; case FileIngestionMethod::Git: // Use NAR; Git is not a serialization method - fsm = FileSerialisationMethod::Recursive; + fsm = FileSerialisationMethod::NixArchive; break; } - auto source = sinkToSource([&](Sink & sink) { - dumpPath(path, sink, fsm, filter); + std::optional storePath; + auto sink = sourceToSink([&](Source & source) { + LengthSource lengthSource(source); + storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair); + if (lengthSource.total >= settings.warnLargePathThreshold) + warn("copied large path '%s' to the store (%s)", path, renderSize(lengthSource.total)); }); - return addToStoreFromDump(*source, name, fsm, method, hashAlgo, references, repair); + dumpPath(path, *sink, fsm, filter); + sink->finish(); + return storePath.value(); } void Store::addMultipleToStore( @@ -351,7 +360,7 @@ ValidPathInfo Store::addToStoreSlow( RegularFileSink fileSink { caHashSink }; TeeSink unusualHashTee { narHashSink, caHashSink }; - auto & narSink = method == FileIngestionMethod::Recursive && hashAlgo != HashAlgorithm::SHA256 + auto & narSink = method == ContentAddressMethod::Raw::NixArchive && hashAlgo != HashAlgorithm::SHA256 ? static_cast(unusualHashTee) : narHashSink; @@ -379,9 +388,9 @@ ValidPathInfo Store::addToStoreSlow( finish. */ auto [narHash, narSize] = narHashSink.finish(); - auto hash = method == FileIngestionMethod::Recursive && hashAlgo == HashAlgorithm::SHA256 + auto hash = method == ContentAddressMethod::Raw::NixArchive && hashAlgo == HashAlgorithm::SHA256 ? narHash - : method == FileIngestionMethod::Git + : method == ContentAddressMethod::Raw::Git ? git::dumpHash(hashAlgo, srcPath).hash : caHashSink.finish().first; @@ -813,14 +822,25 @@ StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag m auto doQuery = [&](const StorePath & path) { checkInterrupt(); queryPathInfo(path, {[path, &state_, &wakeup](std::future> fut) { - auto state(state_.lock()); + bool exists = false; + std::exception_ptr newExc{}; + try { auto info = fut.get(); - state->valid.insert(path); + exists = true; } catch (InvalidPath &) { } catch (...) { - state->exc = std::current_exception(); + newExc = std::current_exception(); } + + auto state(state_.lock()); + + if (exists) + state->valid.insert(path); + + if (newExc) + state->exc = newExc; + assert(state->left); if (!--state->left) wakeup.notify_one(); @@ -913,7 +933,7 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor const Store::Stats & Store::getStats() { { - auto state_(state.lock()); + auto state_(state.readLock()); stats.pathInfoCacheSize = state_->pathInfoCache.size(); } return stats; @@ -1035,7 +1055,7 @@ std::map copyPaths( // not be within our control to change that, and we might still want // to at least copy the output paths. if (e.missingFeature == Xp::CaDerivations) - ignoreException(); + ignoreExceptionExceptInterrupt(); else throw; } @@ -1268,144 +1288,63 @@ Derivation Store::readInvalidDerivation(const StorePath & drvPath) namespace nix { -/* Split URI into protocol+hierarchy part and its parameter set. */ -std::pair splitUriAndParams(const std::string & uri_) -{ - auto uri(uri_); - Store::Params params; - auto q = uri.find('?'); - if (q != std::string::npos) { - params = decodeQuery(uri.substr(q + 1)); - uri = uri_.substr(0, q); - } - return {uri, params}; -} - -static bool isNonUriPath(const std::string & spec) -{ - return - // is not a URL - spec.find("://") == std::string::npos - // Has at least one path separator, and so isn't a single word that - // might be special like "auto" - && spec.find("/") != std::string::npos; -} - -std::shared_ptr openFromNonUri(const std::string & uri, const Store::Params & params) -{ - // TODO reenable on Windows once we have `LocalStore` and - // `UDSRemoteStore`. - if (uri == "" || uri == "auto") { - auto stateDir = getOr(params, "state", settings.nixStateDir); - if (access(stateDir.c_str(), R_OK | W_OK) == 0) - return std::make_shared(params); - else if (pathExists(settings.nixDaemonSocketFile)) - return std::make_shared(params); - #if __linux__ - else if (!pathExists(stateDir) - && params.empty() - && !isRootUser() - && !getEnv("NIX_STORE_DIR").has_value() - && !getEnv("NIX_STATE_DIR").has_value()) - { - /* If /nix doesn't exist, there is no daemon socket, and - we're not root, then automatically set up a chroot - store in ~/.local/share/nix/root. */ - auto chrootStore = getDataDir() + "/nix/root"; - if (!pathExists(chrootStore)) { - try { - createDirs(chrootStore); - } catch (Error & e) { - return std::make_shared(params); - } - warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); - } else - debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); - Store::Params params2; - params2["root"] = chrootStore; - return std::make_shared(params2); - } - #endif - else - return std::make_shared(params); - } else if (uri == "daemon") { - return std::make_shared(params); - } else if (uri == "local") { - return std::make_shared(params); - } else if (isNonUriPath(uri)) { - Store::Params params2 = params; - params2["root"] = absPath(uri); - return std::make_shared(params2); - } else { - return nullptr; - } -} - -// The `parseURL` function supports both IPv6 URIs as defined in -// RFC2732, but also pure addresses. The latter one is needed here to -// connect to a remote store via SSH (it's possible to do e.g. `ssh root@::1`). -// -// This function now ensures that a usable connection string is available: -// * If the store to be opened is not an SSH store, nothing will be done. -// * If the URL looks like `root@[::1]` (which is allowed by the URL parser and probably -// needed to pass further flags), it -// will be transformed into `root@::1` for SSH (same for `[::1]` -> `::1`). -// * If the URL looks like `root@::1` it will be left as-is. -// * In any other case, the string will be left as-is. -static std::string extractConnStr(const std::string &proto, const std::string &connStr) -{ - if (proto.rfind("ssh") != std::string::npos) { - std::smatch result; - std::regex v6AddrRegex("^((.*)@)?\\[(.*)\\]$"); - - if (std::regex_match(connStr, result, v6AddrRegex)) { - if (result[1].matched) { - return result.str(1) + result.str(3); - } - return result.str(3); - } - } - - return connStr; -} - -ref openStore(const std::string & uri_, +ref openStore(const std::string & uri, const Store::Params & extraParams) { - auto params = extraParams; - try { - auto parsedUri = parseURL(uri_); - params.insert(parsedUri.query.begin(), parsedUri.query.end()); + return openStore(StoreReference::parse(uri, extraParams)); +} - auto baseURI = extractConnStr( - parsedUri.scheme, - parsedUri.authority.value_or("") + parsedUri.path - ); +ref openStore(StoreReference && storeURI) +{ + auto & params = storeURI.params; - for (auto implem : *Implementations::registered) { - if (implem.uriSchemes.count(parsedUri.scheme)) { - auto store = implem.create(parsedUri.scheme, baseURI, params); - if (store) { - experimentalFeatureSettings.require(store->experimentalFeature()); - store->init(); - store->warnUnknownSettings(); - return ref(store); - } + auto store = std::visit(overloaded { + [&](const StoreReference::Auto &) -> std::shared_ptr { + auto stateDir = getOr(params, "state", settings.nixStateDir); + if (access(stateDir.c_str(), R_OK | W_OK) == 0) + return std::make_shared(params); + else if (pathExists(settings.nixDaemonSocketFile)) + return std::make_shared(params); + #if __linux__ + else if (!pathExists(stateDir) + && params.empty() + && !isRootUser() + && !getEnv("NIX_STORE_DIR").has_value() + && !getEnv("NIX_STATE_DIR").has_value()) + { + /* If /nix doesn't exist, there is no daemon socket, and + we're not root, then automatically set up a chroot + store in ~/.local/share/nix/root. */ + auto chrootStore = getDataDir() + "/root"; + if (!pathExists(chrootStore)) { + try { + createDirs(chrootStore); + } catch (SystemError & e) { + return std::make_shared(params); + } + warn("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); + } else + debug("'%s' does not exist, so Nix will use '%s' as a chroot store", stateDir, chrootStore); + return std::make_shared("local", chrootStore, params); } - } - } - catch (BadURL &) { - auto [uri, uriParams] = splitUriAndParams(uri_); - params.insert(uriParams.begin(), uriParams.end()); + #endif + else + return std::make_shared(params); + }, + [&](const StoreReference::Specified & g) { + for (auto implem : *Implementations::registered) + if (implem.uriSchemes.count(g.scheme)) + return implem.create(g.scheme, g.authority, params); - if (auto store = openFromNonUri(uri, params)) { - experimentalFeatureSettings.require(store->experimentalFeature()); - store->warnUnknownSettings(); - return ref(store); - } - } + throw Error("don't know how to open Nix store with scheme '%s'", g.scheme); + }, + }, storeURI.variant); - throw Error("don't know how to open Nix store '%s'", uri_); + experimentalFeatureSettings.require(store->experimentalFeature()); + store->warnUnknownSettings(); + store->init(); + + return ref { store }; } std::list> getDefaultSubstituters() diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index ae8c22437..8288cfdf0 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -13,18 +13,15 @@ #include "path-info.hh" #include "repair-flag.hh" #include "store-dir-config.hh" +#include "store-reference.hh" #include "source-path.hh" #include #include -#include #include -#include -#include #include #include #include -#include namespace nix { @@ -65,7 +62,7 @@ MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); MakeError(SubstituterDisabled, Error); -MakeError(InvalidStoreURI, Error); +MakeError(InvalidStoreReference, Error); struct Realisation; struct RealisedPath; @@ -91,7 +88,7 @@ enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; const uint32_t exportMagic = 0x4558494e; -enum BuildMode { bmNormal, bmRepair, bmCheck }; +enum BuildMode : uint8_t { bmNormal, bmRepair, bmCheck }; enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; struct BuildResult; @@ -102,7 +99,7 @@ typedef std::map> StorePathCAMap; struct StoreConfig : public StoreDirConfig { - typedef std::map Params; + using Params = StoreReference::Params; using StoreDirConfig::StoreDirConfig; @@ -204,7 +201,7 @@ protected: LRUCache pathInfoCache; }; - Sync state; + SharedSync state; std::shared_ptr diskCache; @@ -219,6 +216,10 @@ public: virtual ~Store() { } + /** + * @todo move to `StoreConfig` one we store enough information in + * those to recover the scheme and authority in all cases. + */ virtual std::string getUri() = 0; /** @@ -440,7 +441,7 @@ public: virtual StorePath addToStore( std::string_view name, const SourcePath & path, - ContentAddressMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), PathFilter & filter = defaultPathFilter, @@ -454,7 +455,7 @@ public: ValidPathInfo addToStoreSlow( std::string_view name, const SourcePath & path, - ContentAddressMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), std::optional expectedCAHash = {}); @@ -469,7 +470,7 @@ public: * * @param dumpMethod What serialisation format is `dump`, i.e. how * to deserialize it. Must either match hashMethod or be - * `FileSerialisationMethod::Recursive`. + * `FileSerialisationMethod::NixArchive`. * * @param hashMethod How content addressing? Need not match be the * same as `dumpMethod`. @@ -479,8 +480,8 @@ public: virtual StorePath addToStoreFromDump( Source & dump, std::string_view name, - FileSerialisationMethod dumpMethod = FileSerialisationMethod::Recursive, - ContentAddressMethod hashMethod = FileIngestionMethod::Recursive, + FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, + ContentAddressMethod hashMethod = ContentAddressMethod::Raw::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) = 0; @@ -859,34 +860,13 @@ OutputPathMap resolveDerivedPath(Store &, const DerivedPath::Built &, Store * ev /** * @return a Store object to access the Nix store denoted by * ‘uri’ (slight misnomer...). - * - * @param uri Supported values are: - * - * - ‘local’: The Nix store in /nix/store and database in - * /nix/var/nix/db, accessed directly. - * - * - ‘daemon’: The Nix store accessed via a Unix domain socket - * connection to nix-daemon. - * - * - ‘unix://’: The Nix store accessed via a Unix domain socket - * connection to nix-daemon, with the socket located at . - * - * - ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on - * whether the user has write access to the local Nix - * store/database. - * - * - ‘file://’: A binary cache stored in . - * - * - ‘https://’: A binary cache accessed via HTTP. - * - * - ‘s3://’: A writable binary cache stored on Amazon's Simple - * Storage Service. - * - * - ‘ssh://[user@]’: A remote Nix store accessed by running - * ‘nix-store --serve’ via SSH. - * - * You can pass parameters to the store type by appending - * ‘?key=value&key=value&...’ to the URI. + */ +ref openStore(StoreReference && storeURI); + + +/** + * Opens the store at `uri`, where `uri` is in the format expected by `StoreReference::parse` + */ ref openStore(const std::string & uri = settings.storeUri.get(), const Store::Params & extraParams = Store::Params()); @@ -901,7 +881,14 @@ std::list> getDefaultSubstituters(); struct StoreFactory { std::set uriSchemes; - std::function (const std::string & scheme, const std::string & uri, const Store::Params & params)> create; + /** + * The `authorityPath` parameter is `/`, or really + * whatever comes after `://` and before `?`. + */ + std::function ( + std::string_view scheme, + std::string_view authorityPath, + const Store::Params & params)> create; std::function ()> getConfig; }; @@ -914,9 +901,9 @@ struct Implementations { if (!registered) registered = new std::vector(); StoreFactory factory{ - .uriSchemes = T::uriSchemes(), + .uriSchemes = TConfig::uriSchemes(), .create = - ([](const std::string & scheme, const std::string & uri, const Store::Params & params) + ([](auto scheme, auto uri, auto & params) -> std::shared_ptr { return std::make_shared(scheme, uri, params); }), .getConfig = @@ -950,11 +937,6 @@ std::optional decodeValidPathInfo( std::istream & str, std::optional hashGiven = std::nullopt); -/** - * Split URI into protocol+hierarchy part and its parameter set. - */ -std::pair splitUriAndParams(const std::string & uri); - const ContentAddress * getDerivationCA(const BasicDerivation & drv); std::map drvOutputReferences( diff --git a/src/libstore/store-dir-config.hh b/src/libstore/store-dir-config.hh index 643f8854d..64c0dd8b7 100644 --- a/src/libstore/store-dir-config.hh +++ b/src/libstore/store-dir-config.hh @@ -16,6 +16,7 @@ namespace nix { struct SourcePath; MakeError(BadStorePath, Error); +MakeError(BadStorePathName, BadStorePath); struct StoreDirConfig : public Config { @@ -97,7 +98,7 @@ struct StoreDirConfig : public Config std::pair computeStorePath( std::string_view name, const SourcePath & path, - ContentAddressMethod method = FileIngestionMethod::Recursive, + ContentAddressMethod method = FileIngestionMethod::NixArchive, HashAlgorithm hashAlgo = HashAlgorithm::SHA256, const StorePathSet & references = {}, PathFilter & filter = defaultPathFilter) const; diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc new file mode 100644 index 000000000..b4968dfad --- /dev/null +++ b/src/libstore/store-reference.cc @@ -0,0 +1,116 @@ +#include + +#include "error.hh" +#include "url.hh" +#include "store-reference.hh" +#include "file-system.hh" +#include "util.hh" + +namespace nix { + +static bool isNonUriPath(const std::string & spec) +{ + return + // is not a URL + spec.find("://") == std::string::npos + // Has at least one path separator, and so isn't a single word that + // might be special like "auto" + && spec.find("/") != std::string::npos; +} + +std::string StoreReference::render() const +{ + std::string res; + + std::visit( + overloaded{ + [&](const StoreReference::Auto &) { res = "auto"; }, + [&](const StoreReference::Specified & g) { + res = g.scheme; + res += "://"; + res += g.authority; + }, + }, + variant); + + if (!params.empty()) { + res += "?"; + res += encodeQuery(params); + } + + return res; +} + +StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams) +{ + auto params = extraParams; + try { + auto parsedUri = parseURL(uri); + params.insert(parsedUri.query.begin(), parsedUri.query.end()); + + auto baseURI = parsedUri.authority.value_or("") + parsedUri.path; + + return { + .variant = + Specified{ + .scheme = std::move(parsedUri.scheme), + .authority = std::move(baseURI), + }, + .params = std::move(params), + }; + } catch (BadURL &) { + auto [baseURI, uriParams] = splitUriAndParams(uri); + params.insert(uriParams.begin(), uriParams.end()); + + if (baseURI == "" || baseURI == "auto") { + return { + .variant = Auto{}, + .params = std::move(params), + }; + } else if (baseURI == "daemon") { + return { + .variant = + Specified{ + .scheme = "unix", + .authority = "", + }, + .params = std::move(params), + }; + } else if (baseURI == "local") { + return { + .variant = + Specified{ + .scheme = "local", + .authority = "", + }, + .params = std::move(params), + }; + } else if (isNonUriPath(baseURI)) { + return { + .variant = + Specified{ + .scheme = "local", + .authority = absPath(baseURI), + }, + .params = std::move(params), + }; + } + } + + throw UsageError("Cannot parse Nix store '%s'", uri); +} + +/* Split URI into protocol+hierarchy part and its parameter set. */ +std::pair splitUriAndParams(const std::string & uri_) +{ + auto uri(uri_); + StoreReference::Params params; + auto q = uri.find('?'); + if (q != std::string::npos) { + params = decodeQuery(uri.substr(q + 1)); + uri = uri_.substr(0, q); + } + return {uri, params}; +} + +} diff --git a/src/libstore/store-reference.hh b/src/libstore/store-reference.hh new file mode 100644 index 000000000..459cea9c2 --- /dev/null +++ b/src/libstore/store-reference.hh @@ -0,0 +1,91 @@ +#pragma once +///@file + +#include + +#include "types.hh" + +namespace nix { + +/** + * A parsed Store URI (URI is a slight misnomer...), parsed but not yet + * resolved to a specific instance and query parms validated. + * + * Supported values are: + * + * - ‘local’: The Nix store in /nix/store and database in + * /nix/var/nix/db, accessed directly. + * + * - ‘daemon’: The Nix store accessed via a Unix domain socket + * connection to nix-daemon. + * + * - ‘unix://’: The Nix store accessed via a Unix domain socket + * connection to nix-daemon, with the socket located at . + * + * - ‘auto’ or ‘’: Equivalent to ‘local’ or ‘daemon’ depending on + * whether the user has write access to the local Nix + * store/database. + * + * - ‘file://’: A binary cache stored in . + * + * - ‘https://’: A binary cache accessed via HTTP. + * + * - ‘s3://’: A writable binary cache stored on Amazon's Simple + * Storage Service. + * + * - ‘ssh://[user@]’: A remote Nix store accessed by running + * ‘nix-store --serve’ via SSH. + * + * You can pass parameters to the store type by appending + * ‘?key=value&key=value&...’ to the URI. + */ +struct StoreReference +{ + using Params = std::map; + + /** + * Special store reference `""` or `"auto"` + */ + struct Auto + { + inline bool operator==(const Auto & rhs) const = default; + inline auto operator<=>(const Auto & rhs) const = default; + }; + + /** + * General case, a regular `scheme://authority` URL. + */ + struct Specified + { + std::string scheme; + std::string authority = ""; + + bool operator==(const Specified & rhs) const = default; + auto operator<=>(const Specified & rhs) const = default; + }; + + typedef std::variant Variant; + + Variant variant; + + Params params; + + bool operator==(const StoreReference & rhs) const = default; + + /** + * Render the whole store reference as a URI, including parameters. + */ + std::string render() const; + + /** + * Parse a URI into a store reference. + */ + static StoreReference parse(const std::string & uri, const Params & extraParams = Params{}); +}; + +/** + * Split URI into protocol+hierarchy part and its parameter set. + */ +std::pair splitUriAndParams(const std::string & uri); + +} diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 649644146..3c445eb13 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -2,10 +2,8 @@ #include "unix-domain-socket.hh" #include "worker-protocol.hh" -#include #include #include -#include #include #include @@ -19,6 +17,21 @@ namespace nix { +UDSRemoteStoreConfig::UDSRemoteStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , RemoteStoreConfig(params) + , path{authority.empty() ? settings.nixDaemonSocketFile : authority} +{ + if (scheme != UDSRemoteStoreConfig::scheme) { + throw UsageError("Scheme must be 'unix'"); + } +} + + std::string UDSRemoteStoreConfig::doc() { return @@ -27,11 +40,20 @@ std::string UDSRemoteStoreConfig::doc() } +// A bit gross that we now pass empty string but this is knowing that +// empty string will later default to the same nixDaemonSocketFile. Why +// don't we just wire it all through? I believe there are cases where it +// will live reload so we want to continue to account for that. UDSRemoteStore::UDSRemoteStore(const Params & params) + : UDSRemoteStore(scheme, "", params) +{} + + +UDSRemoteStore::UDSRemoteStore(std::string_view scheme, std::string_view authority, const Params & params) : StoreConfig(params) , LocalFSStoreConfig(params) , RemoteStoreConfig(params) - , UDSRemoteStoreConfig(params) + , UDSRemoteStoreConfig(scheme, authority, params) , Store(params) , LocalFSStore(params) , RemoteStore(params) @@ -39,23 +61,15 @@ UDSRemoteStore::UDSRemoteStore(const Params & params) } -UDSRemoteStore::UDSRemoteStore( - const std::string scheme, - std::string socket_path, - const Params & params) - : UDSRemoteStore(params) -{ - path.emplace(socket_path); -} - - std::string UDSRemoteStore::getUri() { - if (path) { - return std::string("unix://") + *path; - } else { - return "daemon"; - } + return path == settings.nixDaemonSocketFile + ? // FIXME: Not clear why we return daemon here and not default + // to settings.nixDaemonSocketFile + // + // unix:// with no path also works. Change what we return? + "daemon" + : std::string(scheme) + "://" + path; } @@ -72,7 +86,7 @@ ref UDSRemoteStore::openConnection() /* Connect to a daemon that does the privileged work for us. */ conn->fd = createUnixDomainSocket(); - nix::connect(toSocket(conn->fd.get()), path ? *path : settings.nixDaemonSocketFile); + nix::connect(toSocket(conn->fd.get()), path); conn->from.fd = conn->fd.get(); conn->to.fd = conn->fd.get(); diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index 8bce8994a..a8e571664 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -9,16 +9,37 @@ namespace nix { struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreConfig { - UDSRemoteStoreConfig(const Params & params) - : StoreConfig(params) - , LocalFSStoreConfig(params) - , RemoteStoreConfig(params) - { - } + // TODO(fzakaria): Delete this constructor once moved over to the factory pattern + // outlined in https://github.com/NixOS/nix/issues/10766 + using LocalFSStoreConfig::LocalFSStoreConfig; + using RemoteStoreConfig::RemoteStoreConfig; + + /** + * @param authority is the socket path. + */ + UDSRemoteStoreConfig( + std::string_view scheme, + std::string_view authority, + const Params & params); const std::string name() override { return "Local Daemon Store"; } std::string doc() override; + + /** + * The path to the unix domain socket. + * + * The default is `settings.nixDaemonSocketFile`, but we don't write + * that below, instead putting in the constructor. + */ + Path path; + +protected: + static constexpr char const * scheme = "unix"; + +public: + static std::set uriSchemes() + { return {scheme}; } }; class UDSRemoteStore : public virtual UDSRemoteStoreConfig @@ -27,14 +48,21 @@ class UDSRemoteStore : public virtual UDSRemoteStoreConfig { public: + /** + * @deprecated This is the old API to construct the store. + */ UDSRemoteStore(const Params & params); - UDSRemoteStore(const std::string scheme, std::string path, const Params & params); + + /** + * @param authority is the socket path. + */ + UDSRemoteStore( + std::string_view scheme, + std::string_view authority, + const Params & params); std::string getUri() override; - static std::set uriSchemes() - { return {"unix"}; } - ref getFSAccessor(bool requireValidPath = true) override { return LocalFSStore::getFSAccessor(requireValidPath); } @@ -60,7 +88,6 @@ private: }; ref openConnection() override; - std::optional path; }; } diff --git a/src/libstore/unix/build/drv-output-substitution-goal.cc b/src/libstore/unix/build/drv-output-substitution-goal.cc deleted file mode 100644 index b30957c84..000000000 --- a/src/libstore/unix/build/drv-output-substitution-goal.cc +++ /dev/null @@ -1,167 +0,0 @@ -#include "drv-output-substitution-goal.hh" -#include "finally.hh" -#include "worker.hh" -#include "substitution-goal.hh" -#include "callback.hh" - -namespace nix { - -DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal( - const DrvOutput & id, - Worker & worker, - RepairFlag repair, - std::optional ca) - : Goal(worker, DerivedPath::Opaque { StorePath::dummy }) - , id(id) -{ - state = &DrvOutputSubstitutionGoal::init; - name = fmt("substitution of '%s'", id.to_string()); - trace("created"); -} - - -void DrvOutputSubstitutionGoal::init() -{ - trace("init"); - - /* If the derivation already exists, we’re done */ - if (worker.store.queryRealisation(id)) { - amDone(ecSuccess); - return; - } - - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - tryNext(); -} - -void DrvOutputSubstitutionGoal::tryNext() -{ - trace("trying next substituter"); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - debug("derivation output '%s' is required, but there is no substituter that can provide it", id.to_string()); - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - amDone(substituterFailed ? ecFailed : ecNoSubstituters); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); - } - - return; - } - - sub = subs.front(); - subs.pop_front(); - - // FIXME: Make async - // outputInfo = sub->queryRealisation(id); - - /* The callback of the curl download below can outlive `this` (if - some other error occurs), so it must not touch `this`. So put - the shared state in a separate refcounted object. */ - downloadState = std::make_shared(); - downloadState->outPipe.create(); - - sub->queryRealisation( - id, - { [downloadState(downloadState)](std::future> res) { - try { - Finally updateStats([&]() { downloadState->outPipe.writeSide.close(); }); - downloadState->promise.set_value(res.get()); - } catch (...) { - downloadState->promise.set_exception(std::current_exception()); - } - } }); - - worker.childStarted(shared_from_this(), {downloadState->outPipe.readSide.get()}, true, false); - - state = &DrvOutputSubstitutionGoal::realisationFetched; -} - -void DrvOutputSubstitutionGoal::realisationFetched() -{ - worker.childTerminated(this); - - try { - outputInfo = downloadState->promise.get_future().get(); - } catch (std::exception & e) { - printError(e.what()); - substituterFailed = true; - } - - if (!outputInfo) { - return tryNext(); - } - - for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { - if (depId != id) { - if (auto localOutputInfo = worker.store.queryRealisation(depId); - localOutputInfo && localOutputInfo->outPath != depPath) { - warn( - "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" - "Local: %s\n" - "Remote: %s", - sub->getUri(), - depId.to_string(), - worker.store.printStorePath(localOutputInfo->outPath), - worker.store.printStorePath(depPath) - ); - tryNext(); - return; - } - addWaitee(worker.makeDrvOutputSubstitutionGoal(depId)); - } - } - - addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath)); - - if (waitees.empty()) outPathValid(); - else state = &DrvOutputSubstitutionGoal::outPathValid; -} - -void DrvOutputSubstitutionGoal::outPathValid() -{ - assert(outputInfo); - trace("output path substituted"); - - if (nrFailed > 0) { - debug("The output path of the derivation output '%s' could not be substituted", id.to_string()); - amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed); - return; - } - - worker.store.registerDrvOutput(*outputInfo); - finished(); -} - -void DrvOutputSubstitutionGoal::finished() -{ - trace("finished"); - amDone(ecSuccess); -} - -std::string DrvOutputSubstitutionGoal::key() -{ - /* "a$" ensures substitution goals happen before derivation - goals. */ - return "a$" + std::string(id.to_string()); -} - -void DrvOutputSubstitutionGoal::work() -{ - (this->*state)(); -} - -void DrvOutputSubstitutionGoal::handleEOF(int fd) -{ - if (fd == downloadState->outPipe.readSide.get()) worker.wakeUp(shared_from_this()); -} - - -} diff --git a/src/libstore/unix/build/goal.cc b/src/libstore/unix/build/goal.cc deleted file mode 100644 index f8db98280..000000000 --- a/src/libstore/unix/build/goal.cc +++ /dev/null @@ -1,109 +0,0 @@ -#include "goal.hh" -#include "worker.hh" - -namespace nix { - - -bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) const { - std::string s1 = a->key(); - std::string s2 = b->key(); - return s1 < s2; -} - - -BuildResult Goal::getBuildResult(const DerivedPath & req) const { - BuildResult res { buildResult }; - - if (auto pbp = std::get_if(&req)) { - auto & bp = *pbp; - - /* Because goals are in general shared between derived paths - that share the same derivation, we need to filter their - results to get back just the results we care about. - */ - - for (auto it = res.builtOutputs.begin(); it != res.builtOutputs.end();) { - if (bp.outputs.contains(it->first)) - ++it; - else - it = res.builtOutputs.erase(it); - } - } - - return res; -} - - -void addToWeakGoals(WeakGoals & goals, GoalPtr p) -{ - if (goals.find(p) != goals.end()) - return; - goals.insert(p); -} - - -void Goal::addWaitee(GoalPtr waitee) -{ - waitees.insert(waitee); - addToWeakGoals(waitee->waiters, shared_from_this()); -} - - -void Goal::waiteeDone(GoalPtr waitee, ExitCode result) -{ - assert(waitees.count(waitee)); - waitees.erase(waitee); - - trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size())); - - if (result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure) ++nrFailed; - - if (result == ecNoSubstituters) ++nrNoSubstituters; - - if (result == ecIncompleteClosure) ++nrIncompleteClosure; - - if (waitees.empty() || (result == ecFailed && !settings.keepGoing)) { - - /* If we failed and keepGoing is not set, we remove all - remaining waitees. */ - for (auto & goal : waitees) { - goal->waiters.extract(shared_from_this()); - } - waitees.clear(); - - worker.wakeUp(shared_from_this()); - } -} - - -void Goal::amDone(ExitCode result, std::optional ex) -{ - trace("done"); - assert(exitCode == ecBusy); - assert(result == ecSuccess || result == ecFailed || result == ecNoSubstituters || result == ecIncompleteClosure); - exitCode = result; - - if (ex) { - if (!waiters.empty()) - logError(ex->info()); - else - this->ex = std::move(*ex); - } - - for (auto & i : waiters) { - GoalPtr goal = i.lock(); - if (goal) goal->waiteeDone(shared_from_this(), result); - } - waiters.clear(); - worker.removeGoal(shared_from_this()); - - cleanup(); -} - - -void Goal::trace(std::string_view s) -{ - debug("%1%: %2%", name, s); -} - -} diff --git a/src/libstore/unix/build/goal.hh b/src/libstore/unix/build/goal.hh deleted file mode 100644 index 9af083230..000000000 --- a/src/libstore/unix/build/goal.hh +++ /dev/null @@ -1,180 +0,0 @@ -#pragma once -///@file - -#include "types.hh" -#include "store-api.hh" -#include "build-result.hh" - -namespace nix { - -/** - * Forward definition. - */ -struct Goal; -class Worker; - -/** - * A pointer to a goal. - */ -typedef std::shared_ptr GoalPtr; -typedef std::weak_ptr WeakGoalPtr; - -struct CompareGoalPtrs { - bool operator() (const GoalPtr & a, const GoalPtr & b) const; -}; - -/** - * Set of goals. - */ -typedef std::set Goals; -typedef std::set> WeakGoals; - -/** - * A map of paths to goals (and the other way around). - */ -typedef std::map WeakGoalMap; - -/** - * Used as a hint to the worker on how to schedule a particular goal. For example, - * builds are typically CPU- and memory-bound, while substitutions are I/O bound. - * Using this information, the worker might decide to schedule more or fewer goals - * of each category in parallel. - */ -enum struct JobCategory { - /** - * A build of a derivation; it will use CPU and disk resources. - */ - Build, - /** - * A substitution an arbitrary store object; it will use network resources. - */ - Substitution, -}; - -struct Goal : public std::enable_shared_from_this -{ - typedef enum {ecBusy, ecSuccess, ecFailed, ecNoSubstituters, ecIncompleteClosure} ExitCode; - - /** - * Backlink to the worker. - */ - Worker & worker; - - /** - * Goals that this goal is waiting for. - */ - Goals waitees; - - /** - * Goals waiting for this one to finish. Must use weak pointers - * here to prevent cycles. - */ - WeakGoals waiters; - - /** - * Number of goals we are/were waiting for that have failed. - */ - size_t nrFailed = 0; - - /** - * Number of substitution goals we are/were waiting for that - * failed because there are no substituters. - */ - size_t nrNoSubstituters = 0; - - /** - * Number of substitution goals we are/were waiting for that - * failed because they had unsubstitutable references. - */ - size_t nrIncompleteClosure = 0; - - /** - * Name of this goal for debugging purposes. - */ - std::string name; - - /** - * Whether the goal is finished. - */ - ExitCode exitCode = ecBusy; - -protected: - /** - * Build result. - */ - BuildResult buildResult; - -public: - - /** - * Project a `BuildResult` with just the information that pertains - * to the given request. - * - * In general, goals may be aliased between multiple requests, and - * the stored `BuildResult` has information for the union of all - * requests. We don't want to leak what the other request are for - * sake of both privacy and determinism, and this "safe accessor" - * ensures we don't. - */ - BuildResult getBuildResult(const DerivedPath &) const; - - /** - * Exception containing an error message, if any. - */ - std::optional ex; - - Goal(Worker & worker, DerivedPath path) - : worker(worker) - { } - - virtual ~Goal() - { - trace("goal destroyed"); - } - - virtual void work() = 0; - - void addWaitee(GoalPtr waitee); - - virtual void waiteeDone(GoalPtr waitee, ExitCode result); - - virtual void handleChildOutput(int fd, std::string_view data) - { - abort(); - } - - virtual void handleEOF(int fd) - { - abort(); - } - - void trace(std::string_view s); - - std::string getName() const - { - return name; - } - - /** - * Callback in case of a timeout. It should wake up its waiters, - * get rid of any running child processes that are being monitored - * by the worker (important!), etc. - */ - virtual void timedOut(Error && ex) = 0; - - virtual std::string key() = 0; - - void amDone(ExitCode result, std::optional ex = {}); - - virtual void cleanup() { } - - /** - * @brief Hint for the scheduler, which concurrency limit applies. - * @see JobCategory - */ - virtual JobCategory jobCategory() const = 0; -}; - -void addToWeakGoals(WeakGoals & goals, GoalPtr p); - -} diff --git a/src/libstore/unix/build/hook-instance.cc b/src/libstore/unix/build/hook-instance.cc index 5d045ec3d..79eb25a91 100644 --- a/src/libstore/unix/build/hook-instance.cc +++ b/src/libstore/unix/build/hook-instance.cc @@ -1,7 +1,10 @@ #include "globals.hh" +#include "config-global.hh" #include "hook-instance.hh" #include "file-system.hh" #include "child.hh" +#include "strings.hh" +#include "executable-path.hh" namespace nix { @@ -14,11 +17,18 @@ HookInstance::HookInstance() if (buildHookArgs.empty()) throw Error("'build-hook' setting is empty"); - auto buildHook = canonPath(buildHookArgs.front()); + std::filesystem::path buildHook = buildHookArgs.front(); buildHookArgs.pop_front(); + try { + buildHook = ExecutablePath::load().findPath(buildHook); + } catch (ExecutableLookupError & e) { + e.addTrace(nullptr, "while resolving the 'build-hook' setting'"); + throw; + } + Strings args; - args.push_back(std::string(baseNameOf(buildHook))); + args.push_back(buildHook.filename().string()); for (auto & arg : buildHookArgs) args.push_back(arg); @@ -57,7 +67,7 @@ HookInstance::HookInstance() if (dup2(builderOut.readSide.get(), 5) == -1) throw SysError("dupping builder's stdout/stderr"); - execv(buildHook.c_str(), stringsToCharPtrs(args).data()); + execv(buildHook.native().c_str(), stringsToCharPtrs(args).data()); throw SysError("executing '%s'", buildHook); }); @@ -81,7 +91,7 @@ HookInstance::~HookInstance() toHook.writeSide = -1; if (pid != -1) pid.kill(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libstore/unix/build/local-derivation-goal.cc b/src/libstore/unix/build/local-derivation-goal.cc index 16095cf5d..e3e3a4c9b 100644 --- a/src/libstore/unix/build/local-derivation-goal.cc +++ b/src/libstore/unix/build/local-derivation-goal.cc @@ -64,6 +64,8 @@ #include #include +#include "strings.hh" + namespace nix { void handleDiffHook( @@ -107,9 +109,9 @@ LocalDerivationGoal::~LocalDerivationGoal() { /* Careful: we should never ever throw an exception from a destructor. */ - try { deleteTmpDir(false); } catch (...) { ignoreException(); } - try { killChild(); } catch (...) { ignoreException(); } - try { stopDaemon(); } catch (...) { ignoreException(); } + try { deleteTmpDir(false); } catch (...) { ignoreExceptionInDestructor(); } + try { killChild(); } catch (...) { ignoreExceptionInDestructor(); } + try { stopDaemon(); } catch (...) { ignoreExceptionInDestructor(); } } @@ -163,7 +165,7 @@ void LocalDerivationGoal::killSandbox(bool getStats) buildResult.cpuSystem = stats.cpuSystem; } #else - abort(); + unreachable(); #endif } @@ -175,7 +177,7 @@ void LocalDerivationGoal::killSandbox(bool getStats) } -void LocalDerivationGoal::tryLocalBuild() +Goal::Co LocalDerivationGoal::tryLocalBuild() { #if __APPLE__ additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or(""); @@ -183,10 +185,10 @@ void LocalDerivationGoal::tryLocalBuild() unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { - state = &DerivationGoal::tryToBuild; worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); - return; + co_await Suspend{}; + co_return tryToBuild(); } assert(derivationType); @@ -240,7 +242,8 @@ void LocalDerivationGoal::tryLocalBuild() actLock = std::make_unique(*logger, lvlWarn, actBuildWaiting, fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); worker.waitForAWhile(shared_from_this()); - return; + co_await Suspend{}; + co_return tryLocalBuild(); } } @@ -255,15 +258,13 @@ void LocalDerivationGoal::tryLocalBuild() outputLocks.unlock(); buildUser.reset(); worker.permanentFailure = true; - done(BuildResult::InputRejected, {}, std::move(e)); - return; + co_return done(BuildResult::InputRejected, {}, std::move(e)); } - /* This state will be reached when we get EOF on the child's - log pipe. */ - state = &DerivationGoal::buildDone; - started(); + co_await Suspend{}; + // after EOF on child + co_return buildDone(); } static void chmod_(const Path & path, mode_t mode) @@ -432,6 +433,41 @@ static void doBind(const Path & source, const Path & target, bool optional = fal }; #endif +/** + * Rethrow the current exception as a subclass of `Error`. + */ +static void rethrowExceptionAsError() +{ + try { + throw; + } catch (Error &) { + throw; + } catch (std::exception & e) { + throw Error(e.what()); + } catch (...) { + throw Error("unknown exception"); + } +} + +/** + * Send the current exception to the parent in the format expected by + * `LocalDerivationGoal::processSandboxSetupMessages()`. + */ +static void handleChildException(bool sendException) +{ + try { + rethrowExceptionAsError(); + } catch (Error & e) { + if (sendException) { + writeFull(STDERR_FILENO, "\1\n"); + FdSink sink(STDERR_FILENO); + sink << e; + sink.flush(); + } else + std::cerr << e.msg(); + } +} + void LocalDerivationGoal::startBuilder() { if ((buildUser && buildUser->getUIDCount() != 1) @@ -443,25 +479,22 @@ void LocalDerivationGoal::startBuilder() #if __linux__ experimentalFeatureSettings.require(Xp::Cgroups); + /* If we're running from the daemon, then this will return the + root cgroup of the service. Otherwise, it will return the + current cgroup. */ + auto rootCgroup = getRootCgroup(); auto cgroupFS = getCgroupFS(); if (!cgroupFS) throw Error("cannot determine the cgroups file system"); - - auto ourCgroups = getCgroups("/proc/self/cgroup"); - auto ourCgroup = ourCgroups[""]; - if (ourCgroup == "") - throw Error("cannot determine cgroup name from /proc/self/cgroup"); - - auto ourCgroupPath = canonPath(*cgroupFS + "/" + ourCgroup); - - if (!pathExists(ourCgroupPath)) - throw Error("expected cgroup directory '%s'", ourCgroupPath); + auto rootCgroupPath = canonPath(*cgroupFS + "/" + rootCgroup); + if (!pathExists(rootCgroupPath)) + throw Error("expected cgroup directory '%s'", rootCgroupPath); static std::atomic counter{0}; cgroup = buildUser - ? fmt("%s/nix-build-uid-%d", ourCgroupPath, buildUser->getUID()) - : fmt("%s/nix-build-pid-%d-%d", ourCgroupPath, getpid(), counter++); + ? fmt("%s/nix-build-uid-%d", rootCgroupPath, buildUser->getUID()) + : fmt("%s/nix-build-pid-%d-%d", rootCgroupPath, getpid(), counter++); debug("using cgroup '%s'", *cgroup); @@ -503,8 +536,24 @@ void LocalDerivationGoal::startBuilder() /* Create a temporary directory where the build will take place. */ - tmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700); + topTmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700); +#if __APPLE__ + if (false) { +#else + if (useChroot) { +#endif + /* If sandboxing is enabled, put the actual TMPDIR underneath + an inaccessible root-owned directory, to prevent outside + access. + On macOS, we don't use an actual chroot, so this isn't + possible. Any mitigation along these lines would have to be + done directly in the sandbox profile. */ + tmpDir = topTmpDir + "/build"; + createDir(tmpDir, 0700); + } else { + tmpDir = topTmpDir; + } chownToBuilder(tmpDir); for (auto & [outputName, status] : initialOutputs) { @@ -672,15 +721,19 @@ void LocalDerivationGoal::startBuilder() environment using bind-mounts. We put it in the Nix store so that the build outputs can be moved efficiently from the chroot to their final location. */ - chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; - deletePath(chrootRootDir); + chrootParentDir = worker.store.Store::toRealPath(drvPath) + ".chroot"; + deletePath(chrootParentDir); /* Clean up the chroot directory automatically. */ - autoDelChroot = std::make_shared(chrootRootDir); + autoDelChroot = std::make_shared(chrootParentDir); - printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootRootDir); + printMsg(lvlChatty, "setting up chroot environment in '%1%'", chrootParentDir); + + if (mkdir(chrootParentDir.c_str(), 0700) == -1) + throw SysError("cannot create '%s'", chrootRootDir); + + chrootRootDir = chrootParentDir + "/root"; - // FIXME: make this 0700 if (mkdir(chrootRootDir.c_str(), buildUser && buildUser->getUIDCount() != 1 ? 0755 : 0750) == -1) throw SysError("cannot create '%1%'", chrootRootDir); @@ -931,32 +984,40 @@ void LocalDerivationGoal::startBuilder() root. */ openSlave(); - /* Drop additional groups here because we can't do it - after we've created the new user namespace. */ - if (setgroups(0, 0) == -1) { - if (errno != EPERM) - throw SysError("setgroups failed"); - if (settings.requireDropSupplementaryGroups) - throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step."); + try { + /* Drop additional groups here because we can't do it + after we've created the new user namespace. */ + if (setgroups(0, 0) == -1) { + if (errno != EPERM) + throw SysError("setgroups failed"); + if (settings.requireDropSupplementaryGroups) + throw Error("setgroups failed. Set the require-drop-supplementary-groups option to false to skip this step."); + } + + ProcessOptions options; + options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; + if (privateNetwork) + options.cloneFlags |= CLONE_NEWNET; + if (usingUserNamespace) + options.cloneFlags |= CLONE_NEWUSER; + + pid_t child = startProcess([&]() { runChild(); }, options); + + writeFull(sendPid.writeSide.get(), fmt("%d\n", child)); + _exit(0); + } catch (...) { + handleChildException(true); + _exit(1); } - - ProcessOptions options; - options.cloneFlags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD; - if (privateNetwork) - options.cloneFlags |= CLONE_NEWNET; - if (usingUserNamespace) - options.cloneFlags |= CLONE_NEWUSER; - - pid_t child = startProcess([&]() { runChild(); }, options); - - writeFull(sendPid.writeSide.get(), fmt("%d\n", child)); - _exit(0); }); sendPid.writeSide.close(); - if (helper.wait() != 0) + if (helper.wait() != 0) { + processSandboxSetupMessages(); + // Only reached if the child process didn't send an exception. throw Error("unable to start build process"); + } userNamespaceSync.readSide = -1; @@ -1032,7 +1093,12 @@ void LocalDerivationGoal::startBuilder() pid.setSeparatePG(true); worker.childStarted(shared_from_this(), {builderOut.get()}, true, true); - /* Check if setting up the build environment failed. */ + processSandboxSetupMessages(); +} + + +void LocalDerivationGoal::processSandboxSetupMessages() +{ std::vector msgs; while (true) { std::string msg = [&]() { @@ -1060,7 +1126,8 @@ void LocalDerivationGoal::startBuilder() } -void LocalDerivationGoal::initTmpDir() { +void LocalDerivationGoal::initTmpDir() +{ /* In a sandbox, for determinism, always use the same temporary directory. */ #if __linux__ @@ -1237,7 +1304,7 @@ bool LocalDerivationGoal::isAllowed(const DerivedPath & req) struct RestrictedStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; - const std::string name() { return "Restricted Store"; } + const std::string name() override { return "Restricted Store"; } }; /* A wrapper around LocalStore that only allows building/querying of @@ -1500,19 +1567,20 @@ void LocalDerivationGoal::startDaemon() throw SysError("accepting connection"); } - closeOnExec(remote.get()); + unix::closeOnExec(remote.get()); debug("received daemon connection"); auto workerThread = std::thread([store, remote{std::move(remote)}]() { - FdSource from(remote.get()); - FdSink to(remote.get()); try { - daemon::processConnection(store, from, to, + daemon::processConnection( + store, + FdSource(remote.get()), + FdSink(remote.get()), NotTrusted, daemon::Recursive); debug("terminated daemon connection"); } catch (SystemError &) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } }); @@ -1680,10 +1748,13 @@ void setupSeccomp() throw SysError("unable to add seccomp rule"); } - /* Prevent builders from creating EAs or ACLs. Not all filesystems + /* Prevent builders from using EAs or ACLs. Not all filesystems support these, and they're not allowed in the Nix store because they're not representable in the NAR serialisation. */ - if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || + if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(getxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lgetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fgetxattr), 0) != 0 || + seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 || seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0) throw SysError("unable to add seccomp rule"); @@ -1721,13 +1792,20 @@ void LocalDerivationGoal::runChild() bool setUser = true; - /* Make the contents of netrc available to builtin:fetchurl - (which may run under a different uid and/or in a sandbox). */ + /* Make the contents of netrc and the CA certificate bundle + available to builtin:fetchurl (which may run under a + different uid and/or in a sandbox). */ std::string netrcData; - try { - if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") - netrcData = readFile(settings.netrcFile); - } catch (SystemError &) { } + std::string caFileData; + if (drv->isBuiltin() && drv->builder == "builtin:fetchurl") { + try { + netrcData = readFile(settings.netrcFile); + } catch (SystemError &) { } + + try { + caFileData = readFile(settings.caFile); + } catch (SystemError &) { } + } #if __linux__ if (useChroot) { @@ -1930,7 +2008,7 @@ void LocalDerivationGoal::runChild() if (chdir(chrootRootDir.c_str()) == -1) throw SysError("cannot change directory to '%1%'", chrootRootDir); - if (mkdir("real-root", 0) == -1) + if (mkdir("real-root", 0500) == -1) throw SysError("cannot create real-root directory"); if (pivot_root(".", "real-root") == -1) @@ -1961,7 +2039,7 @@ void LocalDerivationGoal::runChild() throw SysError("changing into '%1%'", tmpDir); /* Close all other file descriptors. */ - closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}); + unix::closeExtraFDs(); #if __linux__ linux::setPersonality(drv->platform); @@ -2166,7 +2244,7 @@ void LocalDerivationGoal::runChild() worker.store.printStorePath(scratchOutputs.at(e.first))); if (drv->builder == "builtin:fetchurl") - builtinFetchurl(*drv, outputs, netrcData); + builtinFetchurl(*drv, outputs, netrcData, caFileData); else if (drv->builder == "builtin:buildenv") builtinBuildenv(*drv, outputs); else if (drv->builder == "builtin:unpack-channel") @@ -2208,14 +2286,8 @@ void LocalDerivationGoal::runChild() throw SysError("executing '%1%'", drv->builder); - } catch (Error & e) { - if (sendException) { - writeFull(STDERR_FILENO, "\1\n"); - FdSink sink(STDERR_FILENO); - sink << e; - sink.flush(); - } else - std::cerr << e.msg(); + } catch (...) { + handleChildException(sendException); _exit(1); } } @@ -2489,7 +2561,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() auto fim = outputHash.method.getFileIngestionMethod(); switch (fim) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: { HashModuloSink caSink { outputHash.hashAlgo, oldHashPart }; auto fim = outputHash.method.getFileIngestionMethod(); @@ -2531,7 +2603,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() { HashResult narHashAndSize = hashPath( {getFSSourceAccessor(), CanonPath(actualPath)}, - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256); + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); newInfo0.narHash = narHashAndSize.first; newInfo0.narSize = narHashAndSize.second; } @@ -2554,7 +2626,7 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() rewriteOutput(outputRewrites); HashResult narHashAndSize = hashPath( {getFSSourceAccessor(), CanonPath(actualPath)}, - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256); + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first }; newInfo0.narSize = narHashAndSize.second; auto refs = rewriteRefs(); @@ -2904,6 +2976,24 @@ void LocalDerivationGoal::checkOutputs(const std::mapgetStructuredAttrs()) { + if (get(*structuredAttrs, "allowedReferences")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'allowedReferences'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "allowedRequisites")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'allowedRequisites'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "disallowedRequisites")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'disallowedRequisites'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "disallowedReferences")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'disallowedReferences'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "maxSize")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'maxSize'; use 'outputChecks' instead"); + } + if (get(*structuredAttrs, "maxClosureSize")){ + warn("'structuredAttrs' disables the effect of the top-level attribute 'maxClosureSize'; use 'outputChecks' instead"); + } if (auto outputChecks = get(*structuredAttrs, "outputChecks")) { if (auto output = get(*outputChecks, outputName)) { Checks checks; @@ -2952,15 +3042,17 @@ void LocalDerivationGoal::checkOutputs(const std::mapisBuiltin()) { printError("note: keeping build directory '%s'", tmpDir); + chmod(topTmpDir.c_str(), 0755); chmod(tmpDir.c_str(), 0755); } else - deletePath(tmpDir); + deletePath(topTmpDir); + topTmpDir = ""; tmpDir = ""; } } diff --git a/src/libstore/unix/build/local-derivation-goal.hh b/src/libstore/unix/build/local-derivation-goal.hh index f25cb9424..231393308 100644 --- a/src/libstore/unix/build/local-derivation-goal.hh +++ b/src/libstore/unix/build/local-derivation-goal.hh @@ -27,10 +27,16 @@ struct LocalDerivationGoal : public DerivationGoal std::optional cgroup; /** - * The temporary directory. + * The temporary directory used for the build. */ Path tmpDir; + /** + * The top-level temporary directory. `tmpDir` is either equal to + * or a child of this directory. + */ + Path topTmpDir; + /** * The path of the temporary directory in the sandbox. */ @@ -65,6 +71,16 @@ struct LocalDerivationGoal : public DerivationGoal */ bool useChroot = false; + /** + * The parent directory of `chrootRootDir`. It has permission 700 + * and is owned by root to ensure other users cannot mess with + * `chrootRootDir`. + */ + Path chrootParentDir; + + /** + * The root of the chroot environment. + */ Path chrootRootDir; /** @@ -182,7 +198,7 @@ struct LocalDerivationGoal : public DerivationGoal /** * The additional states. */ - void tryLocalBuild() override; + Goal::Co tryLocalBuild() override; /** * Start building a derivation. @@ -194,6 +210,11 @@ struct LocalDerivationGoal : public DerivationGoal */ void initEnv(); + /** + * Process messages send by the sandbox initialization. + */ + void processSandboxSetupMessages(); + /** * Setup tmp dir location. */ diff --git a/src/libstore/unix/build/sandbox-defaults.sb b/src/libstore/unix/build/sandbox-defaults.sb index 2ad5fb616..15cd6daf5 100644 --- a/src/libstore/unix/build/sandbox-defaults.sb +++ b/src/libstore/unix/build/sandbox-defaults.sb @@ -17,6 +17,9 @@ R""( ; Allow POSIX semaphores and shared memory. (allow ipc-posix*) +; Allow SYSV semaphores and shared memory. +(allow ipc-sysv*) + ; Allow socket creation. (allow system-socket) @@ -46,6 +49,7 @@ R""( (if (param "_ALLOW_LOCAL_NETWORKING") (begin (allow network* (remote ip "localhost:*")) + (allow network-inbound (local ip "*:*")) ; required to bind and listen ; Allow access to /etc/resolv.conf (which is a symlink to ; /private/var/run/resolv.conf). diff --git a/src/libstore/unix/build/substitution-goal.cc b/src/libstore/unix/build/substitution-goal.cc deleted file mode 100644 index c7e8e2825..000000000 --- a/src/libstore/unix/build/substitution-goal.cc +++ /dev/null @@ -1,324 +0,0 @@ -#include "worker.hh" -#include "substitution-goal.hh" -#include "nar-info.hh" -#include "finally.hh" -#include "signals.hh" - -namespace nix { - -PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional ca) - : Goal(worker, DerivedPath::Opaque { storePath }) - , storePath(storePath) - , repair(repair) - , ca(ca) -{ - state = &PathSubstitutionGoal::init; - name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath)); - trace("created"); - maintainExpectedSubstitutions = std::make_unique>(worker.expectedSubstitutions); -} - - -PathSubstitutionGoal::~PathSubstitutionGoal() -{ - cleanup(); -} - - -void PathSubstitutionGoal::done( - ExitCode result, - BuildResult::Status status, - std::optional errorMsg) -{ - buildResult.status = status; - if (errorMsg) { - debug(*errorMsg); - buildResult.errorMsg = *errorMsg; - } - amDone(result); -} - - -void PathSubstitutionGoal::work() -{ - (this->*state)(); -} - - -void PathSubstitutionGoal::init() -{ - trace("init"); - - worker.store.addTempRoot(storePath); - - /* If the path already exists we're done. */ - if (!repair && worker.store.isValidPath(storePath)) { - done(ecSuccess, BuildResult::AlreadyValid); - return; - } - - if (settings.readOnlyMode) - throw Error("cannot substitute path '%s' - no write access to the Nix store", worker.store.printStorePath(storePath)); - - subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); - - tryNext(); -} - - -void PathSubstitutionGoal::tryNext() -{ - trace("trying next substituter"); - - cleanup(); - - if (subs.size() == 0) { - /* None left. Terminate this goal and let someone else deal - with it. */ - - /* Hack: don't indicate failure if there were no substituters. - In that case the calling derivation should just do a - build. */ - done( - substituterFailed ? ecFailed : ecNoSubstituters, - BuildResult::NoSubstituters, - fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); - - if (substituterFailed) { - worker.failedSubstitutions++; - worker.updateProgress(); - } - - return; - } - - sub = subs.front(); - subs.pop_front(); - - if (ca) { - subPath = sub->makeFixedOutputPathFromCA( - std::string { storePath.name() }, - ContentAddressWithReferences::withoutRefs(*ca)); - if (sub->storeDir == worker.store.storeDir) - assert(subPath == storePath); - } else if (sub->storeDir != worker.store.storeDir) { - tryNext(); - return; - } - - try { - // FIXME: make async - info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath &) { - tryNext(); - return; - } catch (SubstituterDisabled &) { - if (settings.tryFallback) { - tryNext(); - return; - } - throw; - } catch (Error & e) { - if (settings.tryFallback) { - logError(e.info()); - tryNext(); - return; - } - throw; - } - - if (info->path != storePath) { - if (info->isContentAddressed(*sub) && info->references.empty()) { - auto info2 = std::make_shared(*info); - info2->path = storePath; - info = info2; - } else { - printError("asked '%s' for '%s' but got '%s'", - sub->getUri(), worker.store.printStorePath(storePath), sub->printStorePath(info->path)); - tryNext(); - return; - } - } - - /* Update the total expected download size. */ - auto narInfo = std::dynamic_pointer_cast(info); - - maintainExpectedNar = std::make_unique>(worker.expectedNarSize, info->narSize); - - maintainExpectedDownload = - narInfo && narInfo->fileSize - ? std::make_unique>(worker.expectedDownloadSize, narInfo->fileSize) - : nullptr; - - worker.updateProgress(); - - /* Bail out early if this substituter lacks a valid - signature. LocalStore::addToStore() also checks for this, but - only after we've downloaded the path. */ - if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info)) - { - warn("ignoring substitute for '%s' from '%s', as it's not signed by any of the keys in 'trusted-public-keys'", - worker.store.printStorePath(storePath), sub->getUri()); - tryNext(); - return; - } - - /* To maintain the closure invariant, we first have to realise the - paths referenced by this one. */ - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - addWaitee(worker.makePathSubstitutionGoal(i)); - - if (waitees.empty()) /* to prevent hang (no wake-up event) */ - referencesValid(); - else - state = &PathSubstitutionGoal::referencesValid; -} - - -void PathSubstitutionGoal::referencesValid() -{ - trace("all references realised"); - - if (nrFailed > 0) { - done( - nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed, - BuildResult::DependencyFailed, - fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath))); - return; - } - - for (auto & i : info->references) - if (i != storePath) /* ignore self-references */ - assert(worker.store.isValidPath(i)); - - state = &PathSubstitutionGoal::tryToRun; - worker.wakeUp(shared_from_this()); -} - - -void PathSubstitutionGoal::tryToRun() -{ - trace("trying to run"); - - /* Make sure that we are allowed to start a substitution. Note that even - if maxSubstitutionJobs == 0, we still allow a substituter to run. This - prevents infinite waiting. */ - if (worker.getNrSubstitutions() >= std::max(1U, (unsigned int) settings.maxSubstitutionJobs)) { - worker.waitForBuildSlot(shared_from_this()); - return; - } - - maintainRunningSubstitutions = std::make_unique>(worker.runningSubstitutions); - worker.updateProgress(); - - outPipe.create(); - - promise = std::promise(); - - thr = std::thread([this]() { - try { - ReceiveInterrupts receiveInterrupts; - - /* Wake up the worker loop when we're done. */ - Finally updateStats([this]() { outPipe.writeSide.close(); }); - - Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()}); - PushActivity pact(act.id); - - copyStorePath(*sub, worker.store, - subPath ? *subPath : storePath, repair, sub->isTrusted ? NoCheckSigs : CheckSigs); - - promise.set_value(); - } catch (...) { - promise.set_exception(std::current_exception()); - } - }); - - worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); - - state = &PathSubstitutionGoal::finished; -} - - -void PathSubstitutionGoal::finished() -{ - trace("substitute finished"); - - thr.join(); - worker.childTerminated(this); - - try { - promise.get_future().get(); - } catch (std::exception & e) { - printError(e.what()); - - /* Cause the parent build to fail unless --fallback is given, - or the substitute has disappeared. The latter case behaves - the same as the substitute never having existed in the - first place. */ - try { - throw; - } catch (SubstituteGone &) { - } catch (...) { - substituterFailed = true; - } - - /* Try the next substitute. */ - state = &PathSubstitutionGoal::tryNext; - worker.wakeUp(shared_from_this()); - return; - } - - worker.markContentsGood(storePath); - - printMsg(lvlChatty, "substitution of path '%s' succeeded", worker.store.printStorePath(storePath)); - - maintainRunningSubstitutions.reset(); - - maintainExpectedSubstitutions.reset(); - worker.doneSubstitutions++; - - if (maintainExpectedDownload) { - auto fileSize = maintainExpectedDownload->delta; - maintainExpectedDownload.reset(); - worker.doneDownloadSize += fileSize; - } - - worker.doneNarSize += maintainExpectedNar->delta; - maintainExpectedNar.reset(); - - worker.updateProgress(); - - done(ecSuccess, BuildResult::Substituted); -} - - -void PathSubstitutionGoal::handleChildOutput(int fd, std::string_view data) -{ -} - - -void PathSubstitutionGoal::handleEOF(int fd) -{ - if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); -} - - -void PathSubstitutionGoal::cleanup() -{ - try { - if (thr.joinable()) { - // FIXME: signal worker thread to quit. - thr.join(); - worker.childTerminated(this); - } - - outPipe.close(); - } catch (...) { - ignoreException(); - } -} - - -} diff --git a/src/libstore/unix/builtins/unpack-channel.cc b/src/libstore/unix/builtins/unpack-channel.cc deleted file mode 100644 index a5f2b8e3a..000000000 --- a/src/libstore/unix/builtins/unpack-channel.cc +++ /dev/null @@ -1,33 +0,0 @@ -#include "builtins.hh" -#include "tarfile.hh" - -namespace nix { - -void builtinUnpackChannel( - const BasicDerivation & drv, - const std::map & outputs) -{ - auto getAttr = [&](const std::string & name) { - auto i = drv.env.find(name); - if (i == drv.env.end()) throw Error("attribute '%s' missing", name); - return i->second; - }; - - auto out = outputs.at("out"); - auto channelName = getAttr("channelName"); - auto src = getAttr("src"); - - createDirs(out); - - unpackTarfile(src, out); - - auto entries = std::filesystem::directory_iterator{out}; - auto fileName = entries->path().string(); - auto fileCount = std::distance(std::filesystem::begin(entries), std::filesystem::end(entries)); - - if (fileCount != 1) - throw Error("channel tarball '%s' contains more than one file", src); - std::filesystem::rename(fileName, (out + "/" + channelName)); -} - -} diff --git a/src/libstore/unix/meson.build b/src/libstore/unix/meson.build new file mode 100644 index 000000000..d9d190131 --- /dev/null +++ b/src/libstore/unix/meson.build @@ -0,0 +1,19 @@ +sources += files( + 'build/child.cc', + 'build/hook-instance.cc', + 'build/local-derivation-goal.cc', + 'pathlocks.cc', + 'user-lock.cc', +) + +include_dirs += include_directories( + '.', + 'build', +) + +headers += files( + 'build/child.hh', + 'build/hook-instance.hh', + 'build/local-derivation-goal.hh', + 'user-lock.hh', +) diff --git a/src/libstore/unix/pathlocks.cc b/src/libstore/unix/pathlocks.cc index af21319a7..1ec4579ec 100644 --- a/src/libstore/unix/pathlocks.cc +++ b/src/libstore/unix/pathlocks.cc @@ -45,7 +45,7 @@ bool lockFile(Descriptor desc, LockType lockType, bool wait) if (lockType == ltRead) type = LOCK_SH; else if (lockType == ltWrite) type = LOCK_EX; else if (lockType == ltNone) type = LOCK_UN; - else abort(); + else unreachable(); if (wait) { while (flock(desc, type) != 0) { diff --git a/src/libstore/unix/lock.cc b/src/libstore/unix/user-lock.cc similarity index 99% rename from src/libstore/unix/lock.cc rename to src/libstore/unix/user-lock.cc index 023c74e34..29f4b2cb3 100644 --- a/src/libstore/unix/lock.cc +++ b/src/libstore/unix/user-lock.cc @@ -1,12 +1,13 @@ -#include "lock.hh" +#include +#include +#include + +#include "user-lock.hh" #include "file-system.hh" #include "globals.hh" #include "pathlocks.hh" #include "users.hh" -#include -#include - namespace nix { #if __linux__ diff --git a/src/libstore/unix/lock.hh b/src/libstore/unix/user-lock.hh similarity index 94% rename from src/libstore/unix/lock.hh rename to src/libstore/unix/user-lock.hh index 1c268e1fb..a7caf8518 100644 --- a/src/libstore/unix/lock.hh +++ b/src/libstore/unix/user-lock.hh @@ -1,10 +1,8 @@ #pragma once ///@file -#include "types.hh" - -#include - +#include +#include #include namespace nix { diff --git a/src/libstore/windows/build.cc b/src/libstore/windows/build.cc deleted file mode 100644 index 3eadc5bda..000000000 --- a/src/libstore/windows/build.cc +++ /dev/null @@ -1,37 +0,0 @@ -#include "store-api.hh" -#include "build-result.hh" - -namespace nix { - -void Store::buildPaths(const std::vector & reqs, BuildMode buildMode, std::shared_ptr evalStore) -{ - unsupported("buildPaths"); -} - -std::vector Store::buildPathsWithResults( - const std::vector & reqs, - BuildMode buildMode, - std::shared_ptr evalStore) -{ - unsupported("buildPathsWithResults"); -} - -BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, - BuildMode buildMode) -{ - unsupported("buildDerivation"); -} - - -void Store::ensurePath(const StorePath & path) -{ - unsupported("ensurePath"); -} - - -void Store::repairPath(const StorePath & path) -{ - unsupported("repairPath"); -} - -} diff --git a/src/libstore/windows/meson.build b/src/libstore/windows/meson.build new file mode 100644 index 000000000..b81c5b2af --- /dev/null +++ b/src/libstore/windows/meson.build @@ -0,0 +1,11 @@ +sources += files( + 'pathlocks.cc', +) + +include_dirs += include_directories( + '.', + #'build', +) + +headers += files( +) diff --git a/src/libstore/windows/pathlocks.cc b/src/libstore/windows/pathlocks.cc index 738057f68..00761a8c3 100644 --- a/src/libstore/windows/pathlocks.cc +++ b/src/libstore/windows/pathlocks.cc @@ -9,6 +9,8 @@ namespace nix { +using namespace nix::windows; + void deleteLockFile(const Path & path, Descriptor desc) { @@ -35,8 +37,13 @@ void PathLocks::unlock() AutoCloseFD openLockFile(const Path & path, bool create) { AutoCloseFD desc = CreateFileA( - path.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE, NULL, - create ? OPEN_ALWAYS : OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL | FILE_FLAG_POSIX_SEMANTICS, NULL); + path.c_str(), + GENERIC_READ | GENERIC_WRITE, + FILE_SHARE_READ | FILE_SHARE_WRITE, + NULL, + create ? OPEN_ALWAYS : OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL | FILE_FLAG_POSIX_SEMANTICS, + NULL); if (desc.get() == INVALID_HANDLE_VALUE) warn("%s: %s", path, std::to_string(GetLastError())); diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc new file mode 100644 index 000000000..6585df4be --- /dev/null +++ b/src/libstore/worker-protocol-connection.cc @@ -0,0 +1,325 @@ +#include "worker-protocol-connection.hh" +#include "worker-protocol-impl.hh" +#include "build-result.hh" +#include "derivations.hh" + +namespace nix { + +const std::set WorkerProto::allFeatures{}; + +WorkerProto::BasicClientConnection::~BasicClientConnection() +{ + try { + to.flush(); + } catch (...) { + ignoreExceptionInDestructor(); + } +} + +static Logger::Fields readFields(Source & from) +{ + Logger::Fields fields; + size_t size = readInt(from); + for (size_t n = 0; n < size; n++) { + auto type = (decltype(Logger::Field::type)) readInt(from); + if (type == Logger::Field::tInt) + fields.push_back(readNum(from)); + else if (type == Logger::Field::tString) + fields.push_back(readString(from)); + else + throw Error("got unsupported field type %x from Nix daemon", (int) type); + } + return fields; +} + +std::exception_ptr +WorkerProto::BasicClientConnection::processStderrReturn(Sink * sink, Source * source, bool flush, bool block) +{ + if (flush) + to.flush(); + + std::exception_ptr ex; + + while (true) { + + if (!block && !from.hasData()) + break; + + auto msg = readNum(from); + + if (msg == STDERR_WRITE) { + auto s = readString(from); + if (!sink) + throw Error("no sink"); + (*sink)(s); + } + + else if (msg == STDERR_READ) { + if (!source) + throw Error("no source"); + size_t len = readNum(from); + auto buf = std::make_unique(len); + writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to); + to.flush(); + } + + else if (msg == STDERR_ERROR) { + if (GET_PROTOCOL_MINOR(protoVersion) >= 26) { + ex = std::make_exception_ptr(readError(from)); + } else { + auto error = readString(from); + unsigned int status = readInt(from); + ex = std::make_exception_ptr(Error(status, error)); + } + break; + } + + else if (msg == STDERR_NEXT) + printError(chomp(readString(from))); + + else if (msg == STDERR_START_ACTIVITY) { + auto act = readNum(from); + auto lvl = (Verbosity) readInt(from); + auto type = (ActivityType) readInt(from); + auto s = readString(from); + auto fields = readFields(from); + auto parent = readNum(from); + logger->startActivity(act, lvl, type, s, fields, parent); + } + + else if (msg == STDERR_STOP_ACTIVITY) { + auto act = readNum(from); + logger->stopActivity(act); + } + + else if (msg == STDERR_RESULT) { + auto act = readNum(from); + auto type = (ResultType) readInt(from); + auto fields = readFields(from); + logger->result(act, type, fields); + } + + else if (msg == STDERR_LAST) { + assert(block); + break; + } + + else + throw Error("got unknown message type %x from Nix daemon", msg); + } + + if (!ex) { + return ex; + } else { + try { + std::rethrow_exception(ex); + } catch (const Error & e) { + // Nix versions before #4628 did not have an adequate + // behavior for reporting that the derivation format was + // upgraded. To avoid having to add compatibility logic in + // many places, we expect to catch almost all occurrences of + // the old incomprehensible error here, so that we can + // explain to users what's going on when their daemon is + // older than #4628 (2023). + if (experimentalFeatureSettings.isEnabled(Xp::DynamicDerivations) + && GET_PROTOCOL_MINOR(protoVersion) <= 35) { + auto m = e.msg(); + if (m.find("parsing derivation") != std::string::npos && m.find("expected string") != std::string::npos + && m.find("Derive([") != std::string::npos) + return std::make_exception_ptr(Error( + "%s, this might be because the daemon is too old to understand dependencies on dynamic derivations. Check to see if the raw derivation is in the form '%s'", + std::move(m), + "Drv WithVersion(..)")); + } + return std::current_exception(); + } + } +} + +void WorkerProto::BasicClientConnection::processStderr( + bool * daemonException, Sink * sink, Source * source, bool flush, bool block) +{ + auto ex = processStderrReturn(sink, source, flush, block); + if (ex) { + *daemonException = true; + std::rethrow_exception(ex); + } +} + +static std::set +intersectFeatures(const std::set & a, const std::set & b) +{ + std::set res; + for (auto & x : a) + if (b.contains(x)) + res.insert(x); + return res; +} + +std::tuple> WorkerProto::BasicClientConnection::handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures) +{ + to << WORKER_MAGIC_1 << localVersion; + to.flush(); + + unsigned int magic = readInt(from); + if (magic != WORKER_MAGIC_2) + throw Error("nix-daemon protocol mismatch from"); + auto daemonVersion = readInt(from); + + if (GET_PROTOCOL_MAJOR(daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION)) + throw Error("Nix daemon protocol version not supported"); + if (GET_PROTOCOL_MINOR(daemonVersion) < 10) + throw Error("the Nix daemon version is too old"); + + auto protoVersion = std::min(daemonVersion, localVersion); + + /* Exchange features. */ + std::set daemonFeatures; + if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { + to << supportedFeatures; + to.flush(); + daemonFeatures = readStrings>(from); + } + + return {protoVersion, intersectFeatures(daemonFeatures, supportedFeatures)}; +} + +std::tuple> WorkerProto::BasicServerConnection::handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures) +{ + unsigned int magic = readInt(from); + if (magic != WORKER_MAGIC_1) + throw Error("protocol mismatch"); + to << WORKER_MAGIC_2 << localVersion; + to.flush(); + auto clientVersion = readInt(from); + + auto protoVersion = std::min(clientVersion, localVersion); + + /* Exchange features. */ + std::set clientFeatures; + if (GET_PROTOCOL_MINOR(protoVersion) >= 38) { + clientFeatures = readStrings>(from); + to << supportedFeatures; + to.flush(); + } + + return {protoVersion, intersectFeatures(clientFeatures, supportedFeatures)}; +} + +WorkerProto::ClientHandshakeInfo WorkerProto::BasicClientConnection::postHandshake(const StoreDirConfig & store) +{ + WorkerProto::ClientHandshakeInfo res; + + if (GET_PROTOCOL_MINOR(protoVersion) >= 14) { + // Obsolete CPU affinity. + to << 0; + } + + if (GET_PROTOCOL_MINOR(protoVersion) >= 11) + to << false; // obsolete reserveSpace + + if (GET_PROTOCOL_MINOR(protoVersion) >= 33) + to.flush(); + + return WorkerProto::Serialise::read(store, *this); +} + +void WorkerProto::BasicServerConnection::postHandshake(const StoreDirConfig & store, const ClientHandshakeInfo & info) +{ + if (GET_PROTOCOL_MINOR(protoVersion) >= 14 && readInt(from)) { + // Obsolete CPU affinity. + readInt(from); + } + + if (GET_PROTOCOL_MINOR(protoVersion) >= 11) + readInt(from); // obsolete reserveSpace + + WorkerProto::write(store, *this, info); +} + +UnkeyedValidPathInfo WorkerProto::BasicClientConnection::queryPathInfo( + const StoreDirConfig & store, bool * daemonException, const StorePath & path) +{ + to << WorkerProto::Op::QueryPathInfo << store.printStorePath(path); + try { + processStderr(daemonException); + } catch (Error & e) { + // Ugly backwards compatibility hack. + if (e.msg().find("is not valid") != std::string::npos) + throw InvalidPath(std::move(e.info())); + throw; + } + if (GET_PROTOCOL_MINOR(protoVersion) >= 17) { + bool valid; + from >> valid; + if (!valid) + throw InvalidPath("path '%s' is not valid", store.printStorePath(path)); + } + return WorkerProto::Serialise::read(store, *this); +} + +StorePathSet WorkerProto::BasicClientConnection::queryValidPaths( + const StoreDirConfig & store, bool * daemonException, const StorePathSet & paths, SubstituteFlag maybeSubstitute) +{ + assert(GET_PROTOCOL_MINOR(protoVersion) >= 12); + to << WorkerProto::Op::QueryValidPaths; + WorkerProto::write(store, *this, paths); + if (GET_PROTOCOL_MINOR(protoVersion) >= 27) { + to << maybeSubstitute; + } + processStderr(daemonException); + return WorkerProto::Serialise::read(store, *this); +} + +void WorkerProto::BasicClientConnection::addTempRoot( + const StoreDirConfig & store, bool * daemonException, const StorePath & path) +{ + to << WorkerProto::Op::AddTempRoot << store.printStorePath(path); + processStderr(daemonException); + readInt(from); +} + +void WorkerProto::BasicClientConnection::putBuildDerivationRequest( + const StoreDirConfig & store, + bool * daemonException, + const StorePath & drvPath, + const BasicDerivation & drv, + BuildMode buildMode) +{ + to << WorkerProto::Op::BuildDerivation << store.printStorePath(drvPath); + writeDerivation(to, store, drv); + to << buildMode; +} + +BuildResult +WorkerProto::BasicClientConnection::getBuildDerivationResponse(const StoreDirConfig & store, bool * daemonException) +{ + return WorkerProto::Serialise::read(store, *this); +} + +void WorkerProto::BasicClientConnection::narFromPath( + const StoreDirConfig & store, bool * daemonException, const StorePath & path, std::function fun) +{ + to << WorkerProto::Op::NarFromPath << store.printStorePath(path); + processStderr(daemonException); + + fun(from); +} + +void WorkerProto::BasicClientConnection::importPaths( + const StoreDirConfig & store, bool * daemonException, Source & source) +{ + to << WorkerProto::Op::ImportPaths; + processStderr(daemonException, 0, &source); + auto importedPaths = WorkerProto::Serialise::read(store, *this); + assert(importedPaths.size() <= importedPaths.size()); +} +} diff --git a/src/libstore/worker-protocol-connection.hh b/src/libstore/worker-protocol-connection.hh new file mode 100644 index 000000000..9665067dd --- /dev/null +++ b/src/libstore/worker-protocol-connection.hh @@ -0,0 +1,171 @@ +#pragma once +///@file + +#include "worker-protocol.hh" +#include "store-api.hh" + +namespace nix { + +struct WorkerProto::BasicConnection +{ + /** + * Send with this. + */ + FdSink to; + + /** + * Receive with this. + */ + FdSource from; + + /** + * The protocol version agreed by both sides. + */ + WorkerProto::Version protoVersion; + + /** + * The set of features that both sides support. + */ + std::set features; + + /** + * Coercion to `WorkerProto::ReadConn`. This makes it easy to use the + * factored out serve protocol serializers with a + * `LegacySSHStore::Connection`. + * + * The serve protocol connection types are unidirectional, unlike + * this type. + */ + operator WorkerProto::ReadConn() + { + return WorkerProto::ReadConn{ + .from = from, + .version = protoVersion, + }; + } + + /** + * Coercion to `WorkerProto::WriteConn`. This makes it easy to use the + * factored out serve protocol serializers with a + * `LegacySSHStore::Connection`. + * + * The serve protocol connection types are unidirectional, unlike + * this type. + */ + operator WorkerProto::WriteConn() + { + return WorkerProto::WriteConn{ + .to = to, + .version = protoVersion, + }; + } +}; + +struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection +{ + /** + * Flush to direction + */ + virtual ~BasicClientConnection(); + + virtual void closeWrite() = 0; + + std::exception_ptr processStderrReturn(Sink * sink = 0, Source * source = 0, bool flush = true, bool block = true); + + void + processStderr(bool * daemonException, Sink * sink = 0, Source * source = 0, bool flush = true, bool block = true); + + /** + * Establishes connection, negotiating version. + * + * @return the minimum version supported by both sides and the set + * of protocol features supported by both sides. + * + * @param to Taken by reference to allow for various error handling + * mechanisms. + * + * @param from Taken by reference to allow for various error + * handling mechanisms. + * + * @param localVersion Our version which is sent over + * + * @param features The protocol features that we support + */ + // FIXME: this should probably be a constructor. + static std::tuple> handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures); + + /** + * After calling handshake, must call this to exchange some basic + * information abou the connection. + */ + ClientHandshakeInfo postHandshake(const StoreDirConfig & store); + + void addTempRoot(const StoreDirConfig & remoteStore, bool * daemonException, const StorePath & path); + + StorePathSet queryValidPaths( + const StoreDirConfig & remoteStore, + bool * daemonException, + const StorePathSet & paths, + SubstituteFlag maybeSubstitute); + + UnkeyedValidPathInfo queryPathInfo(const StoreDirConfig & store, bool * daemonException, const StorePath & path); + + void putBuildDerivationRequest( + const StoreDirConfig & store, + bool * daemonException, + const StorePath & drvPath, + const BasicDerivation & drv, + BuildMode buildMode); + + /** + * Get the response, must be paired with + * `putBuildDerivationRequest`. + */ + BuildResult getBuildDerivationResponse(const StoreDirConfig & store, bool * daemonException); + + void narFromPath( + const StoreDirConfig & store, + bool * daemonException, + const StorePath & path, + std::function fun); + + void importPaths(const StoreDirConfig & store, bool * daemonException, Source & source); +}; + +struct WorkerProto::BasicServerConnection : WorkerProto::BasicConnection +{ + /** + * Establishes connection, negotiating version. + * + * @return the version provided by the other side of the + * connection. + * + * @param to Taken by reference to allow for various error handling + * mechanisms. + * + * @param from Taken by reference to allow for various error + * handling mechanisms. + * + * @param localVersion Our version which is sent over + * + * @param features The protocol features that we support + */ + // FIXME: this should probably be a constructor. + static std::tuple> handshake( + BufferedSink & to, + Source & from, + WorkerProto::Version localVersion, + const std::set & supportedFeatures); + + /** + * After calling handshake, must call this to exchange some basic + * information abou the connection. + */ + void postHandshake(const StoreDirConfig & store, const ClientHandshakeInfo & info); +}; + +} diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index a50259d24..f06fb2893 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -14,6 +14,34 @@ namespace nix { /* protocol-specific definitions */ +BuildMode WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) +{ + auto temp = readNum(conn.from); + switch (temp) { + case 0: return bmNormal; + case 1: return bmRepair; + case 2: return bmCheck; + default: throw Error("Invalid build mode"); + } +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const BuildMode & buildMode) +{ + switch (buildMode) { + case bmNormal: + conn.to << uint8_t{0}; + break; + case bmRepair: + conn.to << uint8_t{1}; + break; + case bmCheck: + conn.to << uint8_t{2}; + break; + default: + assert(false); + }; +} + std::optional WorkerProto::Serialise>::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { auto temp = readNum(conn.from); @@ -222,4 +250,35 @@ void WorkerProto::Serialise::write(const StoreDirConfig & } } + +WorkerProto::ClientHandshakeInfo WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) +{ + WorkerProto::ClientHandshakeInfo res; + + if (GET_PROTOCOL_MINOR(conn.version) >= 33) { + res.daemonNixVersion = readString(conn.from); + } + + if (GET_PROTOCOL_MINOR(conn.version) >= 35) { + res.remoteTrustsUs = WorkerProto::Serialise>::read(store, conn); + } else { + // We don't know the answer; protocol to old. + res.remoteTrustsUs = std::nullopt; + } + + return res; +} + +void WorkerProto::Serialise::write(const StoreDirConfig & store, WriteConn conn, const WorkerProto::ClientHandshakeInfo & info) +{ + if (GET_PROTOCOL_MINOR(conn.version) >= 33) { + assert(info.daemonNixVersion); + conn.to << *info.daemonNixVersion; + } + + if (GET_PROTOCOL_MINOR(conn.version) >= 35) { + WorkerProto::write(store, conn, info.remoteTrustsUs); + } +} + } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 91d277b77..c356fa1bf 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -11,7 +11,9 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION (1 << 8 | 37) +/* Note: you generally shouldn't change the protocol version. Define a + new `WorkerProto::Feature` instead. */ +#define PROTOCOL_VERSION (1 << 8 | 38) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -35,6 +37,7 @@ struct BuildResult; struct KeyedBuildResult; struct ValidPathInfo; struct UnkeyedValidPathInfo; +enum BuildMode : uint8_t; enum TrustedFlag : bool; @@ -76,6 +79,20 @@ struct WorkerProto Version version; }; + /** + * Stripped down serialization logic suitable for sharing with Hydra. + * + * @todo remove once Hydra uses Store abstraction consistently. + */ + struct BasicConnection; + struct BasicClientConnection; + struct BasicServerConnection; + + /** + * Extra information provided as part of protocol negotation. + */ + struct ClientHandshakeInfo; + /** * Data type for canonical pairs of serialisers for the worker protocol. * @@ -116,6 +133,10 @@ struct WorkerProto { WorkerProto::Serialise::write(store, conn, t); } + + using Feature = std::string; + + static const std::set allFeatures; }; enum struct WorkerProto::Op : uint64_t @@ -166,6 +187,32 @@ enum struct WorkerProto::Op : uint64_t AddPermRoot = 47, }; +struct WorkerProto::ClientHandshakeInfo +{ + /** + * The version of the Nix daemon that is processing our requests. + * + * Do note, it may or may not communicating with another daemon, + * rather than being an "end" `LocalStore` or similar. + */ + std::optional daemonNixVersion; + + /** + * Whether the remote side trusts us or not. + * + * 3 values: "yes", "no", or `std::nullopt` for "unknown". + * + * Note that the "remote side" might not be just the end daemon, but + * also an intermediary forwarder that can make its own trusting + * decisions. This would be the intersection of all their trust + * decisions, since it takes only one link in the chain to start + * denying operations. + */ + std::optional remoteTrustsUs; + + bool operator == (const ClientHandshakeInfo &) const = default; +}; + /** * Convenience for sending operation codes. * @@ -215,9 +262,13 @@ DECLARE_WORKER_SERIALISER(ValidPathInfo); template<> DECLARE_WORKER_SERIALISER(UnkeyedValidPathInfo); template<> +DECLARE_WORKER_SERIALISER(BuildMode); +template<> DECLARE_WORKER_SERIALISER(std::optional); template<> DECLARE_WORKER_SERIALISER(std::optional); +template<> +DECLARE_WORKER_SERIALISER(WorkerProto::ClientHandshakeInfo); template DECLARE_WORKER_SERIALISER(std::vector); diff --git a/src/libutil-c/.version b/src/libutil-c/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libutil-c/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libutil-c/build-utils-meson b/src/libutil-c/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libutil-c/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build new file mode 100644 index 000000000..b5ed19631 --- /dev/null +++ b/src/libutil-c/meson.build @@ -0,0 +1,81 @@ +project('nix-util-c', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ + dependency('nix-util'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +# TODO rename, because it will conflict with downstream projects +configdata.set_quoted('PACKAGE_VERSION', meson.project_version()) + +config_h = configure_file( + configuration : configdata, + output : 'config-util.h', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + + # From C++ libraries, only for internals + '-include', 'config-util.hh', + + # From C libraries, for our public, installed headers too + '-include', 'config-util.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'nix_api_util.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'nix_api_util.h', +) + +# TODO don't install this once tests don't use it. +headers += files('nix_api_util_internal.h') + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixutilc', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/src/libutil-c/meson.options b/src/libutil-c/meson.options new file mode 100644 index 000000000..04422feaf --- /dev/null +++ b/src/libutil-c/meson.options @@ -0,0 +1 @@ +# vim: filetype=meson diff --git a/src/libutil-c/nix-util-c.pc.in b/src/libutil-c/nix-util-c.pc.in new file mode 100644 index 000000000..0ccae3f8a --- /dev/null +++ b/src/libutil-c/nix-util-c.pc.in @@ -0,0 +1,9 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix libutil C API +Description: Common functions for the Nix C API, such as error handling +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lnixutil +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libutil-c/nix_api_util.cc b/src/libutil-c/nix_api_util.cc index 0a9b49345..992ea0a2a 100644 --- a/src/libutil-c/nix_api_util.cc +++ b/src/libutil-c/nix_api_util.cc @@ -1,5 +1,5 @@ #include "nix_api_util.h" -#include "config.hh" +#include "config-global.hh" #include "error.hh" #include "nix_api_util_internal.h" #include "util.hh" @@ -57,6 +57,12 @@ nix_err nix_set_err_msg(nix_c_context * context, nix_err err, const char * msg) return err; } +void nix_clear_err(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; +} + const char * nix_version_get() { return PACKAGE_VERSION; @@ -106,7 +112,7 @@ const char * nix_err_msg(nix_c_context * context, const nix_c_context * read_con { if (context) context->last_err_code = NIX_OK; - if (read_context->last_err) { + if (read_context->last_err && read_context->last_err_code != NIX_OK) { if (n) *n = read_context->last_err->size(); return read_context->last_err->c_str(); diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index e0ca04e69..6790a6964 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -56,47 +56,51 @@ extern "C" { * - NIX_ERR_KEY: A key error occurred (-3) * - NIX_ERR_NIX_ERROR: A generic Nix error occurred (-4) */ -typedef int nix_err; +enum nix_err { -/** - * @brief No error occurred. - * - * This error code is returned when no error has occurred during the function - * execution. - */ -#define NIX_OK 0 + /** + * @brief No error occurred. + * + * This error code is returned when no error has occurred during the function + * execution. + */ + NIX_OK = 0, -/** - * @brief An unknown error occurred. - * - * This error code is returned when an unknown error occurred during the - * function execution. - */ -#define NIX_ERR_UNKNOWN -1 + /** + * @brief An unknown error occurred. + * + * This error code is returned when an unknown error occurred during the + * function execution. + */ + NIX_ERR_UNKNOWN = -1, -/** - * @brief An overflow error occurred. - * - * This error code is returned when an overflow error occurred during the - * function execution. - */ -#define NIX_ERR_OVERFLOW -2 + /** + * @brief An overflow error occurred. + * + * This error code is returned when an overflow error occurred during the + * function execution. + */ + NIX_ERR_OVERFLOW = -2, -/** - * @brief A key error occurred. - * - * This error code is returned when a key error occurred during the function - * execution. - */ -#define NIX_ERR_KEY -3 + /** + * @brief A key error occurred. + * + * This error code is returned when a key error occurred during the function + * execution. + */ + NIX_ERR_KEY = -3, -/** - * @brief A generic Nix error occurred. - * - * This error code is returned when a generic Nix error occurred during the - * function execution. - */ -#define NIX_ERR_NIX_ERROR -4 + /** + * @brief A generic Nix error occurred. + * + * This error code is returned when a generic Nix error occurred during the + * function execution. + */ + NIX_ERR_NIX_ERROR = -4, + +}; + +typedef enum nix_err nix_err; /** * @brief This object stores error state. @@ -221,7 +225,9 @@ const char * nix_version_get(); * @param[out] n optional: a pointer to an unsigned int that is set to the * length of the error. * @return nullptr if no error message was ever set, - * a borrowed pointer to the error message otherwise. + * a borrowed pointer to the error message otherwise, which is valid + * until the next call to a Nix function, or until the context is + * destroyed. */ const char * nix_err_msg(nix_c_context * context, const nix_c_context * ctx, unsigned int * n); @@ -282,13 +288,34 @@ nix_err nix_err_code(const nix_c_context * read_context); * * All other use is internal to the API. * - * @param context context to write the error message to, or NULL + * @param context context to write the error message to, required unless C++ exceptions are supported * @param err The error code to set and return - * @param msg The error message to set. + * @param msg The error message to set. This string is copied. * @returns the error code set */ nix_err nix_set_err_msg(nix_c_context * context, nix_err err, const char * msg); +/** + * @brief Clear the error message from a nix context. + * + * This is performed implicitly by all functions that accept a context, so + * this won't be necessary in most cases. + * However, if you want to clear the error message without calling another + * function, you can use this. + * + * Example use case: a higher order function that helps with error handling, + * to make it more robust in the following scenario: + * + * 1. A previous call failed, and the error was caught and handled. + * 2. The context is reused with our error handling helper function. + * 3. The callback passed to the helper function doesn't actually make a call to + * a Nix function. + * 4. The handled error is raised again, from an unrelated call. + * + * This failure can be avoided by clearing the error message after handling it. + */ +void nix_clear_err(nix_c_context * context); + /** * @} */ diff --git a/src/libutil-c/nix_api_util_internal.h b/src/libutil-c/nix_api_util_internal.h index aa829feaf..7fa4252ac 100644 --- a/src/libutil-c/nix_api_util_internal.h +++ b/src/libutil-c/nix_api_util_internal.h @@ -10,6 +10,7 @@ struct nix_c_context { nix_err last_err_code = NIX_OK; + /** The last error message. Always check last_err_code. This may not have been cleared, so that clearing is fast. */ std::optional last_err = {}; std::optional info = {}; std::string name = ""; diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix new file mode 100644 index 000000000..ccfafd4d3 --- /dev/null +++ b/src/libutil-c/package.nix @@ -0,0 +1,73 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-util-c"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + (fileset.fileFilter (file: file.hasExt "h") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libutil/.version b/src/libutil/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/libutil/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 04f777d00..20d8a1e09 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -6,7 +6,7 @@ #include // for strcasecmp #include "archive.hh" -#include "config.hh" +#include "config-global.hh" #include "posix-source-accessor.hh" #include "source-path.hh" #include "file-system.hh" @@ -23,7 +23,7 @@ struct ArchiveSettings : Config false, #endif "use-case-hack", - "Whether to enable a Darwin-specific hack for dealing with file name collisions."}; + "Whether to enable a macOS-specific hack for dealing with file name case collisions."}; }; static ArchiveSettings archiveSettings; @@ -82,7 +82,7 @@ void SourceAccessor::dumpPath( name.erase(pos); } if (!unhacked.emplace(name, i.first).second) - throw Error("file name collision in between '%s' and '%s'", + throw Error("file name collision between '%s' and '%s'", (path / unhacked[name]), (path / i.first)); } else @@ -128,9 +128,10 @@ void dumpString(std::string_view s, Sink & sink) } -static SerialisationError badArchive(const std::string & s) +template +static SerialisationError badArchive(std::string_view s, const Args & ... args) { - return SerialisationError("bad archive: " + s); + return SerialisationError("bad archive: " + s, args...); } @@ -165,117 +166,99 @@ struct CaseInsensitiveCompare }; -static void parse(FileSystemObjectSink & sink, Source & source, const Path & path) +static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath & path) { - std::string s; - - s = readString(source); - if (s != "(") throw badArchive("expected open tag"); - - std::map names; - auto getString = [&]() { checkInterrupt(); return readString(source); }; - // For first iteration - s = getString(); + auto expectTag = [&](std::string_view expected) { + auto tag = getString(); + if (tag != expected) + throw badArchive("expected tag '%s', got '%s'", expected, tag); + }; - while (1) { + expectTag("("); - if (s == ")") { - break; - } + expectTag("type"); - else if (s == "type") { - std::string t = getString(); + auto type = getString(); - if (t == "regular") { - sink.createRegularFile(path, [&](auto & crf) { - while (1) { - s = getString(); + if (type == "regular") { + sink.createRegularFile(path, [&](auto & crf) { + auto tag = getString(); - if (s == "contents") { - parseContents(crf, source); - } - - else if (s == "executable") { - auto s2 = getString(); - if (s2 != "") throw badArchive("executable marker has non-empty value"); - crf.isExecutable(); - } - - else break; - } - }); + if (tag == "executable") { + auto s2 = getString(); + if (s2 != "") throw badArchive("executable marker has non-empty value"); + crf.isExecutable(); + tag = getString(); } - else if (t == "directory") { - sink.createDirectory(path); + if (tag == "contents") + parseContents(crf, source); - while (1) { - s = getString(); - - if (s == "entry") { - std::string name, prevName; - - s = getString(); - if (s != "(") throw badArchive("expected open tag"); - - while (1) { - s = getString(); - - if (s == ")") { - break; - } else if (s == "name") { - name = getString(); - if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos || name.find((char) 0) != std::string::npos) - throw Error("NAR contains invalid file name '%1%'", name); - if (name <= prevName) - throw Error("NAR directory is not sorted"); - prevName = name; - if (archiveSettings.useCaseHack) { - auto i = names.find(name); - if (i != names.end()) { - debug("case collision between '%1%' and '%2%'", i->first, name); - name += caseHackSuffix; - name += std::to_string(++i->second); - } else - names[name] = 0; - } - } else if (s == "node") { - if (name.empty()) throw badArchive("entry name missing"); - parse(sink, source, path + "/" + name); - } else - throw badArchive("unknown field " + s); - } - } - - else break; - } - } - - else if (t == "symlink") { - s = getString(); - - if (s != "target") - throw badArchive("expected 'target' got " + s); - - std::string target = getString(); - sink.createSymlink(path, target); - - // for the next iteration - s = getString(); - } - - else throw badArchive("unknown file type " + t); - - } - - else - throw badArchive("unknown field " + s); + expectTag(")"); + }); } + + else if (type == "directory") { + sink.createDirectory(path); + + std::map names; + + std::string prevName; + + while (1) { + auto tag = getString(); + + if (tag == ")") break; + + if (tag != "entry") + throw badArchive("expected tag 'entry' or ')', got '%s'", tag); + + expectTag("("); + + expectTag("name"); + + auto name = getString(); + if (name.empty() || name == "." || name == ".." || name.find('/') != std::string::npos || name.find((char) 0) != std::string::npos) + throw badArchive("NAR contains invalid file name '%1%'", name); + if (name <= prevName) + throw badArchive("NAR directory is not sorted"); + prevName = name; + if (archiveSettings.useCaseHack) { + auto i = names.find(name); + if (i != names.end()) { + debug("case collision between '%1%' and '%2%'", i->first, name); + name += caseHackSuffix; + name += std::to_string(++i->second); + auto j = names.find(name); + if (j != names.end()) + throw badArchive("NAR contains file name '%s' that collides with case-hacked file name '%s'", prevName, j->first); + } else + names[name] = 0; + } + + expectTag("node"); + + parse(sink, source, path / name); + + expectTag(")"); + } + } + + else if (type == "symlink") { + expectTag("target"); + + auto target = getString(); + sink.createSymlink(path, target); + + expectTag(")"); + } + + else throw badArchive("unknown file type '%s'", type); } @@ -290,13 +273,13 @@ void parseDump(FileSystemObjectSink & sink, Source & source) } if (version != narVersionMagic1) throw badArchive("input doesn't look like a Nix archive"); - parse(sink, source, ""); + parse(sink, source, CanonPath::root); } -void restorePath(const Path & path, Source & source) +void restorePath(const std::filesystem::path & path, Source & source, bool startFsync) { - RestoreSink sink; + RestoreSink sink{startFsync}; sink.dstPath = path; parseDump(sink, source); } @@ -315,13 +298,4 @@ void copyNAR(Source & source, Sink & sink) } -void copyPath(const Path & from, const Path & to) -{ - auto source = sinkToSource([&](Sink & sink) { - dumpPath(from, sink); - }); - restorePath(to, *source); -} - - } diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index 28c63bb85..c38fa8a46 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -75,15 +75,13 @@ void dumpString(std::string_view s, Sink & sink); void parseDump(FileSystemObjectSink & sink, Source & source); -void restorePath(const Path & path, Source & source); +void restorePath(const std::filesystem::path & path, Source & source, bool startFsync = false); /** * Read a NAR from 'source' and write it to 'sink'. */ void copyNAR(Source & source, Sink & sink); -void copyPath(const Path & from, const Path & to); - inline constexpr std::string_view narVersionMagic1 = "nix-archive-1"; diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 243e3a5a6..4e87389d6 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -57,8 +57,7 @@ void Completions::add(std::string completion, std::string description) }); } -bool Completion::operator<(const Completion & other) const -{ return completion < other.completion || (completion == other.completion && description < other.description); } +auto Completion::operator<=>(const Completion & other) const noexcept = default; std::string completionMarker = "___COMPLETE___"; @@ -268,8 +267,6 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) verbosity = lvlError; } - bool argsSeen = false; - // Heuristic to see if we're invoked as a shebang script, namely, // if we have at least one argument, it's the name of an // executable file, and it starts with "#!". @@ -296,7 +293,7 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) // We match one space after `nix` so that we preserve indentation. // No space is necessary for an empty line. An empty line has basically no effect. if (std::regex_match(line, match, std::regex("^#!\\s*nix(:? |$)(.*)$"))) - shebangContent += match[2].str() + "\n"; + shebangContent += std::string_view{match[2].first, match[2].second} + "\n"; } for (const auto & word : parseShebangContent(shebangContent)) { cmdline.push_back(word); @@ -336,10 +333,6 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) throw UsageError("unrecognised flag '%1%'", arg); } else { - if (!argsSeen) { - argsSeen = true; - initialFlagsProcessed(); - } pos = rewriteArgs(cmdline, pos); pendingArgs.push_back(*pos++); if (processArgs(pendingArgs, false)) @@ -349,8 +342,7 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) processArgs(pendingArgs, true); - if (!argsSeen) - initialFlagsProcessed(); + initialFlagsProcessed(); /* Now that we are done parsing, make sure that any experimental * feature required by the flags is enabled */ diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 7759b74a9..127a0809e 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -1,8 +1,8 @@ #pragma once ///@file -#include #include +#include #include #include #include @@ -11,6 +11,7 @@ #include "types.hh" #include "experimental-features.hh" +#include "ref.hh" namespace nix { @@ -113,6 +114,16 @@ protected: , arity(1) { } + Handler(std::filesystem::path * dest) + : fun([dest](std::vector ss) { *dest = ss[0]; }) + , arity(1) + { } + + Handler(std::optional * dest) + : fun([dest](std::vector ss) { *dest = ss[0]; }) + , arity(1) + { } + template Handler(T * dest, const T & val) : fun([dest, val](std::vector ss) { *dest = val; }) @@ -283,6 +294,18 @@ public: }); } + /** + * Expect a path argument. + */ + void expectArg(const std::string & label, std::filesystem::path * dest, bool optional = false) + { + expectArgs({ + .label = label, + .optional = optional, + .handler = {dest} + }); + } + /** * Expect 0 or more arguments. */ @@ -380,7 +403,7 @@ struct Completion { std::string completion; std::string description; - bool operator<(const Completion & other) const; + auto operator<=>(const Completion & other) const noexcept; }; /** diff --git a/src/libutil/args/root.hh b/src/libutil/args/root.hh index 5c55c37a5..34a43b538 100644 --- a/src/libutil/args/root.hh +++ b/src/libutil/args/root.hh @@ -29,6 +29,7 @@ struct Completions final : AddCompletions */ class RootArgs : virtual public Args { +protected: /** * @brief The command's "working directory", but only set when top level. * diff --git a/src/libutil/build-utils-meson b/src/libutil/build-utils-meson new file mode 120000 index 000000000..5fff21bab --- /dev/null +++ b/src/libutil/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson \ No newline at end of file diff --git a/src/libutil/canon-path.cc b/src/libutil/canon-path.cc index 27f048697..03db6378a 100644 --- a/src/libutil/canon-path.cc +++ b/src/libutil/canon-path.cc @@ -1,6 +1,7 @@ #include "canon-path.hh" #include "util.hh" #include "file-path-impl.hh" +#include "strings-inline.hh" namespace nix { diff --git a/src/libutil/canon-path.hh b/src/libutil/canon-path.hh index 8f5a1c279..f84347dc4 100644 --- a/src/libutil/canon-path.hh +++ b/src/libutil/canon-path.hh @@ -169,7 +169,7 @@ public: * a directory is always followed directly by its children. For * instance, 'foo' < 'foo/bar' < 'foo!'. */ - bool operator < (const CanonPath & x) const + auto operator <=> (const CanonPath & x) const { auto i = path.begin(); auto j = x.path.begin(); @@ -178,10 +178,9 @@ public: if (c_i == '/') c_i = 0; auto c_j = *j; if (c_j == '/') c_j = 0; - if (c_i < c_j) return true; - if (c_i > c_j) return false; + if (auto cmp = c_i <=> c_j; cmp != 0) return cmp; } - return i == path.end() && j != x.path.end(); + return (i != path.end()) <=> (j != x.path.end()); } /** diff --git a/src/libutil/checked-arithmetic.hh b/src/libutil/checked-arithmetic.hh new file mode 100644 index 000000000..55d6ad205 --- /dev/null +++ b/src/libutil/checked-arithmetic.hh @@ -0,0 +1,182 @@ +#pragma once +/** + * @file Checked arithmetic with classes that make it hard to accidentally make something an unchecked operation. + */ + +#include +#include // IWYU pragma: keep +#include +#include +#include +#include +#include + +namespace nix::checked { + +class DivideByZero : std::exception +{}; + +/** + * Numeric value enforcing checked arithmetic. Performing mathematical operations on such values will return a Result + * type which needs to be checked. + */ +template +struct Checked +{ + using Inner = T; + + // TODO: this must be a "trivial default constructor", which means it + // cannot set the value to NOT DO UB on uninit. + T value; + + Checked() = default; + explicit Checked(T const value) + : value{value} + { + } + Checked(Checked const & other) = default; + Checked(Checked && other) = default; + Checked & operator=(Checked const & other) = default; + + std::strong_ordering operator<=>(Checked const & other) const = default; + std::strong_ordering operator<=>(T const & other) const + { + return value <=> other; + } + + explicit operator T() const + { + return value; + } + + enum class OverflowKind { + NoOverflow, + Overflow, + DivByZero, + }; + + class Result + { + T value; + OverflowKind overflowed_; + + public: + Result(T value, bool overflowed) + : value{value} + , overflowed_{overflowed ? OverflowKind::Overflow : OverflowKind::NoOverflow} + { + } + Result(T value, OverflowKind overflowed) + : value{value} + , overflowed_{overflowed} + { + } + + bool operator==(Result other) const + { + return value == other.value && overflowed_ == other.overflowed_; + } + + std::optional valueChecked() const + { + if (overflowed_ != OverflowKind::NoOverflow) { + return std::nullopt; + } else { + return value; + } + } + + /** + * Returns the result as if the arithmetic were performed as wrapping arithmetic. + * + * \throws DivideByZero if the operation was a divide by zero. + */ + T valueWrapping() const + { + if (overflowed_ == OverflowKind::DivByZero) { + throw DivideByZero{}; + } + return value; + } + + bool overflowed() const + { + return overflowed_ == OverflowKind::Overflow; + } + + bool divideByZero() const + { + return overflowed_ == OverflowKind::DivByZero; + } + }; + + Result operator+(Checked const other) const + { + return (*this) + other.value; + } + Result operator+(T const other) const + { + T result; + bool overflowed = __builtin_add_overflow(value, other, &result); + return Result{result, overflowed}; + } + + Result operator-(Checked const other) const + { + return (*this) - other.value; + } + Result operator-(T const other) const + { + T result; + bool overflowed = __builtin_sub_overflow(value, other, &result); + return Result{result, overflowed}; + } + + Result operator*(Checked const other) const + { + return (*this) * other.value; + } + Result operator*(T const other) const + { + T result; + bool overflowed = __builtin_mul_overflow(value, other, &result); + return Result{result, overflowed}; + } + + Result operator/(Checked const other) const + { + return (*this) / other.value; + } + /** + * Performs a checked division. + * + * If the right hand side is zero, the result is marked as a DivByZero and + * valueWrapping will throw. + */ + Result operator/(T const other) const + { + constexpr T const minV = std::numeric_limits::min(); + + // It's only possible to overflow with signed division since doing so + // requires crossing the two's complement limits by MIN / -1 (since + // two's complement has one more in range in the negative direction + // than in the positive one). + if (std::is_signed() && (value == minV && other == -1)) { + return Result{minV, true}; + } else if (other == 0) { + return Result{0, OverflowKind::DivByZero}; + } else { + T result = value / other; + return Result{result, false}; + } + } +}; + +template +std::ostream & operator<<(std::ostream & ios, Checked v) +{ + ios << v.value; + return ios; +} + +} diff --git a/src/libutil/chunked-vector.hh b/src/libutil/chunked-vector.hh index d914e2542..4709679a6 100644 --- a/src/libutil/chunked-vector.hh +++ b/src/libutil/chunked-vector.hh @@ -6,6 +6,8 @@ #include #include +#include "error.hh" + namespace nix { /** @@ -30,7 +32,7 @@ private: auto & addChunk() { if (size_ >= std::numeric_limits::max() - ChunkSize) - abort(); + unreachable(); chunks.emplace_back(); chunks.back().reserve(ChunkSize); return chunks.back(); diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index cbc2bb4fd..34ba6f453 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -1,17 +1,8 @@ #pragma once ///@file -#define DECLARE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE) \ - PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const; -#define DECLARE_EQUAL(prefix, qualification, my_type) \ - DECLARE_ONE_CMP(prefix, qualification, ==, my_type) -#define DECLARE_LEQ(prefix, qualification, my_type) \ - DECLARE_ONE_CMP(prefix, qualification, <, my_type) -#define DECLARE_NEQ(prefix, qualification, my_type) \ - DECLARE_ONE_CMP(prefix, qualification, !=, my_type) - -#define GENERATE_ONE_CMP(PRE, QUAL, COMPARATOR, MY_TYPE, ...) \ - PRE bool QUAL operator COMPARATOR(const MY_TYPE & other) const { \ +#define GENERATE_ONE_CMP(PRE, RET, QUAL, COMPARATOR, MY_TYPE, ...) \ + PRE RET QUAL operator COMPARATOR(const MY_TYPE & other) const noexcept { \ __VA_OPT__(const MY_TYPE * me = this;) \ auto fields1 = std::tie( __VA_ARGS__ ); \ __VA_OPT__(me = &other;) \ @@ -19,30 +10,9 @@ return fields1 COMPARATOR fields2; \ } #define GENERATE_EQUAL(prefix, qualification, my_type, args...) \ - GENERATE_ONE_CMP(prefix, qualification, ==, my_type, args) -#define GENERATE_LEQ(prefix, qualification, my_type, args...) \ - GENERATE_ONE_CMP(prefix, qualification, <, my_type, args) -#define GENERATE_NEQ(prefix, qualification, my_type, args...) \ - GENERATE_ONE_CMP(prefix, qualification, !=, my_type, args) - -/** - * Declare comparison methods without defining them. - */ -#define DECLARE_CMP(my_type) \ - DECLARE_EQUAL(,,my_type) \ - DECLARE_LEQ(,,my_type) \ - DECLARE_NEQ(,,my_type) - -/** - * @param prefix This is for something before each declaration like - * `template`. - * - * @param my_type the type are defining operators for. - */ -#define DECLARE_CMP_EXT(prefix, qualification, my_type) \ - DECLARE_EQUAL(prefix, qualification, my_type) \ - DECLARE_LEQ(prefix, qualification, my_type) \ - DECLARE_NEQ(prefix, qualification, my_type) + GENERATE_ONE_CMP(prefix, bool, qualification, ==, my_type, args) +#define GENERATE_SPACESHIP(prefix, ret, qualification, my_type, args...) \ + GENERATE_ONE_CMP(prefix, ret, qualification, <=>, my_type, args) /** * Awful hacky generation of the comparison operators by doing a lexicographic @@ -55,15 +25,19 @@ * will generate comparison operators semantically equivalent to: * * ``` - * bool operator<(const ClassName& other) { - * return field1 < other.field1 && field2 < other.field2 && ...; + * auto operator<=>(const ClassName& other) const noexcept { + * if (auto cmp = field1 <=> other.field1; cmp != 0) + * return cmp; + * if (auto cmp = field2 <=> other.field2; cmp != 0) + * return cmp; + * ... + * return 0; * } * ``` */ #define GENERATE_CMP(args...) \ GENERATE_EQUAL(,,args) \ - GENERATE_LEQ(,,args) \ - GENERATE_NEQ(,,args) + GENERATE_SPACESHIP(,auto,,args) /** * @param prefix This is for something before each declaration like @@ -71,7 +45,6 @@ * * @param my_type the type are defining operators for. */ -#define GENERATE_CMP_EXT(prefix, my_type, args...) \ +#define GENERATE_CMP_EXT(prefix, ret, my_type, args...) \ GENERATE_EQUAL(prefix, my_type ::, my_type, args) \ - GENERATE_LEQ(prefix, my_type ::, my_type, args) \ - GENERATE_NEQ(prefix, my_type ::, my_type, args) + GENERATE_SPACESHIP(prefix, ret, my_type ::, my_type, args) diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index d17401f27..d27028565 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -263,8 +263,13 @@ struct BrotliCompressionSink : ChunkedCompressionSink checkInterrupt(); if (!BrotliEncoderCompressStream( - state, data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, &avail_in, &next_in, - &avail_out, &next_out, nullptr)) + state, + data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, + &avail_in, + &next_in, + &avail_out, + &next_out, + nullptr)) throw CompressionError("error while compressing brotli compression"); if (avail_out < sizeof(outbuf) || avail_in == 0) { @@ -280,8 +285,8 @@ struct BrotliCompressionSink : ChunkedCompressionSink ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level) { - std::vector la_supports = {"bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", - "lzip", "lzma", "lzop", "xz", "zstd"}; + std::vector la_supports = { + "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd"}; if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) { return make_ref(nextSink, method, parallel, level); } diff --git a/src/libutil/config-global.cc b/src/libutil/config-global.cc new file mode 100644 index 000000000..3ed1dd1d3 --- /dev/null +++ b/src/libutil/config-global.cc @@ -0,0 +1,69 @@ +#include "config-global.hh" + +#include + +namespace nix { + +bool GlobalConfig::set(const std::string & name, const std::string & value) +{ + for (auto & config : *configRegistrations) + if (config->set(name, value)) + return true; + + unknownSettings.emplace(name, value); + + return false; +} + +void GlobalConfig::getSettings(std::map & res, bool overriddenOnly) +{ + for (auto & config : *configRegistrations) + config->getSettings(res, overriddenOnly); +} + +void GlobalConfig::resetOverridden() +{ + for (auto & config : *configRegistrations) + config->resetOverridden(); +} + +nlohmann::json GlobalConfig::toJSON() +{ + auto res = nlohmann::json::object(); + for (const auto & config : *configRegistrations) + res.update(config->toJSON()); + return res; +} + +std::string GlobalConfig::toKeyValue() +{ + std::string res; + std::map settings; + globalConfig.getSettings(settings); + for (const auto & s : settings) + res += fmt("%s = %s\n", s.first, s.second.value); + return res; +} + +void GlobalConfig::convertToArgs(Args & args, const std::string & category) +{ + for (auto & config : *configRegistrations) + config->convertToArgs(args, category); +} + +GlobalConfig globalConfig; + +GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; + +GlobalConfig::Register::Register(Config * config) +{ + if (!configRegistrations) + configRegistrations = new ConfigRegistrations; + configRegistrations->emplace_back(config); +} + +ExperimentalFeatureSettings experimentalFeatureSettings; + +static GlobalConfig::Register rSettings(&experimentalFeatureSettings); + +} diff --git a/src/libutil/config-global.hh b/src/libutil/config-global.hh new file mode 100644 index 000000000..2caf51524 --- /dev/null +++ b/src/libutil/config-global.hh @@ -0,0 +1,33 @@ +#pragma once +///@file + +#include "config.hh" + +namespace nix { + +struct GlobalConfig : public AbstractConfig +{ + typedef std::vector ConfigRegistrations; + static ConfigRegistrations * configRegistrations; + + bool set(const std::string & name, const std::string & value) override; + + void getSettings(std::map & res, bool overriddenOnly = false) override; + + void resetOverridden() override; + + nlohmann::json toJSON() override; + + std::string toKeyValue() override; + + void convertToArgs(Args & args, const std::string & category) override; + + struct Register + { + Register(Config * config); + }; +}; + +extern GlobalConfig globalConfig; + +} diff --git a/src/libutil/config-impl.hh b/src/libutil/config-impl.hh index 1d349fab5..c3aa61ddb 100644 --- a/src/libutil/config-impl.hh +++ b/src/libutil/config-impl.hh @@ -81,6 +81,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) { args.addFlag({ .longName = name, + .aliases = aliases, .description = fmt("Set the `%s` setting.", name), .category = category, .labels = {"value"}, @@ -91,6 +92,7 @@ void BaseSetting::convertToArg(Args & args, const std::string & category) if (isAppendable()) args.addFlag({ .longName = "extra-" + name, + .aliases = aliases, .description = fmt("Append to the `%s` setting.", name), .category = category, .labels = {"value"}, diff --git a/src/libutil/config.cc b/src/libutil/config.cc index efde8591b..ca8480304 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -1,6 +1,7 @@ #include "config.hh" #include "args.hh" #include "abstract-setting-to-json.hh" +#include "environment-variables.hh" #include "experimental-features.hh" #include "util.hh" #include "file-system.hh" @@ -9,6 +10,8 @@ #include +#include "strings.hh" + namespace nix { Config::Config(StringMap initials) @@ -91,7 +94,14 @@ void Config::getSettings(std::map & res, bool overridd } -static void applyConfigInner(const std::string & contents, const std::string & path, std::vector> & parsedContents) { +/** + * Parse configuration in `contents`, and also the configuration files included from there, with their location specified relative to `path`. + * + * `contents` and `path` represent the file that is being parsed. + * The result is only an intermediate list of key-value pairs of strings. + * More parsing according to the settings-specific semantics is being done by `loadConfFile` in `libstore/globals.cc`. +*/ +static void parseConfigFiles(const std::string & contents, const std::string & path, std::vector> & parsedContents) { unsigned int pos = 0; while (pos < contents.size()) { @@ -107,7 +117,7 @@ static void applyConfigInner(const std::string & contents, const std::string & p if (tokens.empty()) continue; if (tokens.size() < 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); auto include = false; auto ignoreMissing = false; @@ -120,12 +130,12 @@ static void applyConfigInner(const std::string & contents, const std::string & p if (include) { if (tokens.size() != 2) - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); auto p = absPath(tokens[1], dirOf(path)); if (pathExists(p)) { try { std::string includedContents = readFile(p); - applyConfigInner(includedContents, p, parsedContents); + parseConfigFiles(includedContents, p, parsedContents); } catch (SystemError &) { // TODO: Do we actually want to ignore this? Or is it better to fail? } @@ -136,7 +146,7 @@ static void applyConfigInner(const std::string & contents, const std::string & p } if (tokens[1] != "=") - throw UsageError("illegal configuration line '%1%' in '%2%'", line, path); + throw UsageError("syntax error in configuration line '%1%' in '%2%'", line, path); std::string name = std::move(tokens[0]); @@ -153,7 +163,7 @@ static void applyConfigInner(const std::string & contents, const std::string & p void AbstractConfig::applyConfig(const std::string & contents, const std::string & path) { std::vector> parsedContents; - applyConfigInner(contents, path, parsedContents); + parseConfigFiles(contents, path, parsedContents); // First apply experimental-feature related settings for (const auto & [name, value] : parsedContents) @@ -161,9 +171,18 @@ void AbstractConfig::applyConfig(const std::string & contents, const std::string set(name, value); // Then apply other settings - for (const auto & [name, value] : parsedContents) - if (name != "experimental-features" && name != "extra-experimental-features") + // XXX: NIX_PATH must override the regular setting! This is done in `initGC()` + // Environment variables overriding settings should probably be part of the Config mechanism, + // but at the time of writing it's not worth building that for just one thing + for (const auto & [name, value] : parsedContents) { + if (name != "experimental-features" && name != "extra-experimental-features") { + if ((name == "nix-path" || name == "extra-nix-path") + && getEnv("NIX_PATH").has_value()) { + continue; + } set(name, value); + } + } } void Config::resetOverridden() @@ -283,6 +302,7 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & { args.addFlag({ .longName = name, + .aliases = aliases, .description = fmt("Enable the `%s` setting.", name), .category = category, .handler = {[this] { override(true); }}, @@ -290,6 +310,7 @@ template<> void BaseSetting::convertToArg(Args & args, const std::string & }); args.addFlag({ .longName = "no-" + name, + .aliases = aliases, .description = fmt("Disable the `%s` setting.", name), .category = category, .handler = {[this] { override(false); }}, @@ -443,67 +464,6 @@ void OptionalPathSetting::operator =(const std::optional & v) this->assign(v); } -bool GlobalConfig::set(const std::string & name, const std::string & value) -{ - for (auto & config : *configRegistrations) - if (config->set(name, value)) return true; - - unknownSettings.emplace(name, value); - - return false; -} - -void GlobalConfig::getSettings(std::map & res, bool overriddenOnly) -{ - for (auto & config : *configRegistrations) - config->getSettings(res, overriddenOnly); -} - -void GlobalConfig::resetOverridden() -{ - for (auto & config : *configRegistrations) - config->resetOverridden(); -} - -nlohmann::json GlobalConfig::toJSON() -{ - auto res = nlohmann::json::object(); - for (const auto & config : *configRegistrations) - res.update(config->toJSON()); - return res; -} - -std::string GlobalConfig::toKeyValue() -{ - std::string res; - std::map settings; - globalConfig.getSettings(settings); - for (const auto & s : settings) - res += fmt("%s = %s\n", s.first, s.second.value); - return res; -} - -void GlobalConfig::convertToArgs(Args & args, const std::string & category) -{ - for (auto & config : *configRegistrations) - config->convertToArgs(args, category); -} - -GlobalConfig globalConfig; - -GlobalConfig::ConfigRegistrations * GlobalConfig::configRegistrations; - -GlobalConfig::Register::Register(Config * config) -{ - if (!configRegistrations) - configRegistrations = new ConfigRegistrations; - configRegistrations->emplace_back(config); -} - -ExperimentalFeatureSettings experimentalFeatureSettings; - -static GlobalConfig::Register rSettings(&experimentalFeatureSettings); - bool ExperimentalFeatureSettings::isEnabled(const ExperimentalFeature & feature) const { auto & f = experimentalFeatures.get(); diff --git a/src/libutil/config.hh b/src/libutil/config.hh index 07322b60d..c0c59ac68 100644 --- a/src/libutil/config.hh +++ b/src/libutil/config.hh @@ -375,31 +375,6 @@ public: void operator =(const std::optional & v); }; -struct GlobalConfig : public AbstractConfig -{ - typedef std::vector ConfigRegistrations; - static ConfigRegistrations * configRegistrations; - - bool set(const std::string & name, const std::string & value) override; - - void getSettings(std::map & res, bool overriddenOnly = false) override; - - void resetOverridden() override; - - nlohmann::json toJSON() override; - - std::string toKeyValue() override; - - void convertToArgs(Args & args, const std::string & category) override; - - struct Register - { - Register(Config * config); - }; -}; - -extern GlobalConfig globalConfig; - struct ExperimentalFeatureSettings : Config { @@ -418,7 +393,7 @@ struct ExperimentalFeatureSettings : Config { {{#include experimental-features-shortlist.md}} - Experimental features are [further documented in the manual](@docroot@/contributing/experimental-features.md). + Experimental features are [further documented in the manual](@docroot@/development/experimental-features.md). )"}; /** diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index c88013b3c..ac01f441e 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -7,6 +7,7 @@ #include "file-system.hh" #include "processes.hh" #include "signals.hh" +#include #ifdef __APPLE__ # include @@ -14,13 +15,12 @@ #if __linux__ # include -# include # include "cgroup.hh" # include "namespaces.hh" #endif #ifndef _WIN32 -# include +# include #endif namespace nix { @@ -32,11 +32,7 @@ unsigned int getMaxCPU() auto cgroupFS = getCgroupFS(); if (!cgroupFS) return 0; - auto cgroups = getCgroups("/proc/self/cgroup"); - auto cgroup = cgroups[""]; - if (cgroup == "") return 0; - - auto cpuFile = *cgroupFS + "/" + cgroup + "/cpu.max"; + auto cpuFile = *cgroupFS + "/" + getCurrentCgroup() + "/cpu.max"; auto cpuMax = readFile(cpuFile); auto cpuMaxParts = tokenizeString>(cpuMax, " \n"); @@ -49,7 +45,7 @@ unsigned int getMaxCPU() auto period = cpuMaxParts[1]; if (quota != "max") return std::ceil(std::stoi(quota) / std::stof(period)); - } catch (Error &) { ignoreException(lvlDebug); } + } catch (Error &) { ignoreExceptionInDestructor(lvlDebug); } #endif return 0; @@ -59,15 +55,15 @@ unsigned int getMaxCPU() ////////////////////////////////////////////////////////////////////// -#ifndef _WIN32 -rlim_t savedStackSize = 0; +size_t savedStackSize = 0; -void setStackSize(rlim_t stackSize) +void setStackSize(size_t stackSize) { + #ifndef _WIN32 struct rlimit limit; if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) { savedStackSize = limit.rlim_cur; - limit.rlim_cur = std::min(stackSize, limit.rlim_max); + limit.rlim_cur = std::min(static_cast(stackSize), limit.rlim_max); if (setrlimit(RLIMIT_STACK, &limit) != 0) { logger->log( lvlError, @@ -81,8 +77,31 @@ void setStackSize(rlim_t stackSize) ); } } + #else + ULONG_PTR stackLow, stackHigh; + GetCurrentThreadStackLimits(&stackLow, &stackHigh); + ULONG maxStackSize = stackHigh - stackLow; + ULONG currStackSize = 0; + // This retrieves the current promised stack size + SetThreadStackGuarantee(&currStackSize); + if (currStackSize < stackSize) { + savedStackSize = currStackSize; + ULONG newStackSize = std::min(static_cast(stackSize), maxStackSize); + if (SetThreadStackGuarantee(&newStackSize) == 0) { + logger->log( + lvlError, + HintFmt( + "Failed to increase stack size from %1% to %2% (maximum allowed stack size: %3%): %4%", + savedStackSize, + stackSize, + maxStackSize, + std::to_string(GetLastError()) + ).str() + ); + } + } + #endif } -#endif void restoreProcessContext(bool restoreMounts) { @@ -114,7 +133,7 @@ std::optional getSelfExe() { static auto cached = []() -> std::optional { - #if __linux__ + #if __linux__ || __GNU__ return readLink("/proc/self/exe"); #elif __APPLE__ char buf[1024]; diff --git a/src/libutil/current-process.hh b/src/libutil/current-process.hh index a5adb70cf..8286bf89d 100644 --- a/src/libutil/current-process.hh +++ b/src/libutil/current-process.hh @@ -17,12 +17,10 @@ namespace nix { */ unsigned int getMaxCPU(); -#ifndef _WIN32 // TODO implement on Windows, if needed. /** * Change the stack size. */ -void setStackSize(rlim_t stackSize); -#endif +void setStackSize(size_t stackSize); /** * Restore the original inherited Unix process context (such as signal diff --git a/src/libutil/environment-variables.cc b/src/libutil/environment-variables.cc index d43197aa0..5947cf742 100644 --- a/src/libutil/environment-variables.cc +++ b/src/libutil/environment-variables.cc @@ -1,20 +1,23 @@ #include "util.hh" #include "environment-variables.hh" -extern char * * environ __attribute__((weak)); +extern char ** environ __attribute__((weak)); namespace nix { std::optional getEnv(const std::string & key) { char * value = getenv(key.c_str()); - if (!value) return {}; + if (!value) + return {}; return std::string(value); } -std::optional getEnvNonEmpty(const std::string & key) { +std::optional getEnvNonEmpty(const std::string & key) +{ auto value = getEnv(key); - if (value == "") return {}; + if (value == "") + return {}; return value; } diff --git a/src/libutil/environment-variables.hh b/src/libutil/environment-variables.hh index e0649adac..879e1f304 100644 --- a/src/libutil/environment-variables.hh +++ b/src/libutil/environment-variables.hh @@ -9,6 +9,7 @@ #include #include "types.hh" +#include "file-path.hh" namespace nix { @@ -17,6 +18,11 @@ namespace nix { */ std::optional getEnv(const std::string & key); +/** + * Like `getEnv`, but using `OsString` to avoid coercions. + */ +std::optional getEnvOs(const OsString & key); + /** * @return a non empty environment variable. Returns nullopt if the env * variable is set to "" @@ -43,6 +49,11 @@ int unsetenv(const char * name); */ int setEnv(const char * name, const char * value); +/** + * Like `setEnv`, but using `OsString` to avoid coercions. + */ +int setEnvOs(const OsString & name, const OsString & value); + /** * Clear the environment. */ diff --git a/src/libutil/error.cc b/src/libutil/error.cc index fd4f4efd1..ccd008c7c 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -1,3 +1,5 @@ +#include + #include "error.hh" #include "environment-variables.hh" #include "signals.hh" @@ -46,27 +48,22 @@ std::ostream & operator <<(std::ostream & os, const HintFmt & hf) /** * An arbitrarily defined value comparison for the purpose of using traces in the key of a sorted container. */ -inline bool operator<(const Trace& lhs, const Trace& rhs) +inline std::strong_ordering operator<=>(const Trace& lhs, const Trace& rhs) { // `std::shared_ptr` does not have value semantics for its comparison // functions, so we need to check for nulls and compare the dereferenced // values here. if (lhs.pos != rhs.pos) { - if (!lhs.pos) - return true; - if (!rhs.pos) - return false; - if (*lhs.pos != *rhs.pos) - return *lhs.pos < *rhs.pos; + if (auto cmp = bool{lhs.pos} <=> bool{rhs.pos}; cmp != 0) + return cmp; + if (auto cmp = *lhs.pos <=> *rhs.pos; cmp != 0) + return cmp; } // This formats a freshly formatted hint string and then throws it away, which // shouldn't be much of a problem because it only runs when pos is equal, and this function is // used for trace printing, which is infrequent. - return lhs.hint.str() < rhs.hint.str(); + return lhs.hint.str() <=> rhs.hint.str(); } -inline bool operator> (const Trace& lhs, const Trace& rhs) { return rhs < lhs; } -inline bool operator<=(const Trace& lhs, const Trace& rhs) { return !(lhs > rhs); } -inline bool operator>=(const Trace& lhs, const Trace& rhs) { return !(lhs < rhs); } // print lines of code to the ostream, indicating the error column. void printCodeLines(std::ostream & out, @@ -240,7 +237,10 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s break; } case Verbosity::lvlWarn: { - prefix = ANSI_WARNING "warning"; + if (einfo.isFromExpr) + prefix = ANSI_WARNING "evaluation warning"; + else + prefix = ANSI_WARNING "warning"; break; } case Verbosity::lvlInfo: { @@ -432,4 +432,36 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s return out; } +/** Write to stderr in a robust and minimal way, considering that the process + * may be in a bad state. + */ +static void writeErr(std::string_view buf) +{ + while (!buf.empty()) { + auto n = write(STDERR_FILENO, buf.data(), buf.size()); + if (n < 0) { + if (errno == EINTR) continue; + abort(); + } + buf = buf.substr(n); + } +} + +void panic(std::string_view msg) +{ + writeErr("\n\n" ANSI_RED "terminating due to unexpected unrecoverable internal error: " ANSI_NORMAL ); + writeErr(msg); + writeErr("\n"); + abort(); +} + +void panic(const char * file, int line, const char * func) +{ + char buf[512]; + int n = snprintf(buf, sizeof(buf), "Unexpected condition in %s at %s:%d", func, file, line); + if (n < 0) + panic("Unexpected condition and could not format error message"); + panic(std::string_view(buf, std::min(static_cast(sizeof(buf)), n))); +} + } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 0419f36d6..58d902622 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -16,16 +16,12 @@ */ #include "suggestions.hh" -#include "ref.hh" -#include "types.hh" #include "fmt.hh" #include #include #include -#include #include -#include #include #include @@ -79,16 +75,18 @@ struct Trace { TracePrint print = TracePrint::Default; }; -inline bool operator<(const Trace& lhs, const Trace& rhs); -inline bool operator> (const Trace& lhs, const Trace& rhs); -inline bool operator<=(const Trace& lhs, const Trace& rhs); -inline bool operator>=(const Trace& lhs, const Trace& rhs); +inline std::strong_ordering operator<=>(const Trace& lhs, const Trace& rhs); struct ErrorInfo { Verbosity level; HintFmt msg; std::shared_ptr pos; std::list traces; + /** + * Some messages are generated directly by expressions; notably `builtins.warn`, `abort`, `throw`. + * These may be rendered differently, so that users can distinguish them. + */ + bool isFromExpr = false; /** * Exit status. @@ -122,6 +120,8 @@ protected: public: BaseError(const BaseError &) = default; + BaseError& operator=(const BaseError &) = default; + BaseError& operator=(BaseError &&) = default; template BaseError(unsigned int status, const Args & ... args) @@ -150,6 +150,7 @@ public: : err(e) { } + /** The error message without "error: " prefixed to it. */ std::string message() { return err.msg.str(); } @@ -206,11 +207,11 @@ MakeError(SystemError, Error); * * Throw this, but prefer not to catch this, and catch `SystemError` * instead. This allows implementations to freely switch between this - * and `WinError` without breaking catch blocks. + * and `windows::WinError` without breaking catch blocks. * * However, it is permissible to catch this and rethrow so long as * certain conditions are not met (e.g. to catch only if `errNo = - * EFooBar`). In that case, try to also catch the equivalent `WinError` + * EFooBar`). In that case, try to also catch the equivalent `windows::WinError` * code. * * @todo Rename this to `PosixError` or similar. At this point Windows @@ -248,7 +249,9 @@ public: }; #ifdef _WIN32 -class WinError; +namespace windows { + class WinError; +} #endif /** @@ -258,7 +261,7 @@ class WinError; */ using NativeSysError = #ifdef _WIN32 - WinError + windows::WinError #else SysError #endif @@ -270,4 +273,24 @@ using NativeSysError = */ void throwExceptionSelfCheck(); +/** + * Print a message and abort(). + */ +[[noreturn]] +void panic(std::string_view msg); + +/** + * Print a basic error message with source position and abort(). + * Use the unreachable() macro to call this. + */ +[[noreturn]] +void panic(const char * file, int line, const char * func); + +/** + * Print a basic error message with source position and abort(). + * + * @note: This assumes that the logger is operational + */ +#define unreachable() (::nix::panic(__FILE__, __LINE__, __func__)) + } diff --git a/src/libutil/exec.hh b/src/libutil/exec.hh new file mode 100644 index 000000000..cbbe80c4e --- /dev/null +++ b/src/libutil/exec.hh @@ -0,0 +1,15 @@ +#pragma once + +#include "os-string.hh" + +namespace nix { + +/** + * `execvpe` is a GNU extension, so we need to implement it for other POSIX + * platforms. + * + * We use our own implementation unconditionally for consistency. + */ +int execvpe(const OsChar * file0, const OsChar * const argv[], const OsChar * const envp[]); + +} diff --git a/src/libutil/executable-path.cc b/src/libutil/executable-path.cc new file mode 100644 index 000000000..9fb5214b2 --- /dev/null +++ b/src/libutil/executable-path.cc @@ -0,0 +1,97 @@ +#include "environment-variables.hh" +#include "executable-path.hh" +#include "strings-inline.hh" +#include "util.hh" +#include "file-path-impl.hh" + +namespace nix { + +namespace fs { +using namespace std::filesystem; +} + +constexpr static const OsStringView path_var_separator{ + &ExecutablePath::separator, + 1, +}; + +ExecutablePath ExecutablePath::load() +{ + // "If PATH is unset or is set to null, the path search is + // implementation-defined." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + return ExecutablePath::parse(getEnvOs(OS_STR("PATH")).value_or(OS_STR(""))); +} + +ExecutablePath ExecutablePath::parse(const OsString & path) +{ + auto strings = path.empty() ? (std::list{}) + : basicSplitString, OsChar>(path, path_var_separator); + + std::vector ret; + ret.reserve(strings.size()); + + std::transform( + std::make_move_iterator(strings.begin()), + std::make_move_iterator(strings.end()), + std::back_inserter(ret), + [](auto && str) { + return fs::path{ + str.empty() + // "A zero-length prefix is a legacy feature that + // indicates the current working directory. It + // appears as two adjacent characters + // ("::"), as an initial preceding the rest + // of the list, or as a trailing following + // the rest of the list." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + ? OS_STR(".") + : std::move(str), + }; + }); + + return {ret}; +} + +OsString ExecutablePath::render() const +{ + std::vector path2; + for (auto & p : directories) + path2.push_back(p.native()); + return basicConcatStringsSep(path_var_separator, path2); +} + +std::optional +ExecutablePath::findName(const OsString & exe, std::function isExecutable) const +{ + // "If the pathname being sought contains a , the search + // through the path prefixes shall not be performed." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + assert(OsPathTrait::rfindPathSep(exe) == exe.npos); + + for (auto & dir : directories) { + auto candidate = dir / exe; + if (isExecutable(candidate)) + return std::filesystem::canonical(candidate); + } + + return std::nullopt; +} + +fs::path ExecutablePath::findPath(const fs::path & exe, std::function isExecutable) const +{ + // "If the pathname being sought contains a , the search + // through the path prefixes shall not be performed." + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + if (exe.filename() == exe) { + auto resOpt = findName(exe, isExecutable); + if (resOpt) + return *resOpt; + else + throw ExecutableLookupError("Could not find executable '%s'", exe.string()); + } else { + return exe; + } +} + +} // namespace nix diff --git a/src/libutil/executable-path.hh b/src/libutil/executable-path.hh new file mode 100644 index 000000000..c5cfa1c39 --- /dev/null +++ b/src/libutil/executable-path.hh @@ -0,0 +1,81 @@ +#pragma once +///@file + +#include "file-system.hh" + +namespace nix { + +MakeError(ExecutableLookupError, Error); + +/** + * @todo rename, it is not just good for execuatable paths, but also + * other lists of paths. + */ +struct ExecutablePath +{ + std::vector directories; + + constexpr static const OsChar separator = +#ifdef WIN32 + L';' +#else + ':' +#endif + ; + + /** + * Parse `path` into a list of paths. + * + * On Unix we split on `:`, on Windows we split on `;`. + * + * For Unix, this is according to the POSIX spec for `PATH`. + * https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + */ + static ExecutablePath parse(const OsString & path); + + /** + * Load the `PATH` environment variable and `parse` it. + */ + static ExecutablePath load(); + + /** + * Opposite of `parse` + */ + OsString render() const; + + /** + * Search for an executable. + * + * For Unix, this is according to the POSIX spec for `PATH`. + * https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html#tag_08_03 + * + * @param exe This must just be a name, and not contain any `/` (or + * `\` on Windows). in case it does, per the spec no lookup should + * be perfomed, and the path (it is not just a file name) as is. + * This is the caller's respsonsibility. + * + * This is a pure function, except for the default `isExecutable` + * argument, which uses the ambient file system to check if a file is + * executable (and exists). + * + * @return path to a resolved executable + */ + std::optional findName( + const OsString & exe, + std::function isExecutableFile = isExecutableFileAmbient) const; + + /** + * Like the `findName` but also allows a file path as input. + * + * This implements the full POSIX spec: if the path is just a name, + * it searches like the above. Otherwise, it returns the path as is. + * If (in the name case) the search fails, an exception is thrown. + */ + std::filesystem::path findPath( + const std::filesystem::path & exe, + std::function isExecutable = isExecutableFileAmbient) const; + + bool operator==(const ExecutablePath &) const = default; +}; + +} // namespace nix diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 9b7000f9f..a0c955816 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -24,7 +24,7 @@ struct ExperimentalFeatureDetails * feature, we either have no issue at all if few features are not added * at the end of the list, or a proper merge conflict if they are. */ -constexpr size_t numXpFeatures = 1 + static_cast(Xp::VerifiedFetches); +constexpr size_t numXpFeatures = 1 + static_cast(Xp::PipeOperators); constexpr std::array xpFeatureDetails = {{ { @@ -66,7 +66,7 @@ constexpr std::array xpFeatureDetails an impure derivation cannot also be [content-addressed](#xp-feature-ca-derivations). - This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime). + This is a more explicit alternative to using [`builtins.currentTime`](@docroot@/language/builtins.md#builtins-currentTime). )", .trackingUrl = "https://github.com/NixOS/nix/milestone/42", }, @@ -294,6 +294,14 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "https://github.com/NixOS/nix/milestone/48", }, + { + .tag = Xp::PipeOperators, + .name = "pipe-operators", + .description = R"( + Add `|>` and `<|` operators to the Nix language. + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/55", + }, }}; static_assert( diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index 1da2a3ff5..412bf0886 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -1,11 +1,11 @@ #pragma once ///@file -#include "comparator.hh" #include "error.hh" -#include "json-utils.hh" #include "types.hh" +#include + namespace nix { /** @@ -36,6 +36,7 @@ enum struct ExperimentalFeature ConfigurableImpureEnv, MountedSSHStore, VerifiedFetches, + PipeOperators, }; /** @@ -98,10 +99,4 @@ public: void to_json(nlohmann::json &, const ExperimentalFeature &); void from_json(const nlohmann::json &, ExperimentalFeature &); -/** - * It is always rendered as a string - */ -template<> -struct json_avoids_null : std::true_type {}; - } diff --git a/src/libutil/file-content-address.cc b/src/libutil/file-content-address.cc index 769042d00..69301d9c8 100644 --- a/src/libutil/file-content-address.cc +++ b/src/libutil/file-content-address.cc @@ -10,7 +10,7 @@ static std::optional parseFileSerialisationMethodOpt(st if (input == "flat") { return FileSerialisationMethod::Flat; } else if (input == "nar") { - return FileSerialisationMethod::Recursive; + return FileSerialisationMethod::NixArchive; } else { return std::nullopt; } @@ -45,7 +45,7 @@ std::string_view renderFileSerialisationMethod(FileSerialisationMethod method) switch (method) { case FileSerialisationMethod::Flat: return "flat"; - case FileSerialisationMethod::Recursive: + case FileSerialisationMethod::NixArchive: return "nar"; default: assert(false); @@ -57,13 +57,13 @@ std::string_view renderFileIngestionMethod(FileIngestionMethod method) { switch (method) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: return renderFileSerialisationMethod( static_cast(method)); case FileIngestionMethod::Git: return "git"; default: - abort(); + unreachable(); } } @@ -78,7 +78,7 @@ void dumpPath( case FileSerialisationMethod::Flat: path.readFile(sink); break; - case FileSerialisationMethod::Recursive: + case FileSerialisationMethod::NixArchive: path.dumpPath(sink, filter); break; } @@ -88,14 +88,15 @@ void dumpPath( void restorePath( const Path & path, Source & source, - FileSerialisationMethod method) + FileSerialisationMethod method, + bool startFsync) { switch (method) { case FileSerialisationMethod::Flat: - writeFile(path, source); + writeFile(path, source, 0666, startFsync); break; - case FileSerialisationMethod::Recursive: - restorePath(path, source); + case FileSerialisationMethod::NixArchive: + restorePath(path, source, startFsync); break; } } @@ -112,17 +113,19 @@ HashResult hashPath( } -Hash hashPath( +std::pair> hashPath( const SourcePath & path, FileIngestionMethod method, HashAlgorithm ht, PathFilter & filter) { switch (method) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: - return hashPath(path, (FileSerialisationMethod) method, ht, filter).first; + case FileIngestionMethod::NixArchive: { + auto res = hashPath(path, (FileSerialisationMethod) method, ht, filter); + return {res.first, {res.second}}; + } case FileIngestionMethod::Git: - return git::dumpHash(ht, path, filter).hash; + return {git::dumpHash(ht, path, filter).hash, std::nullopt}; } assert(false); } diff --git a/src/libutil/file-content-address.hh b/src/libutil/file-content-address.hh index c19de27ed..0c584ea8a 100644 --- a/src/libutil/file-content-address.hh +++ b/src/libutil/file-content-address.hh @@ -2,8 +2,6 @@ ///@file #include "source-accessor.hh" -#include "fs-sink.hh" -#include "util.hh" namespace nix { @@ -35,14 +33,14 @@ enum struct FileSerialisationMethod : uint8_t { * See `file-system-object/content-address.md#serial-nix-archive` in * the manual. */ - Recursive, + NixArchive, }; /** * Parse a `FileSerialisationMethod` by name. Choice of: * * - `flat`: `FileSerialisationMethod::Flat` - * - `nar`: `FileSerialisationMethod::Recursive` + * - `nar`: `FileSerialisationMethod::NixArchive` * * Opposite of `renderFileSerialisationMethod`. */ @@ -72,7 +70,8 @@ void dumpPath( void restorePath( const Path & path, Source & source, - FileSerialisationMethod method); + FileSerialisationMethod method, + bool startFsync = false); /** @@ -107,16 +106,18 @@ enum struct FileIngestionMethod : uint8_t { Flat, /** - * Hash `FileSerialisationMethod::Recursive` serialisation. + * Hash `FileSerialisationMethod::NixArchive` serialisation. * * See `file-system-object/content-address.md#serial-flat` in the * manual. */ - Recursive, + NixArchive, /** * Git hashing. * + * Part of `ExperimentalFeature::GitHashing`. + * * See `file-system-object/content-address.md#serial-git` in the * manual. */ @@ -127,7 +128,7 @@ enum struct FileIngestionMethod : uint8_t { * Parse a `FileIngestionMethod` by name. Choice of: * * - `flat`: `FileIngestionMethod::Flat` - * - `nar`: `FileIngestionMethod::Recursive` + * - `nar`: `FileIngestionMethod::NixArchive` * - `git`: `FileIngestionMethod::Git` * * Opposite of `renderFileIngestionMethod`. @@ -143,14 +144,15 @@ std::string_view renderFileIngestionMethod(FileIngestionMethod method); /** * Compute the hash of the given file system object according to the - * given method. + * given method, and for some ingestion methods, the size of the + * serialisation. * * Unlike the other `hashPath`, this works on an arbitrary * `FileIngestionMethod` instead of `FileSerialisationMethod`, but - * doesn't return the size as this is this is not a both simple and + * may not return the size as this is this is not a both simple and * useful defined for a merkle format. */ -Hash hashPath( +std::pair> hashPath( const SourcePath & path, FileIngestionMethod method, HashAlgorithm ha, PathFilter & filter = defaultPathFilter); diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index 3bbfc50ee..3d8d70fdb 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -2,6 +2,7 @@ #include "signals.hh" #include "finally.hh" #include "serialise.hh" +#include "util.hh" #include #include @@ -65,7 +66,7 @@ AutoCloseFD::~AutoCloseFD() try { close(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -92,7 +93,7 @@ void AutoCloseFD::close() } } -void AutoCloseFD::fsync() +void AutoCloseFD::fsync() const { if (fd != INVALID_DESCRIPTOR) { int result; @@ -111,6 +112,18 @@ void AutoCloseFD::fsync() } + +void AutoCloseFD::startFsync() const +{ +#if __linux__ + if (fd != -1) { + /* Ignore failure, since fsync must be run later anyway. This is just a performance optimization. */ + ::sync_file_range(fd, 0, 0, SYNC_FILE_RANGE_WRITE); + } +#endif +} + + AutoCloseFD::operator bool() const { return fd != INVALID_DESCRIPTOR; diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh index 84786e95a..b54bab83c 100644 --- a/src/libutil/file-descriptor.hh +++ b/src/libutil/file-descriptor.hh @@ -128,7 +128,18 @@ public: explicit operator bool() const; Descriptor release(); void close(); - void fsync(); + + /** + * Perform a blocking fsync operation. + */ + void fsync() const; + + /** + * Asynchronously flush to disk without blocking, if available on + * the platform. This is just a performance optimization, and + * fsync must be run later even if this is called. + */ + void startFsync() const; }; class Pipe @@ -140,25 +151,29 @@ public: }; #ifndef _WIN32 // Not needed on Windows, where we don't fork +namespace unix { /** - * Close all file descriptors except those listed in the given set. + * Close all file descriptors except stdio fds (ie 0, 1, 2). * Good practice in child processes. */ -void closeMostFDs(const std::set & exceptions); +void closeExtraFDs(); /** * Set the close-on-exec flag for the given file descriptor. */ void closeOnExec(Descriptor fd); +} // namespace unix #endif -#ifdef _WIN32 -# if _WIN32_WINNT >= 0x0600 +#if defined(_WIN32) && _WIN32_WINNT >= 0x0600 +namespace windows { + Path handleToPath(Descriptor handle); std::wstring handleToFileName(Descriptor handle); -# endif + +} // namespace windows #endif MakeError(EndOfFile, Error); diff --git a/src/libutil/file-path-impl.hh b/src/libutil/file-path-impl.hh index 4c90150fd..d7c823fd0 100644 --- a/src/libutil/file-path-impl.hh +++ b/src/libutil/file-path-impl.hh @@ -91,13 +91,10 @@ struct WindowsPathTrait }; -/** - * @todo Revisit choice of `char` or `wchar_t` for `WindowsPathTrait` - * argument. - */ -using NativePathTrait = +template +using OsPathTrait = #ifdef _WIN32 - WindowsPathTrait + WindowsPathTrait #else UnixPathTrait #endif diff --git a/src/libutil/file-path.hh b/src/libutil/file-path.hh index 6589c4060..8e4a88b9d 100644 --- a/src/libutil/file-path.hh +++ b/src/libutil/file-path.hh @@ -1,10 +1,10 @@ #pragma once ///@file -#include #include #include "types.hh" +#include "os-string.hh" namespace nix { @@ -22,39 +22,26 @@ typedef std::set PathSetNG; * * @todo drop `NG` suffix and replace the one in `types.hh`. */ -struct PathViewNG : std::basic_string_view +struct PathViewNG : OsStringView { - using string_view = std::basic_string_view; + using string_view = OsStringView; using string_view::string_view; PathViewNG(const std::filesystem::path & path) - : std::basic_string_view(path.native()) + : OsStringView{path.native()} { } - PathViewNG(const std::filesystem::path::string_type & path) - : std::basic_string_view(path) + PathViewNG(const OsString & path) + : OsStringView{path} { } const string_view & native() const { return *this; } string_view & native() { return *this; } }; -std::string os_string_to_string(PathViewNG::string_view path); - -std::filesystem::path::string_type string_to_os_string(std::string_view s); - std::optional maybePath(PathView path); std::filesystem::path pathNG(PathView path); -/** - * Create string literals with the native character width of paths - */ -#ifndef _WIN32 -# define PATHNG_LITERAL(s) s -#else -# define PATHNG_LITERAL(s) L ## s -#endif - } diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 919bf5d50..224b78b23 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -5,12 +5,14 @@ #include "signals.hh" #include "finally.hh" #include "serialise.hh" +#include "util.hh" #include #include #include #include #include +#include #include #include @@ -23,10 +25,12 @@ # include #endif -namespace fs = std::filesystem; +#include "strings-inline.hh" namespace nix { +namespace fs { using namespace std::filesystem; } + /** * Treat the string as possibly an absolute path, by inspecting the * start of it. Return whether it was probably intended to be @@ -70,6 +74,10 @@ Path absPath(PathView path, std::optional dir, bool resolveSymlinks) return canonPath(path, resolveSymlinks); } +std::filesystem::path absPath(const std::filesystem::path & path, bool resolveSymlinks) +{ + return absPath(path.string(), std::nullopt, resolveSymlinks); +} Path canonPath(PathView path, bool resolveSymlinks) { @@ -90,7 +98,7 @@ Path canonPath(PathView path, bool resolveSymlinks) arbitrary (but high) limit to prevent infinite loops. */ unsigned int followCount = 0, maxFollow = 1024; - auto ret = canonPathInner( + auto ret = canonPathInner>( path, [&followCount, &temp, maxFollow, resolveSymlinks] (std::string & result, std::string_view & remaining) { @@ -120,7 +128,7 @@ Path canonPath(PathView path, bool resolveSymlinks) Path dirOf(const PathView path) { - Path::size_type pos = NativePathTrait::rfindPathSep(path); + Path::size_type pos = OsPathTrait::rfindPathSep(path); if (pos == path.npos) return "."; return fs::path{path}.parent_path().string(); @@ -133,10 +141,10 @@ std::string_view baseNameOf(std::string_view path) return ""; auto last = path.size() - 1; - while (last > 0 && NativePathTrait::isPathSep(path[last])) + while (last > 0 && OsPathTrait::isPathSep(path[last])) last -= 1; - auto pos = NativePathTrait::rfindPathSep(path, last); + auto pos = OsPathTrait::rfindPathSep(path, last); if (pos == path.npos) pos = 0; else @@ -203,10 +211,10 @@ bool pathExists(const Path & path) return maybeLstat(path).has_value(); } -bool pathAccessible(const Path & path) +bool pathAccessible(const std::filesystem::path & path) { try { - return pathExists(path); + return pathExists(path.string()); } catch (SysError & e) { // swallow EPERM if (e.errNo == EPERM) return false; @@ -235,6 +243,11 @@ std::string readFile(const Path & path) return readFile(fd.get()); } +std::string readFile(const std::filesystem::path & path) +{ + return readFile(os_string_to_string(PathViewNG { path })); +} + void readFile(const Path & path, Sink & sink) { @@ -316,6 +329,50 @@ void syncParent(const Path & path) } +void recursiveSync(const Path & path) +{ + /* If it's a file, just fsync and return. */ + auto st = lstat(path); + if (S_ISREG(st.st_mode)) { + AutoCloseFD fd = toDescriptor(open(path.c_str(), O_RDONLY, 0)); + if (!fd) + throw SysError("opening file '%1%'", path); + fd.fsync(); + return; + } + + /* Otherwise, perform a depth-first traversal of the directory and + fsync all the files. */ + std::deque dirsToEnumerate; + dirsToEnumerate.push_back(path); + std::vector dirsToFsync; + while (!dirsToEnumerate.empty()) { + auto currentDir = dirsToEnumerate.back(); + dirsToEnumerate.pop_back(); + for (auto & entry : std::filesystem::directory_iterator(currentDir)) { + auto st = entry.symlink_status(); + if (fs::is_directory(st)) { + dirsToEnumerate.emplace_back(entry.path()); + } else if (fs::is_regular_file(st)) { + AutoCloseFD fd = toDescriptor(open(entry.path().string().c_str(), O_RDONLY, 0)); + if (!fd) + throw SysError("opening file '%1%'", entry.path()); + fd.fsync(); + } + } + dirsToFsync.emplace_back(std::move(currentDir)); + } + + /* Fsync all the directories. */ + for (auto dir = dirsToFsync.rbegin(); dir != dirsToFsync.rend(); ++dir) { + AutoCloseFD fd = toDescriptor(open(dir->string().c_str(), O_RDONLY, 0)); + if (!fd) + throw SysError("opening directory '%1%'", *dir); + fd.fsync(); + } +} + + static void _deletePath(Descriptor parentfd, const fs::path & path, uint64_t & bytesFreed) { #ifndef _WIN32 @@ -412,31 +469,23 @@ void deletePath(const fs::path & path) deletePath(path, dummy); } - -Paths createDirs(const Path & path) +void createDir(const Path & path, mode_t mode) { - Paths created; - if (path == "/") return created; - - struct stat st; - if (STAT(path.c_str(), &st) == -1) { - created = createDirs(dirOf(path)); - if (mkdir(path.c_str() -#ifndef _WIN32 // TODO abstract mkdir perms for Windows - , 0777 + if (mkdir(path.c_str() +#ifndef _WIN32 + , mode #endif - ) == -1 && errno != EEXIST) - throw SysError("creating directory '%1%'", path); - st = STAT(path); - created.push_back(path); + ) == -1) + throw SysError("creating directory '%1%'", path); +} + +void createDirs(const Path & path) +{ + try { + fs::create_directories(path); + } catch (fs::filesystem_error & e) { + throw SysError("creating directory '%1%'", path); } - - if (S_ISLNK(st.st_mode) && stat(path.c_str(), &st) == -1) - throw SysError("statting symlink '%1%'", path); - - if (!S_ISDIR(st.st_mode)) throw Error("'%1%' is not a directory", path); - - return created; } @@ -469,7 +518,7 @@ AutoDelete::~AutoDelete() } } } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } @@ -546,7 +595,7 @@ std::pair createTempFile(const Path & prefix) if (!fd) throw SysError("creating temporary file '%s'", tmpl); #ifndef _WIN32 - closeOnExec(fd.get()); + unix::closeOnExec(fd.get()); #endif return {std::move(fd), tmpl}; } @@ -556,47 +605,88 @@ void createSymlink(const Path & target, const Path & link) fs::create_symlink(target, link); } -void replaceSymlink(const Path & target, const Path & link) +void replaceSymlink(const fs::path & target, const fs::path & link) { for (unsigned int n = 0; true; n++) { - Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); + auto tmp = link.parent_path() / fs::path{fmt(".%d_%s", n, link.filename().string())}; + tmp = tmp.lexically_normal(); try { - createSymlink(target, tmp); + fs::create_symlink(target, tmp); } catch (fs::filesystem_error & e) { if (e.code() == std::errc::file_exists) continue; throw; } - std::filesystem::rename(tmp, link); + fs::rename(tmp, link); break; } } -#ifndef _WIN32 -static void setWriteTime(const fs::path & p, const struct stat & st) +void setWriteTime( + const fs::path & path, + time_t accessedTime, + time_t modificationTime, + std::optional optIsSymlink) { - struct timeval times[2]; - times[0] = { - .tv_sec = st.st_atime, - .tv_usec = 0, +#ifndef _WIN32 + struct timeval times[2] = { + { + .tv_sec = accessedTime, + .tv_usec = 0, + }, + { + .tv_sec = modificationTime, + .tv_usec = 0, + }, }; - times[1] = { - .tv_sec = st.st_mtime, - .tv_usec = 0, - }; - if (lutimes(p.c_str(), times) != 0) - throw SysError("changing modification time of '%s'", p); -} #endif + auto nonSymlink = [&]{ + bool isSymlink = optIsSymlink + ? *optIsSymlink + : fs::is_symlink(path); + + if (!isSymlink) { +#ifdef _WIN32 + // FIXME use `fs::last_write_time`. + // + // Would be nice to use std::filesystem unconditionally, but + // doesn't support access time just modification time. + // + // System clock vs File clock issues also make that annoying. + warn("Changing file times is not yet implemented on Windows, path is '%s'", path); +#else + if (utimes(path.c_str(), times) == -1) { + + throw SysError("changing modification time of '%s' (not a symlink)", path); + } +#endif + } else { + throw Error("Cannot modification time of symlink '%s'", path); + } + }; + +#if HAVE_LUTIMES + if (lutimes(path.c_str(), times) == -1) { + if (errno == ENOSYS) + nonSymlink(); + else + throw SysError("changing modification time of '%s'", path); + } +#else + nonSymlink(); +#endif +} + +void setWriteTime(const fs::path & path, const struct stat & st) +{ + setWriteTime(path, st.st_atime, st.st_mtime, S_ISLNK(st.st_mode)); +} + void copyFile(const fs::path & from, const fs::path & to, bool andDelete) { -#ifndef _WIN32 - // TODO: Rewrite the `is_*` to use `symlink_status()` - auto statOfFrom = lstat(from.c_str()); -#endif auto fromStatus = fs::symlink_status(from); // Mark the directory as writable so that we can delete its children @@ -616,9 +706,7 @@ void copyFile(const fs::path & from, const fs::path & to, bool andDelete) throw Error("file '%s' has an unsupported type", from); } -#ifndef _WIN32 - setWriteTime(to, statOfFrom); -#endif + setWriteTime(to, lstat(from.string().c_str())); if (andDelete) { if (!fs::is_symlink(fromStatus)) fs::permissions(from, fs::perms::owner_write, fs::perm_options::add | fs::perm_options::nofollow); @@ -653,4 +741,18 @@ void moveFile(const Path & oldName, const Path & newName) ////////////////////////////////////////////////////////////////////// +bool isExecutableFileAmbient(const fs::path & exe) { + // Check file type, because directory being executable means + // something completely different. + // `is_regular_file` follows symlinks before checking. + return std::filesystem::is_regular_file(exe) + && access(exe.string().c_str(), +#ifdef WIN32 + 0 // TODO do better +#else + X_OK +#endif + ) == 0; +} + } diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 933e88441..eb3e4ec66 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -20,8 +20,6 @@ #endif #include -#include - #include #include #include @@ -48,16 +46,33 @@ struct Source; * @return An absolutized path, resolving paths relative to the * specified directory, or the current directory otherwise. The path * is also canonicalised. + * + * In the process of being deprecated for `std::filesystem::absolute`. */ Path absPath(PathView path, std::optional dir = {}, bool resolveSymlinks = false); +inline Path absPath(const Path & path, + std::optional dir = {}, + bool resolveSymlinks = false) +{ + return absPath(PathView{path}, dir, resolveSymlinks); +} + +std::filesystem::path absPath(const std::filesystem::path & path, + bool resolveSymlinks = false); + /** * Canonicalise a path by removing all `.` or `..` components and * double or trailing slashes. Optionally resolves all symlink * components such that each component of the resulting path is *not* * a symbolic link. + * + * In the process of being deprecated for + * `std::filesystem::path::lexically_normal` (for the `resolveSymlinks = + * false` case), and `std::filesystem::weakly_canonical` (for the + * `resolveSymlinks = true` case). */ Path canonPath(PathView path, bool resolveSymlinks = false); @@ -66,12 +81,18 @@ Path canonPath(PathView path, bool resolveSymlinks = false); * everything before the final `/`. If the path is the root or an * immediate child thereof (e.g., `/foo`), this means `/` * is returned. + * + * In the process of being deprecated for + * `std::filesystem::path::parent_path`. */ Path dirOf(const PathView path); /** * @return the base name of the given canonical path, i.e., everything * following the final `/` (trailing slashes are removed). + * + * In the process of being deprecated for + * `std::filesystem::path::filename`. */ std::string_view baseNameOf(std::string_view path); @@ -100,20 +121,42 @@ std::optional maybeLstat(const Path & path); /** * @return true iff the given path exists. + * + * In the process of being deprecated for `fs::symlink_exists`. */ bool pathExists(const Path & path); +namespace fs { + +/** + * ``` + * symlink_exists(p) = std::filesystem::exists(std::filesystem::symlink_status(p)) + * ``` + * Missing convenience analogous to + * ``` + * std::filesystem::exists(p) = std::filesystem::exists(std::filesystem::status(p)) + * ``` + */ +inline bool symlink_exists(const std::filesystem::path & path) { + return std::filesystem::exists(std::filesystem::symlink_status(path)); +} + +} // namespace fs + /** * A version of pathExists that returns false on a permission error. * Useful for inferring default paths across directories that might not * be readable. * @return true iff the given path can be accessed and exists */ -bool pathAccessible(const Path & path); +bool pathAccessible(const std::filesystem::path & path); /** * Read the contents (target) of a symbolic link. The result is not * in any way canonicalised. + * + * In the process of being deprecated for + * `std::filesystem::read_symlink`. */ Path readLink(const Path & path); @@ -126,20 +169,34 @@ Descriptor openDirectory(const std::filesystem::path & path); * Read the contents of a file into a string. */ std::string readFile(const Path & path); +std::string readFile(const std::filesystem::path & path); void readFile(const Path & path, Sink & sink); /** * Write a string to a file. */ void writeFile(const Path & path, std::string_view s, mode_t mode = 0666, bool sync = false); +static inline void writeFile(const std::filesystem::path & path, std::string_view s, mode_t mode = 0666, bool sync = false) +{ + return writeFile(path.string(), s, mode, sync); +} void writeFile(const Path & path, Source & source, mode_t mode = 0666, bool sync = false); +static inline void writeFile(const std::filesystem::path & path, Source & source, mode_t mode = 0666, bool sync = false) +{ + return writeFile(path.string(), source, mode, sync); +} /** - * Flush a file's parent directory to disk + * Flush a path's parent directory to disk. */ void syncParent(const Path & path); +/** + * Flush a file or entire directory tree to disk. + */ +void recursiveSync(const Path & path); + /** * Delete a path; i.e., in the case of a directory, it is deleted * recursively. It's not an error if the path does not exist. The @@ -150,24 +207,63 @@ void deletePath(const std::filesystem::path & path); void deletePath(const std::filesystem::path & path, uint64_t & bytesFreed); /** - * Create a directory and all its parents, if necessary. Returns the - * list of created directories, in order of creation. + * Create a directory and all its parents, if necessary. + * + * In the process of being deprecated for + * `std::filesystem::create_directories`. */ -Paths createDirs(const Path & path); -inline Paths createDirs(PathView path) +void createDirs(const Path & path); +inline void createDirs(PathView path) { return createDirs(Path(path)); } +/** + * Create a single directory. + */ +void createDir(const Path & path, mode_t mode = 0755); + +/** + * Set the access and modification times of the given path, not + * following symlinks. + * + * @param accessTime Specified in seconds. + * + * @param modificationTime Specified in seconds. + * + * @param isSymlink Whether the file in question is a symlink. Used for + * fallback code where we don't have `lutimes` or similar. if + * `std::optional` is passed, the information will be recomputed if it + * is needed. Race conditions are possible so be careful! + */ +void setWriteTime( + const std::filesystem::path & path, + time_t accessedTime, + time_t modificationTime, + std::optional isSymlink = std::nullopt); + +/** + * Convenience wrapper that takes all arguments from the `struct stat`. + */ +void setWriteTime(const std::filesystem::path & path, const struct stat & st); + /** * Create a symlink. + * + * In the process of being deprecated for + * `std::filesystem::create_symlink`. */ void createSymlink(const Path & target, const Path & link); /** * Atomically create or replace a symlink. */ -void replaceSymlink(const Path & target, const Path & link); +void replaceSymlink(const std::filesystem::path & target, const std::filesystem::path & link); + +inline void replaceSymlink(const Path & target, const Path & link) +{ + return replaceSymlink(std::filesystem::path{target}, std::filesystem::path{link}); +} /** * Similar to 'renameFile', but fallback to a copy+remove if `src` and `dst` @@ -237,6 +333,12 @@ std::pair createTempFile(const Path & prefix = "nix"); */ Path defaultTempDir(); +/** + * Interpret `exe` as a location in the ambient file system and return + * whether it resolves to a file that is executable. + */ +bool isExecutableFileAmbient(const std::filesystem::path & exe); + /** * Used in various places. */ diff --git a/src/libutil/finally.hh b/src/libutil/finally.hh index f9f0195a1..bda4227e6 100644 --- a/src/libutil/finally.hh +++ b/src/libutil/finally.hh @@ -2,6 +2,8 @@ ///@file #include +#include +#include /** * A trivial class to run a function at the end of a scope. @@ -21,5 +23,25 @@ public: Finally(Finally &&other) : fun(std::move(other.fun)) { other.movedFrom = true; } - ~Finally() { if (!movedFrom) fun(); } + ~Finally() noexcept(false) + { + try { + if (!movedFrom) + fun(); + } catch (...) { + // finally may only throw an exception if exception handling is not already + // in progress. if handling *is* in progress we have to return cleanly here + // but are still prohibited from doing so since eating the exception would, + // in almost all cases, mess up error handling even more. the only good way + // to handle this is to abort entirely and leave a message, so we'll assert + // (and rethrow anyway, just as a defense against possible NASSERT builds.) + if (std::uncaught_exceptions()) { + assert(false && + "Finally function threw an exception during exception handling. " + "this is not what you want, please use some other methods (like " + "std::promise or async) instead."); + } + throw; + } + } }; diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index c178257d4..850b7162d 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -111,6 +111,8 @@ std::ostream & operator<<(std::ostream & out, const Magenta & y) /** * Values wrapped in this class are printed without coloring. * + * Specifically, the color is reset to normal before printing the value. + * * By default, arguments to `HintFmt` are printed in magenta (see `Magenta`). */ template @@ -182,6 +184,8 @@ public: return *this; } + HintFmt & operator=(HintFmt const & rhs) = default; + std::string str() const { return fmt.str(); diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 91070ea89..72e5c731f 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -1,7 +1,7 @@ #include #include "error.hh" -#include "config.hh" +#include "config-global.hh" #include "fs-sink.hh" #if _WIN32 @@ -14,7 +14,7 @@ namespace nix { void copyRecursive( SourceAccessor & accessor, const CanonPath & from, - FileSystemObjectSink & sink, const Path & to) + FileSystemObjectSink & sink, const CanonPath & to) { auto stat = accessor.lstat(from); @@ -43,7 +43,7 @@ void copyRecursive( for (auto & [name, _] : accessor.readDirectory(from)) { copyRecursive( accessor, from / name, - sink, to + "/" + name); + sink, to / name); break; } break; @@ -53,7 +53,7 @@ void copyRecursive( throw Error("file '%1%' has an unsupported type", from); default: - abort(); + unreachable(); } } @@ -68,35 +68,49 @@ static RestoreSinkSettings restoreSinkSettings; static GlobalConfig::Register r1(&restoreSinkSettings); - -void RestoreSink::createDirectory(const Path & path) +static std::filesystem::path append(const std::filesystem::path & src, const CanonPath & path) { - Path p = dstPath + path; - if ( -#ifndef _WIN32 // TODO abstract mkdir perms for Windows - mkdir(p.c_str(), 0777) == -1 -#else - !CreateDirectoryW(pathNG(p).c_str(), NULL) -#endif - ) - throw NativeSysError("creating directory '%1%'", p); + auto dst = src; + if (!path.rel().empty()) + dst /= path.rel(); + return dst; +} + +void RestoreSink::createDirectory(const CanonPath & path) +{ + auto p = append(dstPath, path); + if (!std::filesystem::create_directory(p)) + throw Error("path '%s' already exists", p.string()); }; struct RestoreRegularFile : CreateRegularFileSink { AutoCloseFD fd; + bool startFsync = false; + + ~RestoreRegularFile() + { + /* Initiate an fsync operation without waiting for the + result. The real fsync should be run before registering a + store path, but this is a performance optimization to allow + the disk write to start early. */ + if (fd && startFsync) + fd.startFsync(); + } void operator () (std::string_view data) override; void isExecutable() override; void preallocateContents(uint64_t size) override; }; -void RestoreSink::createRegularFile(const Path & path, std::function func) +void RestoreSink::createRegularFile(const CanonPath & path, std::function func) { - Path p = dstPath + path; + auto p = append(dstPath, path); + RestoreRegularFile crf; + crf.startFsync = startFsync; crf.fd = #ifdef _WIN32 - CreateFileW(pathNG(path).c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL) + CreateFileW(p.c_str(), GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL) #else open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666) #endif @@ -141,14 +155,14 @@ void RestoreRegularFile::operator () (std::string_view data) writeFull(fd.get(), data); } -void RestoreSink::createSymlink(const Path & path, const std::string & target) +void RestoreSink::createSymlink(const CanonPath & path, const std::string & target) { - Path p = dstPath + path; - nix::createSymlink(target, p); + auto p = append(dstPath, path); + nix::createSymlink(target, p.string()); } -void RegularFileSink::createRegularFile(const Path & path, std::function func) +void RegularFileSink::createRegularFile(const CanonPath & path, std::function func) { struct CRF : CreateRegularFileSink { RegularFileSink & back; @@ -163,7 +177,7 @@ void RegularFileSink::createRegularFile(const Path & path, std::function func) +void NullFileSystemObjectSink::createRegularFile(const CanonPath & path, std::function func) { struct : CreateRegularFileSink { void operator () (std::string_view data) override {} diff --git a/src/libutil/fs-sink.hh b/src/libutil/fs-sink.hh index ae577819a..5c5073731 100644 --- a/src/libutil/fs-sink.hh +++ b/src/libutil/fs-sink.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "types.hh" #include "serialise.hh" #include "source-accessor.hh" #include "file-system.hh" @@ -28,17 +27,30 @@ struct FileSystemObjectSink { virtual ~FileSystemObjectSink() = default; - virtual void createDirectory(const Path & path) = 0; + virtual void createDirectory(const CanonPath & path) = 0; /** * This function in general is no re-entrant. Only one file can be * written at a time. */ virtual void createRegularFile( - const Path & path, + const CanonPath & path, std::function) = 0; - virtual void createSymlink(const Path & path, const std::string & target) = 0; + virtual void createSymlink(const CanonPath & path, const std::string & target) = 0; +}; + +/** + * An extension of `FileSystemObjectSink` that supports file types + * that are not supported by Nix's FSO model. + */ +struct ExtendedFileSystemObjectSink : virtual FileSystemObjectSink +{ + /** + * Create a hard link. The target must be the path of a previously + * encountered file relative to the root of the FSO. + */ + virtual void createHardlink(const CanonPath & path, const CanonPath & target) = 0; }; /** @@ -46,17 +58,17 @@ struct FileSystemObjectSink */ void copyRecursive( SourceAccessor & accessor, const CanonPath & sourcePath, - FileSystemObjectSink & sink, const Path & destPath); + FileSystemObjectSink & sink, const CanonPath & destPath); /** * Ignore everything and do nothing */ struct NullFileSystemObjectSink : FileSystemObjectSink { - void createDirectory(const Path & path) override { } - void createSymlink(const Path & path, const std::string & target) override { } + void createDirectory(const CanonPath & path) override { } + void createSymlink(const CanonPath & path, const std::string & target) override { } void createRegularFile( - const Path & path, + const CanonPath & path, std::function) override; }; @@ -65,15 +77,20 @@ struct NullFileSystemObjectSink : FileSystemObjectSink */ struct RestoreSink : FileSystemObjectSink { - Path dstPath; + std::filesystem::path dstPath; + bool startFsync = false; - void createDirectory(const Path & path) override; + explicit RestoreSink(bool startFsync) + : startFsync{startFsync} + { } + + void createDirectory(const CanonPath & path) override; void createRegularFile( - const Path & path, + const CanonPath & path, std::function) override; - void createSymlink(const Path & path, const std::string & target) override; + void createSymlink(const CanonPath & path, const std::string & target) override; }; /** @@ -88,18 +105,18 @@ struct RegularFileSink : FileSystemObjectSink RegularFileSink(Sink & sink) : sink(sink) { } - void createDirectory(const Path & path) override + void createDirectory(const CanonPath & path) override { regular = false; } - void createSymlink(const Path & path, const std::string & target) override + void createSymlink(const CanonPath & path, const std::string & target) override { regular = false; } void createRegularFile( - const Path & path, + const CanonPath & path, std::function) override; }; diff --git a/src/libutil/git.cc b/src/libutil/git.cc index 8c538c988..af91fa643 100644 --- a/src/libutil/git.cc +++ b/src/libutil/git.cc @@ -53,7 +53,7 @@ static std::string getString(Source & source, int n) void parseBlob( FileSystemObjectSink & sink, - const Path & sinkPath, + const CanonPath & sinkPath, Source & source, BlobMode blobMode, const ExperimentalFeatureSettings & xpSettings) @@ -116,7 +116,7 @@ void parseBlob( void parseTree( FileSystemObjectSink & sink, - const Path & sinkPath, + const CanonPath & sinkPath, Source & source, std::function hook, const ExperimentalFeatureSettings & xpSettings) @@ -147,7 +147,7 @@ void parseTree( Hash hash(HashAlgorithm::SHA1); std::copy(hashs.begin(), hashs.end(), hash.hash); - hook(name, TreeEntry { + hook(CanonPath{name}, TreeEntry { .mode = mode, .hash = hash, }); @@ -171,7 +171,7 @@ ObjectType parseObjectType( void parse( FileSystemObjectSink & sink, - const Path & sinkPath, + const CanonPath & sinkPath, Source & source, BlobMode rootModeIfBlob, std::function hook, @@ -201,14 +201,14 @@ std::optional convertMode(SourceAccessor::Type type) case SourceAccessor::tRegular: return Mode::Regular; case SourceAccessor::tDirectory: return Mode::Directory; case SourceAccessor::tMisc: return std::nullopt; - default: abort(); + default: unreachable(); } } void restore(FileSystemObjectSink & sink, Source & source, std::function hook) { - parse(sink, "", source, BlobMode::Regular, [&](Path name, TreeEntry entry) { + parse(sink, CanonPath::root, source, BlobMode::Regular, [&](CanonPath name, TreeEntry entry) { auto [accessor, from] = hook(entry.hash); auto stat = accessor->lstat(from); auto gotOpt = convertMode(stat.type); diff --git a/src/libutil/git.hh b/src/libutil/git.hh index a65edb964..1dbdb7335 100644 --- a/src/libutil/git.hh +++ b/src/libutil/git.hh @@ -39,7 +39,8 @@ struct TreeEntry Mode mode; Hash hash; - GENERATE_CMP(TreeEntry, me->mode, me->hash); + bool operator ==(const TreeEntry &) const = default; + auto operator <=>(const TreeEntry &) const = default; }; /** @@ -64,7 +65,7 @@ using Tree = std::map; * Implementations may seek to memoize resources (bandwidth, storage, * etc.) for the same Git hash. */ -using SinkHook = void(const Path & name, TreeEntry entry); +using SinkHook = void(const CanonPath & name, TreeEntry entry); /** * Parse the "blob " or "tree " prefix. @@ -89,13 +90,13 @@ enum struct BlobMode : RawMode }; void parseBlob( - FileSystemObjectSink & sink, const Path & sinkPath, + FileSystemObjectSink & sink, const CanonPath & sinkPath, Source & source, BlobMode blobMode, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); void parseTree( - FileSystemObjectSink & sink, const Path & sinkPath, + FileSystemObjectSink & sink, const CanonPath & sinkPath, Source & source, std::function hook, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); @@ -108,7 +109,7 @@ void parseTree( * a blob, this is ignored. */ void parse( - FileSystemObjectSink & sink, const Path & sinkPath, + FileSystemObjectSink & sink, const CanonPath & sinkPath, Source & source, BlobMode rootModeIfBlob, std::function hook, diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index d4c9d6533..748176d33 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -25,7 +25,7 @@ static size_t regularHashSize(HashAlgorithm type) { case HashAlgorithm::SHA256: return sha256HashSize; case HashAlgorithm::SHA512: return sha512HashSize; } - abort(); + unreachable(); } @@ -41,7 +41,7 @@ Hash::Hash(HashAlgorithm algo) : algo(algo) } -bool Hash::operator == (const Hash & h2) const +bool Hash::operator == (const Hash & h2) const noexcept { if (hashSize != h2.hashSize) return false; for (unsigned int i = 0; i < hashSize; i++) @@ -50,21 +50,14 @@ bool Hash::operator == (const Hash & h2) const } -bool Hash::operator != (const Hash & h2) const +std::strong_ordering Hash::operator <=> (const Hash & h) const noexcept { - return !(*this == h2); -} - - -bool Hash::operator < (const Hash & h) const -{ - if (hashSize < h.hashSize) return true; - if (hashSize > h.hashSize) return false; + if (auto cmp = hashSize <=> h.hashSize; cmp != 0) return cmp; for (unsigned int i = 0; i < hashSize; i++) { - if (hash[i] < h.hash[i]) return true; - if (hash[i] > h.hash[i]) return false; + if (auto cmp = hash[i] <=> h.hash[i]; cmp != 0) return cmp; } - return false; + if (auto cmp = algo <=> h.algo; cmp != 0) return cmp; + return std::strong_ordering::equivalent; } @@ -252,7 +245,12 @@ Hash::Hash(std::string_view rest, HashAlgorithm algo, bool isSRI) } else if (isSRI || rest.size() == base64Len()) { - auto d = base64Decode(rest); + std::string d; + try { + d = base64Decode(rest); + } catch (Error & e) { + e.addTrace({}, "While decoding hash '%s'", rest); + } if (d.size() != hashSize) throw BadHash("invalid %s hash '%s'", isSRI ? "SRI" : "base-64", rest); assert(hashSize); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index e14aae43c..dc95b9f2f 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -86,19 +86,14 @@ private: public: /** - * Check whether two hash are equal. + * Check whether two hashes are equal. */ - bool operator == (const Hash & h2) const; + bool operator == (const Hash & h2) const noexcept; /** - * Check whether two hash are not equal. + * Compare how two hashes are ordered. */ - bool operator != (const Hash & h2) const; - - /** - * For sorting. - */ - bool operator < (const Hash & h) const; + std::strong_ordering operator <=> (const Hash & h2) const noexcept; /** * Returns the length of a base-16 representation of this hash. diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc index 1b911bf75..dff068e07 100644 --- a/src/libutil/json-utils.cc +++ b/src/libutil/json-utils.cc @@ -39,12 +39,9 @@ std::optional optionalValueAt(const nlohmann::json::object_t & m } -std::optional getNullable(const nlohmann::json & value) +const nlohmann::json * getNullable(const nlohmann::json & value) { - if (value.is_null()) - return std::nullopt; - - return value.get(); + return value.is_null() ? nullptr : &value; } /** diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index 08c98cc8c..a61c9cada 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -3,12 +3,13 @@ #include #include -#include #include "types.hh" namespace nix { +enum struct ExperimentalFeature; + const nlohmann::json * get(const nlohmann::json & map, const std::string & key); nlohmann::json * get(nlohmann::json & map, const std::string & key); @@ -29,7 +30,7 @@ std::optional optionalValueAt(const nlohmann::json::object_t & v * Downcast the json object, failing with a nice error if the conversion fails. * See https://json.nlohmann.me/features/types/ */ -std::optional getNullable(const nlohmann::json & value); +const nlohmann::json * getNullable(const nlohmann::json & value); const nlohmann::json::object_t & getObject(const nlohmann::json & value); const nlohmann::json::array_t & getArray(const nlohmann::json & value); const nlohmann::json::string_t & getString(const nlohmann::json & value); @@ -71,6 +72,12 @@ struct json_avoids_null> : std::true_type {}; template struct json_avoids_null> : std::true_type {}; +/** + * `ExperimentalFeature` is always rendered as a string. + */ +template<> +struct json_avoids_null : std::true_type {}; + } namespace nlohmann { diff --git a/src/libutil/linux/cgroup.cc b/src/libutil/linux/cgroup.cc index ec4077478..ad3e8a017 100644 --- a/src/libutil/linux/cgroup.cc +++ b/src/libutil/linux/cgroup.cc @@ -1,4 +1,5 @@ #include "cgroup.hh" +#include "signals.hh" #include "util.hh" #include "file-system.hh" #include "finally.hh" @@ -65,6 +66,7 @@ static CgroupStats destroyCgroup(const std::filesystem::path & cgroup, bool retu /* Otherwise, manually kill every process in the subcgroups and this cgroup. */ for (auto & entry : std::filesystem::directory_iterator{cgroup}) { + checkInterrupt(); if (entry.symlink_status().type() != std::filesystem::file_type::directory) continue; destroyCgroup(cgroup / entry.path().filename(), false); } @@ -142,4 +144,23 @@ CgroupStats destroyCgroup(const Path & cgroup) return destroyCgroup(cgroup, true); } +std::string getCurrentCgroup() +{ + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) + throw Error("cannot determine the cgroups file system"); + + auto ourCgroups = getCgroups("/proc/self/cgroup"); + auto ourCgroup = ourCgroups[""]; + if (ourCgroup == "") + throw Error("cannot determine cgroup name from /proc/self/cgroup"); + return ourCgroup; +} + +std::string getRootCgroup() +{ + static std::string rootCgroup = getCurrentCgroup(); + return rootCgroup; +} + } diff --git a/src/libutil/linux/cgroup.hh b/src/libutil/linux/cgroup.hh index 783a0ab87..87d135ba6 100644 --- a/src/libutil/linux/cgroup.hh +++ b/src/libutil/linux/cgroup.hh @@ -25,4 +25,13 @@ struct CgroupStats */ CgroupStats destroyCgroup(const Path & cgroup); +std::string getCurrentCgroup(); + +/** + * Get the cgroup that should be used as the parent when creating new + * sub-cgroups. The first time this is called, the current cgroup will be + * returned, and then all subsequent calls will return the original cgroup. + */ +std::string getRootCgroup(); + } diff --git a/src/libutil/linux/meson.build b/src/libutil/linux/meson.build new file mode 100644 index 000000000..a1ded76ca --- /dev/null +++ b/src/libutil/linux/meson.build @@ -0,0 +1,11 @@ +sources += files( + 'cgroup.cc', + 'namespaces.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'cgroup.hh', + 'namespaces.hh', +) diff --git a/src/libutil/linux/namespaces.cc b/src/libutil/linux/namespaces.cc index f8289ef39..c5e21dffc 100644 --- a/src/libutil/linux/namespaces.cc +++ b/src/libutil/linux/namespaces.cc @@ -118,7 +118,7 @@ void saveMountNamespace() void restoreMountNamespace() { try { - auto savedCwd = absPath("."); + auto savedCwd = std::filesystem::current_path(); if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) throw SysError("restoring parent mount namespace"); @@ -137,10 +137,10 @@ void restoreMountNamespace() } } -void unshareFilesystem() +void tryUnshareFilesystem() { - if (unshare(CLONE_FS) != 0 && errno != EPERM) - throw SysError("unsharing filesystem state in download thread"); + if (unshare(CLONE_FS) != 0 && errno != EPERM && errno != ENOSYS) + throw SysError("unsharing filesystem state"); } } diff --git a/src/libutil/linux/namespaces.hh b/src/libutil/linux/namespaces.hh index ef3c9123f..208920b80 100644 --- a/src/libutil/linux/namespaces.hh +++ b/src/libutil/linux/namespaces.hh @@ -20,11 +20,13 @@ void saveMountNamespace(); void restoreMountNamespace(); /** - * Cause this thread to not share any FS attributes with the main + * Cause this thread to try to not share any FS attributes with the main * thread, because this causes setns() in restoreMountNamespace() to * fail. + * + * This is best effort -- EPERM and ENOSYS failures are just ignored. */ -void unshareFilesystem(); +void tryUnshareFilesystem(); bool userNamespacesSupported(); diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 5cd8d4ac8..e9b498e65 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -40,3 +40,5 @@ $(foreach i, $(wildcard $(d)/signature/*.hh), \ ifeq ($(HAVE_LIBCPUID), 1) libutil_LDFLAGS += -lcpuid endif + +$(eval $(call install-file-in, $(buildprefix)$(d)/nix-util.pc, $(libdir)/pkgconfig, 0644)) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 2511c8849..3d7371457 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -3,11 +3,12 @@ #include "environment-variables.hh" #include "terminal.hh" #include "util.hh" -#include "config.hh" +#include "config-global.hh" #include "source-path.hh" #include "position.hh" #include +#include #include #include @@ -84,10 +85,10 @@ public: void logEI(const ErrorInfo & ei) override { - std::stringstream oss; + std::ostringstream oss; showErrorInfo(oss, ei, loggerSettings.showTrace.get()); - log(ei.level, oss.str()); + log(ei.level, toView(oss)); } void startActivity(ActivityId act, Verbosity lvl, ActivityType type, @@ -188,7 +189,7 @@ struct JSONLogger : Logger { else if (f.type == Logger::Field::tString) arr.push_back(f.s); else - abort(); + unreachable(); } void write(const nlohmann::json & json) @@ -345,7 +346,7 @@ Activity::~Activity() try { logger.stopActivity(id); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 9e81132e3..250f92099 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "types.hh" #include "error.hh" #include "config.hh" diff --git a/src/libutil/lru-cache.hh b/src/libutil/lru-cache.hh index 0e19517ed..6e14cac35 100644 --- a/src/libutil/lru-cache.hh +++ b/src/libutil/lru-cache.hh @@ -89,7 +89,7 @@ public: return i->second.second; } - size_t size() + size_t size() const { return data.size(); } diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index b7207cffb..c4eee1031 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -124,9 +124,9 @@ SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents using File = MemorySourceAccessor::File; -void MemorySink::createDirectory(const Path & path) +void MemorySink::createDirectory(const CanonPath & path) { - auto * f = dst.open(CanonPath{path}, File { File::Directory { } }); + auto * f = dst.open(path, File { File::Directory { } }); if (!f) throw Error("file '%s' cannot be made because some parent file is not a directory", path); @@ -146,9 +146,9 @@ struct CreateMemoryRegularFile : CreateRegularFileSink { void preallocateContents(uint64_t size) override; }; -void MemorySink::createRegularFile(const Path & path, std::function func) +void MemorySink::createRegularFile(const CanonPath & path, std::function func) { - auto * f = dst.open(CanonPath{path}, File { File::Regular {} }); + auto * f = dst.open(path, File { File::Regular {} }); if (!f) throw Error("file '%s' cannot be made because some parent file is not a directory", path); if (auto * rp = std::get_if(&f->raw)) { @@ -173,9 +173,9 @@ void CreateMemoryRegularFile::operator () (std::string_view data) regularFile.contents += data; } -void MemorySink::createSymlink(const Path & path, const std::string & target) +void MemorySink::createSymlink(const CanonPath & path, const std::string & target) { - auto * f = dst.open(CanonPath{path}, File { File::Symlink { } }); + auto * f = dst.open(path, File { File::Symlink { } }); if (!f) throw Error("file '%s' cannot be made because some parent file is not a directory", path); if (auto * s = std::get_if(&f->raw)) diff --git a/src/libutil/memory-source-accessor.hh b/src/libutil/memory-source-accessor.hh index c8f793922..012a388c0 100644 --- a/src/libutil/memory-source-accessor.hh +++ b/src/libutil/memory-source-accessor.hh @@ -15,11 +15,15 @@ struct MemorySourceAccessor : virtual SourceAccessor * defining what a "file system object" is in Nix. */ struct File { + bool operator == (const File &) const noexcept; + std::strong_ordering operator <=> (const File &) const noexcept; + struct Regular { bool executable = false; std::string contents; - GENERATE_CMP(Regular, me->executable, me->contents); + bool operator == (const Regular &) const = default; + auto operator <=> (const Regular &) const = default; }; struct Directory { @@ -27,13 +31,16 @@ struct MemorySourceAccessor : virtual SourceAccessor std::map> contents; - GENERATE_CMP(Directory, me->contents); + bool operator == (const Directory &) const noexcept; + // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. + bool operator < (const Directory &) const noexcept; }; struct Symlink { std::string target; - GENERATE_CMP(Symlink, me->target); + bool operator == (const Symlink &) const = default; + auto operator <=> (const Symlink &) const = default; }; using Raw = std::variant; @@ -41,14 +48,15 @@ struct MemorySourceAccessor : virtual SourceAccessor MAKE_WRAPPER_CONSTRUCTOR(File); - GENERATE_CMP(File, me->raw); - Stat lstat() const; }; File root { File::Directory {} }; - GENERATE_CMP(MemorySourceAccessor, me->root); + bool operator == (const MemorySourceAccessor &) const noexcept = default; + bool operator < (const MemorySourceAccessor & other) const noexcept { + return root < other.root; + } std::string readFile(const CanonPath & path) override; bool pathExists(const CanonPath & path) override; @@ -72,6 +80,20 @@ struct MemorySourceAccessor : virtual SourceAccessor SourcePath addFile(CanonPath path, std::string && contents); }; + +inline bool MemorySourceAccessor::File::Directory::operator == ( + const MemorySourceAccessor::File::Directory &) const noexcept = default; +inline bool MemorySourceAccessor::File::Directory::operator < ( + const MemorySourceAccessor::File::Directory & other) const noexcept +{ + return contents < other.contents; +} + +inline bool MemorySourceAccessor::File::operator == ( + const MemorySourceAccessor::File &) const noexcept = default; +inline std::strong_ordering MemorySourceAccessor::File::operator <=> ( + const MemorySourceAccessor::File &) const noexcept = default; + /** * Write to a `MemorySourceAccessor` at the given path */ @@ -81,13 +103,13 @@ struct MemorySink : FileSystemObjectSink MemorySink(MemorySourceAccessor & dst) : dst(dst) { } - void createDirectory(const Path & path) override; + void createDirectory(const CanonPath & path) override; void createRegularFile( - const Path & path, + const CanonPath & path, std::function) override; - void createSymlink(const Path & path, const std::string & target) override; + void createSymlink(const CanonPath & path, const std::string & target) override; }; } diff --git a/src/libutil/meson.build b/src/libutil/meson.build new file mode 100644 index 000000000..57b741a50 --- /dev/null +++ b/src/libutil/meson.build @@ -0,0 +1,284 @@ +project('nix-util', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +configdata = configuration_data() + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +# Check for each of these functions, and create a define like `#define +# HAVE_LUTIMES 1`. The `#define` is unconditional, 0 for not found and 1 +# for found. One therefore uses it with `#if` not `#ifdef`. +check_funcs = [ + 'close_range', + # Optionally used for changing the mtime of symlinks. + 'lutimes', + # Optionally used for creating pipes on Unix + 'pipe2', + # Optionally used to preallocate files to be large enough before + # writing to them. + 'posix_fallocate', + # Optionally used to get more information about processes failing due + # to a signal on Unix. + 'strsignal', + # Optionally used to try to close more file descriptors (e.g. before + # forking) on Unix. + 'sysconf', +] +foreach funcspec : check_funcs + define_name = 'HAVE_' + funcspec.underscorify().to_upper() + define_value = cxx.has_function(funcspec).to_int() + configdata.set(define_name, define_value) +endforeach + +subdir('build-utils-meson/threads') + +# Check if -latomic is needed +# This is needed for std::atomic on some platforms +# We did not manage to test this reliably on all platforms, so we hardcode +# it for now. +if host_machine.cpu_family() == 'arm' + deps_other += cxx.find_library('atomic') +endif + +if host_machine.system() == 'windows' + socket = cxx.find_library('ws2_32') + deps_other += socket +elif host_machine.system() == 'sunos' + socket = cxx.find_library('socket') + network_service_library = cxx.find_library('nsl') + deps_other += [socket, network_service_library] +endif + +boost = dependency( + 'boost', + modules : ['context', 'coroutine'], + include_type: 'system', +) +# boost is a public dependency, but not a pkg-config dependency unfortunately, so we +# put in `deps_other`. +deps_other += boost + +openssl = dependency( + 'libcrypto', + 'openssl', + version : '>= 1.1.1', +) +deps_private += openssl + +libarchive = dependency('libarchive', version : '>= 3.1.2') +deps_public += libarchive +if get_option('default_library') == 'static' + # Workaround until https://github.com/libarchive/libarchive/issues/1446 is fixed + add_project_arguments('-lz', language : 'cpp') +endif + +sodium = dependency('libsodium', 'sodium') +deps_private += sodium + +brotli = [ + dependency('libbrotlicommon'), + dependency('libbrotlidec'), + dependency('libbrotlienc'), +] +deps_private += brotli + +cpuid_required = get_option('cpuid') +if host_machine.cpu_family() != 'x86_64' and cpuid_required.enabled() + warning('Force-enabling seccomp on non-x86_64 does not make sense') +endif +cpuid = dependency('libcpuid', 'cpuid', required : cpuid_required) +configdata.set('HAVE_LIBCPUID', cpuid.found().to_int()) +deps_private += cpuid + +nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') +deps_public += nlohmann_json + +config_h = configure_file( + configuration : configdata, + output : 'config-util.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'archive.cc', + 'args.cc', + 'canon-path.cc', + 'compression.cc', + 'compute-levels.cc', + 'config.cc', + 'config-global.cc', + 'current-process.cc', + 'english.cc', + 'environment-variables.cc', + 'error.cc', + 'executable-path.cc', + 'exit.cc', + 'experimental-features.cc', + 'file-content-address.cc', + 'file-descriptor.cc', + 'file-system.cc', + 'fs-sink.cc', + 'git.cc', + 'hash.cc', + 'hilite.cc', + 'json-utils.cc', + 'logging.cc', + 'memory-source-accessor.cc', + 'position.cc', + 'posix-source-accessor.cc', + 'references.cc', + 'serialise.cc', + 'signature/local-keys.cc', + 'signature/signer.cc', + 'source-accessor.cc', + 'source-path.cc', + 'strings.cc', + 'suggestions.cc', + 'tarfile.cc', + 'terminal.cc', + 'thread-pool.cc', + 'unix-domain-socket.cc', + 'url.cc', + 'users.cc', + 'util.cc', + 'xml-writer.cc', +) + +include_dirs = [include_directories('.')] + +headers = [config_h] + files( + 'abstract-setting-to-json.hh', + 'ansicolor.hh', + 'archive.hh', + 'args.hh', + 'args/root.hh', + 'callback.hh', + 'canon-path.hh', + 'checked-arithmetic.hh', + 'chunked-vector.hh', + 'closure.hh', + 'comparator.hh', + 'compression.hh', + 'compute-levels.hh', + 'config-global.hh', + 'config-impl.hh', + 'config.hh', + 'current-process.hh', + 'english.hh', + 'environment-variables.hh', + 'error.hh', + 'exec.hh', + 'executable-path.hh', + 'exit.hh', + 'experimental-features.hh', + 'file-content-address.hh', + 'file-descriptor.hh', + 'file-path-impl.hh', + 'file-path.hh', + 'file-system.hh', + 'finally.hh', + 'fmt.hh', + 'fs-sink.hh', + 'git.hh', + 'hash.hh', + 'hilite.hh', + 'json-impls.hh', + 'json-utils.hh', + 'logging.hh', + 'lru-cache.hh', + 'memory-source-accessor.hh', + 'muxable-pipe.hh', + 'os-string.hh', + 'pool.hh', + 'position.hh', + 'posix-source-accessor.hh', + 'processes.hh', + 'ref.hh', + 'references.hh', + 'regex-combinators.hh', + 'repair-flag.hh', + 'serialise.hh', + 'signals.hh', + 'signature/local-keys.hh', + 'signature/signer.hh', + 'source-accessor.hh', + 'source-path.hh', + 'split.hh', + 'std-hash.hh', + 'strings.hh', + 'strings-inline.hh', + 'suggestions.hh', + 'sync.hh', + 'tarfile.hh', + 'terminal.hh', + 'thread-pool.hh', + 'topo-sort.hh', + 'types.hh', + 'unix-domain-socket.hh', + 'url-parts.hh', + 'url.hh', + 'users.hh', + 'util.hh', + 'variant-wrapper.hh', + 'xml-writer.hh', +) + +if host_machine.system() == 'linux' + subdir('linux') +endif + +if host_machine.system() == 'windows' + subdir('windows') +else + subdir('unix') +endif + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nixutil', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] +if host_machine.system() == 'windows' + # `libraries_private` cannot contain ad-hoc dependencies (from + # `find_library), so we need to do this manually + libraries_private += ['-lws2_32'] +endif + +subdir('build-utils-meson/export') diff --git a/src/libutil/meson.options b/src/libutil/meson.options new file mode 100644 index 000000000..21883af01 --- /dev/null +++ b/src/libutil/meson.options @@ -0,0 +1,5 @@ +# vim: filetype=meson + +option('cpuid', type : 'feature', + description : 'determine microarchitecture levels with libcpuid (only relevant on x86_64)', +) diff --git a/src/libutil/muxable-pipe.hh b/src/libutil/muxable-pipe.hh new file mode 100644 index 000000000..53ac39170 --- /dev/null +++ b/src/libutil/muxable-pipe.hh @@ -0,0 +1,82 @@ +#pragma once +///@file + +#include "file-descriptor.hh" +#ifdef _WIN32 +# include "windows-async-pipe.hh" +#endif + +#ifndef _WIN32 +# include +#else +# include +# include "windows-error.hh" +#endif + +namespace nix { + +/** + * An "muxable pipe" is a type of pipe supporting endpoints that wait + * for events on multiple pipes at once. + * + * On Unix, this is just a regular anonymous pipe. On Windows, this has + * to be a named pipe because we need I/O Completion Ports to wait on + * multiple pipes. + */ +using MuxablePipe = +#ifndef _WIN32 + Pipe +#else + windows::AsyncPipe +#endif + ; + +/** + * Use pool() (Unix) / I/O Completion Ports (Windows) to wait for the + * input side of any logger pipe to become `available'. Note that + * `available' (i.e., non-blocking) includes EOF. + */ +struct MuxablePipePollState +{ +#ifndef _WIN32 + std::vector pollStatus; + std::map fdToPollStatus; +#else + OVERLAPPED_ENTRY oentries[0x20] = {0}; + ULONG removed; + bool gotEOF = false; + +#endif + + /** + * Check for ready (Unix) / completed (Windows) operations + */ + void poll( +#ifdef _WIN32 + HANDLE ioport, +#endif + std::optional timeout); + + using CommChannel = +#ifndef _WIN32 + Descriptor +#else + windows::AsyncPipe * +#endif + ; + + /** + * Process for ready (Unix) / completed (Windows) operations, + * calling the callbacks as needed. + * + * @param handleRead callback to be passed read data. + * + * @param handleEOF callback for when the `MuxablePipe` has closed. + */ + void iterate( + std::set & channels, + std::function handleRead, + std::function handleEOF); +}; + +} diff --git a/src/libutil/nix-util.pc.in b/src/libutil/nix-util.pc.in new file mode 100644 index 000000000..85bb1e70e --- /dev/null +++ b/src/libutil/nix-util.pc.in @@ -0,0 +1,9 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Package Manager +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lnixutil +Cflags: -I${includedir}/nix -std=c++2a diff --git a/src/libutil/os-string.hh b/src/libutil/os-string.hh new file mode 100644 index 000000000..3e24763fb --- /dev/null +++ b/src/libutil/os-string.hh @@ -0,0 +1,52 @@ +#pragma once +///@file + +#include +#include +#include + +namespace nix { + +/** + * Named because it is similar to the Rust type, except it is in the + * native encoding not WTF-8. + * + * Same as `std::filesystem::path::value_type`, but manually defined to + * avoid including a much more complex header. + */ +using OsChar = +#if defined(_WIN32) && !defined(__CYGWIN__) + wchar_t +#else + char +#endif + ; + +/** + * Named because it is similar to the Rust type, except it is in the + * native encoding not WTF-8. + * + * Same as `std::filesystem::path::string_type`, but manually defined + * for the same reason as `OsChar`. + */ +using OsString = std::basic_string; + +/** + * `std::string_view` counterpart for `OsString`. + */ +using OsStringView = std::basic_string_view; + +std::string os_string_to_string(OsStringView path); + +OsString string_to_os_string(std::string_view s); + +/** + * Create string literals with the native character width of paths + */ +#ifndef _WIN32 +# define OS_STR(s) s +#else +# define OS_STR(s) L##s +#endif + +} diff --git a/src/libutil/package.nix b/src/libutil/package.nix new file mode 100644 index 000000000..4ce1a75b0 --- /dev/null +++ b/src/libutil/package.nix @@ -0,0 +1,99 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, boost +, brotli +, libarchive +, libcpuid +, libsodium +, nlohmann_json +, openssl + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-util"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + ./linux/meson.build + ./unix/meson.build + ./windows/meson.build + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + brotli + libsodium + openssl + ] ++ lib.optional stdenv.hostPlatform.isx86_64 libcpuid + ; + + propagatedBuildInputs = [ + boost + libarchive + nlohmann_json + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + # + # TODO: change release process to add `pre` in `.version`, remove it + # before tagging, and restore after. + '' + chmod u+w ./.version + echo ${version} > ../../.version + ''; + + mesonFlags = [ + (lib.mesonEnable "cpuid" stdenv.hostPlatform.isx86_64) + ]; + + env = { + # Needed for Meson to find Boost. + # https://github.com/NixOS/nixpkgs/issues/86131. + BOOST_INCLUDEDIR = "${lib.getDev boost}/include"; + BOOST_LIBRARYDIR = "${lib.getLib boost}/lib"; + } // lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/libutil/position.cc b/src/libutil/position.cc index 724e560b7..5a2529262 100644 --- a/src/libutil/position.cc +++ b/src/libutil/position.cc @@ -17,12 +17,6 @@ Pos::operator std::shared_ptr() const return std::make_shared(&*this); } -bool Pos::operator<(const Pos &rhs) const -{ - return std::forward_as_tuple(line, column, origin) - < std::forward_as_tuple(rhs.line, rhs.column, rhs.origin); -} - std::optional Pos::getCodeLines() const { if (line == 0) @@ -116,4 +110,50 @@ void Pos::LinesIterator::bump(bool atFirst) input.remove_prefix(eol); } +std::optional Pos::getSnippetUpTo(const Pos & end) const { + assert(this->origin == end.origin); + + if (end.line < this->line) + return std::nullopt; + + if (auto source = getSource()) { + + auto firstLine = LinesIterator(*source); + for (uint32_t i = 1; i < this->line; ++i) { + ++firstLine; + } + + auto lastLine = LinesIterator(*source); + for (uint32_t i = 1; i < end.line; ++i) { + ++lastLine; + } + + LinesIterator linesEnd; + + std::string result; + for (auto i = firstLine; i != linesEnd; ++i) { + auto firstColumn = i == firstLine ? (this->column ? this->column - 1 : 0) : 0; + if (firstColumn > i->size()) + firstColumn = i->size(); + + auto lastColumn = i == lastLine ? (end.column ? end.column - 1 : 0) : std::numeric_limits::max(); + if (lastColumn < firstColumn) + lastColumn = firstColumn; + if (lastColumn > i->size()) + lastColumn = i->size(); + + result += i->substr(firstColumn, lastColumn - firstColumn); + + if (i == lastLine) { + break; + } else { + result += '\n'; + } + } + return result; + } + return std::nullopt; +} + + } diff --git a/src/libutil/position.hh b/src/libutil/position.hh index 9bdf3b4b5..25217069c 100644 --- a/src/libutil/position.hh +++ b/src/libutil/position.hh @@ -7,6 +7,7 @@ #include #include +#include #include "source-path.hh" @@ -22,21 +23,17 @@ struct Pos struct Stdin { ref source; - bool operator==(const Stdin & rhs) const + bool operator==(const Stdin & rhs) const noexcept { return *source == *rhs.source; } - bool operator!=(const Stdin & rhs) const - { return *source != *rhs.source; } - bool operator<(const Stdin & rhs) const - { return *source < *rhs.source; } + std::strong_ordering operator<=>(const Stdin & rhs) const noexcept + { return *source <=> *rhs.source; } }; struct String { ref source; - bool operator==(const String & rhs) const + bool operator==(const String & rhs) const noexcept { return *source == *rhs.source; } - bool operator!=(const String & rhs) const - { return *source != *rhs.source; } - bool operator<(const String & rhs) const - { return *source < *rhs.source; } + std::strong_ordering operator<=>(const String & rhs) const noexcept + { return *source <=> *rhs.source; } }; typedef std::variant Origin; @@ -65,8 +62,16 @@ struct Pos std::optional getCodeLines() const; bool operator==(const Pos & rhs) const = default; - bool operator!=(const Pos & rhs) const = default; - bool operator<(const Pos & rhs) const; + auto operator<=>(const Pos & rhs) const = default; + + std::optional getSnippetUpTo(const Pos & end) const; + + /** + * Get the SourcePath, if the source was loaded from a file. + */ + std::optional getSourcePath() const { + return *std::get_if(&origin); + } struct LinesIterator { using difference_type = size_t; diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 225fc852c..f26f74d58 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -20,7 +20,7 @@ PosixSourceAccessor::PosixSourceAccessor() SourcePath PosixSourceAccessor::createAtRoot(const std::filesystem::path & path) { - std::filesystem::path path2 = absPath(path.string()); + std::filesystem::path path2 = absPath(path); return { make_ref(path2.root_path()), CanonPath { path2.relative_path().string() }, @@ -90,14 +90,14 @@ bool PosixSourceAccessor::pathExists(const CanonPath & path) std::optional PosixSourceAccessor::cachedLstat(const CanonPath & path) { - static Sync>> _cache; + static SharedSync>> _cache; // Note: we convert std::filesystem::path to Path because the // former is not hashable on libc++. Path absPath = makeAbsPath(path).string(); { - auto cache(_cache.lock()); + auto cache(_cache.readLock()); auto i = cache->find(absPath); if (i != cache->end()) return i->second; } @@ -132,22 +132,24 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & { assertNoSymlinks(path); DirEntries res; - for (auto & entry : std::filesystem::directory_iterator{makeAbsPath(path)}) { - auto type = [&]() -> std::optional { - std::filesystem::file_type nativeType; - try { - nativeType = entry.symlink_status().type(); - } catch (std::filesystem::filesystem_error & e) { - // We cannot always stat the child. (Ideally there is no - // stat because the native directory entry has the type - // already, but this isn't always the case.) - if (e.code() == std::errc::permission_denied || e.code() == std::errc::operation_not_permitted) - return std::nullopt; - else throw; - } + try { + for (auto & entry : std::filesystem::directory_iterator{makeAbsPath(path)}) { + checkInterrupt(); + auto type = [&]() -> std::optional { + std::filesystem::file_type nativeType; + try { + nativeType = entry.symlink_status().type(); + } catch (std::filesystem::filesystem_error & e) { + // We cannot always stat the child. (Ideally there is no + // stat because the native directory entry has the type + // already, but this isn't always the case.) + if (e.code() == std::errc::permission_denied || e.code() == std::errc::operation_not_permitted) + return std::nullopt; + else throw; + } - // cannot exhaustively enumerate because implementation-specific - // additional file types are allowed. + // cannot exhaustively enumerate because implementation-specific + // additional file types are allowed. #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wswitch-enum" switch (nativeType) { @@ -157,8 +159,11 @@ SourceAccessor::DirEntries PosixSourceAccessor::readDirectory(const CanonPath & default: return tMisc; } #pragma GCC diagnostic pop - }(); - res.emplace(entry.path().filename().string(), type); + }(); + res.emplace(entry.path().filename().string(), type); + } + } catch (std::filesystem::filesystem_error & e) { + throw SysError("reading directory %1%", showPath(path)); } return res; } diff --git a/src/libutil/processes.hh b/src/libutil/processes.hh index e319f79e0..bbbe7dcab 100644 --- a/src/libutil/processes.hh +++ b/src/libutil/processes.hh @@ -3,6 +3,7 @@ #include "types.hh" #include "error.hh" +#include "file-descriptor.hh" #include "logging.hh" #include "ansicolor.hh" @@ -12,8 +13,6 @@ #include #include -#include - #include #include #include @@ -25,26 +24,36 @@ namespace nix { struct Sink; struct Source; -#ifndef _WIN32 class Pid { +#ifndef _WIN32 pid_t pid = -1; bool separatePG = false; int killSignal = SIGKILL; +#else + AutoCloseFD pid = INVALID_DESCRIPTOR; +#endif public: Pid(); +#ifndef _WIN32 Pid(pid_t pid); - ~Pid(); void operator =(pid_t pid); operator pid_t(); +#else + Pid(AutoCloseFD pid); + void operator =(AutoCloseFD pid); +#endif + ~Pid(); int kill(); int wait(); + // TODO: Implement for Windows +#ifndef _WIN32 void setSeparatePG(bool separatePG); void setKillSignal(int signal); pid_t release(); -}; #endif +}; #ifndef _WIN32 @@ -118,8 +127,6 @@ public: { } }; -#ifndef _WIN32 - /** * Convert the exit status of a child as returned by wait() into an * error string. @@ -128,6 +135,4 @@ std::string statusToString(int status); bool statusOk(int status); -#endif - } diff --git a/src/libutil/ref.hh b/src/libutil/ref.hh index 5d0c3696d..3d0e64ab4 100644 --- a/src/libutil/ref.hh +++ b/src/libutil/ref.hh @@ -1,9 +1,7 @@ #pragma once ///@file -#include #include -#include #include namespace nix { @@ -25,14 +23,14 @@ public: : p(r.p) { } - explicit ref(const std::shared_ptr & p) + explicit ref(const std::shared_ptr & p) : p(p) { if (!p) throw std::invalid_argument("null pointer cast to ref"); } - explicit ref(T * p) + explicit ref(T * p) : p(p) { if (!p) @@ -77,6 +75,8 @@ public: return ref((std::shared_ptr) p); } + ref & operator=(ref const & rhs) = default; + bool operator == (const ref & other) const { return p == other.p; @@ -87,9 +87,9 @@ public: return p != other.p; } - bool operator < (const ref & other) const + auto operator <=> (const ref & other) const { - return p < other.p; + return p <=> other.p; } private: diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 5ea27ccbe..168d2ed32 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -1,5 +1,6 @@ #include "serialise.hh" #include "signals.hh" +#include "util.hh" #include #include @@ -9,7 +10,10 @@ #ifdef _WIN32 # include +# include # include "windows-error.hh" +#else +# include #endif @@ -49,7 +53,7 @@ void BufferedSink::flush() FdSink::~FdSink() { - try { flush(); } catch (...) { ignoreException(); } + try { flush(); } catch (...) { ignoreExceptionInDestructor(); } } @@ -136,7 +140,7 @@ size_t FdSource::readUnbuffered(char * data, size_t len) checkInterrupt(); if (!::ReadFile(fd, data, len, &n, NULL)) { _good = false; - throw WinError("ReadFile when FdSource::readUnbuffered"); + throw windows::WinError("ReadFile when FdSource::readUnbuffered"); } #else ssize_t n; @@ -158,6 +162,30 @@ bool FdSource::good() } +bool FdSource::hasData() +{ + if (BufferedSource::hasData()) return true; + + while (true) { + fd_set fds; + FD_ZERO(&fds); + int fd_ = fromDescriptorReadOnly(fd); + FD_SET(fd_, &fds); + + struct timeval timeout; + timeout.tv_sec = 0; + timeout.tv_usec = 0; + + auto n = select(fd_ + 1, &fds, nullptr, nullptr, &timeout); + if (n < 0) { + if (errno == EINTR) continue; + throw SysError("polling file descriptor"); + } + return FD_ISSET(fd, &fds); + } +} + + size_t StringSource::read(char * data, size_t len) { if (pos == s.size()) throw EndOfFile("end of string reached"); @@ -171,55 +199,6 @@ size_t StringSource::read(char * data, size_t len) #error Coroutines are broken in this version of Boost! #endif -/* A concrete datatype allow virtual dispatch of stack allocation methods. */ -struct VirtualStackAllocator { - StackAllocator *allocator = StackAllocator::defaultAllocator; - - boost::context::stack_context allocate() { - return allocator->allocate(); - } - - void deallocate(boost::context::stack_context sctx) { - allocator->deallocate(sctx); - } -}; - - -/* This class reifies the default boost coroutine stack allocation strategy with - a virtual interface. */ -class DefaultStackAllocator : public StackAllocator { - boost::coroutines2::default_stack stack; - - boost::context::stack_context allocate() { - return stack.allocate(); - } - - void deallocate(boost::context::stack_context sctx) { - stack.deallocate(sctx); - } -}; - -static DefaultStackAllocator defaultAllocatorSingleton; - -StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton; - - -std::shared_ptr (*create_coro_gc_hook)() = []() -> std::shared_ptr { - return {}; -}; - -/* This class is used for entry and exit hooks on coroutines */ -class CoroutineContext { - /* Disable GC when entering the coroutine without the boehm patch, - * since it doesn't find the main thread stack in this case. - * std::shared_ptr performs type-erasure, so it will call the right - * deleter. */ - const std::shared_ptr coro_gc_hook = create_coro_gc_hook(); -public: - CoroutineContext() {}; - ~CoroutineContext() {}; -}; - std::unique_ptr sourceToSink(std::function fun) { struct SourceToSink : FinishSink @@ -241,14 +220,12 @@ std::unique_ptr sourceToSink(std::function fun) cur = in; if (!coro) { - CoroutineContext ctx; - coro = coro_t::push_type(VirtualStackAllocator{}, [&](coro_t::pull_type & yield) { - LambdaSource source([&](char *out, size_t out_len) { + coro = coro_t::push_type([&](coro_t::pull_type & yield) { + LambdaSource source([&](char * out, size_t out_len) { if (cur.empty()) { yield(); - if (yield.get()) { - return (size_t)0; - } + if (yield.get()) + throw EndOfFile("coroutine has finished"); } size_t n = std::min(cur.size(), out_len); @@ -260,23 +237,17 @@ std::unique_ptr sourceToSink(std::function fun) }); } - if (!*coro) { abort(); } + if (!*coro) { unreachable(); } if (!cur.empty()) { - CoroutineContext ctx; (*coro)(false); } } void finish() override { - if (!coro) return; - if (!*coro) abort(); - { - CoroutineContext ctx; + if (coro && *coro) (*coro)(true); - } - if (*coro) abort(); } }; @@ -307,8 +278,7 @@ std::unique_ptr sinkToSource( size_t read(char * data, size_t len) override { if (!coro) { - CoroutineContext ctx; - coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) { + coro = coro_t::pull_type([&](coro_t::push_type & yield) { LambdaSink sink([&](std::string_view data) { if (!data.empty()) yield(std::string(data)); }); @@ -316,11 +286,10 @@ std::unique_ptr sinkToSource( }); } - if (!*coro) { eof(); abort(); } + if (!*coro) { eof(); unreachable(); } if (pos == cur.size()) { if (!cur.empty()) { - CoroutineContext ctx; (*coro)(); } cur = coro->get(); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 6249ddaf5..d9e34e1e0 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -104,6 +104,9 @@ struct BufferedSource : Source size_t read(char * data, size_t len) override; + /** + * Return true if the buffer is not empty. + */ bool hasData(); protected: @@ -159,15 +162,16 @@ struct FdSource : BufferedSource FdSource(Descriptor fd) : fd(fd) { } FdSource(FdSource &&) = default; - FdSource & operator=(FdSource && s) - { - fd = s.fd; - s.fd = INVALID_DESCRIPTOR; - read = s.read; - return *this; - } + FdSource & operator=(FdSource && s) = default; bool good() override; + + /** + * Return true if the buffer is not empty after a non-blocking + * read. + */ + bool hasData(); + protected: size_t readUnbuffered(char * data, size_t len) override; private: @@ -210,7 +214,7 @@ struct TeeSink : Sink { Sink & sink1, & sink2; TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { } - virtual void operator () (std::string_view data) + virtual void operator () (std::string_view data) override { sink1(data); sink2(data); @@ -227,7 +231,7 @@ struct TeeSource : Source Sink & sink; TeeSource(Source & orig, Sink & sink) : orig(orig), sink(sink) { } - size_t read(char * data, size_t len) + size_t read(char * data, size_t len) override { size_t n = orig.read(data, len); sink({data, n}); @@ -244,7 +248,7 @@ struct SizedSource : Source size_t remain; SizedSource(Source & orig, size_t size) : orig(orig), remain(size) { } - size_t read(char * data, size_t len) + size_t read(char * data, size_t len) override { if (this->remain <= 0) { throw EndOfFile("sized: unexpected end-of-file"); @@ -283,6 +287,26 @@ struct LengthSink : Sink } }; +/** + * A wrapper source that counts the number of bytes read from it. + */ +struct LengthSource : Source +{ + Source & next; + + LengthSource(Source & next) : next(next) + { } + + uint64_t total = 0; + + size_t read(char * data, size_t len) override + { + auto n = next.read(data, len); + total += n; + return n; + } +}; + /** * Convert a function into a sink. */ @@ -469,13 +493,17 @@ struct FramedSource : Source ~FramedSource() { - if (!eof) { - while (true) { - auto n = readInt(from); - if (!n) break; - std::vector data(n); - from(data.data(), n); + try { + if (!eof) { + while (true) { + auto n = readInt(from); + if (!n) break; + std::vector data(n); + from(data.data(), n); + } } + } catch (...) { + ignoreExceptionInDestructor(); } } @@ -504,15 +532,16 @@ struct FramedSource : Source /** * Write as chunks in the format expected by FramedSource. * - * The exception_ptr reference can be used to terminate the stream when you - * detect that an error has occurred on the remote end. + * The `checkError` function can be used to terminate the stream when you + * detect that an error has occurred. It does so by throwing an exception. */ struct FramedSink : nix::BufferedSink { BufferedSink & to; - std::exception_ptr & ex; + std::function checkError; - FramedSink(BufferedSink & to, std::exception_ptr & ex) : to(to), ex(ex) + FramedSink(BufferedSink & to, std::function && checkError) + : to(to), checkError(checkError) { } ~FramedSink() @@ -521,45 +550,18 @@ struct FramedSink : nix::BufferedSink to << 0; to.flush(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } void writeUnbuffered(std::string_view data) override { - /* Don't send more data if the remote has - encountered an error. */ - if (ex) { - auto ex2 = ex; - ex = nullptr; - std::rethrow_exception(ex2); - } + /* Don't send more data if an error has occured. */ + checkError(); + to << data.size(); to(data); }; }; -/** - * Stack allocation strategy for sinkToSource. - * Mutable to avoid a boehm gc dependency in libutil. - * - * boost::context doesn't provide a virtual class, so we define our own. - */ -struct StackAllocator { - virtual boost::context::stack_context allocate() = 0; - virtual void deallocate(boost::context::stack_context sctx) = 0; - - /** - * The stack allocator to use in sinkToSource and potentially elsewhere. - * It is reassigned by the initGC() method in libexpr. - */ - static StackAllocator *defaultAllocator; -}; - -/* Disabling GC when entering a coroutine (without the boehm patch). - mutable to avoid boehm gc dependency in libutil. - */ -extern std::shared_ptr (*create_coro_gc_hook)(); - - } diff --git a/src/libutil/signature/local-keys.cc b/src/libutil/signature/local-keys.cc index 858b036f5..70bcb5f33 100644 --- a/src/libutil/signature/local-keys.cc +++ b/src/libutil/signature/local-keys.cc @@ -14,17 +14,25 @@ BorrowedCryptoValue BorrowedCryptoValue::parse(std::string_view s) return {s.substr(0, colon), s.substr(colon + 1)}; } -Key::Key(std::string_view s) +Key::Key(std::string_view s, bool sensitiveValue) { auto ss = BorrowedCryptoValue::parse(s); name = ss.name; key = ss.payload; - if (name == "" || key == "") - throw Error("secret key is corrupt"); + try { + if (name == "" || key == "") + throw FormatError("key is corrupt"); - key = base64Decode(key); + key = base64Decode(key); + } catch (Error & e) { + std::string extra; + if (!sensitiveValue) + extra = fmt(" with raw value '%s'", key); + e.addTrace({}, "while decoding key named '%s'%s", name, extra); + throw; + } } std::string Key::to_string() const @@ -33,7 +41,7 @@ std::string Key::to_string() const } SecretKey::SecretKey(std::string_view s) - : Key(s) + : Key{s, true} { if (key.size() != crypto_sign_SECRETKEYBYTES) throw Error("secret key is not valid"); @@ -66,7 +74,7 @@ SecretKey SecretKey::generate(std::string_view name) } PublicKey::PublicKey(std::string_view s) - : Key(s) + : Key{s, false} { if (key.size() != crypto_sign_PUBLICKEYBYTES) throw Error("public key is not valid"); @@ -83,7 +91,12 @@ bool PublicKey::verifyDetached(std::string_view data, std::string_view sig) cons bool PublicKey::verifyDetachedAnon(std::string_view data, std::string_view sig) const { - auto sig2 = base64Decode(sig); + std::string sig2; + try { + sig2 = base64Decode(sig); + } catch (Error & e) { + e.addTrace({}, "while decoding signature '%s'", sig); + } if (sig2.size() != crypto_sign_BYTES) throw Error("signature is not valid"); diff --git a/src/libutil/signature/local-keys.hh b/src/libutil/signature/local-keys.hh index 4aafc1239..9977f0dac 100644 --- a/src/libutil/signature/local-keys.hh +++ b/src/libutil/signature/local-keys.hh @@ -31,15 +31,19 @@ struct Key std::string name; std::string key; - /** - * Construct Key from a string in the format - * ‘:’. - */ - Key(std::string_view s); - std::string to_string() const; protected: + + /** + * Construct Key from a string in the format + * ‘:’. + * + * @param sensitiveValue Avoid displaying the raw Base64 in error + * messages to avoid leaking private keys. + */ + Key(std::string_view s, bool sensitiveValue); + Key(std::string_view name, std::string && key) : name(name), key(std::move(key)) { } }; diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index 66093d2cc..e797951c7 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -53,7 +53,7 @@ SourceAccessor::Stat SourceAccessor::lstat(const CanonPath & path) if (auto st = maybeLstat(path)) return *st; else - throw Error("path '%s' does not exist", showPath(path)); + throw FileNotFound("path '%s' does not exist", showPath(path)); } void SourceAccessor::setPathDisplay(std::string displayPrefix, std::string displaySuffix) diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index d7fb0af5f..b16960d4a 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -4,6 +4,7 @@ #include "canon-path.hh" #include "hash.hh" +#include "ref.hh" namespace nix { @@ -29,6 +30,8 @@ enum class SymlinkResolution { Full, }; +MakeError(FileNotFound, Error); + /** * A read-only filesystem abstraction. This is used by the Nix * evaluator and elsewhere for accessing sources in various @@ -149,9 +152,9 @@ struct SourceAccessor : std::enable_shared_from_this return number == x.number; } - bool operator < (const SourceAccessor & x) const + auto operator <=> (const SourceAccessor & x) const { - return number < x.number; + return number <=> x.number; } void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); diff --git a/src/libutil/source-path.cc b/src/libutil/source-path.cc index 023b5ed4b..759d3c355 100644 --- a/src/libutil/source-path.cc +++ b/src/libutil/source-path.cc @@ -47,19 +47,14 @@ SourcePath SourcePath::operator / (const CanonPath & x) const SourcePath SourcePath::operator / (std::string_view c) const { return {accessor, path / c}; } -bool SourcePath::operator==(const SourcePath & x) const +bool SourcePath::operator==(const SourcePath & x) const noexcept { return std::tie(*accessor, path) == std::tie(*x.accessor, x.path); } -bool SourcePath::operator!=(const SourcePath & x) const +std::strong_ordering SourcePath::operator<=>(const SourcePath & x) const noexcept { - return std::tie(*accessor, path) != std::tie(*x.accessor, x.path); -} - -bool SourcePath::operator<(const SourcePath & x) const -{ - return std::tie(*accessor, path) < std::tie(*x.accessor, x.path); + return std::tie(*accessor, path) <=> std::tie(*x.accessor, x.path); } std::ostream & operator<<(std::ostream & str, const SourcePath & path) diff --git a/src/libutil/source-path.hh b/src/libutil/source-path.hh index 83ec6295d..fc2288f74 100644 --- a/src/libutil/source-path.hh +++ b/src/libutil/source-path.hh @@ -8,6 +8,7 @@ #include "ref.hh" #include "canon-path.hh" #include "source-accessor.hh" +#include "std-hash.hh" namespace nix { @@ -103,9 +104,8 @@ struct SourcePath */ SourcePath operator / (std::string_view c) const; - bool operator==(const SourcePath & x) const; - bool operator!=(const SourcePath & x) const; - bool operator<(const SourcePath & x) const; + bool operator==(const SourcePath & x) const noexcept; + std::strong_ordering operator<=>(const SourcePath & x) const noexcept; /** * Convenience wrapper around `SourceAccessor::resolveSymlinks()`. @@ -115,8 +115,21 @@ struct SourcePath { return {accessor, accessor->resolveSymlinks(path, mode)}; } + + friend class std::hash; }; std::ostream & operator << (std::ostream & str, const SourcePath & path); } + +template<> +struct std::hash +{ + std::size_t operator()(const nix::SourcePath & s) const noexcept + { + std::size_t hash = 0; + hash_combine(hash, s.accessor->number, s.path); + return hash; + } +}; diff --git a/src/libutil/std-hash.hh b/src/libutil/std-hash.hh new file mode 100644 index 000000000..c359d11ca --- /dev/null +++ b/src/libutil/std-hash.hh @@ -0,0 +1,24 @@ +#pragma once + +//!@file Hashing utilities for use with unordered_map, etc. (ie low level implementation logic, not domain logic like +//! Nix hashing) + +#include + +namespace nix { + +/** + * hash_combine() from Boost. Hash several hashable values together + * into a single hash. + */ +inline void hash_combine(std::size_t & seed) {} + +template +inline void hash_combine(std::size_t & seed, const T & v, Rest... rest) +{ + std::hash hasher; + seed ^= hasher(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); + hash_combine(seed, rest...); +} + +} // namespace nix diff --git a/src/libutil/strings-inline.hh b/src/libutil/strings-inline.hh new file mode 100644 index 000000000..25b8e0ff6 --- /dev/null +++ b/src/libutil/strings-inline.hh @@ -0,0 +1,106 @@ +#pragma once + +#include "strings.hh" + +namespace nix { + +template +C basicTokenizeString(std::basic_string_view s, std::basic_string_view separators) +{ + C result; + auto pos = s.find_first_not_of(separators, 0); + while (pos != s.npos) { + auto end = s.find_first_of(separators, pos + 1); + if (end == s.npos) + end = s.size(); + result.insert(result.end(), std::basic_string(s, pos, end - pos)); + pos = s.find_first_not_of(separators, end); + } + return result; +} + +template +C tokenizeString(std::string_view s, std::string_view separators) +{ + return basicTokenizeString(s, separators); +} + +template +C basicSplitString(std::basic_string_view s, std::basic_string_view separators) +{ + C result; + size_t pos = 0; + while (pos <= s.size()) { + auto end = s.find_first_of(separators, pos); + if (end == s.npos) + end = s.size(); + result.insert(result.end(), std::basic_string(s, pos, end - pos)); + pos = end + 1; + } + + return result; +} + +template +C splitString(std::string_view s, std::string_view separators) +{ + return basicSplitString(s, separators); +} + +template +std::basic_string basicConcatStringsSep(const std::basic_string_view sep, const C & ss) +{ + size_t size = 0; + bool tail = false; + // need a cast to string_view since this is also called with Symbols + for (const auto & s : ss) { + if (tail) + size += sep.size(); + size += std::basic_string_view{s}.size(); + tail = true; + } + std::basic_string s; + s.reserve(size); + tail = false; + for (auto & i : ss) { + if (tail) + s += sep; + s += i; + tail = true; + } + return s; +} + +template +std::string concatStringsSep(const std::string_view sep, const C & ss) +{ + return basicConcatStringsSep(sep, ss); +} + +template +std::string dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss) +{ + size_t size = 0; + + // TODO? remove to make sure we don't rely on the empty item ignoring behavior, + // or just get rid of this function by understanding the remaining calls. + // for (auto & i : ss) { + // // Make sure we don't rely on the empty item ignoring behavior + // assert(!i.empty()); + // break; + // } + + // need a cast to string_view since this is also called with Symbols + for (const auto & s : ss) + size += sep.size() + std::string_view(s).size(); + std::string s; + s.reserve(size); + for (auto & i : ss) { + if (s.size() != 0) + s += sep; + s += i; + } + return s; +} + +} // namespace nix diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc new file mode 100644 index 000000000..d1c9f700c --- /dev/null +++ b/src/libutil/strings.cc @@ -0,0 +1,50 @@ +#include +#include + +#include "strings-inline.hh" +#include "os-string.hh" + +namespace nix { + +struct view_stringbuf : public std::stringbuf +{ + inline std::string_view toView() + { + auto begin = pbase(); + return {begin, begin + pubseekoff(0, std::ios_base::cur, std::ios_base::out)}; + } +}; + +std::string_view toView(const std::ostringstream & os) +{ + auto buf = static_cast(os.rdbuf()); + return buf->toView(); +} + +template std::list tokenizeString(std::string_view s, std::string_view separators); +template std::set tokenizeString(std::string_view s, std::string_view separators); +template std::vector tokenizeString(std::string_view s, std::string_view separators); + +template std::list splitString(std::string_view s, std::string_view separators); +template std::set splitString(std::string_view s, std::string_view separators); +template std::vector splitString(std::string_view s, std::string_view separators); + +template std::list +basicSplitString(std::basic_string_view s, std::basic_string_view separators); + +template std::string concatStringsSep(std::string_view, const std::list &); +template std::string concatStringsSep(std::string_view, const std::set &); +template std::string concatStringsSep(std::string_view, const std::vector &); + +typedef std::string_view strings_2[2]; +template std::string concatStringsSep(std::string_view, const strings_2 &); +typedef std::string_view strings_3[3]; +template std::string concatStringsSep(std::string_view, const strings_3 &); +typedef std::string_view strings_4[4]; +template std::string concatStringsSep(std::string_view, const strings_4 &); + +template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list &); +template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); +template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); + +} // namespace nix diff --git a/src/libutil/strings.hh b/src/libutil/strings.hh new file mode 100644 index 000000000..533126be1 --- /dev/null +++ b/src/libutil/strings.hh @@ -0,0 +1,74 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace nix { + +/* + * workaround for unavailable view() method (C++20) of std::ostringstream under MacOS with clang-16 + */ +std::string_view toView(const std::ostringstream & os); + +/** + * String tokenizer. + * + * See also `basicSplitString()`, which preserves empty strings between separators, as well as at the start and end. + */ +template +C basicTokenizeString(std::basic_string_view s, std::basic_string_view separators); + +/** + * Like `basicTokenizeString` but specialized to the default `char` + */ +template +C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); + +extern template std::list tokenizeString(std::string_view s, std::string_view separators); +extern template std::set tokenizeString(std::string_view s, std::string_view separators); +extern template std::vector tokenizeString(std::string_view s, std::string_view separators); + +/** + * Split a string, preserving empty strings between separators, as well as at the start and end. + * + * Returns a non-empty collection of strings. + */ +template +C basicSplitString(std::basic_string_view s, std::basic_string_view separators); +template +C splitString(std::string_view s, std::string_view separators); + +extern template std::list splitString(std::string_view s, std::string_view separators); +extern template std::set splitString(std::string_view s, std::string_view separators); +extern template std::vector splitString(std::string_view s, std::string_view separators); + +/** + * Concatenate the given strings with a separator between the elements. + */ +template +std::string concatStringsSep(const std::string_view sep, const C & ss); + +extern template std::string concatStringsSep(std::string_view, const std::list &); +extern template std::string concatStringsSep(std::string_view, const std::set &); +extern template std::string concatStringsSep(std::string_view, const std::vector &); + +/** + * Ignore any empty strings at the start of the list, and then concatenate the + * given strings with a separator between the elements. + * + * @deprecated This function exists for historical reasons. You probably just + * want to use `concatStringsSep`. + */ +template +[[deprecated( + "Consider removing the empty string dropping behavior. If acceptable, use concatStringsSep instead.")]] std::string +dropEmptyInitThenConcatStringsSep(const std::string_view sep, const C & ss); + +extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::list &); +extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::set &); +extern template std::string dropEmptyInitThenConcatStringsSep(std::string_view, const std::vector &); + +} diff --git a/src/libutil/suggestions.cc b/src/libutil/suggestions.cc index e67e986fb..84c8e296f 100644 --- a/src/libutil/suggestions.cc +++ b/src/libutil/suggestions.cc @@ -38,8 +38,8 @@ int levenshteinDistance(std::string_view first, std::string_view second) } Suggestions Suggestions::bestMatches ( - std::set allMatches, - std::string query) + const std::set & allMatches, + std::string_view query) { std::set res; for (const auto & possibleMatch : allMatches) { diff --git a/src/libutil/suggestions.hh b/src/libutil/suggestions.hh index 9abf5ee5f..e39ab400c 100644 --- a/src/libutil/suggestions.hh +++ b/src/libutil/suggestions.hh @@ -1,7 +1,6 @@ #pragma once ///@file -#include "comparator.hh" #include "types.hh" #include @@ -20,7 +19,8 @@ public: std::string to_string() const; - GENERATE_CMP(Suggestion, me->distance, me->suggestion) + bool operator ==(const Suggestion &) const = default; + auto operator <=>(const Suggestion &) const = default; }; class Suggestions { @@ -35,8 +35,8 @@ public: ) const; static Suggestions bestMatches ( - std::set allMatches, - std::string query + const std::set & allMatches, + std::string_view query ); Suggestions& operator+=(const Suggestions & other); diff --git a/src/libutil/sync.hh b/src/libutil/sync.hh index 47e4512b1..d340f3d97 100644 --- a/src/libutil/sync.hh +++ b/src/libutil/sync.hh @@ -3,9 +3,12 @@ #include #include +#include #include #include +#include "error.hh" + namespace nix { /** @@ -24,8 +27,8 @@ namespace nix { * Here, "data" is automatically unlocked when "data_" goes out of * scope. */ -template -class Sync +template +class SyncBase { private: M mutex; @@ -33,23 +36,22 @@ private: public: - Sync() { } - Sync(const T & data) : data(data) { } - Sync(T && data) noexcept : data(std::move(data)) { } + SyncBase() { } + SyncBase(const T & data) : data(data) { } + SyncBase(T && data) noexcept : data(std::move(data)) { } + template class Lock { - private: - Sync * s; - std::unique_lock lk; - friend Sync; - Lock(Sync * s) : s(s), lk(s->mutex) { } + protected: + SyncBase * s; + L lk; + friend SyncBase; + Lock(SyncBase * s) : s(s), lk(s->mutex) { } public: - Lock(Lock && l) : s(l.s) { abort(); } + Lock(Lock && l) : s(l.s) { unreachable(); } Lock(const Lock & l) = delete; ~Lock() { } - T * operator -> () { return &s->data; } - T & operator * () { return s->data; } void wait(std::condition_variable & cv) { @@ -83,7 +85,34 @@ public: } }; - Lock lock() { return Lock(this); } + struct WriteLock : Lock + { + T * operator -> () { return &WriteLock::s->data; } + T & operator * () { return WriteLock::s->data; } + }; + + /** + * Acquire write (exclusive) access to the inner value. + */ + WriteLock lock() { return WriteLock(this); } + + struct ReadLock : Lock + { + const T * operator -> () { return &ReadLock::s->data; } + const T & operator * () { return ReadLock::s->data; } + }; + + /** + * Acquire read access to the inner value. When using + * `std::shared_mutex`, this will use a shared lock. + */ + ReadLock readLock() const { return ReadLock(const_cast(this)); } }; +template +using Sync = SyncBase, std::unique_lock>; + +template +using SharedSync = SyncBase, std::shared_lock>; + } diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 6bb2bd2f3..a8a22d283 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -8,6 +8,10 @@ namespace nix { +namespace fs { +using namespace std::filesystem; +} + namespace { int callback_open(struct archive *, void * self) @@ -67,6 +71,17 @@ int getArchiveFilterCodeByName(const std::string & method) return code; } +static void enableSupportedFormats(struct archive * archive) +{ + archive_read_support_format_tar(archive); + archive_read_support_format_zip(archive); + + /* Enable support for empty files so we don't throw an exception + for empty HTTP 304 "Not modified" responses. See + downloadTarball(). */ + archive_read_support_format_empty(archive); +} + TarArchive::TarArchive(Source & source, bool raw, std::optional compression_method) : archive{archive_read_new()} , source{&source} @@ -78,9 +93,9 @@ TarArchive::TarArchive(Source & source, bool raw, std::optional com archive_read_support_filter_by_code(archive, getArchiveFilterCodeByName(*compression_method)); } - if (!raw) { - archive_read_support_format_all(archive); - } else { + if (!raw) + enableSupportedFormats(archive); + else { archive_read_support_format_raw(archive); archive_read_support_format_empty(archive); } @@ -91,14 +106,14 @@ TarArchive::TarArchive(Source & source, bool raw, std::optional com "Failed to open archive (%s)"); } -TarArchive::TarArchive(const Path & path) +TarArchive::TarArchive(const fs::path & path) : archive{archive_read_new()} , buffer(defaultBufferSize) { archive_read_support_filter_all(archive); - archive_read_support_format_all(archive); + enableSupportedFormats(archive); archive_read_set_option(archive, NULL, "mac-ext", NULL); - check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s"); + check(archive_read_open_filename(archive, path.string().c_str(), 16384), "failed to open archive: %s"); } void TarArchive::close() @@ -112,7 +127,7 @@ TarArchive::~TarArchive() archive_read_free(this->archive); } -static void extract_archive(TarArchive & archive, const Path & destDir) +static void extract_archive(TarArchive & archive, const fs::path & destDir) { int flags = ARCHIVE_EXTRACT_TIME | ARCHIVE_EXTRACT_SECURE_SYMLINKS | ARCHIVE_EXTRACT_SECURE_NODOTDOT; @@ -129,7 +144,7 @@ static void extract_archive(TarArchive & archive, const Path & destDir) else archive.check(r); - archive_entry_copy_pathname(entry, (destDir + "/" + name).c_str()); + archive_entry_copy_pathname(entry, (destDir / name).string().c_str()); // sources can and do contain dirs with no rx bits if (archive_entry_filetype(entry) == AE_IFDIR && (archive_entry_mode(entry) & 0500) != 0500) @@ -138,7 +153,7 @@ static void extract_archive(TarArchive & archive, const Path & destDir) // Patch hardlink path const char * original_hardlink = archive_entry_hardlink(entry); if (original_hardlink) { - archive_entry_copy_hardlink(entry, (destDir + "/" + original_hardlink).c_str()); + archive_entry_copy_hardlink(entry, (destDir / original_hardlink).string().c_str()); } archive.check(archive_read_extract(archive.archive, entry, flags)); @@ -147,23 +162,23 @@ static void extract_archive(TarArchive & archive, const Path & destDir) archive.close(); } -void unpackTarfile(Source & source, const Path & destDir) +void unpackTarfile(Source & source, const fs::path & destDir) { auto archive = TarArchive(source); - createDirs(destDir); + fs::create_directories(destDir); extract_archive(archive, destDir); } -void unpackTarfile(const Path & tarFile, const Path & destDir) +void unpackTarfile(const fs::path & tarFile, const fs::path & destDir) { auto archive = TarArchive(tarFile); - createDirs(destDir); + fs::create_directories(destDir); extract_archive(archive, destDir); } -time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSink) +time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & parseSink) { time_t lastModified = 0; @@ -176,6 +191,7 @@ time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSin auto path = archive_entry_pathname(entry); if (!path) throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); + auto cpath = CanonPath{path}; if (r == ARCHIVE_WARN) warn(archive_error_string(archive.archive)); else @@ -183,14 +199,19 @@ time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSin lastModified = std::max(lastModified, archive_entry_mtime(entry)); - switch (archive_entry_filetype(entry)) { + if (auto target = archive_entry_hardlink(entry)) { + parseSink.createHardlink(cpath, CanonPath(target)); + continue; + } + + switch (auto type = archive_entry_filetype(entry)) { case AE_IFDIR: - parseSink.createDirectory(path); + parseSink.createDirectory(cpath); break; case AE_IFREG: { - parseSink.createRegularFile(path, [&](auto & crf) { + parseSink.createRegularFile(cpath, [&](auto & crf) { if (archive_entry_mode(entry) & S_IXUSR) crf.isExecutable(); @@ -214,13 +235,13 @@ time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSin case AE_IFLNK: { auto target = archive_entry_symlink(entry); - parseSink.createSymlink(path, target); + parseSink.createSymlink(cpath, target); break; } default: - throw Error("file '%s' in tarball has unsupported file type", path); + throw Error("file '%s' in tarball has unsupported file type %d", path, type); } } diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index 705d211e4..5e29c6bba 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -15,7 +15,7 @@ struct TarArchive void check(int err, const std::string & reason = "failed to extract archive (%s)"); - explicit TarArchive(const Path & path); + explicit TarArchive(const std::filesystem::path & path); /// @brief Create a generic archive from source. /// @param source - Input byte stream. @@ -37,10 +37,10 @@ struct TarArchive int getArchiveFilterCodeByName(const std::string & method); -void unpackTarfile(Source & source, const Path & destDir); +void unpackTarfile(Source & source, const std::filesystem::path & destDir); -void unpackTarfile(const Path & tarFile, const Path & destDir); +void unpackTarfile(const std::filesystem::path & tarFile, const std::filesystem::path & destDir); -time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSink); +time_t unpackTarfileToSink(TarArchive & archive, ExtendedFileSystemObjectSink & parseSink); } diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc index 4dc280f8c..5d5ff7dcb 100644 --- a/src/libutil/terminal.cc +++ b/src/libutil/terminal.cc @@ -4,6 +4,8 @@ #if _WIN32 # include +# define WIN32_LEAN_AND_MEAN +# include # define isatty _isatty #else # include @@ -97,17 +99,26 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int w static Sync> windowSize{{0, 0}}; -#ifndef _WIN32 void updateWindowSize() { + #ifndef _WIN32 struct winsize ws; if (ioctl(2, TIOCGWINSZ, &ws) == 0) { auto windowSize_(windowSize.lock()); windowSize_->first = ws.ws_row; windowSize_->second = ws.ws_col; } + #else + CONSOLE_SCREEN_BUFFER_INFO info; + // From https://stackoverflow.com/a/12642749 + if (GetConsoleScreenBufferInfo(GetStdHandle(STD_OUTPUT_HANDLE), &info) != 0) { + auto windowSize_(windowSize.lock()); + // From https://github.com/libuv/libuv/blob/v1.48.0/src/win/tty.c#L1130 + windowSize_->first = info.srWindow.Bottom - info.srWindow.Top + 1; + windowSize_->second = info.dwSize.X; + } + #endif } -#endif std::pair getWindowSize() diff --git a/src/libutil/terminal.hh b/src/libutil/terminal.hh index 628833283..7ff05a487 100644 --- a/src/libutil/terminal.hh +++ b/src/libutil/terminal.hh @@ -1,7 +1,8 @@ #pragma once ///@file -#include "types.hh" +#include +#include namespace nix { /** @@ -21,16 +22,13 @@ std::string filterANSIEscapes(std::string_view s, bool filterAll = false, unsigned int width = std::numeric_limits::max()); -#ifndef _WIN32 - /** - * Recalculate the window size, updating a global variable. Used in the - * `SIGWINCH` signal handler. + * Recalculate the window size, updating a global variable. + * + * Used in the `SIGWINCH` signal handler on Unix, for example. */ void updateWindowSize(); -#endif - /** * @return the number of rows and columns of the terminal. * diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index 0f6349642..0355e1f07 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -111,9 +111,8 @@ void ThreadPool::doWork(bool mainThread) try { std::rethrow_exception(exc); } catch (std::exception & e) { - if (!dynamic_cast(&e) && - !dynamic_cast(&e)) - ignoreException(); + if (!dynamic_cast(&e)) + ignoreExceptionExceptInterrupt(); } catch (...) { } } diff --git a/src/libutil/types.hh b/src/libutil/types.hh index c86f52175..325e3ea73 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -1,12 +1,10 @@ #pragma once ///@file -#include "ref.hh" #include #include #include -#include #include #include #include diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix-domain-socket.cc index 87914bb83..1707fdb75 100644 --- a/src/libutil/unix-domain-socket.cc +++ b/src/libutil/unix-domain-socket.cc @@ -24,7 +24,7 @@ AutoCloseFD createUnixDomainSocket() if (!fdSocket) throw SysError("cannot create Unix domain socket"); #ifndef _WIN32 - closeOnExec(fdSocket.get()); + unix::closeOnExec(fdSocket.get()); #endif return fdSocket; } diff --git a/src/libutil/unix/environment-variables.cc b/src/libutil/unix/environment-variables.cc index 9c6fd3b18..cd7c8f5e5 100644 --- a/src/libutil/unix/environment-variables.cc +++ b/src/libutil/unix/environment-variables.cc @@ -9,4 +9,14 @@ int setEnv(const char * name, const char * value) return ::setenv(name, value, 1); } +std::optional getEnvOs(const std::string & key) +{ + return getEnv(key); +} + +int setEnvOs(const OsString & name, const OsString & value) +{ + return setEnv(name.c_str(), value.c_str()); +} + } diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc index 84a33af81..2c1126e09 100644 --- a/src/libutil/unix/file-descriptor.cc +++ b/src/libutil/unix/file-descriptor.cc @@ -110,8 +110,8 @@ void Pipe::create() if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); #else if (pipe(fds) != 0) throw SysError("creating pipe"); - closeOnExec(fds[0]); - closeOnExec(fds[1]); + unix::closeOnExec(fds[0]); + unix::closeOnExec(fds[1]); #endif readSide = fds[0]; writeSide = fds[1]; @@ -120,13 +120,38 @@ void Pipe::create() ////////////////////////////////////////////////////////////////////// -void closeMostFDs(const std::set & exceptions) +#if __linux__ || __FreeBSD__ +static int unix_close_range(unsigned int first, unsigned int last, int flags) { +#if !HAVE_CLOSE_RANGE + return syscall(SYS_close_range, first, last, (unsigned int)flags); +#else + return close_range(first, last, flags); +#endif +} +#endif + +void unix::closeExtraFDs() +{ + constexpr int MAX_KEPT_FD = 2; + static_assert(std::max({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO}) == MAX_KEPT_FD); + +#if __linux__ || __FreeBSD__ + // first try to close_range everything we don't care about. if this + // returns an error with these parameters we're running on a kernel + // that does not implement close_range (i.e. pre 5.9) and fall back + // to the old method. we should remove that though, in some future. + if (unix_close_range(MAX_KEPT_FD + 1, ~0U, 0) == 0) { + return; + } +#endif + #if __linux__ try { for (auto & s : std::filesystem::directory_iterator{"/proc/self/fd"}) { + checkInterrupt(); auto fd = std::stoi(s.path().filename()); - if (!exceptions.count(fd)) { + if (fd > MAX_KEPT_FD) { debug("closing leaked FD %d", fd); close(fd); } @@ -138,14 +163,15 @@ void closeMostFDs(const std::set & exceptions) #endif int maxFD = 0; +#if HAVE_SYSCONF maxFD = sysconf(_SC_OPEN_MAX); - for (int fd = 0; fd < maxFD; ++fd) - if (!exceptions.count(fd)) - close(fd); /* ignore result */ +#endif + for (int fd = MAX_KEPT_FD + 1; fd < maxFD; ++fd) + close(fd); /* ignore result */ } -void closeOnExec(int fd) +void unix::closeOnExec(int fd) { int prev; if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || diff --git a/src/libutil/unix/file-path.cc b/src/libutil/unix/file-path.cc index 294048a2f..cccee86a1 100644 --- a/src/libutil/unix/file-path.cc +++ b/src/libutil/unix/file-path.cc @@ -8,16 +8,6 @@ namespace nix { -std::string os_string_to_string(PathViewNG::string_view path) -{ - return std::string { path }; -} - -std::filesystem::path::string_type string_to_os_string(std::string_view s) -{ - return std::string { s }; -} - std::optional maybePath(PathView path) { return { path }; diff --git a/src/libutil/unix/meson.build b/src/libutil/unix/meson.build new file mode 100644 index 000000000..1c5bf27fb --- /dev/null +++ b/src/libutil/unix/meson.build @@ -0,0 +1,18 @@ +sources += files( + 'environment-variables.cc', + 'file-descriptor.cc', + 'file-path.cc', + 'file-system.cc', + 'muxable-pipe.cc', + 'os-string.cc', + 'processes.cc', + 'signals.cc', + 'users.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'monitor-fd.hh', + 'signals-impl.hh', +) diff --git a/src/libutil/unix/monitor-fd.hh b/src/libutil/unix/monitor-fd.hh index 103894de9..b6610feff 100644 --- a/src/libutil/unix/monitor-fd.hh +++ b/src/libutil/unix/monitor-fd.hh @@ -40,7 +40,9 @@ public: #endif ; auto count = poll(fds, 1, -1); - if (count == -1) abort(); // can't happen + if (count == -1) + unreachable(); + /* This shouldn't happen, but can on macOS due to a bug. See rdar://37550628. diff --git a/src/libutil/unix/muxable-pipe.cc b/src/libutil/unix/muxable-pipe.cc new file mode 100644 index 000000000..0104663c3 --- /dev/null +++ b/src/libutil/unix/muxable-pipe.cc @@ -0,0 +1,47 @@ +#include + +#include "logging.hh" +#include "util.hh" +#include "muxable-pipe.hh" + +namespace nix { + +void MuxablePipePollState::poll(std::optional timeout) +{ + if (::poll(pollStatus.data(), pollStatus.size(), timeout ? *timeout : -1) == -1) { + if (errno == EINTR) + return; + throw SysError("waiting for input"); + } +} + +void MuxablePipePollState::iterate( + std::set & channels, + std::function handleRead, + std::function handleEOF) +{ + std::set fds2(channels); + std::vector buffer(4096); + for (auto & k : fds2) { + const auto fdPollStatusId = get(fdToPollStatus, k); + assert(fdPollStatusId); + assert(*fdPollStatusId < pollStatus.size()); + if (pollStatus.at(*fdPollStatusId).revents) { + ssize_t rd = ::read(fromDescriptorReadOnly(k), buffer.data(), buffer.size()); + // FIXME: is there a cleaner way to handle pt close + // than EIO? Is this even standard? + if (rd == 0 || (rd == -1 && errno == EIO)) { + handleEOF(k); + channels.erase(k); + } else if (rd == -1) { + if (errno != EINTR) + throw SysError("read failed"); + } else { + std::string_view data((char *) buffer.data(), rd); + handleRead(k, data); + } + } + } +} + +} diff --git a/src/libutil/unix/os-string.cc b/src/libutil/unix/os-string.cc new file mode 100644 index 000000000..8378afde2 --- /dev/null +++ b/src/libutil/unix/os-string.cc @@ -0,0 +1,21 @@ +#include +#include +#include +#include + +#include "file-path.hh" +#include "util.hh" + +namespace nix { + +std::string os_string_to_string(PathViewNG::string_view path) +{ + return std::string{path}; +} + +std::filesystem::path::string_type string_to_os_string(std::string_view s) +{ + return std::string{s}; +} + +} diff --git a/src/libutil/unix/processes.cc b/src/libutil/unix/processes.cc index 1af559a21..43d9179d9 100644 --- a/src/libutil/unix/processes.cc +++ b/src/libutil/unix/processes.cc @@ -1,5 +1,6 @@ #include "current-process.hh" #include "environment-variables.hh" +#include "executable-path.hh" #include "signals.hh" #include "processes.hh" #include "finally.hh" @@ -182,7 +183,7 @@ static pid_t doFork(bool allowVfork, ChildWrapperFunction & fun) #endif if (pid != 0) return pid; fun(); - abort(); + unreachable(); } @@ -419,4 +420,12 @@ bool statusOk(int status) return WIFEXITED(status) && WEXITSTATUS(status) == 0; } +int execvpe(const char * file0, const char * const argv[], const char * const envp[]) +{ + auto file = ExecutablePath::load().findPath(file0); + // `const_cast` is safe. See the note in + // https://pubs.opengroup.org/onlinepubs/9799919799/functions/exec.html + return execve(file.c_str(), const_cast(argv), const_cast(envp)); +} + } diff --git a/src/libutil/unix/signals-impl.hh b/src/libutil/unix/signals-impl.hh index 7ac8c914d..2193922be 100644 --- a/src/libutil/unix/signals-impl.hh +++ b/src/libutil/unix/signals-impl.hh @@ -84,6 +84,12 @@ static inline bool getInterrupted() return unix::_isInterrupted; } +/** + * Throw `Interrupted` exception if the process has been interrupted. + * + * Call this in long-running loops and between slow operations to terminate + * them as needed. + */ void inline checkInterrupt() { using namespace unix; diff --git a/src/libutil/unix/signals.cc b/src/libutil/unix/signals.cc index 7e30687d8..d0608dace 100644 --- a/src/libutil/unix/signals.cc +++ b/src/libutil/unix/signals.cc @@ -91,7 +91,7 @@ void unix::triggerInterrupt() try { callback(); } catch (...) { - ignoreException(); + ignoreExceptionInDestructor(); } } } diff --git a/src/libutil/unix/users.cc b/src/libutil/unix/users.cc index 58063a953..107a6e04f 100644 --- a/src/libutil/unix/users.cc +++ b/src/libutil/unix/users.cc @@ -9,6 +9,8 @@ namespace nix { +namespace fs { using namespace std::filesystem; } + std::string getUserName() { auto pw = getpwuid(geteuid()); diff --git a/src/libutil/url.cc b/src/libutil/url.cc index c6561441d..9ed49dcbe 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -79,10 +79,14 @@ std::map decodeQuery(const std::string & query) for (auto s : tokenizeString(query, "&")) { auto e = s.find('='); - if (e != std::string::npos) - result.emplace( - s.substr(0, e), - percentDecode(std::string_view(s).substr(e + 1))); + if (e == std::string::npos) { + warn("dubious URI query '%s' is missing equal sign '%s', ignoring", s, "="); + continue; + } + + result.emplace( + s.substr(0, e), + percentDecode(std::string_view(s).substr(e + 1))); } return result; @@ -132,7 +136,7 @@ std::string ParsedURL::to_string() const + (fragment.empty() ? "" : "#" + percentEncode(fragment)); } -bool ParsedURL::operator ==(const ParsedURL & other) const +bool ParsedURL::operator ==(const ParsedURL & other) const noexcept { return scheme == other.scheme @@ -171,16 +175,16 @@ std::string fixGitURL(const std::string & url) std::regex scpRegex("([^/]*)@(.*):(.*)"); if (!hasPrefix(url, "/") && std::regex_match(url, scpRegex)) return std::regex_replace(url, scpRegex, "ssh://$1@$2/$3"); - else { - if (url.find("://") == std::string::npos) { - return (ParsedURL { - .scheme = "file", - .authority = "", - .path = url - }).to_string(); - } else - return url; + if (hasPrefix(url, "file:")) + return url; + if (url.find("://") == std::string::npos) { + return (ParsedURL { + .scheme = "file", + .authority = "", + .path = url + }).to_string(); } + return url; } // https://www.rfc-editor.org/rfc/rfc3986#section-3.1 diff --git a/src/libutil/url.hh b/src/libutil/url.hh index 24806bbff..738ee9f82 100644 --- a/src/libutil/url.hh +++ b/src/libutil/url.hh @@ -18,7 +18,7 @@ struct ParsedURL std::string to_string() const; - bool operator ==(const ParsedURL & other) const; + bool operator ==(const ParsedURL & other) const noexcept; /** * Remove `.` and `..` path elements. @@ -33,6 +33,8 @@ std::string percentEncode(std::string_view s, std::string_view keep=""); std::map decodeQuery(const std::string & query); +std::string encodeQuery(const std::map & query); + ParsedURL parseURL(const std::string & url); /** diff --git a/src/libutil/users.cc b/src/libutil/users.cc index d546e364f..b4bc67cbc 100644 --- a/src/libutil/users.cc +++ b/src/libutil/users.cc @@ -7,15 +7,33 @@ namespace nix { Path getCacheDir() { - auto cacheDir = getEnv("XDG_CACHE_HOME"); - return cacheDir ? *cacheDir : getHome() + "/.cache"; + auto dir = getEnv("NIX_CACHE_HOME"); + if (dir) { + return *dir; + } else { + auto xdgDir = getEnv("XDG_CACHE_HOME"); + if (xdgDir) { + return *xdgDir + "/nix"; + } else { + return getHome() + "/.cache/nix"; + } + } } Path getConfigDir() { - auto configDir = getEnv("XDG_CONFIG_HOME"); - return configDir ? *configDir : getHome() + "/.config"; + auto dir = getEnv("NIX_CONFIG_HOME"); + if (dir) { + return *dir; + } else { + auto xdgDir = getEnv("XDG_CONFIG_HOME"); + if (xdgDir) { + return *xdgDir + "/nix"; + } else { + return getHome() + "/.config/nix"; + } + } } std::vector getConfigDirs() @@ -23,6 +41,9 @@ std::vector getConfigDirs() Path configHome = getConfigDir(); auto configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); std::vector result = tokenizeString>(configDirs, ":"); + for (auto& p : result) { + p += "/nix"; + } result.insert(result.begin(), configHome); return result; } @@ -30,19 +51,37 @@ std::vector getConfigDirs() Path getDataDir() { - auto dataDir = getEnv("XDG_DATA_HOME"); - return dataDir ? *dataDir : getHome() + "/.local/share"; + auto dir = getEnv("NIX_DATA_HOME"); + if (dir) { + return *dir; + } else { + auto xdgDir = getEnv("XDG_DATA_HOME"); + if (xdgDir) { + return *xdgDir + "/nix"; + } else { + return getHome() + "/.local/share/nix"; + } + } } Path getStateDir() { - auto stateDir = getEnv("XDG_STATE_HOME"); - return stateDir ? *stateDir : getHome() + "/.local/state"; + auto dir = getEnv("NIX_STATE_HOME"); + if (dir) { + return *dir; + } else { + auto xdgDir = getEnv("XDG_STATE_HOME"); + if (xdgDir) { + return *xdgDir + "/nix"; + } else { + return getHome() + "/.local/state/nix"; + } + } } Path createNixStateDir() { - Path dir = getStateDir() + "/nix"; + Path dir = getStateDir(); createDirs(dir); return dir; } diff --git a/src/libutil/users.hh b/src/libutil/users.hh index 153cc73fd..d22c3311d 100644 --- a/src/libutil/users.hh +++ b/src/libutil/users.hh @@ -24,12 +24,12 @@ Path getHomeOf(uid_t userId); Path getHome(); /** - * @return $XDG_CACHE_HOME or $HOME/.cache. + * @return $NIX_CACHE_HOME or $XDG_CACHE_HOME/nix or $HOME/.cache/nix. */ Path getCacheDir(); /** - * @return $XDG_CONFIG_HOME or $HOME/.config. + * @return $NIX_CONFIG_HOME or $XDG_CONFIG_HOME/nix or $HOME/.config/nix. */ Path getConfigDir(); @@ -39,12 +39,12 @@ Path getConfigDir(); std::vector getConfigDirs(); /** - * @return $XDG_DATA_HOME or $HOME/.local/share. + * @return $NIX_DATA_HOME or $XDG_DATA_HOME/nix or $HOME/.local/share/nix. */ Path getDataDir(); /** - * @return $XDG_STATE_HOME or $HOME/.local/state. + * @return $NIX_STATE_HOME or $XDG_STATE_HOME/nix or $HOME/.local/state/nix. */ Path getStateDir(); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 103ce4232..ed5c7e4f1 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1,5 +1,7 @@ #include "util.hh" #include "fmt.hh" +#include "file-path.hh" +#include "signals.hh" #include #include @@ -7,6 +9,8 @@ #include #include +#include +#include #ifdef NDEBUG #error "Nix may not be built with assertions disabled (i.e. with -DNDEBUG)." @@ -51,24 +55,6 @@ std::vector stringsToCharPtrs(const Strings & ss) ////////////////////////////////////////////////////////////////////// -template C tokenizeString(std::string_view s, std::string_view separators) -{ - C result; - auto pos = s.find_first_not_of(separators, 0); - while (pos != s.npos) { - auto end = s.find_first_of(separators, pos + 1); - if (end == s.npos) end = s.size(); - result.insert(result.end(), std::string(s, pos, end - pos)); - pos = s.find_first_not_of(separators, end); - } - return result; -} - -template Strings tokenizeString(std::string_view s, std::string_view separators); -template StringSet tokenizeString(std::string_view s, std::string_view separators); -template std::vector tokenizeString(std::string_view s, std::string_view separators); - - std::string chomp(std::string_view s) { size_t i = s.find_last_not_of(" \n\r\t"); @@ -111,6 +97,58 @@ std::string rewriteStrings(std::string s, const StringMap & rewrites) return s; } +template +std::optional string2Int(const std::string_view s) +{ + if (s.substr(0, 1) == "-" && !std::numeric_limits::is_signed) + return std::nullopt; + try { + return boost::lexical_cast(s.data(), s.size()); + } catch (const boost::bad_lexical_cast &) { + return std::nullopt; + } +} + +// Explicitly instantiated in one place for faster compilation +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); +template std::optional string2Int(const std::string_view s); + +template +std::optional string2Float(const std::string_view s) +{ + try { + return boost::lexical_cast(s.data(), s.size()); + } catch (const boost::bad_lexical_cast &) { + return std::nullopt; + } +} + +template std::optional string2Float(const std::string_view s); +template std::optional string2Float(const std::string_view s); + + +std::string renderSize(uint64_t value, bool align) +{ + static const std::array prefixes{{ + 'K', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y' + }}; + size_t power = 0; + double res = value; + while (res > 1024 && power < prefixes.size()) { + ++power; + res /= 1024; + } + return fmt(align ? "%6.1f %ciB" : "%.1f %ciB", power == 0 ? res / 1024 : res, prefixes.at(power)); +} + bool hasPrefix(std::string_view s, std::string_view prefix) { @@ -145,7 +183,7 @@ std::string shellEscape(const std::string_view s) } -void ignoreException(Verbosity lvl) +void ignoreExceptionInDestructor(Verbosity lvl) { /* Make sure no exceptions leave this function. printError() also throws when remote is closed. */ @@ -158,6 +196,17 @@ void ignoreException(Verbosity lvl) } catch (...) { } } +void ignoreExceptionExceptInterrupt(Verbosity lvl) +{ + try { + throw; + } catch (const Interrupted & e) { + throw; + } catch (std::exception & e) { + printMsg(lvl, "error (ignored): %1%", e.what()); + } +} + constexpr char base64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; @@ -207,7 +256,7 @@ std::string base64Decode(std::string_view s) char digit = base64DecodeChars[(unsigned char) c]; if (digit == npos) - throw Error("invalid character in Base64 string: '%c'", c); + throw FormatError("invalid character in Base64 string: '%c'", c); bits += 6; d = d << 6 | digit; diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 8b049875a..0fb6ff837 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -5,13 +5,14 @@ #include "error.hh" #include "logging.hh" -#include #include #include #include #include +#include "strings.hh" + namespace nix { void initLibUtil(); @@ -27,36 +28,11 @@ std::vector stringsToCharPtrs(const Strings & ss); MakeError(FormatError, Error); -/** - * String tokenizer. - */ -template C tokenizeString(std::string_view s, std::string_view separators = " \t\n\r"); - - -/** - * Concatenate the given strings with a separator between the - * elements. - */ -template -std::string concatStringsSep(const std::string_view sep, const C & ss) -{ - size_t size = 0; - // need a cast to string_view since this is also called with Symbols - for (const auto & s : ss) size += sep.size() + std::string_view(s).size(); - std::string s; - s.reserve(size); - for (auto & i : ss) { - if (s.size() != 0) s += sep; - s += i; - } - return s; -} - -template -auto concatStrings(Parts && ... parts) +template +auto concatStrings(Parts &&... parts) -> std::enable_if_t<(... && std::is_convertible_v), std::string> { - std::string_view views[sizeof...(parts)] = { parts... }; + std::string_view views[sizeof...(parts)] = {parts...}; return concatStringsSep({}, views); } @@ -102,16 +78,7 @@ std::string rewriteStrings(std::string s, const StringMap & rewrites); * Parse a string into an integer. */ template -std::optional string2Int(const std::string_view s) -{ - if (s.substr(0, 1) == "-" && !std::numeric_limits::is_signed) - return std::nullopt; - try { - return boost::lexical_cast(s.data(), s.size()); - } catch (const boost::bad_lexical_cast &) { - return std::nullopt; - } -} +std::optional string2Int(const std::string_view s); /** * Like string2Int(), but support an optional suffix 'K', 'M', 'G' or @@ -137,18 +104,18 @@ N string2IntWithUnitPrefix(std::string_view s) throw UsageError("'%s' is not an integer", s); } +/** + * Pretty-print a byte value, e.g. 12433615056 is rendered as `11.6 + * GiB`. If `align` is set, the number will be right-justified by + * padding with spaces on the left. + */ +std::string renderSize(uint64_t value, bool align = false); + /** * Parse a string into a float. */ template -std::optional string2Float(const std::string_view s) -{ - try { - return boost::lexical_cast(s.data(), s.size()); - } catch (const boost::bad_lexical_cast &) { - return std::nullopt; - } -} +std::optional string2Float(const std::string_view s); /** @@ -189,9 +156,26 @@ std::string toLower(std::string s); std::string shellEscape(const std::string_view s); -/* Exception handling in destructors: print an error message, then - ignore the exception. */ -void ignoreException(Verbosity lvl = lvlError); +/** + * Exception handling in destructors: print an error message, then + * ignore the exception. + * + * If you're not in a destructor, you usually want to use `ignoreExceptionExceptInterrupt()`. + * + * This function might also be used in callbacks whose caller may not handle exceptions, + * but ideally we propagate the exception using an exception_ptr in such cases. + * See e.g. `PackBuilderContext` + */ +void ignoreExceptionInDestructor(Verbosity lvl = lvlError); + +/** + * Not destructor-safe. + * Print an error message, then ignore the exception. + * If the exception is an `Interrupted` exception, rethrow it. + * + * This may be used in a few places where Interrupt can't happen, but that's ok. + */ +void ignoreExceptionExceptInterrupt(Verbosity lvl = lvlError); @@ -205,9 +189,13 @@ constexpr char treeNull[] = " "; /** - * Base64 encoding/decoding. + * Encode arbitrary bytes as Base64. */ std::string base64Encode(std::string_view s); + +/** + * Decode arbitrary bytes to Base64. + */ std::string base64Decode(std::string_view s); diff --git a/src/libutil/windows/environment-variables.cc b/src/libutil/windows/environment-variables.cc index 25ab9d63a..525d08c64 100644 --- a/src/libutil/windows/environment-variables.cc +++ b/src/libutil/windows/environment-variables.cc @@ -4,7 +4,30 @@ namespace nix { -int unsetenv(const char *name) +std::optional getEnvOs(const OsString & key) +{ + // Determine the required buffer size for the environment variable value + DWORD bufferSize = GetEnvironmentVariableW(key.c_str(), nullptr, 0); + if (bufferSize == 0) { + return std::nullopt; + } + + // Allocate a buffer to hold the environment variable value + std::wstring value{L'\0', bufferSize}; + + // Retrieve the environment variable value + DWORD resultSize = GetEnvironmentVariableW(key.c_str(), &value[0], bufferSize); + if (resultSize == 0) { + return std::nullopt; + } + + // Resize the string to remove the extra null characters + value.resize(resultSize); + + return value; +} + +int unsetenv(const char * name) { return -SetEnvironmentVariableA(name, nullptr); } @@ -14,4 +37,9 @@ int setEnv(const char * name, const char * value) return -SetEnvironmentVariableA(name, value); } +int setEnvOs(const OsString & name, const OsString & value) +{ + return -SetEnvironmentVariableW(name.c_str(), value.c_str()); +} + } diff --git a/src/libutil/windows/file-descriptor.cc b/src/libutil/windows/file-descriptor.cc index 26f769b66..16773e3ea 100644 --- a/src/libutil/windows/file-descriptor.cc +++ b/src/libutil/windows/file-descriptor.cc @@ -14,6 +14,8 @@ namespace nix { +using namespace nix::windows; + std::string readFile(HANDLE handle) { LARGE_INTEGER li; @@ -120,7 +122,7 @@ void Pipe::create() #if _WIN32_WINNT >= 0x0600 -std::wstring handleToFileName(HANDLE handle) { +std::wstring windows::handleToFileName(HANDLE handle) { std::vector buf(0x100); DWORD dw = GetFinalPathNameByHandleW(handle, buf.data(), buf.size(), FILE_NAME_OPENED); if (dw == 0) { @@ -139,7 +141,7 @@ std::wstring handleToFileName(HANDLE handle) { } -Path handleToPath(HANDLE handle) { +Path windows::handleToPath(HANDLE handle) { return os_string_to_string(handleToFileName(handle)); } diff --git a/src/libutil/windows/file-path.cc b/src/libutil/windows/file-path.cc index 3114ac4df..7405c426b 100644 --- a/src/libutil/windows/file-path.cc +++ b/src/libutil/windows/file-path.cc @@ -9,18 +9,6 @@ namespace nix { -std::string os_string_to_string(PathViewNG::string_view path) -{ - std::wstring_convert> converter; - return converter.to_bytes(std::filesystem::path::string_type { path }); -} - -std::filesystem::path::string_type string_to_os_string(std::string_view s) -{ - std::wstring_convert> converter; - return converter.from_bytes(std::string { s }); -} - std::optional maybePath(PathView path) { if (path.length() >= 3 && (('A' <= path[0] && path[0] <= 'Z') || ('a' <= path[0] && path[0] <= 'z')) && path[1] == ':' && WindowsPathTrait::isPathSep(path[2])) { diff --git a/src/libutil/windows/file-system.cc b/src/libutil/windows/file-system.cc index 8002dd75e..b15355efe 100644 --- a/src/libutil/windows/file-system.cc +++ b/src/libutil/windows/file-system.cc @@ -5,8 +5,13 @@ namespace nix { Descriptor openDirectory(const std::filesystem::path & path) { return CreateFileW( - path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, NULL, OPEN_EXISTING, - FILE_FLAG_BACKUP_SEMANTICS, NULL); + path.c_str(), + GENERIC_READ, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + FILE_FLAG_BACKUP_SEMANTICS, + NULL); } } diff --git a/src/libutil/windows/meson.build b/src/libutil/windows/meson.build new file mode 100644 index 000000000..1c645fe05 --- /dev/null +++ b/src/libutil/windows/meson.build @@ -0,0 +1,20 @@ +sources += files( + 'environment-variables.cc', + 'file-descriptor.cc', + 'file-path.cc', + 'file-system.cc', + 'muxable-pipe.cc', + 'os-string.cc', + 'processes.cc', + 'users.cc', + 'windows-async-pipe.cc', + 'windows-error.cc', +) + +include_dirs += include_directories('.') + +headers += files( + 'signals-impl.hh', + 'windows-async-pipe.hh', + 'windows-error.hh', +) diff --git a/src/libutil/windows/muxable-pipe.cc b/src/libutil/windows/muxable-pipe.cc new file mode 100644 index 000000000..91a321f7c --- /dev/null +++ b/src/libutil/windows/muxable-pipe.cc @@ -0,0 +1,70 @@ +#include +#include "windows-error.hh" + +#include "logging.hh" +#include "util.hh" +#include "muxable-pipe.hh" + +namespace nix { + +void MuxablePipePollState::poll(HANDLE ioport, std::optional timeout) +{ + /* We are on at least Windows Vista / Server 2008 and can get many + (countof(oentries)) statuses in one API call. */ + if (!GetQueuedCompletionStatusEx( + ioport, oentries, sizeof(oentries) / sizeof(*oentries), &removed, timeout ? *timeout : INFINITE, false)) { + windows::WinError winError("GetQueuedCompletionStatusEx"); + if (winError.lastError != WAIT_TIMEOUT) + throw winError; + assert(removed == 0); + } else { + assert(0 < removed && removed <= sizeof(oentries) / sizeof(*oentries)); + } +} + +void MuxablePipePollState::iterate( + std::set & channels, + std::function handleRead, + std::function handleEOF) +{ + auto p = channels.begin(); + while (p != channels.end()) { + decltype(p) nextp = p; + ++nextp; + for (ULONG i = 0; i < removed; i++) { + if (oentries[i].lpCompletionKey == ((ULONG_PTR) ((*p)->readSide.get()) ^ 0x5555)) { + printMsg(lvlVomit, "read %s bytes", oentries[i].dwNumberOfBytesTransferred); + if (oentries[i].dwNumberOfBytesTransferred > 0) { + std::string data{ + (char *) (*p)->buffer.data(), + oentries[i].dwNumberOfBytesTransferred, + }; + handleRead((*p)->readSide.get(), data); + } + + if (gotEOF) { + handleEOF((*p)->readSide.get()); + nextp = channels.erase(p); // no need to maintain `channels`? + } else { + BOOL rc = ReadFile( + (*p)->readSide.get(), (*p)->buffer.data(), (*p)->buffer.size(), &(*p)->got, &(*p)->overlapped); + if (rc) { + // here is possible (but not obligatory) to call + // `handleRead` and repeat ReadFile immediately + } else { + windows::WinError winError("ReadFile(%s, ..)", (*p)->readSide.get()); + if (winError.lastError == ERROR_BROKEN_PIPE) { + handleEOF((*p)->readSide.get()); + nextp = channels.erase(p); // no need to maintain `channels` ? + } else if (winError.lastError != ERROR_IO_PENDING) + throw winError; + } + } + break; + } + } + p = nextp; + } +} + +} diff --git a/src/libutil/windows/os-string.cc b/src/libutil/windows/os-string.cc new file mode 100644 index 000000000..7507f9030 --- /dev/null +++ b/src/libutil/windows/os-string.cc @@ -0,0 +1,24 @@ +#include +#include +#include +#include + +#include "file-path.hh" +#include "file-path-impl.hh" +#include "util.hh" + +namespace nix { + +std::string os_string_to_string(PathViewNG::string_view path) +{ + std::wstring_convert> converter; + return converter.to_bytes(std::filesystem::path::string_type{path}); +} + +std::filesystem::path::string_type string_to_os_string(std::string_view s) +{ + std::wstring_convert> converter; + return converter.from_bytes(std::string{s}); +} + +} diff --git a/src/libutil/windows/processes.cc b/src/libutil/windows/processes.cc index 5ef4ed1e4..7f34c5632 100644 --- a/src/libutil/windows/processes.cc +++ b/src/libutil/windows/processes.cc @@ -1,9 +1,16 @@ #include "current-process.hh" #include "environment-variables.hh" +#include "error.hh" +#include "executable-path.hh" +#include "file-descriptor.hh" +#include "file-path.hh" #include "signals.hh" #include "processes.hh" #include "finally.hh" #include "serialise.hh" +#include "file-system.hh" +#include "util.hh" +#include "windows-error.hh" #include #include @@ -16,33 +23,366 @@ #include #include -#ifdef __APPLE__ -# include -#endif - -#ifdef __linux__ -# include -# include -#endif - +#define WIN32_LEAN_AND_MEAN +#include namespace nix { -std::string runProgram(Path program, bool lookupPath, const Strings & args, - const std::optional & input, bool isInteractive) +using namespace nix::windows; + +Pid::Pid() {} + +Pid::Pid(AutoCloseFD pid) + : pid(std::move(pid)) { - throw UnimplementedError("Cannot shell out to git on Windows yet"); } +Pid::~Pid() +{ + if (pid.get() != INVALID_DESCRIPTOR) + kill(); +} + +void Pid::operator=(AutoCloseFD pid) +{ + if (this->pid.get() != INVALID_DESCRIPTOR && this->pid.get() != pid.get()) + kill(); + this->pid = std::move(pid); +} + +// TODO: Implement (not needed for process spawning yet) +int Pid::kill() +{ + assert(pid.get() != INVALID_DESCRIPTOR); + + debug("killing process %1%", pid.get()); + + throw UnimplementedError("Pid::kill unimplemented"); +} + +int Pid::wait() +{ + // https://github.com/nix-windows/nix/blob/windows-meson/src/libutil/util.cc#L1938 + assert(pid.get() != INVALID_DESCRIPTOR); + DWORD status = WaitForSingleObject(pid.get(), INFINITE); + if (status != WAIT_OBJECT_0) { + debug("WaitForSingleObject returned %1%", status); + } + + DWORD exitCode = 0; + if (GetExitCodeProcess(pid.get(), &exitCode) == FALSE) { + debug("GetExitCodeProcess failed on pid %1%", pid.get()); + } + + pid.close(); + return exitCode; +} + +// TODO: Merge this with Unix's runProgram since it's identical logic. +std::string runProgram( + Path program, bool lookupPath, const Strings & args, const std::optional & input, bool isInteractive) +{ + auto res = runProgram(RunOptions{ + .program = program, .lookupPath = lookupPath, .args = args, .input = input, .isInteractive = isInteractive}); + + if (!statusOk(res.first)) + throw ExecError(res.first, "program '%1%' %2%", program, statusToString(res.first)); + + return res.second; +} + +std::optional getProgramInterpreter(const Path & program) +{ + // These extensions are automatically handled by Windows and don't require an interpreter. + static constexpr const char * exts[] = {".exe", ".cmd", ".bat"}; + for (const auto ext : exts) { + if (hasSuffix(program, ext)) { + return {}; + } + } + // TODO: Open file and read the shebang + throw UnimplementedError("getProgramInterpreter unimplemented"); +} + +// TODO: Not sure if this is needed in the unix version but it might be useful as a member func +void setFDInheritable(AutoCloseFD & fd, bool inherit) +{ + if (fd.get() != INVALID_DESCRIPTOR) { + if (!SetHandleInformation(fd.get(), HANDLE_FLAG_INHERIT, inherit ? HANDLE_FLAG_INHERIT : 0)) { + throw WinError("Couldn't disable inheriting of handle"); + } + } +} + +AutoCloseFD nullFD() +{ + // Create null handle to discard reads / writes + // https://stackoverflow.com/a/25609668 + // https://github.com/nix-windows/nix/blob/windows-meson/src/libutil/util.cc#L2228 + AutoCloseFD nul = CreateFileW( + L"NUL", + GENERIC_READ | GENERIC_WRITE, + // We don't care who reads / writes / deletes this file since it's NUL anyways + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + NULL, + OPEN_EXISTING, + 0, + NULL); + if (!nul.get()) { + throw WinError("Couldn't open NUL device"); + } + // Let this handle be inheritable by child processes + setFDInheritable(nul, true); + return nul; +} + +// Adapted from +// https://blogs.msdn.microsoft.com/twistylittlepassagesallalike/2011/04/23/everyone-quotes-command-line-arguments-the-wrong-way/ +std::string windowsEscape(const std::string & str, bool cmd) +{ + // TODO: This doesn't handle cmd.exe escaping. + if (cmd) { + throw UnimplementedError("cmd.exe escaping is not implemented"); + } + + if (str.find_first_of(" \t\n\v\"") == str.npos && !str.empty()) { + // No need to escape this one, the nonempty contents don't have a special character + return str; + } + std::string buffer; + // Add the opening quote + buffer += '"'; + for (auto iter = str.begin();; ++iter) { + size_t backslashes = 0; + while (iter != str.end() && *iter == '\\') { + ++iter; + ++backslashes; + } + + // We only escape backslashes if: + // - They come immediately before the closing quote + // - They come immediately before a quote in the middle of the string + // Both of these cases break the escaping if not handled. Otherwise backslashes are fine as-is + if (iter == str.end()) { + // Need to escape each backslash + buffer.append(backslashes * 2, '\\'); + // Exit since we've reached the end of the string + break; + } else if (*iter == '"') { + // Need to escape each backslash and the intermediate quote character + buffer.append(backslashes * 2, '\\'); + buffer += "\\\""; + } else { + // Don't escape the backslashes since they won't break the delimiter + buffer.append(backslashes, '\\'); + buffer += *iter; + } + } + // Add the closing quote + return buffer + '"'; +} + +Pid spawnProcess(const Path & realProgram, const RunOptions & options, Pipe & out, Pipe & in) +{ + // Setup pipes. + if (options.standardOut) { + // Don't inherit the read end of the output pipe + setFDInheritable(out.readSide, false); + } else { + out.writeSide = nullFD(); + } + if (options.standardIn) { + // Don't inherit the write end of the input pipe + setFDInheritable(in.writeSide, false); + } else { + in.readSide = nullFD(); + } + + STARTUPINFOW startInfo = {0}; + startInfo.cb = sizeof(startInfo); + startInfo.dwFlags = STARTF_USESTDHANDLES; + startInfo.hStdInput = in.readSide.get(); + startInfo.hStdOutput = out.writeSide.get(); + startInfo.hStdError = out.writeSide.get(); + + std::string envline; + // Retain the current processes' environment variables. + for (const auto & envVar : getEnv()) { + envline += (envVar.first + '=' + envVar.second + '\0'); + } + // Also add new ones specified in options. + if (options.environment) { + for (const auto & envVar : *options.environment) { + envline += (envVar.first + '=' + envVar.second + '\0'); + } + } + + std::string cmdline = windowsEscape(realProgram, false); + for (const auto & arg : options.args) { + // TODO: This isn't the right way to escape windows command + // See https://learn.microsoft.com/en-us/windows/win32/api/shellapi/nf-shellapi-commandlinetoargvw + cmdline += ' ' + windowsEscape(arg, false); + } + + PROCESS_INFORMATION procInfo = {0}; + if (CreateProcessW( + // EXE path is provided in the cmdline + NULL, + string_to_os_string(cmdline).data(), + NULL, + NULL, + TRUE, + CREATE_UNICODE_ENVIRONMENT | CREATE_SUSPENDED, + string_to_os_string(envline).data(), + options.chdir.has_value() ? string_to_os_string(*options.chdir).data() : NULL, + &startInfo, + &procInfo) + == 0) { + throw WinError("CreateProcessW failed (%1%)", cmdline); + } + + // Convert these to use RAII + AutoCloseFD process = procInfo.hProcess; + AutoCloseFD thread = procInfo.hThread; + + // Add current process and child to job object so child terminates when parent terminates + // TODO: This spawns one job per child process. We can probably keep this as a global, and + // add children a single job so we don't use so many jobs at once. + Descriptor job = CreateJobObjectW(NULL, NULL); + if (job == NULL) { + TerminateProcess(procInfo.hProcess, 0); + throw WinError("Couldn't create job object for child process"); + } + if (AssignProcessToJobObject(job, procInfo.hProcess) == FALSE) { + TerminateProcess(procInfo.hProcess, 0); + throw WinError("Couldn't assign child process to job object"); + } + if (ResumeThread(procInfo.hThread) == (DWORD) -1) { + TerminateProcess(procInfo.hProcess, 0); + throw WinError("Couldn't resume child process thread"); + } + + return process; +} + +// TODO: Merge this with Unix's runProgram since it's identical logic. // Output = error code + "standard out" output stream std::pair runProgram(RunOptions && options) { - throw UnimplementedError("Cannot shell out to git on Windows yet"); + StringSink sink; + options.standardOut = &sink; + + int status = 0; + + try { + runProgram2(options); + } catch (ExecError & e) { + status = e.status; + } + + return {status, std::move(sink.s)}; } void runProgram2(const RunOptions & options) { - throw UnimplementedError("Cannot shell out to git on Windows yet"); + checkInterrupt(); + + assert(!(options.standardIn && options.input)); + + std::unique_ptr source_; + Source * source = options.standardIn; + + if (options.input) { + source_ = std::make_unique(*options.input); + source = source_.get(); + } + + /* Create a pipe. */ + Pipe out, in; + // TODO: I copied this from unix but this is handled again in spawnProcess, so might be weird to split it up like + // this + if (options.standardOut) + out.create(); + if (source) + in.create(); + + Path realProgram = options.program; + // TODO: Implement shebang / program interpreter lookup on Windows + auto interpreter = getProgramInterpreter(realProgram); + + std::optional>> resumeLoggerDefer; + if (options.isInteractive) { + logger->pause(); + resumeLoggerDefer.emplace([]() { logger->resume(); }); + } + + Pid pid = spawnProcess(interpreter.has_value() ? *interpreter : realProgram, options, out, in); + + // TODO: This is identical to unix, deduplicate? + out.writeSide.close(); + + std::thread writerThread; + + std::promise promise; + + Finally doJoin([&] { + if (writerThread.joinable()) + writerThread.join(); + }); + + if (source) { + in.readSide.close(); + writerThread = std::thread([&] { + try { + std::vector buf(8 * 1024); + while (true) { + size_t n; + try { + n = source->read(buf.data(), buf.size()); + } catch (EndOfFile &) { + break; + } + writeFull(in.writeSide.get(), {buf.data(), n}); + } + promise.set_value(); + } catch (...) { + promise.set_exception(std::current_exception()); + } + in.writeSide.close(); + }); + } + + if (options.standardOut) + drainFD(out.readSide.get(), *options.standardOut); + + /* Wait for the child to finish. */ + int status = pid.wait(); + + /* Wait for the writer thread to finish. */ + if (source) + promise.get_future().get(); + + if (status) + throw ExecError(status, "program '%1%' %2%", options.program, statusToString(status)); +} + +std::string statusToString(int status) +{ + if (status != 0) + return fmt("with exit code %d", status); + else + return "succeeded"; +} + +bool statusOk(int status) +{ + return status == 0; +} + +int execvpe(const wchar_t * file0, const wchar_t * const argv[], const wchar_t * const envp[]) +{ + auto file = ExecutablePath::load().findPath(file0); + return _wexecve(file.c_str(), argv, envp); } } diff --git a/src/libutil/windows/users.cc b/src/libutil/windows/users.cc index 1792ff1a1..db6c42df3 100644 --- a/src/libutil/windows/users.cc +++ b/src/libutil/windows/users.cc @@ -9,6 +9,8 @@ namespace nix { +using namespace nix::windows; + std::string getUserName() { // Get the required buffer size diff --git a/src/libutil/windows/windows-async-pipe.cc b/src/libutil/windows/windows-async-pipe.cc new file mode 100644 index 000000000..4fa57ca36 --- /dev/null +++ b/src/libutil/windows/windows-async-pipe.cc @@ -0,0 +1,49 @@ +#include "windows-async-pipe.hh" +#include "windows-error.hh" + +namespace nix::windows { + +void AsyncPipe::createAsyncPipe(HANDLE iocp) +{ + // std::cerr << (format("-----AsyncPipe::createAsyncPipe(%x)") % iocp) << std::endl; + + buffer.resize(0x1000); + memset(&overlapped, 0, sizeof(overlapped)); + + std::string pipeName = fmt("\\\\.\\pipe\\nix-%d-%p", GetCurrentProcessId(), (void *) this); + + readSide = CreateNamedPipeA( + pipeName.c_str(), + PIPE_ACCESS_INBOUND | FILE_FLAG_OVERLAPPED, + PIPE_TYPE_BYTE, + PIPE_UNLIMITED_INSTANCES, + 0, + 0, + INFINITE, + NULL); + if (!readSide) + throw WinError("CreateNamedPipeA(%s)", pipeName); + + HANDLE hIocp = CreateIoCompletionPort(readSide.get(), iocp, (ULONG_PTR) (readSide.get()) ^ 0x5555, 0); + if (hIocp != iocp) + throw WinError("CreateIoCompletionPort(%x[%s], %x, ...) returned %x", readSide.get(), pipeName, iocp, hIocp); + + if (!ConnectNamedPipe(readSide.get(), &overlapped) && GetLastError() != ERROR_IO_PENDING) + throw WinError("ConnectNamedPipe(%s)", pipeName); + + SECURITY_ATTRIBUTES psa2 = {0}; + psa2.nLength = sizeof(SECURITY_ATTRIBUTES); + psa2.bInheritHandle = TRUE; + + writeSide = CreateFileA(pipeName.c_str(), GENERIC_WRITE, 0, &psa2, OPEN_EXISTING, 0, NULL); + if (!readSide) + throw WinError("CreateFileA(%s)", pipeName); +} + +void AsyncPipe::close() +{ + readSide.close(); + writeSide.close(); +} + +} diff --git a/src/libutil/windows/windows-async-pipe.hh b/src/libutil/windows/windows-async-pipe.hh new file mode 100644 index 000000000..8f554e403 --- /dev/null +++ b/src/libutil/windows/windows-async-pipe.hh @@ -0,0 +1,27 @@ +#pragma once +///@file + +#include "file-descriptor.hh" + +namespace nix::windows { + +/*** + * An "async pipe" is a pipe that supports I/O Completion Ports so + * multiple pipes can be listened too. + * + * Unfortunately, only named pipes support that on windows, so we use + * those with randomized temp file names. + */ +class AsyncPipe +{ +public: + AutoCloseFD writeSide, readSide; + OVERLAPPED overlapped; + DWORD got; + std::vector buffer; + + void createAsyncPipe(HANDLE iocp); + void close(); +}; + +} diff --git a/src/libutil/windows/windows-error.cc b/src/libutil/windows/windows-error.cc index 26faaae6d..aead4af23 100644 --- a/src/libutil/windows/windows-error.cc +++ b/src/libutil/windows/windows-error.cc @@ -4,7 +4,7 @@ #define WIN32_LEAN_AND_MEAN #include -namespace nix { +namespace nix::windows { std::string WinError::renderError(DWORD lastError) { diff --git a/src/libutil/windows/windows-error.hh b/src/libutil/windows/windows-error.hh index fdfd0f52c..624b4c4cb 100644 --- a/src/libutil/windows/windows-error.hh +++ b/src/libutil/windows/windows-error.hh @@ -5,7 +5,7 @@ #include "error.hh" -namespace nix { +namespace nix::windows { /** * Windows Error type. diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index b601604dc..7d32a6f97 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -26,6 +26,7 @@ #include "legacy.hh" #include "users.hh" #include "network-proxy.hh" +#include "compatibility-settings.hh" using namespace nix; using namespace std::string_literals; @@ -35,7 +36,7 @@ extern char * * environ __attribute__((weak)); /* Recreate the effect of the perl shellwords function, breaking up a * string into arguments like a shell word, including escapes */ -static std::vector shellwords(const std::string & s) +static std::vector shellwords(std::string_view s) { std::regex whitespace("^\\s+"); auto begin = s.cbegin(); @@ -50,7 +51,7 @@ static std::vector shellwords(const std::string & s) auto it = begin; for (; it != s.cend(); ++it) { if (st == sBegin) { - std::smatch match; + std::cmatch match; if (regex_search(it, s.cend(), match, whitespace)) { cur.append(begin, it); res.push_back(cur); @@ -90,24 +91,50 @@ static std::vector shellwords(const std::string & s) return res; } +/** + * Like `resolveExprPath`, but prefers `shell.nix` instead of `default.nix`, + * and if `path` was a directory, it checks eagerly whether `shell.nix` or + * `default.nix` exist, throwing an error if they don't. + */ +static SourcePath resolveShellExprPath(SourcePath path) +{ + auto resolvedOrDir = resolveExprPath(path, false); + if (resolvedOrDir.resolveSymlinks().lstat().type == SourceAccessor::tDirectory) { + if ((resolvedOrDir / "shell.nix").pathExists()) { + if (compatibilitySettings.nixShellAlwaysLooksForShellNix) { + return resolvedOrDir / "shell.nix"; + } else { + warn("Skipping '%1%', because the setting '%2%' is disabled. This is a deprecated behavior. Consider enabling '%2%'.", + resolvedOrDir / "shell.nix", + "nix-shell-always-looks-for-shell-nix"); + } + } + if ((resolvedOrDir / "default.nix").pathExists()) { + return resolvedOrDir / "default.nix"; + } + throw Error("neither '%s' nor '%s' found in '%s'", "shell.nix", "default.nix", resolvedOrDir); + } + return resolvedOrDir; +} + static void main_nix_build(int argc, char * * argv) { auto dryRun = false; - auto runEnv = std::regex_search(argv[0], std::regex("nix-shell$")); + auto isNixShell = std::regex_search(argv[0], std::regex("nix-shell$")); auto pure = false; auto fromArgs = false; auto packages = false; // Same condition as bash uses for interactive shells auto interactive = isatty(STDIN_FILENO) && isatty(STDERR_FILENO); Strings attrPaths; - Strings left; + Strings remainingArgs; BuildMode buildMode = bmNormal; bool readStdin = false; std::string envCommand; // interactive shell Strings envExclude; - auto myName = runEnv ? "nix-shell" : "nix-build"; + auto myName = isNixShell ? "nix-shell" : "nix-build"; auto inShebang = false; std::string script; @@ -132,11 +159,11 @@ static void main_nix_build(int argc, char * * argv) // Heuristic to see if we're invoked as a shebang script, namely, // if we have at least one argument, it's the name of an // executable file, and it starts with "#!". - if (runEnv && argc > 1) { + if (isNixShell && argc > 1) { script = argv[1]; try { auto lines = tokenizeString(readFile(script), "\n"); - if (std::regex_search(lines.front(), std::regex("^#!"))) { + if (!lines.empty() && std::regex_search(lines.front(), std::regex("^#!"))) { lines.pop_front(); inShebang = true; for (int i = 2; i < argc; ++i) @@ -146,7 +173,7 @@ static void main_nix_build(int argc, char * * argv) line = chomp(line); std::smatch match; if (std::regex_match(line, match, std::regex("^#!\\s*nix-shell\\s+(.*)$"))) - for (const auto & word : shellwords(match[1].str())) + for (const auto & word : shellwords({match[1].first, match[1].second})) args.push_back(word); } } @@ -156,6 +183,9 @@ static void main_nix_build(int argc, char * * argv) struct MyArgs : LegacyArgs, MixEvalArgs { using LegacyArgs::LegacyArgs; + void setBaseDir(Path baseDir) { + commandBaseDir = baseDir; + } }; MyArgs myArgs(myName, [&](Strings::iterator & arg, const Strings::iterator & end) { @@ -186,9 +216,9 @@ static void main_nix_build(int argc, char * * argv) dryRun = true; else if (*arg == "--run-env") // obsolete - runEnv = true; + isNixShell = true; - else if (runEnv && (*arg == "--command" || *arg == "--run")) { + else if (isNixShell && (*arg == "--command" || *arg == "--run")) { if (*arg == "--run") interactive = false; envCommand = getArg(*arg, arg, end) + "\nexit"; @@ -206,7 +236,7 @@ static void main_nix_build(int argc, char * * argv) else if (*arg == "--pure") pure = true; else if (*arg == "--impure") pure = false; - else if (runEnv && (*arg == "--packages" || *arg == "-p")) + else if (isNixShell && (*arg == "--packages" || *arg == "-p")) packages = true; else if (inShebang && *arg == "-i") { @@ -230,9 +260,9 @@ static void main_nix_build(int argc, char * * argv) // read the shebang to understand which packages to read from. Since // this is handled via nix-shell -p, we wrap our ruby script execution // in ruby -e 'load' which ignores the shebangs. - envCommand = fmt("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%", execArgs, interpreter, shellEscape(script), joined.str()); + envCommand = fmt("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%", execArgs, interpreter, shellEscape(script), toView(joined)); } else { - envCommand = fmt("exec %1% %2% %3% %4%", execArgs, interpreter, shellEscape(script), joined.str()); + envCommand = fmt("exec %1% %2% %3% %4%", execArgs, interpreter, shellEscape(script), toView(joined)); } } @@ -246,7 +276,7 @@ static void main_nix_build(int argc, char * * argv) return false; else - left.push_back(*arg); + remainingArgs.push_back(*arg); return true; }); @@ -259,14 +289,17 @@ static void main_nix_build(int argc, char * * argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.lookupPath, evalStore, store); + auto state = std::make_unique(myArgs.lookupPath, evalStore, fetchSettings, evalSettings, store); state->repair = myArgs.repair; if (myArgs.repair) buildMode = bmRepair; + if (inShebang && compatibilitySettings.nixShellShebangArgumentsRelativeToScript) { + myArgs.setBaseDir(absPath(dirOf(script))); + } auto autoArgs = myArgs.getAutoArgs(*state); auto autoArgsWithInNixShell = autoArgs; - if (runEnv) { + if (isNixShell) { auto newArgs = state->buildBindings(autoArgsWithInNixShell->size() + 1); newArgs.alloc("inNixShell").mkBool(true); for (auto & i : *autoArgs) newArgs.insert(i); @@ -276,19 +309,26 @@ static void main_nix_build(int argc, char * * argv) if (packages) { std::ostringstream joined; joined << "{...}@args: with import args; (pkgs.runCommandCC or pkgs.runCommand) \"shell\" { buildInputs = [ "; - for (const auto & i : left) + for (const auto & i : remainingArgs) joined << '(' << i << ") "; joined << "]; } \"\""; fromArgs = true; - left = {joined.str()}; - } else if (!fromArgs) { - if (left.empty() && runEnv && pathExists("shell.nix")) - left = {"shell.nix"}; - if (left.empty()) - left = {"default.nix"}; + remainingArgs = {joined.str()}; + } else if (!fromArgs && remainingArgs.empty()) { + if (isNixShell && !compatibilitySettings.nixShellAlwaysLooksForShellNix && std::filesystem::exists("shell.nix")) { + // If we're in 2.3 compatibility mode, we need to look for shell.nix + // now, because it won't be done later. + remainingArgs = {"shell.nix"}; + } else { + remainingArgs = {"."}; + + // Instead of letting it throw later, we throw here to give a more relevant error message + if (isNixShell && !std::filesystem::exists("shell.nix") && !std::filesystem::exists("default.nix")) + throw Error("no argument specified and no '%s' or '%s' file found in the working directory", "shell.nix", "default.nix"); + } } - if (runEnv) + if (isNixShell) setEnv("IN_NIX_SHELL", pure ? "pure" : "impure"); PackageInfos drvs; @@ -299,9 +339,14 @@ static void main_nix_build(int argc, char * * argv) if (readStdin) exprs = {state->parseStdin()}; else - for (auto i : left) { + for (auto i : remainingArgs) { + auto baseDir = inShebang && !packages ? absPath(dirOf(script)) : i; + if (fromArgs) - exprs.push_back(state->parseExprFromString(std::move(i), state->rootPath("."))); + exprs.push_back(state->parseExprFromString( + std::move(i), + (inShebang && compatibilitySettings.nixShellShebangArgumentsRelativeToScript) ? lookupFileArg(*state, baseDir) : state->rootPath(".") + )); else { auto absolute = i; try { @@ -310,14 +355,18 @@ static void main_nix_build(int argc, char * * argv) auto [path, outputNames] = parsePathWithOutputs(absolute); if (evalStore->isStorePath(path) && hasSuffix(path, ".drv")) drvs.push_back(PackageInfo(*state, evalStore, absolute)); - else + else { /* If we're in a #! script, interpret filenames relative to the script. */ - exprs.push_back( - state->parseExprFromFile( - resolveExprPath( - lookupFileArg(*state, - inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i)))); + auto baseDir = inShebang && !packages ? absPath(i, absPath(dirOf(script))) : i; + + auto sourcePath = lookupFileArg(*state, + baseDir); + auto resolvedPath = + isNixShell ? resolveShellExprPath(sourcePath) : resolveExprPath(sourcePath); + + exprs.push_back(state->parseExprFromFile(resolvedPath)); + } } } @@ -330,7 +379,7 @@ static void main_nix_build(int argc, char * * argv) std::function takesNixShellAttr; takesNixShellAttr = [&](const Value & v) { - if (!runEnv) { + if (!isNixShell) { return false; } bool add = false; @@ -381,7 +430,7 @@ static void main_nix_build(int argc, char * * argv) store->buildPaths(paths, buildMode, evalStore); }; - if (runEnv) { + if (isNixShell) { if (drvs.size() != 1) throw UsageError("nix-shell requires a single derivation"); @@ -477,9 +526,7 @@ static void main_nix_build(int argc, char * * argv) // Set the environment. auto env = getEnv(); - auto tmp = getEnvNonEmpty("TMPDIR"); - if (!tmp) - tmp = getEnvNonEmpty("XDG_RUNTIME_DIR").value_or("/tmp"); + auto tmp = getEnvNonEmpty("TMPDIR").value_or("/tmp"); if (pure) { decltype(env) newEnv; @@ -491,7 +538,7 @@ static void main_nix_build(int argc, char * * argv) env["__ETC_PROFILE_SOURCED"] = "1"; } - env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = *tmp; + env["NIX_BUILD_TOP"] = env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmp; env["NIX_STORE"] = store->storeDir; env["NIX_BUILD_CORES"] = std::to_string(settings.buildCores); diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 9f7f557b5..56d1d7abb 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -7,6 +7,7 @@ #include "eval-settings.hh" // for defexpr #include "users.hh" #include "tarball.hh" +#include "self-exe.hh" #include #include @@ -17,7 +18,7 @@ using namespace nix; typedef std::map Channels; static Channels channels; -static Path channelsList; +static std::filesystem::path channelsList; // Reads the list of channels. static void readChannels() @@ -41,7 +42,7 @@ static void writeChannels() { auto channelsFD = AutoCloseFD{open(channelsList.c_str(), O_WRONLY | O_CLOEXEC | O_CREAT | O_TRUNC, 0644)}; if (!channelsFD) - throw SysError("opening '%1%' for writing", channelsList); + throw SysError("opening '%1%' for writing", channelsList.string()); for (const auto & channel : channels) writeFull(channelsFD.get(), channel.second + " " + channel.first + "\n"); } @@ -67,7 +68,7 @@ static void removeChannel(const std::string & name) channels.erase(name); writeChannels(); - runProgram(settings.nixBinDir + "/nix-env", true, { "--profile", profile, "--uninstall", name }); + runProgram(getNixBin("nix-env").string(), true, { "--profile", profile, "--uninstall", name }); } static Path nixDefExpr; @@ -118,7 +119,7 @@ static void update(const StringSet & channelNames) bool unpacked = false; if (std::regex_search(filename, std::regex("\\.tar\\.(gz|bz2|xz)$"))) { - runProgram(settings.nixBinDir + "/nix-build", false, { "--no-out-link", "--expr", "import " + unpackChannelPath + + runProgram(getNixBin("nix-build").string(), false, { "--no-out-link", "--expr", "import " + unpackChannelPath + "{ name = \"" + cname + "\"; channelName = \"" + name + "\"; src = builtins.storePath \"" + filename + "\"; }" }); unpacked = true; } @@ -143,7 +144,7 @@ static void update(const StringSet & channelNames) for (auto & expr : exprs) envArgs.push_back(std::move(expr)); envArgs.push_back("--quiet"); - runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + runProgram(getNixBin("nix-env").string(), false, envArgs); // Make the channels appear in nix-env. struct stat st; @@ -244,7 +245,7 @@ static int main_nix_channel(int argc, char ** argv) case cListGenerations: if (!args.empty()) throw UsageError("'--list-generations' expects no arguments"); - std::cout << runProgram(settings.nixBinDir + "/nix-env", false, {"--profile", profile, "--list-generations"}) << std::flush; + std::cout << runProgram(getNixBin("nix-env").string(), false, {"--profile", profile, "--list-generations"}) << std::flush; break; case cRollback: if (args.size() > 1) @@ -256,7 +257,7 @@ static int main_nix_channel(int argc, char ** argv) } else { envArgs.push_back("--rollback"); } - runProgram(settings.nixBinDir + "/nix-env", false, envArgs); + runProgram(getNixBin("nix-env").string(), false, envArgs); break; } diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc index 91209c978..20d5161df 100644 --- a/src/nix-collect-garbage/nix-collect-garbage.cc +++ b/src/nix-collect-garbage/nix-collect-garbage.cc @@ -11,6 +11,8 @@ #include #include +namespace nix::fs { using namespace std::filesystem; } + using namespace nix; std::string deleteOlderThan; @@ -21,23 +23,23 @@ bool dryRun = false; * Of course, this makes rollbacks to before this point in time * impossible. */ -void removeOldGenerations(std::string dir) +void removeOldGenerations(fs::path dir) { - if (access(dir.c_str(), R_OK) != 0) return; + if (access(dir.string().c_str(), R_OK) != 0) return; - bool canWrite = access(dir.c_str(), W_OK) == 0; + bool canWrite = access(dir.string().c_str(), W_OK) == 0; - for (auto & i : std::filesystem::directory_iterator{dir}) { + for (auto & i : fs::directory_iterator{dir}) { checkInterrupt(); auto path = i.path().string(); auto type = i.symlink_status().type(); - if (type == std::filesystem::file_type::symlink && canWrite) { + if (type == fs::file_type::symlink && canWrite) { std::string link; try { link = readLink(path); - } catch (std::filesystem::filesystem_error & e) { + } catch (fs::filesystem_error & e) { if (e.code() == std::errc::no_such_file_or_directory) continue; throw; } @@ -49,7 +51,7 @@ void removeOldGenerations(std::string dir) } else deleteOldGenerations(path, dryRun); } - } else if (type == std::filesystem::file_type::directory) { + } else if (type == fs::file_type::directory) { removeOldGenerations(path); } } @@ -81,8 +83,11 @@ static int main_nix_collect_garbage(int argc, char * * argv) }); if (removeOld) { - std::set dirsToClean = { - profilesDir(), settings.nixStateDir + "/profiles", dirOf(getDefaultProfile())}; + std::set dirsToClean = { + profilesDir(), + fs::path{settings.nixStateDir} / "profiles", + fs::path{getDefaultProfile()}.parent_path(), + }; for (auto & dir : dirsToClean) removeOldGenerations(dir); } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index b5e13cc23..ba2baccee 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -204,15 +204,15 @@ static void loadDerivations(EvalState & state, const SourcePath & nixExprPath, } -static long getPriority(EvalState & state, PackageInfo & drv) +static NixInt getPriority(EvalState & state, PackageInfo & drv) { - return drv.queryMetaInt("priority", 0); + return drv.queryMetaInt("priority", NixInt(0)); } -static long comparePriorities(EvalState & state, PackageInfo & drv1, PackageInfo & drv2) +static std::strong_ordering comparePriorities(EvalState & state, PackageInfo & drv1, PackageInfo & drv2) { - return getPriority(state, drv2) - getPriority(state, drv1); + return getPriority(state, drv2) <=> getPriority(state, drv1); } @@ -280,7 +280,7 @@ std::vector pickNewestOnly(EvalState & state, std::vector matches) auto & oneDrv = match.packageInfo; const auto drvName = DrvName { oneDrv.queryName() }; - long comparison = 1; + std::strong_ordering comparison = std::strong_ordering::greater; const auto itOther = newest.find(drvName.name); @@ -288,9 +288,9 @@ std::vector pickNewestOnly(EvalState & state, std::vector matches) auto & newestDrv = itOther->second.packageInfo; comparison = - oneDrv.querySystem() == newestDrv.querySystem() ? 0 : - oneDrv.querySystem() == settings.thisSystem ? 1 : - newestDrv.querySystem() == settings.thisSystem ? -1 : 0; + oneDrv.querySystem() == newestDrv.querySystem() ? std::strong_ordering::equal : + oneDrv.querySystem() == settings.thisSystem ? std::strong_ordering::greater : + newestDrv.querySystem() == settings.thisSystem ? std::strong_ordering::less : std::strong_ordering::equal; if (comparison == 0) comparison = comparePriorities(state, oneDrv, newestDrv); if (comparison == 0) @@ -625,13 +625,13 @@ static void upgradeDerivations(Globals & globals, continue; DrvName newName(j->queryName()); if (newName.name == drvName.name) { - int d = compareVersions(drvName.version, newName.version); + std::strong_ordering d = compareVersions(drvName.version, newName.version); if ((upgradeType == utLt && d < 0) || (upgradeType == utLeq && d <= 0) || (upgradeType == utEq && d == 0) || upgradeType == utAlways) { - long d2 = -1; + std::strong_ordering d2 = std::strong_ordering::less; if (bestElem != availElems.end()) { d2 = comparePriorities(*globals.state, *bestElem, *j); if (d2 == 0) d2 = compareVersions(bestVersion, newName.version); @@ -902,7 +902,7 @@ static VersionDiff compareVersionAgainstSet( for (auto & i : elems) { DrvName name2(i.queryName()); if (name.name == name2.name) { - int d = compareVersions(name.version, name2.version); + std::strong_ordering d = compareVersions(name.version, name2.version); if (d < 0) { diff = cvGreater; version = name2.version; @@ -1159,7 +1159,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) case cvEqual: ch = '='; break; case cvGreater: ch = '<'; break; case cvUnavail: ch = '-'; break; - default: abort(); + default: unreachable(); } if (xmlOutput) { @@ -1525,7 +1525,7 @@ static int main_nix_env(int argc, char * * argv) auto store = openStore(); - globals.state = std::shared_ptr(new EvalState(myArgs.lookupPath, store)); + globals.state = std::shared_ptr(new EvalState(myArgs.lookupPath, store, fetchSettings, evalSettings)); globals.state->repair = myArgs.repair; globals.instSource.nixExprPath = std::make_shared( diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 6cbbacb15..ee62077c0 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -9,8 +9,9 @@ #include "eval-inline.hh" #include "profiles.hh" #include "print-ambiguous.hh" -#include +#include +#include namespace nix { @@ -110,11 +111,9 @@ bool createUserEnv(EvalState & state, PackageInfos & elems, auto manifestFile = ({ std::ostringstream str; printAmbiguous(manifest, state.symbols, str, nullptr, std::numeric_limits::max()); - // TODO with C++20 we can use str.view() instead and avoid copy. - std::string str2 = str.str(); - StringSource source { str2 }; + StringSource source { toView(str) }; state.store->addToStoreFromDump( - source, "env-manifest.nix", FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, references); + source, "env-manifest.nix", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, references); }); /* Get the environment builder expression. */ @@ -140,6 +139,7 @@ bool createUserEnv(EvalState & state, PackageInfos & elems, NixStringContext context; auto & aDrvPath(*topLevel.attrs()->find(state.sDrvPath)); auto topLevelDrv = state.coerceToStorePath(aDrvPath.pos, *aDrvPath.value, context, ""); + topLevelDrv.requireDerivation(); auto & aOutPath(*topLevel.attrs()->find(state.sOutPath)); auto topLevelOut = state.coerceToStorePath(aOutPath.pos, *aOutPath.value, context, ""); diff --git a/src/nix-expr-test-support b/src/nix-expr-test-support new file mode 120000 index 000000000..427b80dff --- /dev/null +++ b/src/nix-expr-test-support @@ -0,0 +1 @@ +../tests/unit/libexpr-support \ No newline at end of file diff --git a/src/nix-expr-tests b/src/nix-expr-tests new file mode 120000 index 000000000..3af7110d3 --- /dev/null +++ b/src/nix-expr-tests @@ -0,0 +1 @@ +../tests/unit/libexpr \ No newline at end of file diff --git a/src/nix-fetchers-tests b/src/nix-fetchers-tests new file mode 120000 index 000000000..80e4b68ae --- /dev/null +++ b/src/nix-fetchers-tests @@ -0,0 +1 @@ +../tests/unit/libfetchers \ No newline at end of file diff --git a/src/nix-flake-tests b/src/nix-flake-tests new file mode 120000 index 000000000..bb2d49400 --- /dev/null +++ b/src/nix-flake-tests @@ -0,0 +1 @@ +../tests/unit/libflake \ No newline at end of file diff --git a/src/nix-functional-tests b/src/nix-functional-tests new file mode 120000 index 000000000..ed0cdf60b --- /dev/null +++ b/src/nix-functional-tests @@ -0,0 +1 @@ +../tests/functional \ No newline at end of file diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 35664374c..c48549511 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -157,7 +157,7 @@ static int main_nix_instantiate(int argc, char * * argv) auto store = openStore(); auto evalStore = myArgs.evalStoreUrl ? openStore(*myArgs.evalStoreUrl) : store; - auto state = std::make_unique(myArgs.lookupPath, evalStore, store); + auto state = std::make_unique(myArgs.lookupPath, evalStore, fetchSettings, evalSettings, store); state->repair = myArgs.repair; Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix-manual b/src/nix-manual new file mode 120000 index 000000000..492c97408 --- /dev/null +++ b/src/nix-manual @@ -0,0 +1 @@ +../doc/manual/ \ No newline at end of file diff --git a/src/nix-store-test-support b/src/nix-store-test-support new file mode 120000 index 000000000..af4befd90 --- /dev/null +++ b/src/nix-store-test-support @@ -0,0 +1 @@ +../tests/unit/libstore-support \ No newline at end of file diff --git a/src/nix-store-tests b/src/nix-store-tests new file mode 120000 index 000000000..fc9b910af --- /dev/null +++ b/src/nix-store-tests @@ -0,0 +1 @@ +../tests/unit/libstore \ No newline at end of file diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index b23d99ad6..b4de42ba1 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -2,12 +2,11 @@ #include "derivations.hh" #include "dotgraph.hh" #include "globals.hh" -#include "build-result.hh" #include "store-cast.hh" #include "local-fs-store.hh" #include "log-store.hh" #include "serve-protocol.hh" -#include "serve-protocol-impl.hh" +#include "serve-protocol-connection.hh" #include "shared.hh" #include "graphml.hh" #include "legacy.hh" @@ -22,12 +21,14 @@ #include #include -#include #include #include #include +#include "build-result.hh" +#include "exit.hh" +#include "serve-protocol-impl.hh" namespace nix_store { @@ -193,10 +194,10 @@ static void opAdd(Strings opFlags, Strings opArgs) store. */ static void opAddFixed(Strings opFlags, Strings opArgs) { - auto method = FileIngestionMethod::Flat; + ContentAddressMethod method = ContentAddressMethod::Raw::Flat; for (auto & i : opFlags) - if (i == "--recursive") method = FileIngestionMethod::Recursive; + if (i == "--recursive") method = ContentAddressMethod::Raw::NixArchive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.empty()) @@ -222,7 +223,7 @@ static void opPrintFixedPath(Strings opFlags, Strings opArgs) auto method = FileIngestionMethod::Flat; for (auto i : opFlags) - if (i == "--recursive") method = FileIngestionMethod::Recursive; + if (i == "--recursive") method = FileIngestionMethod::NixArchive; else throw UsageError("unknown flag '%1%'", i); if (opArgs.size() != 3) @@ -479,7 +480,7 @@ static void opQuery(Strings opFlags, Strings opArgs) } default: - abort(); + unreachable(); } } @@ -562,7 +563,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) if (!hashGiven) { HashResult hash = hashPath( {store->getFSAccessor(false), CanonPath { store->printStorePath(info->path) }}, - FileSerialisationMethod::Recursive, HashAlgorithm::SHA256); + FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); info->narHash = hash.first; info->narSize = hash.second; } diff --git a/src/nix-util-test-support b/src/nix-util-test-support new file mode 120000 index 000000000..4b25930eb --- /dev/null +++ b/src/nix-util-test-support @@ -0,0 +1 @@ +../tests/unit/libutil-support \ No newline at end of file diff --git a/src/nix-util-tests b/src/nix-util-tests new file mode 120000 index 000000000..e1138411a --- /dev/null +++ b/src/nix-util-tests @@ -0,0 +1 @@ +../tests/unit/libutil \ No newline at end of file diff --git a/src/nix/.version b/src/nix/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/nix/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index af6743375..5c08f7616 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -12,7 +12,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand { Path path; std::optional namePart; - ContentAddressMethod caMethod = FileIngestionMethod::Recursive; + ContentAddressMethod caMethod = ContentAddressMethod::Raw::NixArchive; HashAlgorithm hashAlgo = HashAlgorithm::SHA256; CmdAddToStore() @@ -68,7 +68,7 @@ struct CmdAddFile : CmdAddToStore { CmdAddFile() { - caMethod = FileIngestionMethod::Flat; + caMethod = ContentAddressMethod::Raw::Flat; } std::string description() override diff --git a/src/nix/build-remote b/src/nix/build-remote new file mode 120000 index 000000000..2cea44d46 --- /dev/null +++ b/src/nix/build-remote @@ -0,0 +1 @@ +../build-remote \ No newline at end of file diff --git a/src/nix/build-utils-meson b/src/nix/build-utils-meson new file mode 120000 index 000000000..91937f183 --- /dev/null +++ b/src/nix/build-utils-meson @@ -0,0 +1 @@ +../../build-utils-meson/ \ No newline at end of file diff --git a/src/nix/build.cc b/src/nix/build.cc index 479100186..da9132d02 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -43,22 +43,22 @@ static nlohmann::json builtPathsWithResultToJSON(const std::vector& buildables, LocalFSStore& store2) +static void createOutLinks(const std::filesystem::path& outLink, const std::vector& buildables, LocalFSStore& store2) { for (const auto & [_i, buildable] : enumerate(buildables)) { auto i = _i; std::visit(overloaded { [&](const BuiltPath::Opaque & bo) { - std::string symlink = outLink; + auto symlink = outLink; if (i) symlink += fmt("-%d", i); - store2.addPermRoot(bo.path, absPath(symlink)); + store2.addPermRoot(bo.path, absPath(symlink.string())); }, [&](const BuiltPath::Built & bfd) { for (auto & output : bfd.outputs) { - std::string symlink = outLink; + auto symlink = outLink; if (i) symlink += fmt("-%d", i); if (output.first != "out") symlink += fmt("-%s", output.first); - store2.addPermRoot(output.second, absPath(symlink)); + store2.addPermRoot(output.second, absPath(symlink.string())); } }, }, buildable.path.raw()); diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 2e50392f7..5b7862c4e 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -6,6 +6,8 @@ #include "local-fs-store.hh" #include "eval-inline.hh" +namespace nix::fs { using namespace std::filesystem; } + using namespace nix; struct CmdBundle : InstallableValueCommand @@ -76,7 +78,9 @@ struct CmdBundle : InstallableValueCommand auto val = installable->toValue(*evalState).first; - auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = parseFlakeRefWithFragmentAndExtendedOutputsSpec(bundler, absPath(".")); + auto [bundlerFlakeRef, bundlerName, extendedOutputsSpec] = + parseFlakeRefWithFragmentAndExtendedOutputsSpec( + fetchSettings, bundler, fs::current_path().string()); const flake::LockFlags lockFlags{ .writeLockFile = false }; InstallableFlake bundler{this, evalState, std::move(bundlerFlakeRef), bundlerName, std::move(extendedOutputsSpec), @@ -100,6 +104,8 @@ struct CmdBundle : InstallableValueCommand NixStringContext context2; auto drvPath = evalState->coerceToStorePath(attr1->pos, *attr1->value, context2, ""); + drvPath.requireDerivation(); + auto attr2 = vRes->attrs()->get(evalState->sOutPath); if (!attr2) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); @@ -113,8 +119,6 @@ struct CmdBundle : InstallableValueCommand }, }); - auto outPathS = store->printStorePath(outPath); - if (!outLink) { auto * attr = vRes->attrs()->get(evalState->sName); if (!attr) diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index 9575bf338..a72b06542 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -1,12 +1,16 @@ #include #include "command.hh" +#include "exit.hh" #include "logging.hh" #include "serve-protocol.hh" #include "shared.hh" #include "store-api.hh" #include "local-fs-store.hh" #include "worker-protocol.hh" +#include "executable-path.hh" + +namespace nix::fs { using namespace std::filesystem; } using namespace nix; @@ -22,17 +26,17 @@ std::string formatProtocol(unsigned int proto) return "unknown"; } -bool checkPass(const std::string & msg) { +bool checkPass(std::string_view msg) { notice(ANSI_GREEN "[PASS] " ANSI_NORMAL + msg); return true; } -bool checkFail(const std::string & msg) { +bool checkFail(std::string_view msg) { notice(ANSI_RED "[FAIL] " ANSI_NORMAL + msg); return false; } -void checkInfo(const std::string & msg) { +void checkInfo(std::string_view msg) { notice(ANSI_BLUE "[INFO] " ANSI_NORMAL + msg); } @@ -74,18 +78,20 @@ struct CmdConfigCheck : StoreCommand bool checkNixInPath() { - PathSet dirs; + std::set dirs; - for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) - if (pathExists(dir + "/nix-env")) - dirs.insert(dirOf(canonPath(dir + "/nix-env", true))); + for (auto & dir : ExecutablePath::load().directories) { + auto candidate = dir / "nix-env"; + if (fs::exists(candidate)) + dirs.insert(fs::canonical(candidate).parent_path() ); + } if (dirs.size() != 1) { - std::stringstream ss; + std::ostringstream ss; ss << "Multiple versions of nix found in PATH:\n"; for (auto & dir : dirs) ss << " " << dir << "\n"; - return checkFail(ss.str()); + return checkFail(toView(ss)); } return checkPass("PATH contains only one nix version."); @@ -93,18 +99,25 @@ struct CmdConfigCheck : StoreCommand bool checkProfileRoots(ref store) { - PathSet dirs; + std::set dirs; - for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) { - Path profileDir = dirOf(dir); + for (auto & dir : ExecutablePath::load().directories) { + auto profileDir = dir.parent_path(); try { - Path userEnv = canonPath(profileDir, true); + auto userEnv = fs::weakly_canonical(profileDir); - if (store->isStorePath(userEnv) && hasSuffix(userEnv, "user-environment")) { - while (profileDir.find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir)) - profileDir = absPath(readLink(profileDir), dirOf(profileDir)); + auto noContainsProfiles = [&]{ + for (auto && part : profileDir) + if (part == "profiles") return false; + return true; + }; - if (profileDir.find("/profiles/") == std::string::npos) + if (store->isStorePath(userEnv.string()) && hasSuffix(userEnv.string(), "user-environment")) { + while (noContainsProfiles() && std::filesystem::is_symlink(profileDir)) + profileDir = fs::weakly_canonical( + profileDir.parent_path() / fs::read_symlink(profileDir)); + + if (noContainsProfiles()) dirs.insert(dir); } } catch (SystemError &) { @@ -112,14 +125,14 @@ struct CmdConfigCheck : StoreCommand } if (!dirs.empty()) { - std::stringstream ss; + std::ostringstream ss; ss << "Found profiles outside of " << settings.nixStateDir << "/profiles.\n" << "The generation this profile points to might not have a gcroot and could be\n" << "garbage collected, resulting in broken symlinks.\n\n"; for (auto & dir : dirs) ss << " " << dir << "\n"; ss << "\n"; - return checkFail(ss.str()); + return checkFail(toView(ss)); } return checkPass("All profiles are gcroots."); @@ -132,13 +145,13 @@ struct CmdConfigCheck : StoreCommand : PROTOCOL_VERSION; if (clientProto != storeProto) { - std::stringstream ss; + std::ostringstream ss; ss << "Warning: protocol version of this client does not match the store.\n" << "While this is not necessarily a problem it's recommended to keep the client in\n" << "sync with the daemon.\n\n" << "Client protocol: " << formatProtocol(clientProto) << "\n" << "Store protocol: " << formatProtocol(storeProto) << "\n\n"; - return checkFail(ss.str()); + return checkFail(toView(ss)); } return checkPass("Client protocol matches store protocol."); diff --git a/src/nix/config.cc b/src/nix/config.cc index 52706afcf..07f975a00 100644 --- a/src/nix/config.cc +++ b/src/nix/config.cc @@ -2,6 +2,7 @@ #include "common-args.hh" #include "shared.hh" #include "store-api.hh" +#include "config-global.hh" #include diff --git a/src/nix/derivation-add.md b/src/nix/derivation-add.md index 331cbdd88..35507d9ad 100644 --- a/src/nix/derivation-add.md +++ b/src/nix/derivation-add.md @@ -3,7 +3,7 @@ R""( # Description This command reads from standard input a JSON representation of a -[store derivation] to which an [*installable*](./nix.md#installables) evaluates. +[store derivation]. Store derivations are used internally by Nix. They are store paths with extension `.drv` that represent the build-time dependency graph to which diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 08d44d7aa..c7a733025 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -1,3 +1,4 @@ +#include "config-global.hh" #include "eval.hh" #include "installable-flake.hh" #include "command-installable-value.hh" @@ -14,9 +15,14 @@ #include #include +#include #include #include +#include "strings.hh" + +namespace nix::fs { using namespace std::filesystem; } + using namespace nix; struct DevelopSettings : Config @@ -237,7 +243,7 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore auto getEnvShPath = ({ StringSource source { getEnvSh }; evalStore->addToStoreFromDump( - source, "get-env.sh", FileSerialisationMethod::Flat, TextIngestionMethod {}, HashAlgorithm::SHA256, {}); + source, "get-env.sh", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256, {}); }); drv.args = {store->printStorePath(getEnvShPath)}; @@ -337,7 +343,7 @@ struct Common : InstallableCommand, MixProfile ref store, const BuildEnvironment & buildEnvironment, const std::filesystem::path & tmpDir, - const std::filesystem::path & outputsDir = std::filesystem::path { absPath(".") } / "outputs") + const std::filesystem::path & outputsDir = fs::path { fs::current_path() } / "outputs") { // A list of colon-separated environment variables that should be // prepended to, rather than overwritten, in order to keep the shell usable. @@ -411,7 +417,7 @@ struct Common : InstallableCommand, MixProfile if (buildEnvironment.providesStructuredAttrs()) { fixupStructuredAttrs( - PATHNG_LITERAL("sh"), + OS_STR("sh"), "NIX_ATTRS_SH_FILE", buildEnvironment.getAttrsSH(), rewrites, @@ -419,7 +425,7 @@ struct Common : InstallableCommand, MixProfile tmpDir ); fixupStructuredAttrs( - PATHNG_LITERAL("json"), + OS_STR("json"), "NIX_ATTRS_JSON_FILE", buildEnvironment.getAttrsJSON(), rewrites, @@ -443,10 +449,10 @@ struct Common : InstallableCommand, MixProfile const BuildEnvironment & buildEnvironment, const std::filesystem::path & tmpDir) { - auto targetFilePath = tmpDir / PATHNG_LITERAL(".attrs."); + auto targetFilePath = tmpDir / OS_STR(".attrs."); targetFilePath += ext; - writeFile(targetFilePath.string(), content); + writeFile(targetFilePath, content); auto fileInBuilderEnv = buildEnvironment.vars.find(envVar); assert(fileInBuilderEnv != buildEnvironment.vars.end()); @@ -610,7 +616,7 @@ struct CmdDevelop : Common, MixEnvironment } else { - script = "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n" + script; + script = "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\nshopt -u expand_aliases\n" + script + "\nshopt -s expand_aliases\n"; if (developSettings.bashPrompt != "") script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n", shellEscape(developSettings.bashPrompt.get())); @@ -666,7 +672,7 @@ struct CmdDevelop : Common, MixEnvironment throw Error("package 'nixpkgs#bashInteractive' does not provide a 'bin/bash'"); } catch (Error &) { - ignoreException(); + ignoreExceptionExceptInterrupt(); } // Override SHELL with the one chosen for this environment. @@ -695,7 +701,11 @@ struct CmdDevelop : Common, MixEnvironment } } - runProgramInStore(store, UseLookupPath::Use, shell, args, buildEnvironment.getSystem()); + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + getEvalState()->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::Use, shell, args, buildEnvironment.getSystem()); #endif } }; diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index c7c37b66f..2bc7fe82b 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -6,6 +6,8 @@ #include +#include "strings.hh" + namespace nix { struct Info @@ -23,15 +25,17 @@ GroupedPaths getClosureInfo(ref store, const StorePath & toplevel) GroupedPaths groupedPaths; - for (auto & path : closure) { + for (auto const & path : closure) { /* Strip the output name. Unfortunately this is ambiguous (we can't distinguish between output names like "bin" and version suffixes like "unstable"). */ static std::regex regex("(.*)-([a-z]+|lib32|lib64)"); - std::smatch match; - std::string name(path.name()); + std::cmatch match; + std::string name{path.name()}; + std::string_view const origName = path.name(); std::string outputName; - if (std::regex_match(name, match, regex)) { + + if (std::regex_match(origName.begin(), origName.end(), match, regex)) { name = match[1]; outputName = match[2]; } diff --git a/src/nix/doc b/src/nix/doc new file mode 120000 index 000000000..7e57b0f58 --- /dev/null +++ b/src/nix/doc @@ -0,0 +1 @@ +../../doc \ No newline at end of file diff --git a/src/nix/env.cc b/src/nix/env.cc new file mode 100644 index 000000000..832320320 --- /dev/null +++ b/src/nix/env.cc @@ -0,0 +1,116 @@ +#include +#include + +#include "command.hh" +#include "eval.hh" +#include "run.hh" +#include "strings.hh" +#include "executable-path.hh" + +using namespace nix; + +struct CmdEnv : NixMultiCommand +{ + CmdEnv() + : NixMultiCommand("env", RegisterCommand::getCommandsFor({"env"})) + { + } + + std::string description() override + { + return "manipulate the process environment"; + } + + Category category() override + { + return catUtility; + } +}; + +static auto rCmdEnv = registerCommand("env"); + +struct CmdShell : InstallablesCommand, MixEnvironment +{ + + using InstallablesCommand::run; + + std::vector command = {getEnv("SHELL").value_or("bash")}; + + CmdShell() + { + addFlag( + {.longName = "command", + .shortName = 'c', + .description = "Command and arguments to be executed, defaulting to `$SHELL`", + .labels = {"command", "args"}, + .handler = {[&](std::vector ss) { + if (ss.empty()) + throw UsageError("--command requires at least one argument"); + command = ss; + }}}); + } + + std::string description() override + { + return "run a shell in which the specified packages are available"; + } + + std::string doc() override + { + return +#include "shell.md" + ; + } + + void run(ref store, Installables && installables) override + { + auto outPaths = + Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, installables); + + auto accessor = store->getFSAccessor(); + + std::unordered_set done; + std::queue todo; + for (auto & path : outPaths) + todo.push(path); + + setEnviron(); + + std::vector pathAdditions; + + while (!todo.empty()) { + auto path = todo.front(); + todo.pop(); + if (!done.insert(path).second) + continue; + + if (true) + pathAdditions.push_back(store->printStorePath(path) + "/bin"); + + auto propPath = accessor->resolveSymlinks( + CanonPath(store->printStorePath(path)) / "nix-support" / "propagated-user-env-packages"); + if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) { + for (auto & p : tokenizeString(accessor->readFile(propPath))) + todo.push(store->parseStorePath(p)); + } + } + + // TODO: split losslessly; empty means . + auto unixPath = ExecutablePath::load(); + unixPath.directories.insert(unixPath.directories.begin(), pathAdditions.begin(), pathAdditions.end()); + auto unixPathString = unixPath.render(); + setEnvOs(OS_STR("PATH"), unixPathString.c_str()); + + Strings args; + for (auto & arg : command) + args.push_back(arg); + + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + getEvalState()->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::Use, *command.begin(), args); + } +}; + +static auto rCmdShell = registerCommand2({"env", "shell"}); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 494735516..04b18ff41 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -11,11 +11,13 @@ using namespace nix; +namespace nix::fs { using namespace std::filesystem; } + struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption { bool raw = false; std::optional apply; - std::optional writeTo; + std::optional writeTo; CmdEval() : InstallableValueCommand() { @@ -75,30 +77,26 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption if (writeTo) { stopProgressBar(); - if (pathExists(*writeTo)) - throw Error("path '%s' already exists", *writeTo); + if (fs::symlink_exists(*writeTo)) + throw Error("path '%s' already exists", writeTo->string()); - std::function recurse; + std::function recurse; - recurse = [&](Value & v, const PosIdx pos, const Path & path) + recurse = [&](Value & v, const PosIdx pos, const fs::path & path) { state->forceValue(v, pos); if (v.type() == nString) // FIXME: disallow strings with contexts? - writeFile(path, v.string_view()); + writeFile(path.string(), v.string_view()); else if (v.type() == nAttrs) { - if (mkdir(path.c_str() -#ifndef _WIN32 // TODO abstract mkdir perms for Windows - , 0777 -#endif - ) == -1) - throw SysError("creating directory '%s'", path); + // Directory should not already exist + assert(fs::create_directory(path.string())); for (auto & attr : *v.attrs()) { std::string_view name = state->symbols[attr.name]; try { if (name == "." || name == "..") throw Error("invalid file name '%s'", name); - recurse(*attr.value, attr.pos, concatStrings(path, "/", name)); + recurse(*attr.value, attr.pos, path / name); } catch (Error & e) { e.addTrace( state->positions[attr.pos], diff --git a/src/nix/eval.md b/src/nix/eval.md index 48d5aa597..bd5b035e1 100644 --- a/src/nix/eval.md +++ b/src/nix/eval.md @@ -50,8 +50,9 @@ R""( # Description -This command evaluates the given Nix expression and prints the -result on standard output. +This command evaluates the given Nix expression, and prints the result on standard output. + +It also evaluates any nested attribute values and list items. # Output format diff --git a/src/nix/flake-archive.md b/src/nix/flake-archive.md index 85bbeeb16..18c735b11 100644 --- a/src/nix/flake-archive.md +++ b/src/nix/flake-archive.md @@ -22,8 +22,20 @@ R""( # nix flake archive --json --dry-run nixops ``` +* Upload all flake inputs to a different machine for remote evaluation + + ``` + # nix flake archive --to ssh://some-machine + ``` + + On the remote machine the flake can then be accessed via its store path. That's computed like this: + + ``` + # nix flake metadata --json | jq -r '.path' + ``` + # Description -FIXME +Copy a flake and all its inputs to a store. This is useful i.e. to evaluate flakes on a different host. )"" diff --git a/src/nix/flake-lock.md b/src/nix/flake-lock.md index 6d10258e3..d13666a4c 100644 --- a/src/nix/flake-lock.md +++ b/src/nix/flake-lock.md @@ -30,9 +30,9 @@ R""( # Description -This command adds inputs to the lock file of a flake (`flake.lock`) -so that it contains a lock for every flake input specified in -`flake.nix`. Existing lock file entries are not updated. +This command updates the lock file of a flake (`flake.lock`) +so that it contains an up-to-date lock for every flake input specified in +`flake.nix`. Lock file entries are aready up-to-date are not modified. If you want to update existing lock entries, use [`nix flake update`](@docroot@/command-ref/new-cli/nix3-flake-update.md) diff --git a/src/nix/flake-metadata.md b/src/nix/flake-metadata.md index 5a009409b..adfd3dc96 100644 --- a/src/nix/flake-metadata.md +++ b/src/nix/flake-metadata.md @@ -2,10 +2,10 @@ R""( # Examples -* Show what `nixpkgs` resolves to: +* Show what `dwarffs` resolves to: ```console - # nix flake metadata nixpkgs + # nix flake metadata dwarffs Resolved URL: github:edolstra/dwarffs Locked URL: github:edolstra/dwarffs/f691e2c991e75edb22836f1dbe632c40324215c5 Description: A filesystem that fetches DWARF debug info from the Internet on demand diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md index 63df3b12a..8b0159ff7 100644 --- a/src/nix/flake-update.md +++ b/src/nix/flake-update.md @@ -25,6 +25,19 @@ R""( → 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' (2023-07-05) ``` +* Update multiple inputs: + + ```console + # nix flake update nixpkgs nixpkgs-unstable + warning: updating lock file '/home/myself/repos/testflake/flake.lock': + • Updated input 'nixpkgs': + 'github:nixos/nixpkgs/8f7492cce28977fbf8bd12c72af08b1f6c7c3e49' (2024-09-14) + → 'github:nixos/nixpkgs/086b448a5d54fd117f4dc2dee55c9f0ff461bdc1' (2024-09-16) + • Updated input 'nixpkgs-unstable': + 'github:nixos/nixpkgs/345c263f2f53a3710abe117f28a5cb86d0ba4059' (2024-09-13) + → 'github:nixos/nixpkgs/99dc8785f6a0adac95f5e2ab05cc2e1bf666d172' (2024-09-16) + ``` + * Update only a single input of a flake in a different directory: ```console diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 78a8a55c3..640a80aed 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -7,6 +7,7 @@ #include "eval-settings.hh" #include "flake/flake.hh" #include "get-drvs.hh" +#include "signals.hh" #include "store-api.hh" #include "derivations.hh" #include "outputs-spec.hh" @@ -16,11 +17,16 @@ #include "eval-cache.hh" #include "markdown.hh" #include "users.hh" +#include "terminal.hh" +#include #include -#include #include +#include "strings-inline.hh" + +namespace nix::fs { using namespace std::filesystem; } + using namespace nix; using namespace nix::flake; using json = nlohmann::json; @@ -47,19 +53,19 @@ public: FlakeRef getFlakeRef() { - return parseFlakeRef(flakeUrl, absPath(".")); //FIXME + return parseFlakeRef(fetchSettings, flakeUrl, fs::current_path().string()); //FIXME } LockedFlake lockFlake() { - return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); + return flake::lockFlake(flakeSettings, *getEvalState(), getFlakeRef(), lockFlags); } std::vector getFlakeRefsForCompletion() override { return { // Like getFlakeRef but with expandTilde calld first - parseFlakeRef(expandTilde(flakeUrl), absPath(".")) + parseFlakeRef(fetchSettings, expandTilde(flakeUrl), fs::current_path().string()) }; } }; @@ -164,7 +170,7 @@ struct CmdFlakeLock : FlakeCommand }; static void enumerateOutputs(EvalState & state, Value & vFlake, - std::function callback) + std::function callback) { auto pos = vFlake.determinePos(noPos); state.forceAttrs(vFlake, pos, "while evaluating a flake to get its outputs"); @@ -208,7 +214,7 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON auto & flake = lockedFlake.flake; // Currently, all flakes are in the Nix store via the rootFS accessor. - auto storePath = store->printStorePath(store->toStorePath(flake.path.path.abs()).first); + auto storePath = store->printStorePath(sourcePathToStorePath(store, flake.path).first); if (json) { nlohmann::json j; @@ -232,6 +238,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["lastModified"] = *lastModified; j["path"] = storePath; j["locks"] = lockedFlake.lockFile.toJSON().first; + if (auto fingerprint = lockedFlake.getFingerprint(store)) + j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false); logger->cout("%s", j.dump()); } else { logger->cout( @@ -264,6 +272,10 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout( ANSI_BOLD "Last modified:" ANSI_NORMAL " %s", std::put_time(std::localtime(&*lastModified), "%F %T")); + if (auto fingerprint = lockedFlake.getFingerprint(store)) + logger->cout( + ANSI_BOLD "Fingerprint:" ANSI_NORMAL " %s", + fingerprint->to_string(HashFormat::Base16, false)); if (!lockedFlake.lockFile.root->inputs.empty()) logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL); @@ -360,9 +372,11 @@ struct CmdFlakeCheck : FlakeCommand auto reportError = [&](const Error & e) { try { throw e; + } catch (Interrupted & e) { + throw; } catch (Error & e) { if (settings.keepGoing) { - ignoreException(); + ignoreExceptionExceptInterrupt(); hasErrors = true; } else @@ -386,15 +400,15 @@ struct CmdFlakeCheck : FlakeCommand || (hasPrefix(name, "_") && name.substr(1) == expected); }; - auto checkSystemName = [&](const std::string & system, const PosIdx pos) { + auto checkSystemName = [&](std::string_view system, const PosIdx pos) { // FIXME: what's the format of "system"? if (system.find('-') == std::string::npos) reportError(Error("'%s' is not a valid system type, at %s", system, resolve(pos))); }; - auto checkSystemType = [&](const std::string & system, const PosIdx pos) { + auto checkSystemType = [&](std::string_view system, const PosIdx pos) { if (!checkAllSystems && system != localSystem) { - omittedSystems.insert(system); + omittedSystems.insert(std::string(system)); return false; } else { return true; @@ -429,21 +443,46 @@ struct CmdFlakeCheck : FlakeCommand auto checkApp = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { - #if 0 - // FIXME - auto app = App(*state, v); - for (auto & i : app.context) { - auto [drvPathS, outputName] = NixStringContextElem::parse(i); - store->parseStorePath(drvPathS); + Activity act(*logger, lvlInfo, actUnknown, fmt("checking app '%s'", attrPath)); + state->forceAttrs(v, pos, ""); + if (auto attr = v.attrs()->get(state->symbols.create("type"))) + state->forceStringNoCtx(*attr->value, attr->pos, ""); + else + throw Error("app '%s' lacks attribute 'type'", attrPath); + + if (auto attr = v.attrs()->get(state->symbols.create("program"))) { + if (attr->name == state->symbols.create("program")) { + NixStringContext context; + state->forceString(*attr->value, context, attr->pos, ""); + } + } else + throw Error("app '%s' lacks attribute 'program'", attrPath); + + if (auto attr = v.attrs()->get(state->symbols.create("meta"))) { + state->forceAttrs(*attr->value, attr->pos, ""); + if (auto dAttr = attr->value->attrs()->get(state->symbols.create("description"))) + state->forceStringNoCtx(*dAttr->value, dAttr->pos, ""); + else + logWarning({ + .msg = HintFmt("app '%s' lacks attribute 'meta.description'", attrPath), + }); + } else + logWarning({ + .msg = HintFmt("app '%s' lacks attribute 'meta'", attrPath), + }); + + for (auto & attr : *v.attrs()) { + std::string_view name(state->symbols[attr.name]); + if (name != "type" && name != "program" && name != "meta") + throw Error("app '%s' has unsupported attribute '%s'", attrPath, name); } - #endif } catch (Error & e) { e.addTrace(resolve(pos), HintFmt("while checking the app definition '%s'", attrPath)); reportError(e); } }; - auto checkOverlay = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + auto checkOverlay = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking overlay '%s'", attrPath)); @@ -462,7 +501,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - auto checkModule = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + auto checkModule = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking NixOS module '%s'", attrPath)); @@ -473,9 +512,9 @@ struct CmdFlakeCheck : FlakeCommand } }; - std::function checkHydraJobs; + std::function checkHydraJobs; - checkHydraJobs = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + checkHydraJobs = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking Hydra job '%s'", attrPath)); @@ -516,7 +555,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - auto checkTemplate = [&](const std::string & attrPath, Value & v, const PosIdx pos) { + auto checkTemplate = [&](std::string_view attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking template '%s'", attrPath)); @@ -572,7 +611,7 @@ struct CmdFlakeCheck : FlakeCommand enumerateOutputs(*state, *vFlake, - [&](const std::string & name, Value & vOutput, const PosIdx pos) { + [&](std::string_view name, Value & vOutput, const PosIdx pos) { Activity act(*logger, lvlInfo, actUnknown, fmt("checking flake output '%s'", name)); @@ -596,7 +635,7 @@ struct CmdFlakeCheck : FlakeCommand if (name == "checks") { state->forceAttrs(vOutput, pos, ""); for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; + std::string_view attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); if (checkSystemType(attr_name, attr.pos)) { state->forceAttrs(*attr.value, attr.pos, ""); @@ -621,7 +660,7 @@ struct CmdFlakeCheck : FlakeCommand const auto & attr_name = state->symbols[attr.name]; checkSystemName(attr_name, attr.pos); if (checkSystemType(attr_name, attr.pos)) { - checkApp( + checkDerivation( fmt("%s.%s", name, attr_name), *attr.value, attr.pos); }; @@ -770,6 +809,8 @@ struct CmdFlakeCheck : FlakeCommand || name == "flakeModules" || name == "herculesCI" || name == "homeConfigurations" + || name == "homeModule" + || name == "homeModules" || name == "nixopsConfigurations" ) // Known but unchecked community attribute @@ -794,6 +835,7 @@ struct CmdFlakeCheck : FlakeCommand throw Error("some errors were encountered during the evaluation"); if (!omittedSystems.empty()) { + // TODO: empty system is not visible; render all as nix strings? warn( "The check omitted these incompatible systems: %s\n" "Use '--all-systems' to check all.", @@ -839,7 +881,8 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand auto evalState = getEvalState(); - auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath(".")); + auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment( + fetchSettings, templateUrl, fs::current_path().string()); auto installable = InstallableFlake(nullptr, evalState, std::move(templateFlakeRef), templateName, ExtendedOutputsSpec::Default(), @@ -858,26 +901,28 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand "If you've set '%s' to a string, try using a path instead.", templateDir, templateDirAttr->getAttrPathStr()).debugThrow(); - std::vector changedFiles; - std::vector conflictedFiles; + std::vector changedFiles; + std::vector conflictedFiles; - std::function copyDir; - copyDir = [&](const Path & from, const Path & to) + std::function copyDir; + copyDir = [&](const fs::path & from, const fs::path & to) { - createDirs(to); + fs::create_directories(to); - for (auto & entry : std::filesystem::directory_iterator{from}) { - auto from2 = entry.path().string(); - auto to2 = to + "/" + entry.path().filename().string(); - auto st = lstat(from2); - if (S_ISDIR(st.st_mode)) + for (auto & entry : fs::directory_iterator{from}) { + checkInterrupt(); + auto from2 = entry.path(); + auto to2 = to / entry.path().filename(); + auto st = entry.symlink_status(); + auto to_st = fs::symlink_status(to2); + if (fs::is_directory(st)) copyDir(from2, to2); - else if (S_ISREG(st.st_mode)) { - auto contents = readFile(from2); - if (pathExists(to2)) { - auto contents2 = readFile(to2); + else if (fs::is_regular_file(st)) { + auto contents = readFile(from2.string()); + if (fs::exists(to_st)) { + auto contents2 = readFile(to2.string()); if (contents != contents2) { - printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2, from2); + printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2.string()); conflictedFiles.push_back(to2); } else { notice("skipping identical file: %s", from2); @@ -886,18 +931,18 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand } else writeFile(to2, contents); } - else if (S_ISLNK(st.st_mode)) { - auto target = readLink(from2); - if (pathExists(to2)) { - if (readLink(to2) != target) { - printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2, from2); + else if (fs::is_symlink(st)) { + auto target = fs::read_symlink(from2); + if (fs::exists(to_st)) { + if (fs::read_symlink(to2) != target) { + printError("refusing to overwrite existing file '%s'\n please merge it manually with '%s'", to2.string(), from2.string()); conflictedFiles.push_back(to2); } else { notice("skipping identical file: %s", from2); } continue; } else - createSymlink(target, to2); + fs::create_symlink(target, to2); } else throw Error("file '%s' has unsupported type", from2); @@ -908,9 +953,9 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand copyDir(templateDir, flakeDir); - if (!changedFiles.empty() && pathExists(flakeDir + "/.git")) { + if (!changedFiles.empty() && fs::exists(std::filesystem::path{flakeDir} / ".git")) { Strings args = { "-C", flakeDir, "add", "--intent-to-add", "--force", "--" }; - for (auto & s : changedFiles) args.push_back(s); + for (auto & s : changedFiles) args.emplace_back(s.string()); runProgram("git", true, args); } auto welcomeText = cursor->maybeGetAttr("welcomeText"); @@ -1036,7 +1081,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun StorePathSet sources; - auto storePath = store->toStorePath(flake.flake.path.path.abs()).first; + auto storePath = sourcePathToStorePath(store, flake.flake.path).first; sources.insert(storePath); @@ -1176,7 +1221,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON // If we don't recognize it, it's probably content return true; } catch (EvalError & e) { - // Some attrs may contain errors, eg. legacyPackages of + // Some attrs may contain errors, e.g. legacyPackages of // nixpkgs. We still want to recurse into it, instead of // skipping it at all. return true; @@ -1230,25 +1275,97 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto showDerivation = [&]() { auto name = visitor.getAttr(state->sName)->getString(); + std::optional description; + if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) { + if (auto aDescription = aMeta->maybeGetAttr(state->sDescription)) + description = aDescription->getString(); + } + if (json) { - std::optional description; - if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) { - if (auto aDescription = aMeta->maybeGetAttr(state->sDescription)) - description = aDescription->getString(); - } j.emplace("type", "derivation"); j.emplace("name", name); - if (description) - j.emplace("description", *description); + j.emplace("description", description ? *description : ""); } else { - logger->cout("%s: %s '%s'", - headerPrefix, + auto type = attrPath.size() == 2 && attrPathS[0] == "devShell" ? "development environment" : attrPath.size() >= 2 && attrPathS[0] == "devShells" ? "development environment" : attrPath.size() == 3 && attrPathS[0] == "checks" ? "derivation" : attrPath.size() >= 1 && attrPathS[0] == "hydraJobs" ? "derivation" : - "package", - name); + "package"; + if (description && !description->empty()) { + + // Takes a string and returns the # of characters displayed + auto columnLengthOfString = [](std::string_view s) -> unsigned int { + unsigned int columnCount = 0; + for (auto i = s.begin(); i < s.end();) { + // Test first character to determine if it is one of + // treeConn, treeLast, treeLine + if (*i == -30) { + i += 3; + ++columnCount; + } + // Escape sequences + // https://en.wikipedia.org/wiki/ANSI_escape_code + else if (*i == '\e') { + // Eat '[' + if (*(++i) == '[') { + ++i; + // Eat parameter bytes + while(*i >= 0x30 && *i <= 0x3f) ++i; + + // Eat intermediate bytes + while(*i >= 0x20 && *i <= 0x2f) ++i; + + // Eat final byte + if(*i >= 0x40 && *i <= 0x73) ++i; + } + else { + // Eat Fe Escape sequence + if (*i >= 0x40 && *i <= 0x5f) ++i; + } + } + else { + ++i; + ++columnCount; + } + } + + return columnCount; + }; + + // Maximum length to print + size_t maxLength = getWindowSize().second > 0 ? getWindowSize().second : 80; + + // Trim the description and only use the first line + auto trimmed = trim(*description); + auto newLinePos = trimmed.find('\n'); + auto length = newLinePos != std::string::npos ? newLinePos : trimmed.length(); + + auto beginningOfLine = fmt("%s: %s '%s'", headerPrefix, type, name); + auto line = fmt("%s: %s '%s' - '%s'", headerPrefix, type, name, trimmed.substr(0, length)); + + // If we are already over the maximum length then do not trim + // and don't print the description (preserves existing behavior) + if (columnLengthOfString(beginningOfLine) >= maxLength) { + logger->cout("%s", beginningOfLine); + } + // If the entire line fits then print that + else if (columnLengthOfString(line) < maxLength) { + logger->cout("%s", line); + } + // Otherwise we need to truncate + else { + auto lineLength = columnLengthOfString(line); + auto chopOff = lineLength - maxLength; + line.resize(line.length() - chopOff); + line = line.replace(line.length() - 3, 3, "..."); + + logger->cout("%s", line); + } + } + else { + logger->cout("%s: %s '%s'", headerPrefix, type, name); + } } }; @@ -1327,12 +1444,19 @@ struct CmdFlakeShow : FlakeCommand, MixJSON (attrPath.size() == 3 && attrPathS[0] == "apps")) { auto aType = visitor.maybeGetAttr("type"); + std::optional description; + if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) { + if (auto aDescription = aMeta->maybeGetAttr(state->sDescription)) + description = aDescription->getString(); + } if (!aType || aType->getString() != "app") state->error("not an app definition").debugThrow(); if (json) { j.emplace("type", "app"); + if (description) + j.emplace("description", *description); } else { - logger->cout("%s: app", headerPrefix); + logger->cout("%s: app: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description ? *description : "no description"); } } diff --git a/src/nix/flake.md b/src/nix/flake.md index 661dd2f73..2b999431c 100644 --- a/src/nix/flake.md +++ b/src/nix/flake.md @@ -120,7 +120,7 @@ Contrary to URL-like references, path-like flake references can contain arbitrar ### Examples -* `.`: The flake to which the current directory belongs to. +* `.`: The flake to which the current directory belongs. * `/home/alice/src/patchelf`: A flake in some other directory. * `./../sub directory/with Ûñî©ôδ€`: A flake in another relative directory that has Unicode characters in its name. @@ -134,7 +134,9 @@ The following generic flake reference attributes are supported: repository or tarball. The default is the root directory of the flake. -* `narHash`: The hash of the NAR serialisation (in SRI format) of the +* `narHash`: The hash of the + [Nix Archive (NAR) serialisation][Nix Archive] + (in SRI format) of the contents of the flake. This is useful for flake types such as tarballs that lack a unique content identifier such as a Git commit hash. @@ -257,6 +259,8 @@ Currently the `type` attribute can be one of the following: `.tgz`, `.tar.gz`, `.tar.xz`, `.tar.bz2` or `.tar.zst`), then the `tarball+` can be dropped. + This can also be used to set the location of gitea/forgejo branches. [See here](@docroot@/protocols/tarball-fetcher.md#gitea-and-forgejo-support) + * `file`: Plain files or directory tarballs, either over http(s) or from the local disk. @@ -423,8 +427,9 @@ The following attributes are supported in `flake.nix`: * `lastModified`: The commit time of the revision `rev` as an integer denoting the number of seconds since 1970. - * `narHash`: The SHA-256 (in SRI format) of the NAR serialization of - the flake's source tree. + * `narHash`: The SHA-256 (in SRI format) of the + [Nix Archive (NAR) serialisation][Nix Archive] + NAR serialization of the flake's source tree. The value returned by the `outputs` function must be an attribute set. The attributes can have arbitrary values; however, various @@ -560,8 +565,9 @@ or NixOS modules, which are composed into the top-level flake's Inputs specified in `flake.nix` are typically "unlocked" in the sense that they don't specify an exact revision. To ensure reproducibility, Nix will automatically generate and use a *lock file* called -`flake.lock` in the flake's directory. The lock file contains a graph -structure isomorphic to the graph of dependencies of the root +`flake.lock` in the flake's directory. +The lock file is a UTF-8 JSON file. +It contains a graph structure isomorphic to the graph of dependencies of the root flake. Each node in the graph (except the root node) maps the (usually) unlocked input specifications in `flake.nix` to locked input specifications. Each node also contains some metadata, such as the @@ -703,4 +709,5 @@ will not look at the lock files of dependencies. However, lock file generation itself *does* use the lock files of dependencies by default. +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive )"" diff --git a/src/nix/fmt.cc b/src/nix/fmt.cc index 4b0fbb89d..f444d6add 100644 --- a/src/nix/fmt.cc +++ b/src/nix/fmt.cc @@ -1,5 +1,6 @@ #include "command.hh" #include "installable-value.hh" +#include "eval.hh" #include "run.hh" using namespace nix; @@ -39,17 +40,15 @@ struct CmdFmt : SourceExprCommand { Strings programArgs{app.program}; // Propagate arguments from the CLI - if (args.empty()) { - // Format the current flake out of the box - programArgs.push_back("."); - } else { - // User wants more power, let them decide which paths to include/exclude - for (auto &i : args) { - programArgs.push_back(i); - } + for (auto &i : args) { + programArgs.push_back(i); } - runProgramInStore(store, UseLookupPath::DontUse, app.program, programArgs); + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + evalState->evalCaches.clear(); + + execProgramInStore(store, UseLookupPath::DontUse, app.program, programArgs); }; }; diff --git a/src/nix/fmt.md b/src/nix/fmt.md index 1c78bb36f..a2afde61c 100644 --- a/src/nix/fmt.md +++ b/src/nix/fmt.md @@ -1,5 +1,14 @@ R""( +# Description + +`nix fmt` calls the formatter specified in the flake. + +Flags can be forwarded to the formatter by using `--` followed by the flags. + +Any arguments will be forwarded to the formatter. Typically these are the files to format. + + # Examples With [nixpkgs-fmt](https://github.com/nix-community/nixpkgs-fmt): @@ -13,10 +22,6 @@ With [nixpkgs-fmt](https://github.com/nix-community/nixpkgs-fmt): } ``` -- Format the current flake: `$ nix fmt` - -- Format a specific folder or file: `$ nix fmt ./folder ./file.nix` - With [nixfmt](https://github.com/serokell/nixfmt): ```nix @@ -28,8 +33,6 @@ With [nixfmt](https://github.com/serokell/nixfmt): } ``` -- Format specific files: `$ nix fmt ./file1.nix ./file2.nix` - With [Alejandra](https://github.com/kamadorueda/alejandra): ```nix @@ -41,13 +44,4 @@ With [Alejandra](https://github.com/kamadorueda/alejandra): } ``` -- Format the current flake: `$ nix fmt` - -- Format a specific folder or file: `$ nix fmt ./folder ./file.nix` - -# Description - -`nix fmt` will rewrite all Nix files (\*.nix) to a canonical format -using the formatter specified in your flake. - )"" diff --git a/src/nix/hash.cc b/src/nix/hash.cc index f969886ea..62266fda1 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -68,7 +68,7 @@ struct CmdHashBase : Command switch (mode) { case FileIngestionMethod::Flat: return "print cryptographic hash of a regular file"; - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: return "print cryptographic hash of the NAR serialisation of a path"; case FileIngestionMethod::Git: return "print cryptographic hash of the Git serialisation of a path"; @@ -91,7 +91,7 @@ struct CmdHashBase : Command Hash h { HashAlgorithm::SHA256 }; // throwaway def to appease C++ switch (mode) { case FileIngestionMethod::Flat: - case FileIngestionMethod::Recursive: + case FileIngestionMethod::NixArchive: { auto hashSink = makeSink(); dumpPath(path2, *hashSink, (FileSerialisationMethod) mode); @@ -126,7 +126,7 @@ struct CmdHashBase : Command struct CmdHashPath : CmdHashBase { CmdHashPath() - : CmdHashBase(FileIngestionMethod::Recursive) + : CmdHashBase(FileIngestionMethod::NixArchive) { addFlag(flag::hashAlgo("algo", &hashAlgo)); addFlag(flag::fileIngestionMethod(&mode)); @@ -181,7 +181,7 @@ struct CmdToBase : Command void run() override { - warn("The old format conversion sub commands of `nix hash` where deprecated in favor of `nix hash convert`."); + warn("The old format conversion sub commands of `nix hash` were deprecated in favor of `nix hash convert`."); for (auto s : args) logger->cout(Hash::parseAny(s, hashAlgo).to_string(hashFormat, hashFormat == HashFormat::SRI)); } @@ -311,7 +311,7 @@ static int compatNixHash(int argc, char * * argv) }); if (op == opHash) { - CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive); + CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::NixArchive); if (!hashAlgo.has_value()) hashAlgo = HashAlgorithm::MD5; cmd.hashAlgo = hashAlgo.value(); cmd.hashFormat = hashFormat; diff --git a/src/nix/help-stores.md b/src/nix/help-stores.md new file mode 120000 index 000000000..5c5624f5e --- /dev/null +++ b/src/nix/help-stores.md @@ -0,0 +1 @@ +../../doc/manual/src/store/types/index.md.in \ No newline at end of file diff --git a/src/nix/local.mk b/src/nix/local.mk index 305b0e9df..b57f6b3e2 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -24,12 +24,19 @@ ifdef HOST_UNIX INCLUDE_nix += -I $(d)/unix endif -nix_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libmain) -I src/libcmd -I doc/manual $(INCLUDE_nix) +nix_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libflake) $(INCLUDE_libmain) -I src/libcmd -I doc/manual $(INCLUDE_nix) -nix_LIBS = libexpr libmain libfetchers libstore libutil libcmd +nix_CXXFLAGS += -DNIX_BIN_DIR=\"$(NIX_ROOT)$(bindir)\" + +nix_LIBS = libexpr libmain libfetchers libflake libstore libutil libcmd nix_LDFLAGS = $(THREAD_LDFLAGS) $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) $(LOWDOWN_LIBS) +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + nix_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif + $(foreach name, \ nix-build nix-channel nix-collect-garbage nix-copy-closure nix-daemon nix-env nix-hash nix-instantiate nix-prefetch-url nix-shell nix-store, \ $(eval $(call install-symlink, nix, $(bindir)/$(name)))) @@ -37,27 +44,16 @@ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh -src/nix/develop.cc: src/nix/get-env.sh.gen.hh +$(d)/develop.cc: $(d)/get-env.sh.gen.hh src/nix-channel/nix-channel.cc: src/nix-channel/unpack-channel.nix.gen.hh -src/nix/main.cc: \ +$(d)/main.cc: \ doc/manual/generate-manpage.nix.gen.hh \ doc/manual/utils.nix.gen.hh doc/manual/generate-settings.nix.gen.hh \ doc/manual/generate-store-info.nix.gen.hh \ - src/nix/generated-doc/help-stores.md + $(d)/help-stores.md.gen.hh -src/nix/generated-doc/files/%.md: doc/manual/src/command-ref/files/%.md - @mkdir -p $$(dirname $@) - @cp $< $@ +$(d)/profile.cc: $(d)/profile.md -src/nix/profile.cc: src/nix/profile.md src/nix/generated-doc/files/profiles.md.gen.hh - -src/nix/generated-doc/help-stores.md: doc/manual/src/store/types/index.md.in - @mkdir -p $$(dirname $@) - @echo 'R"(' >> $@.tmp - @echo >> $@.tmp - @cat $^ >> $@.tmp - @echo >> $@.tmp - @echo ')"' >> $@.tmp - @mv $@.tmp $@ +$(d)/profile.md: $(d)/profiles.md.gen.hh diff --git a/src/nix/main.cc b/src/nix/main.cc index bc13a4df5..eff2d60a4 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -1,5 +1,3 @@ -#include - #include "args/root.hh" #include "current-process.hh" #include "command.hh" @@ -18,6 +16,10 @@ #include "terminal.hh" #include "users.hh" #include "network-proxy.hh" +#include "eval-cache.hh" +#include "flake/flake.hh" +#include "self-exe.hh" +#include "json-utils.hh" #include #include @@ -40,8 +42,23 @@ extern std::string chrootHelperName; void chrootHelper(int argc, char * * argv); #endif +#include "strings.hh" + namespace nix { +enum struct AliasStatus { + /** Aliases that don't go away */ + AcceptedShorthand, + /** Aliases that will go away */ + Deprecated, +}; + +/** An alias, except for the original syntax, which is in the map key. */ +struct AliasInfo { + AliasStatus status; + std::vector replacement; +}; + /* Check if we have a non-loopback/link-local network interface. */ static bool haveInternet() { @@ -134,29 +151,30 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs }); } - std::map> aliases = { - {"add-to-store", {"store", "add-path"}}, - {"cat-nar", {"nar", "cat"}}, - {"cat-store", {"store", "cat"}}, - {"copy-sigs", {"store", "copy-sigs"}}, - {"dev-shell", {"develop"}}, - {"diff-closures", {"store", "diff-closures"}}, - {"dump-path", {"store", "dump-path"}}, - {"hash-file", {"hash", "file"}}, - {"hash-path", {"hash", "path"}}, - {"ls-nar", {"nar", "ls"}}, - {"ls-store", {"store", "ls"}}, - {"make-content-addressable", {"store", "make-content-addressed"}}, - {"optimise-store", {"store", "optimise"}}, - {"ping-store", {"store", "ping"}}, - {"sign-paths", {"store", "sign"}}, - {"show-derivation", {"derivation", "show"}}, - {"show-config", {"config", "show"}}, - {"to-base16", {"hash", "to-base16"}}, - {"to-base32", {"hash", "to-base32"}}, - {"to-base64", {"hash", "to-base64"}}, - {"verify", {"store", "verify"}}, - {"doctor", {"config", "check"}}, + std::map aliases = { + {"add-to-store", { AliasStatus::Deprecated, {"store", "add-path"}}}, + {"cat-nar", { AliasStatus::Deprecated, {"nar", "cat"}}}, + {"cat-store", { AliasStatus::Deprecated, {"store", "cat"}}}, + {"copy-sigs", { AliasStatus::Deprecated, {"store", "copy-sigs"}}}, + {"dev-shell", { AliasStatus::Deprecated, {"develop"}}}, + {"diff-closures", { AliasStatus::Deprecated, {"store", "diff-closures"}}}, + {"dump-path", { AliasStatus::Deprecated, {"store", "dump-path"}}}, + {"hash-file", { AliasStatus::Deprecated, {"hash", "file"}}}, + {"hash-path", { AliasStatus::Deprecated, {"hash", "path"}}}, + {"ls-nar", { AliasStatus::Deprecated, {"nar", "ls"}}}, + {"ls-store", { AliasStatus::Deprecated, {"store", "ls"}}}, + {"make-content-addressable", { AliasStatus::Deprecated, {"store", "make-content-addressed"}}}, + {"optimise-store", { AliasStatus::Deprecated, {"store", "optimise"}}}, + {"ping-store", { AliasStatus::Deprecated, {"store", "info"}}}, + {"sign-paths", { AliasStatus::Deprecated, {"store", "sign"}}}, + {"shell", { AliasStatus::AcceptedShorthand, {"env", "shell"}}}, + {"show-derivation", { AliasStatus::Deprecated, {"derivation", "show"}}}, + {"show-config", { AliasStatus::Deprecated, {"config", "show"}}}, + {"to-base16", { AliasStatus::Deprecated, {"hash", "to-base16"}}}, + {"to-base32", { AliasStatus::Deprecated, {"hash", "to-base32"}}}, + {"to-base64", { AliasStatus::Deprecated, {"hash", "to-base64"}}}, + {"verify", { AliasStatus::Deprecated, {"store", "verify"}}}, + {"doctor", { AliasStatus::Deprecated, {"config", "check"}}}, }; bool aliasUsed = false; @@ -167,10 +185,13 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs auto arg = *pos; auto i = aliases.find(arg); if (i == aliases.end()) return pos; - warn("'%s' is a deprecated alias for '%s'", - arg, concatStringsSep(" ", i->second)); + auto & info = i->second; + if (info.status == AliasStatus::Deprecated) { + warn("'%s' is a deprecated alias for '%s'", + arg, concatStringsSep(" ", info.replacement)); + } pos = args.erase(pos); - for (auto j = i->second.rbegin(); j != i->second.rend(); ++j) + for (auto j = info.replacement.rbegin(); j != info.replacement.rend(); ++j) pos = args.insert(pos, *j); aliasUsed = true; return pos; @@ -224,7 +245,7 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) evalSettings.restrictEval = false; evalSettings.pureEval = false; - EvalState state({}, openStore("dummy://")); + EvalState state({}, openStore("dummy://"), fetchSettings, evalSettings); auto vGenerateManpage = state.allocValue(); state.eval(state.parseExprFromString( @@ -315,7 +336,7 @@ struct CmdHelpStores : Command std::string doc() override { return - #include "generated-doc/help-stores.md" + #include "help-stores.md.gen.hh" ; } @@ -344,6 +365,18 @@ void mainWrapped(int argc, char * * argv) initNix(); initGC(); + flake::initLib(flakeSettings); + + /* Set the build hook location + + For builds we perform a self-invocation, so Nix has to be + self-aware. That is, it has to know where it is installed. We + don't think it's sentient. + */ + settings.buildHook.setDefault(Strings { + getNixBin({}).string(), + "__build-remote", + }); #if __linux__ if (isRootUser()) { @@ -400,36 +433,29 @@ void mainWrapped(int argc, char * * argv) Xp::FetchTree, }; evalSettings.pureEval = false; - EvalState state({}, openStore("dummy://")); - auto res = nlohmann::json::object(); - res["builtins"] = ({ - auto builtinsJson = nlohmann::json::object(); - for (auto & builtin : *state.baseEnv.values[0]->attrs()) { - auto b = nlohmann::json::object(); - if (!builtin.value->isPrimOp()) continue; - auto primOp = builtin.value->primOp(); - if (!primOp->doc) continue; - b["arity"] = primOp->arity; - b["args"] = primOp->args; - b["doc"] = trim(stripIndentation(primOp->doc)); + EvalState state({}, openStore("dummy://"), fetchSettings, evalSettings); + auto builtinsJson = nlohmann::json::object(); + for (auto & builtin : *state.baseEnv.values[0]->attrs()) { + auto b = nlohmann::json::object(); + if (!builtin.value->isPrimOp()) continue; + auto primOp = builtin.value->primOp(); + if (!primOp->doc) continue; + b["args"] = primOp->args; + b["doc"] = trim(stripIndentation(primOp->doc)); + if (primOp->experimentalFeature) b["experimental-feature"] = primOp->experimentalFeature; - builtinsJson[state.symbols[builtin.name]] = std::move(b); - } - std::move(builtinsJson); - }); - res["constants"] = ({ - auto constantsJson = nlohmann::json::object(); - for (auto & [name, info] : state.constantInfos) { - auto c = nlohmann::json::object(); - if (!info.doc) continue; - c["doc"] = trim(stripIndentation(info.doc)); - c["type"] = showType(info.type, false); - c["impure-only"] = info.impureOnly; - constantsJson[name] = std::move(c); - } - std::move(constantsJson); - }); - logger->cout("%s", res); + builtinsJson.emplace(state.symbols[builtin.name], std::move(b)); + } + for (auto & [name, info] : state.constantInfos) { + auto b = nlohmann::json::object(); + if (!info.doc) continue; + b["doc"] = trim(stripIndentation(info.doc)); + b["type"] = showType(info.type, false); + if (info.impureOnly) + b["impure-only"] = true; + builtinsJson[name] = std::move(b); + } + logger->cout("%s", builtinsJson); return; } @@ -515,18 +541,24 @@ void mainWrapped(int argc, char * * argv) if (args.command->second->forceImpureByDefault() && !evalSettings.pureEval.overridden) { evalSettings.pureEval = false; } - args.command->second->run(); + + try { + args.command->second->run(); + } catch (eval_cache::CachedEvalError & e) { + /* Evaluate the original attribute that resulted in this + cached error so that we can show the original error to the + user. */ + e.force(); + } } } int main(int argc, char * * argv) { -#ifndef _WIN32 // TODO implement on Windows // Increase the default stack size for the evaluator and for // libstdc++'s std::regex. nix::setStackSize(64 * 1024 * 1024); -#endif return nix::handleExceptions(argv[0], [&]() { nix::mainWrapped(argc, argv); diff --git a/src/nix/meson.build b/src/nix/meson.build new file mode 100644 index 000000000..55089d821 --- /dev/null +++ b/src/nix/meson.build @@ -0,0 +1,263 @@ +project('nix', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + 'localstatedir=/nix/var', + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +nix_store = dependency('nix-store') + +deps_private_maybe_subproject = [ + dependency('nix-util'), + nix_store, + dependency('nix-expr'), + dependency('nix-flake'), + dependency('nix-fetchers'), + dependency('nix-main'), + dependency('nix-cmd'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +subdir('build-utils-meson/export-all-symbols') + +configdata = configuration_data() + +fs = import('fs') + +bindir = get_option('bindir') +if not fs.is_absolute(bindir) + bindir = get_option('prefix') / bindir +endif +configdata.set_quoted('NIX_BIN_DIR', bindir) + +config_h = configure_file( + configuration : configdata, + output : 'config-nix-cli.hh', +) + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + #'-include', 'config-fetchers.hh', + '-include', 'config-main.hh', + '-include', 'config-cmd.hh', + '-include', 'config-nix-cli.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') +subdir('build-utils-meson/generate-header') + +nix_sources = [config_h] + files( + 'add-to-store.cc', + 'app.cc', + 'self-exe.cc', + 'build.cc', + 'bundle.cc', + 'cat.cc', + 'config-check.cc', + 'config.cc', + 'copy.cc', + 'derivation-add.cc', + 'derivation-show.cc', + 'derivation.cc', + 'develop.cc', + 'diff-closures.cc', + 'dump-path.cc', + 'edit.cc', + 'env.cc', + 'eval.cc', + 'flake.cc', + 'fmt.cc', + 'hash.cc', + 'log.cc', + 'ls.cc', + 'main.cc', + 'make-content-addressed.cc', + 'nar.cc', + 'optimise-store.cc', + 'path-from-hash-part.cc', + 'path-info.cc', + 'prefetch.cc', + 'profile.cc', + 'realisation.cc', + 'registry.cc', + 'repl.cc', + 'run.cc', + 'search.cc', + 'sigs.cc', + 'store-copy-log.cc', + 'store-delete.cc', + 'store-gc.cc', + 'store-info.cc', + 'store-repair.cc', + 'store.cc', + 'upgrade-nix.cc', + 'verify.cc', + 'why-depends.cc', +) + +if host_machine.system() != 'windows' + nix_sources += files( + 'unix/daemon.cc', + ) +endif + +nix_sources += [ + gen_header.process('doc/manual/generate-manpage.nix'), + gen_header.process('doc/manual/generate-settings.nix'), + gen_header.process('doc/manual/generate-store-info.nix'), + gen_header.process('doc/manual/utils.nix'), + gen_header.process('get-env.sh'), + gen_header.process('profiles.md'), + gen_header.process('help-stores.md'), +] + +# The rest of the subdirectories aren't separate components, +# just source files in another directory, so we process them here. + +build_remote_sources = files( + 'build-remote/build-remote.cc', +) +nix_build_sources = files( + 'nix-build/nix-build.cc', +) +nix_channel_sources = files( + 'nix-channel/nix-channel.cc', +) +unpack_channel_gen = gen_header.process('nix-channel/unpack-channel.nix') +nix_collect_garbage_sources = files( + 'nix-collect-garbage/nix-collect-garbage.cc', +) +nix_copy_closure_sources = files( + 'nix-copy-closure/nix-copy-closure.cc', +) +nix_env_buildenv_gen = gen_header.process('nix-env/buildenv.nix') +nix_env_sources = files( + 'nix-env/nix-env.cc', + 'nix-env/user-env.cc', +) +nix_instantiate_sources = files( + 'nix-instantiate/nix-instantiate.cc', +) +nix_store_sources = files( + 'nix-store/dotgraph.cc', + 'nix-store/graphml.cc', + 'nix-store/nix-store.cc', +) + +# Hurray for Meson list flattening! +sources = [ + nix_sources, + nix_build_sources, + unpack_channel_gen, + nix_collect_garbage_sources, + nix_copy_closure_sources, + nix_env_buildenv_gen, + nix_env_sources, + nix_instantiate_sources, + nix_store_sources, +] + +if host_machine.system() != 'windows' + sources += [ + build_remote_sources, + nix_channel_sources, + ] +endif + +include_dirs = [include_directories('.')] + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + link_args: linker_export_flags, + install : true, +) + +meson.override_find_program('nix', this_exe) + +nix_symlinks = [ + 'nix-build', + 'nix-channel', + 'nix-collect-garbage', + 'nix-copy-closure', + 'nix-daemon', + 'nix-env', + 'nix-hash', + 'nix-instantiate', + 'nix-prefetch-url', + 'nix-shell', + 'nix-store', +] + +executable_suffix = '' +if host_machine.system() == 'windows' + executable_suffix = '.exe' +endif + +foreach linkname : nix_symlinks + install_symlink( + linkname + executable_suffix, + # TODO(Qyriad): should these continue to be relative symlinks? + pointing_to : fs.name(this_exe), + install_dir : get_option('bindir'), + # The 'runtime' tag is what executables default to, which we want to emulate here. + install_tag : 'runtime' + ) + t = custom_target( + command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'], + output: linkname + executable_suffix, + # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working) + build_by_default: true + ) + # TODO(Ericson3214): Dosen't yet work + #meson.override_find_program(linkname, t) +endforeach + +install_symlink( + 'build-remote', + pointing_to : '..' / '..'/ get_option('bindir') / fs.name(this_exe), + install_dir : get_option('libexecdir') / fs.name(this_exe), + # The 'runtime' tag is what executables default to, which we want to emulate here. + install_tag : 'runtime' +) + +custom_target( + command: ['ln', '-sf', fs.name(this_exe), '@OUTPUT@'], + output: 'build-remote' + executable_suffix, + # TODO(Ericson2314): Don't do this once we have the `meson.override_find_program` working) + build_by_default: true +) +# TODO(Ericson3214): Dosen't yet work +#meson.override_find_program(linkname, t) + +localstatedir = nix_store.get_variable( + 'localstatedir', + default_value : get_option('localstatedir'), +) +assert(localstatedir == get_option('localstatedir')) +store_dir = nix_store.get_variable('storedir') +subdir('scripts') +subdir('misc') diff --git a/src/nix/meson.options b/src/nix/meson.options new file mode 100644 index 000000000..8430dd669 --- /dev/null +++ b/src/nix/meson.options @@ -0,0 +1,6 @@ +# vim: filetype=meson + +# A relative path means it gets appended to prefix. +option('profile-dir', type : 'string', value : 'etc/profile.d', + description : 'the path to install shell profile files', +) diff --git a/src/nix/misc b/src/nix/misc new file mode 120000 index 000000000..2825552c9 --- /dev/null +++ b/src/nix/misc @@ -0,0 +1 @@ +../../misc \ No newline at end of file diff --git a/src/nix/nar-cat.md b/src/nix/nar-cat.md index 55c481a28..1131eb2bf 100644 --- a/src/nix/nar-cat.md +++ b/src/nix/nar-cat.md @@ -2,7 +2,7 @@ R""( # Examples -* List a file in a NAR and pipe it through `gunzip`: +* List a file in a [Nix Archive (NAR)][Nix Archive] and pipe it through `gunzip`: ```console # nix nar cat ./hello.nar /share/man/man1/hello.1.gz | gunzip @@ -16,4 +16,5 @@ R""( This command prints on standard output the contents of the regular file *path* inside the NAR file *nar*. +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive )"" diff --git a/src/nix/nar-dump-path.md b/src/nix/nar-dump-path.md index de82202de..4676e4fef 100644 --- a/src/nix/nar-dump-path.md +++ b/src/nix/nar-dump-path.md @@ -2,7 +2,7 @@ R""( # Examples -* To serialise directory `foo` as a NAR: +* To serialise directory `foo` as a [Nix Archive (NAR)][Nix Archive]: ```console # nix nar pack ./foo > foo.nar @@ -10,8 +10,10 @@ R""( # Description -This command generates a NAR file containing the serialisation of +This command generates a [Nix Archive (NAR)][Nix Archive] file containing the serialisation of *path*, which must contain only regular files, directories and symbolic links. The NAR is written to standard output. +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + )"" diff --git a/src/nix/nar-ls.md b/src/nix/nar-ls.md index 5a03c5d82..27c4b97e6 100644 --- a/src/nix/nar-ls.md +++ b/src/nix/nar-ls.md @@ -2,7 +2,7 @@ R""( # Examples -* To list a specific file in a NAR: +* To list a specific file in a [NAR][Nix Archive]: ```console # nix nar ls --long ./hello.nar /bin/hello @@ -19,6 +19,8 @@ R""( # Description -This command shows information about a *path* inside NAR file *nar*. +This command shows information about a *path* inside [Nix Archive (NAR)][Nix Archive] file *nar*. + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive )"" diff --git a/src/nix/nar.md b/src/nix/nar.md index a83b5c764..b0f70ce93 100644 --- a/src/nix/nar.md +++ b/src/nix/nar.md @@ -3,11 +3,14 @@ R""( # Description `nix nar` provides several subcommands for creating and inspecting -*Nix Archives* (NARs). +[*Nix Archives* (NARs)][Nix Archive]. # File format -For the definition of the NAR file format, see Figure 5.2 in -https://edolstra.github.io/pubs/phd-thesis.pdf. +For the definition of the Nix Archive file format, see +[within the protocols chapter](@docroot@/protocols/nix-archive.md) +of the manual. + +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive )"" diff --git a/src/nix/nix-build b/src/nix/nix-build new file mode 120000 index 000000000..2954d8ac7 --- /dev/null +++ b/src/nix/nix-build @@ -0,0 +1 @@ +../nix-build \ No newline at end of file diff --git a/src/nix/nix-channel b/src/nix/nix-channel new file mode 120000 index 000000000..29b759473 --- /dev/null +++ b/src/nix/nix-channel @@ -0,0 +1 @@ +../nix-channel \ No newline at end of file diff --git a/src/nix/nix-collect-garbage b/src/nix/nix-collect-garbage new file mode 120000 index 000000000..b037fc1b0 --- /dev/null +++ b/src/nix/nix-collect-garbage @@ -0,0 +1 @@ +../nix-collect-garbage \ No newline at end of file diff --git a/src/nix/nix-copy-closure b/src/nix/nix-copy-closure new file mode 120000 index 000000000..9063c583a --- /dev/null +++ b/src/nix/nix-copy-closure @@ -0,0 +1 @@ +../nix-copy-closure \ No newline at end of file diff --git a/src/nix/nix-env b/src/nix/nix-env new file mode 120000 index 000000000..f2f19f580 --- /dev/null +++ b/src/nix/nix-env @@ -0,0 +1 @@ +../nix-env \ No newline at end of file diff --git a/src/nix/nix-instantiate b/src/nix/nix-instantiate new file mode 120000 index 000000000..2d7502ffa --- /dev/null +++ b/src/nix/nix-instantiate @@ -0,0 +1 @@ +../nix-instantiate \ No newline at end of file diff --git a/src/nix/nix-store b/src/nix/nix-store new file mode 120000 index 000000000..e6efcac42 --- /dev/null +++ b/src/nix/nix-store @@ -0,0 +1 @@ +../nix-store/ \ No newline at end of file diff --git a/src/nix/nix.md b/src/nix/nix.md index 4464bef37..b88bd9a94 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -50,7 +50,7 @@ manual](https://nixos.org/manual/nix/stable/). > **Warning** \ > Installables are part of the unstable -> [`nix-command` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-nix-command), +> [`nix-command` experimental feature](@docroot@/development/experimental-features.md#xp-feature-nix-command), > and subject to change without notice. Many `nix` subcommands operate on one or more *installables*. @@ -59,9 +59,13 @@ These are command line arguments that represent something that can be realised i The following types of installable are supported by most commands: - [Flake output attribute](#flake-output-attribute) (experimental) + - This is the default - [Store path](#store-path) + - This is assumed if the argument is a Nix store path or a symlink to a Nix store path - [Nix file](#nix-file), optionally qualified by an attribute path + - Specified with `--file`/`-f` - [Nix expression](#nix-expression), optionally qualified by an attribute path + - Specified with `--expr` For most commands, if no installable is specified, `.` is assumed. That is, Nix will operate on the default flake output attribute of the flake in the current directory. @@ -70,9 +74,9 @@ That is, Nix will operate on the default flake output attribute of the flake in > **Warning** \ > Flake output attribute installables depend on both the -> [`flakes`](@docroot@/contributing/experimental-features.md#xp-feature-flakes) +> [`flakes`](@docroot@/development/experimental-features.md#xp-feature-flakes) > and -> [`nix-command`](@docroot@/contributing/experimental-features.md#xp-feature-nix-command) +> [`nix-command`](@docroot@/development/experimental-features.md#xp-feature-nix-command) > experimental features, and subject to change without notice. Example: `nixpkgs#hello` @@ -178,9 +182,10 @@ that contains programs, and a `dev` output that provides development artifacts like C/C++ header files. The outputs on which `nix` commands operate are determined as follows: -* You can explicitly specify the desired outputs using the syntax - *installable*`^`*output1*`,`*...*`,`*outputN*. For example, you can - obtain the `dev` and `static` outputs of the `glibc` package: +* You can explicitly specify the desired outputs using the syntax *installable*`^`*output1*`,`*...*`,`*outputN* — that is, a caret followed immediately by a comma-separated list of derivation outputs to select. + For installables specified as [Flake output attributes](#flake-output-attribute) or [Store paths](#store-path), the output is specified in the same argument: + + For example, you can obtain the `dev` and `static` outputs of the `glibc` package: ```console # nix build 'nixpkgs#glibc^dev,static' @@ -195,6 +200,19 @@ operate are determined as follows: … ``` + For `--expr` and `-f`/`--file`, the derivation output is specified as part of the attribute path: + + ```console + $ nix build -f '' 'glibc^dev,static' + $ nix build --impure --expr 'import { }' 'glibc^dev,static' + ``` + + This syntax is the same even if the actual attribute path is empty: + + ```console + $ nix build --impure --expr 'let pkgs = import { }; in pkgs.glibc' '^dev,static' + ``` + * You can also specify that *all* outputs should be used using the syntax *installable*`^*`. For example, the following shows the size of all outputs of the `glibc` package in the binary cache: @@ -284,7 +302,7 @@ or with an **expression**: terraform "$@" ``` -or with cascading interpreters. Note that the `#! nix` lines don't need to follow after the first line, to accomodate other interpreters. +or with cascading interpreters. Note that the `#! nix` lines don't need to follow after the first line, to accommodate other interpreters. ``` #!/usr/bin/env nix diff --git a/src/nix/package.nix b/src/nix/package.nix new file mode 100644 index 000000000..3e19c6dca --- /dev/null +++ b/src/nix/package.nix @@ -0,0 +1,129 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-store +, nix-expr +, nix-main +, nix-cmd + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix"; + inherit version; + + workDir = ./.; + fileset = fileset.unions ([ + ../../build-utils-meson + ./build-utils-meson + ../../.version + ./.version + ./meson.build + ./meson.options + + # Symbolic links to other dirs + ## exes + ./build-remote + ./doc + ./nix-build + ./nix-channel + ./nix-collect-garbage + ./nix-copy-closure + ./nix-env + ./nix-instantiate + ./nix-store + ## dirs + ./scripts + ../../scripts + ./misc + ../../misc + + # Doc nix files for --help + ../../doc/manual/generate-manpage.nix + ../../doc/manual/utils.nix + ../../doc/manual/generate-settings.nix + ../../doc/manual/generate-store-info.nix + + # Other files to be included as string literals + ../nix-channel/unpack-channel.nix + ../nix-env/buildenv.nix + ./get-env.sh + ./help-stores.md + ../../doc/manual/src/store/types/index.md.in + ./profiles.md + ../../doc/manual/src/command-ref/files/profiles.md + + # Files + ] ++ lib.concatMap + (dir: [ + (fileset.fileFilter (file: file.hasExt "cc") dir) + (fileset.fileFilter (file: file.hasExt "hh") dir) + (fileset.fileFilter (file: file.hasExt "md") dir) + ]) + [ + ./. + ../build-remote + ../nix-build + ../nix-channel + ../nix-collect-garbage + ../nix-copy-closure + ../nix-env + ../nix-instantiate + ../nix-store + ] + ); + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-store + nix-expr + nix-main + nix-cmd + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 921b25d7f..e7cfb6e7a 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -9,6 +9,8 @@ #include +#include "strings.hh" + using namespace nix; using nlohmann::json; @@ -139,21 +141,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON void printSize(uint64_t value) { - if (!humanReadable) { + if (humanReadable) + std::cout << fmt("\t%s", renderSize(value, true)); + else std::cout << fmt("\t%11d", value); - return; - } - - static const std::array idents{{ - ' ', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y' - }}; - size_t power = 0; - double res = value; - while (res > 1024 && power < idents.size()) { - ++power; - res /= 1024; - } - std::cout << fmt("\t%6.1f%c", res, idents.at(power)); } void run(ref store, StorePaths && storePaths) override diff --git a/src/nix/path-info.md b/src/nix/path-info.md index 789984559..2e39225b8 100644 --- a/src/nix/path-info.md +++ b/src/nix/path-info.md @@ -26,8 +26,8 @@ R""( ```console # nix path-info --recursive --size --closure-size --human-readable nixpkgs#rustc - /nix/store/01rrgsg5zk3cds0xgdsq40zpk6g51dz9-ncurses-6.2-dev 386.7K 69.1M - /nix/store/0q783wnvixpqz6dxjp16nw296avgczam-libpfm-4.11.0 5.9M 37.4M + /nix/store/01rrgsg5zk3cds0xgdsq40zpk6g51dz9-ncurses-6.2-dev 386.7 KiB 69.1 MiB + /nix/store/0q783wnvixpqz6dxjp16nw296avgczam-libpfm-4.11.0 5.9 MiB 37.4 MiB … ``` diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 3ce52acc5..db7d9e4ef 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -57,7 +57,9 @@ std::tuple prefetchFile( bool unpack, bool executable) { - auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + ContentAddressMethod method = unpack || executable + ? ContentAddressMethod::Raw::NixArchive + : ContentAddressMethod::Raw::Flat; /* Figure out a name in the Nix store. */ if (!name) { @@ -73,11 +75,10 @@ std::tuple prefetchFile( the store. */ if (expectedHash) { hashAlgo = expectedHash->algo; - storePath = store->makeFixedOutputPath(*name, FixedOutputInfo { - .method = ingestionMethod, - .hash = *expectedHash, - .references = {}, - }); + storePath = store->makeFixedOutputPathFromCA(*name, ContentAddressWithReferences::fromParts( + method, + *expectedHash, + {})); if (store->isValidPath(*storePath)) hash = expectedHash; else @@ -113,14 +114,15 @@ std::tuple prefetchFile( createDirs(unpacked); unpackTarfile(tmpFile.string(), unpacked); + auto entries = std::filesystem::directory_iterator{unpacked}; /* If the archive unpacks to a single file/directory, then use that as the top-level. */ - auto entries = std::filesystem::directory_iterator{unpacked}; - auto file_count = std::distance(entries, std::filesystem::directory_iterator{}); - if (file_count == 1) - tmpFile = entries->path(); - else + tmpFile = entries->path(); + auto fileCount = std::distance(entries, std::filesystem::directory_iterator{}); + if (fileCount != 1) { + /* otherwise, use the directory itself */ tmpFile = unpacked; + } } Activity act(*logger, lvlChatty, actUnknown, @@ -128,7 +130,7 @@ std::tuple prefetchFile( auto info = store->addToStoreSlow( *name, PosixSourceAccessor::createAtRoot(tmpFile), - ingestionMethod, hashAlgo, {}, expectedHash); + method, hashAlgo, {}, expectedHash); storePath = info.path; assert(info.ca); hash = info.ca->hash; @@ -193,7 +195,7 @@ static int main_nix_prefetch_url(int argc, char * * argv) startProgressBar(); auto store = openStore(); - auto state = std::make_unique(myArgs.lookupPath, store); + auto state = std::make_unique(myArgs.lookupPath, store, fetchSettings, evalSettings); Bindings & autoArgs = *myArgs.getAutoArgs(*state); diff --git a/src/nix/profile.cc b/src/nix/profile.cc index a5a40e4f6..324fd6330 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -17,6 +17,8 @@ #include #include +#include "strings.hh" + using namespace nix; struct ProfileElementSource @@ -27,7 +29,9 @@ struct ProfileElementSource std::string attrPath; ExtendedOutputsSpec outputs; - bool operator < (const ProfileElementSource & other) const + // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. + //auto operator <=> (const ProfileElementSource & other) const + auto operator < (const ProfileElementSource & other) const { return std::tuple(originalRef.to_string(), attrPath, outputs) < @@ -56,7 +60,7 @@ struct ProfileElement StringSet names; for (auto & path : storePaths) names.insert(DrvName(path.name()).name); - return concatStringsSep(", ", names); + return dropEmptyInitThenConcatStringsSep(", ", names); } /** @@ -118,12 +122,12 @@ struct ProfileManifest ProfileManifest() { } - ProfileManifest(EvalState & state, const Path & profile) + ProfileManifest(EvalState & state, const std::filesystem::path & profile) { - auto manifestPath = profile + "/manifest.json"; + auto manifestPath = profile / "manifest.json"; - if (pathExists(manifestPath)) { - auto json = nlohmann::json::parse(readFile(manifestPath)); + if (std::filesystem::exists(manifestPath)) { + auto json = nlohmann::json::parse(readFile(manifestPath.string())); auto version = json.value("version", 0); std::string sUrl; @@ -154,8 +158,8 @@ struct ProfileManifest } if (e.value(sUrl, "") != "") { element.source = ProfileElementSource { - parseFlakeRef(e[sOriginalUrl]), - parseFlakeRef(e[sUrl]), + parseFlakeRef(fetchSettings, e[sOriginalUrl]), + parseFlakeRef(fetchSettings, e[sUrl]), e["attrPath"], e["outputs"].get() }; @@ -172,12 +176,12 @@ struct ProfileManifest } } - else if (pathExists(profile + "/manifest.nix")) { + else if (std::filesystem::exists(profile / "manifest.nix")) { // FIXME: needed because of pure mode; ugly. - state.allowPath(state.store->followLinksToStore(profile)); - state.allowPath(state.store->followLinksToStore(profile + "/manifest.nix")); + state.allowPath(state.store->followLinksToStore(profile.string())); + state.allowPath(state.store->followLinksToStore((profile / "manifest.nix").string())); - auto packageInfos = queryInstalled(state, state.store->followLinksToStore(profile)); + auto packageInfos = queryInstalled(state, state.store->followLinksToStore(profile.string())); for (auto & packageInfo : packageInfos) { ProfileElement element; @@ -258,7 +262,7 @@ struct ProfileManifest *store, "profile", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = narHash, .references = { .others = std::move(references), diff --git a/src/nix/profile.md b/src/nix/profile.md index 9b2f86f4a..83a0b5f29 100644 --- a/src/nix/profile.md +++ b/src/nix/profile.md @@ -11,7 +11,7 @@ them to be rolled back easily. )"" -#include "generated-doc/files/profiles.md.gen.hh" +#include "profiles.md.gen.hh" R""( diff --git a/src/nix/profiles.md b/src/nix/profiles.md new file mode 120000 index 000000000..c67a86194 --- /dev/null +++ b/src/nix/profiles.md @@ -0,0 +1 @@ +../../doc/manual/src/command-ref/files/profiles.md \ No newline at end of file diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 812429240..ee4516230 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -33,9 +33,9 @@ public: { if (registry) return registry; if (registry_path.empty()) { - registry = fetchers::getUserRegistry(); + registry = fetchers::getUserRegistry(fetchSettings); } else { - registry = fetchers::getCustomRegistry(registry_path); + registry = fetchers::getCustomRegistry(fetchSettings, registry_path); } return registry; } @@ -68,7 +68,7 @@ struct CmdRegistryList : StoreCommand { using namespace fetchers; - auto registries = getRegistries(store); + auto registries = getRegistries(fetchSettings, store); for (auto & registry : registries) { for (auto & entry : registry->entries) { @@ -109,8 +109,8 @@ struct CmdRegistryAdd : MixEvalArgs, Command, RegistryCommand void run() override { - auto fromRef = parseFlakeRef(fromUrl); - auto toRef = parseFlakeRef(toUrl); + auto fromRef = parseFlakeRef(fetchSettings, fromUrl); + auto toRef = parseFlakeRef(fetchSettings, toUrl); auto registry = getRegistry(); fetchers::Attrs extraAttrs; if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir; @@ -144,7 +144,7 @@ struct CmdRegistryRemove : RegistryCommand, Command void run() override { auto registry = getRegistry(); - registry->remove(parseFlakeRef(url).input); + registry->remove(parseFlakeRef(fetchSettings, url).input); registry->write(getRegistryPath()); } }; @@ -185,8 +185,8 @@ struct CmdRegistryPin : RegistryCommand, EvalCommand { if (locked.empty()) locked = url; auto registry = getRegistry(); - auto ref = parseFlakeRef(url); - auto lockedRef = parseFlakeRef(locked); + auto ref = parseFlakeRef(fetchSettings, url); + auto lockedRef = parseFlakeRef(fetchSettings, locked); registry->remove(ref.input); auto resolved = lockedRef.resolve(store).input.getAccessor(store).second; if (!resolved.isLocked()) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index a2f3e033e..5a570749f 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -1,12 +1,32 @@ #include "eval.hh" #include "eval-settings.hh" +#include "config-global.hh" #include "globals.hh" #include "command.hh" #include "installable-value.hh" #include "repl.hh" +#include "processes.hh" +#include "self-exe.hh" namespace nix { +void runNix(Path program, const Strings & args, + const std::optional & input = {}) +{ + auto subprocessEnv = getEnv(); + subprocessEnv["NIX_CONFIG"] = globalConfig.toKeyValue(); + //isInteractive avoid grabling interactive commands + runProgram2(RunOptions { + .program = getNixBin(program).string(), + .args = args, + .environment = subprocessEnv, + .input = input, + .isInteractive = true, + }); + + return; +} + struct CmdRepl : RawInstallablesCommand { CmdRepl() { @@ -81,7 +101,8 @@ struct CmdRepl : RawInstallablesCommand lookupPath, openStore(), state, - getValues + getValues, + runNix ); repl->autoArgs = getAutoArgs(*repl->state); repl->initEnv(); diff --git a/src/nix/run.cc b/src/nix/run.cc index cc999ddf4..956563591 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -3,6 +3,7 @@ #include "command-installable-value.hh" #include "common-args.hh" #include "shared.hh" +#include "signals.hh" #include "store-api.hh" #include "derivations.hh" #include "local-fs-store.hh" @@ -10,6 +11,7 @@ #include "source-accessor.hh" #include "progress-bar.hh" #include "eval.hh" +#include #if __linux__ # include @@ -18,13 +20,15 @@ #include +namespace nix::fs { using namespace std::filesystem; } + using namespace nix; std::string chrootHelperName = "__run_in_chroot"; namespace nix { -void runProgramInStore(ref store, +void execProgramInStore(ref store, UseLookupPath useLookupPath, const std::string & program, const Strings & args, @@ -71,84 +75,7 @@ void runProgramInStore(ref store, } -struct CmdShell : InstallablesCommand, MixEnvironment -{ - - using InstallablesCommand::run; - - std::vector command = { getEnv("SHELL").value_or("bash") }; - - CmdShell() - { - addFlag({ - .longName = "command", - .shortName = 'c', - .description = "Command and arguments to be executed, defaulting to `$SHELL`", - .labels = {"command", "args"}, - .handler = {[&](std::vector ss) { - if (ss.empty()) throw UsageError("--command requires at least one argument"); - command = ss; - }} - }); - } - - std::string description() override - { - return "run a shell in which the specified packages are available"; - } - - std::string doc() override - { - return - #include "shell.md" - ; - } - - void run(ref store, Installables && installables) override - { - auto outPaths = Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, installables); - - auto accessor = store->getFSAccessor(); - - std::unordered_set done; - std::queue todo; - for (auto & path : outPaths) todo.push(path); - - setEnviron(); - - std::vector pathAdditions; - - while (!todo.empty()) { - auto path = todo.front(); - todo.pop(); - if (!done.insert(path).second) continue; - - if (true) - pathAdditions.push_back(store->printStorePath(path) + "/bin"); - - auto propPath = accessor->resolveSymlinks( - CanonPath(store->printStorePath(path)) / "nix-support" / "propagated-user-env-packages"); - if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) { - for (auto & p : tokenizeString(accessor->readFile(propPath))) - todo.push(store->parseStorePath(p)); - } - } - - auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); - unixPath.insert(unixPath.begin(), pathAdditions.begin(), pathAdditions.end()); - auto unixPathString = concatStringsSep(":", unixPath); - setEnv("PATH", unixPathString.c_str()); - - Strings args; - for (auto & arg : command) args.push_back(arg); - - runProgramInStore(store, UseLookupPath::Use, *command.begin(), args); - } -}; - -static auto rCmdShell = registerCommand("shell"); - -struct CmdRun : InstallableValueCommand +struct CmdRun : InstallableValueCommand, MixEnvironment { using InstallableCommand::run; @@ -204,7 +131,13 @@ struct CmdRun : InstallableValueCommand Strings allArgs{app.program}; for (auto & i : args) allArgs.push_back(i); - runProgramInStore(store, UseLookupPath::DontUse, app.program, allArgs); + // Release our references to eval caches to ensure they are persisted to disk, because + // we are about to exec out of this process without running C++ destructors. + state->evalCaches.clear(); + + setEnviron(); + + execProgramInStore(store, UseLookupPath::DontUse, app.program, allArgs); } }; @@ -241,24 +174,25 @@ void chrootHelper(int argc, char * * argv) if (!pathExists(storeDir)) { // FIXME: Use overlayfs? - Path tmpDir = createTempDir(); + fs::path tmpDir = createTempDir(); createDirs(tmpDir + storeDir); if (mount(realStoreDir.c_str(), (tmpDir + storeDir).c_str(), "", MS_BIND, 0) == -1) throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); - for (auto entry : std::filesystem::directory_iterator{"/"}) { - auto src = entry.path().string(); - Path dst = tmpDir + "/" + entry.path().filename().string(); + for (auto entry : fs::directory_iterator{"/"}) { + checkInterrupt(); + auto src = entry.path(); + fs::path dst = tmpDir / entry.path().filename(); if (pathExists(dst)) continue; - auto st = lstat(src); - if (S_ISDIR(st.st_mode)) { + auto st = entry.symlink_status(); + if (fs::is_directory(st)) { if (mkdir(dst.c_str(), 0700) == -1) throw SysError("creating directory '%s'", dst); if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) throw SysError("mounting '%s' on '%s'", src, dst); - } else if (S_ISLNK(st.st_mode)) + } else if (fs::is_symlink(st)) createSymlink(readLink(src), dst); } @@ -275,9 +209,9 @@ void chrootHelper(int argc, char * * argv) if (mount(realStoreDir.c_str(), storeDir.c_str(), "", MS_BIND, 0) == -1) throw SysError("mounting '%s' on '%s'", realStoreDir, storeDir); - writeFile("/proc/self/setgroups", "deny"); - writeFile("/proc/self/uid_map", fmt("%d %d %d", uid, uid, 1)); - writeFile("/proc/self/gid_map", fmt("%d %d %d", gid, gid, 1)); + writeFile(fs::path{"/proc/self/setgroups"}, "deny"); + writeFile(fs::path{"/proc/self/uid_map"}, fmt("%d %d %d", uid, uid, 1)); + writeFile(fs::path{"/proc/self/gid_map"}, fmt("%d %d %d", gid, gid, 1)); #if __linux__ if (system != "") diff --git a/src/nix/run.hh b/src/nix/run.hh index 2fe6ed86a..51517fdc9 100644 --- a/src/nix/run.hh +++ b/src/nix/run.hh @@ -10,7 +10,7 @@ enum struct UseLookupPath { DontUse }; -void runProgramInStore(ref store, +void execProgramInStore(ref store, UseLookupPath useLookupPath, const std::string & program, const Strings & args, diff --git a/src/nix/run.md b/src/nix/run.md index 250ea65aa..eb96e6b31 100644 --- a/src/nix/run.md +++ b/src/nix/run.md @@ -80,6 +80,7 @@ An app is specified by a flake output attribute named apps.x86_64-linux.blender_2_79 = { type = "app"; program = "${self.packages.x86_64-linux.blender_2_79}/bin/blender"; + meta.description = "Run Blender, a free and open-source 3D creation suite."; }; ``` @@ -90,4 +91,6 @@ The only supported attributes are: * `program` (required): The full path of the executable to run. It must reside in the Nix store. +* `meta.description` (optional): A description of the app. + )"" diff --git a/src/nix/scripts b/src/nix/scripts new file mode 120000 index 000000000..c5efc95eb --- /dev/null +++ b/src/nix/scripts @@ -0,0 +1 @@ +../../scripts \ No newline at end of file diff --git a/src/nix/search.cc b/src/nix/search.cc index 97ef1375e..c8d0b9e96 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -10,11 +10,14 @@ #include "eval-cache.hh" #include "attr-path.hh" #include "hilite.hh" +#include "strings-inline.hh" #include #include #include +#include "strings.hh" + using namespace nix; using json = nlohmann::json; diff --git a/src/nix/self-exe.cc b/src/nix/self-exe.cc new file mode 100644 index 000000000..77d20a835 --- /dev/null +++ b/src/nix/self-exe.cc @@ -0,0 +1,41 @@ +#include "current-process.hh" +#include "file-system.hh" +#include "globals.hh" +#include "self-exe.hh" + +namespace nix { + +namespace fs { +using namespace std::filesystem; +} + +fs::path getNixBin(std::optional binaryNameOpt) +{ + auto getBinaryName = [&] { return binaryNameOpt ? *binaryNameOpt : "nix"; }; + + // If the environment variable is set, use it unconditionally. + if (auto envOpt = getEnvNonEmpty("NIX_BIN_DIR")) + return fs::path{*envOpt} / std::string{getBinaryName()}; + + // Try OS tricks, if available, to get to the path of this Nix, and + // see if we can find the right executable next to that. + if (auto selfOpt = getSelfExe()) { + fs::path path{*selfOpt}; + if (binaryNameOpt) + path = path.parent_path() / std::string{*binaryNameOpt}; + if (fs::exists(path)) + return path; + } + + // If `nix` exists at the hardcoded fallback path, use it. + { + auto path = fs::path{NIX_BIN_DIR} / std::string{getBinaryName()}; + if (fs::exists(path)) + return path; + } + + // return just the name, hoping the exe is on the `PATH` + return getBinaryName(); +} + +} diff --git a/src/nix/self-exe.hh b/src/nix/self-exe.hh new file mode 100644 index 000000000..3161553ec --- /dev/null +++ b/src/nix/self-exe.hh @@ -0,0 +1,31 @@ +#pragma once +///@file + +#include + +namespace nix { + +/** + * Get a path to the given Nix binary. + * + * Normally, nix is installed according to `NIX_BIN_DIR`, which is set + * at compile time, but can be overridden. + * + * However, it may not have been installed at all. For example, if it's + * a static build, there's a good chance that it has been moved out of + * its installation directory. That makes `NIX_BIN_DIR` useless. + * Instead, we'll query the OS for the path to the current executable, + * using `getSelfExe()`. + * + * As a last resort, we rely on `PATH`. Hopefully we find a `nix` there + * that's compatible. If you're porting Nix to a new platform, that + * might be good enough for a while, but you'll want to improve + * `getSelfExe()` to work on your platform. + * + * @param binary_name the exact binary name we're looking up. Might be + * `nix-*` instead of `nix` for the legacy CLI commands. Optional to use + * current binary name. + */ +std::filesystem::path getNixBin(std::optional binary_name = {}); + +} diff --git a/src/nix/shell.md b/src/nix/shell.md index 7c315fb3f..677151a85 100644 --- a/src/nix/shell.md +++ b/src/nix/shell.md @@ -48,7 +48,7 @@ R""( # Description `nix shell` runs a command in an environment in which the `$PATH` variable -provides the specified [*installables*](./nix.md#installable). If no command is specified, it starts the +provides the specified [*installables*](./nix.md#installables). If no command is specified, it starts the default shell of your user account specified by `$SHELL`. # Use as a `#!`-interpreter diff --git a/src/nix/store-dump-path.md b/src/nix/store-dump-path.md index 56e2174b6..21467ff32 100644 --- a/src/nix/store-dump-path.md +++ b/src/nix/store-dump-path.md @@ -17,7 +17,9 @@ R""( # Description -This command generates a NAR file containing the serialisation of the +This command generates a [Nix Archive (NAR)][Nix Archive] file containing the serialisation of the store path [*installable*](./nix.md#installables). The NAR is written to standard output. +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + )"" diff --git a/src/nix/unix/daemon.cc b/src/nix/unix/daemon.cc index de77a7b6b..746963a01 100644 --- a/src/nix/unix/daemon.cc +++ b/src/nix/unix/daemon.cc @@ -10,6 +10,7 @@ #include "serialise.hh" #include "archive.hh" #include "globals.hh" +#include "config-global.hh" #include "derivations.hh" #include "finally.hh" #include "legacy.hh" @@ -32,6 +33,10 @@ #include #include +#if __linux__ +#include "cgroup.hh" +#endif + #if __APPLE__ || __FreeBSD__ #include #endif @@ -202,7 +207,11 @@ static PeerInfo getPeerInfo(int remote) #if defined(SO_PEERCRED) - ucred cred; +# if defined(__OpenBSD__) + struct sockpeercred cred; +# else + ucred cred; +# endif socklen_t credLen = sizeof(cred); if (getsockopt(remote, SOL_SOCKET, SO_PEERCRED, &cred, &credLen) == -1) throw SysError("getting peer credentials"); @@ -210,9 +219,9 @@ static PeerInfo getPeerInfo(int remote) #elif defined(LOCAL_PEERCRED) -#if !defined(SOL_LOCAL) -#define SOL_LOCAL 0 -#endif +# if !defined(SOL_LOCAL) +# define SOL_LOCAL 0 +# endif xucred cred; socklen_t credLen = sizeof(cred); @@ -295,7 +304,7 @@ static void daemonLoop(std::optional forceTrustClientOpt) if (getEnv("LISTEN_PID") != std::to_string(getpid()) || listenFds != "1") throw Error("unexpected systemd environment variables"); fdSocket = SD_LISTEN_FDS_START; - closeOnExec(fdSocket.get()); + unix::closeOnExec(fdSocket.get()); } // Otherwise, create and bind to a Unix domain socket. @@ -307,6 +316,27 @@ static void daemonLoop(std::optional forceTrustClientOpt) // Get rid of children automatically; don't let them become zombies. setSigChldAction(true); + #if __linux__ + if (settings.useCgroups) { + experimentalFeatureSettings.require(Xp::Cgroups); + + // This also sets the root cgroup to the current one. + auto rootCgroup = getRootCgroup(); + auto cgroupFS = getCgroupFS(); + if (!cgroupFS) + throw Error("cannot determine the cgroups file system"); + auto rootCgroupPath = canonPath(*cgroupFS + "/" + rootCgroup); + if (!pathExists(rootCgroupPath)) + throw Error("expected cgroup directory '%s'", rootCgroupPath); + auto daemonCgroupPath = rootCgroupPath + "/nix-daemon"; + // Create new sub-cgroup for the daemon. + if (mkdir(daemonCgroupPath.c_str(), 0755) != 0 && errno != EEXIST) + throw SysError("creating cgroup '%s'", daemonCgroupPath); + // Move daemon into the new cgroup. + writeFile(daemonCgroupPath + "/cgroup.procs", fmt("%d", getpid())); + } + #endif + // Loop accepting connections. while (1) { @@ -323,7 +353,7 @@ static void daemonLoop(std::optional forceTrustClientOpt) throw SysError("accepting connection"); } - closeOnExec(remote.get()); + unix::closeOnExec(remote.get()); PeerInfo peer { .pidKnown = false }; TrustedFlag trusted; @@ -365,9 +395,12 @@ static void daemonLoop(std::optional forceTrustClientOpt) } // Handle the connection. - FdSource from(remote.get()); - FdSink to(remote.get()); - processConnection(openUncachedStore(), from, to, trusted, NotRecursive); + processConnection( + openUncachedStore(), + FdSource(remote.get()), + FdSink(remote.get()), + trusted, + NotRecursive); exit(0); }, options); @@ -432,9 +465,11 @@ static void forwardStdioConnection(RemoteStore & store) { */ static void processStdioConnection(ref store, TrustedFlag trustClient) { - FdSource from(STDIN_FILENO); - FdSink to(STDOUT_FILENO); - processConnection(store, from, to, trustClient, NotRecursive); + processConnection( + store, + FdSource(STDIN_FILENO), + FdSink(STDOUT_FILENO), + trustClient, NotRecursive); } /** diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 17d1edb97..f54cc59d0 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -8,6 +8,8 @@ #include "attr-path.hh" #include "names.hh" #include "progress-bar.hh" +#include "executable-path.hh" +#include "self-exe.hh" using namespace nix; @@ -92,7 +94,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand { Activity act(*logger, lvlInfo, actUnknown, fmt("installing '%s' into profile '%s'...", store->printStorePath(storePath), profileDir)); - runProgram(settings.nixBinDir + "/nix-env", false, + runProgram(getNixBin("nix-env").string(), false, {"--profile", profileDir, "-i", store->printStorePath(storePath), "--no-sandbox"}); } @@ -102,23 +104,17 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand /* Return the profile in which Nix is installed. */ Path getProfileDir(ref store) { - Path where; - - for (auto & dir : tokenizeString(getEnv("PATH").value_or(""), ":")) - if (pathExists(dir + "/nix-env")) { - where = dir; - break; - } - - if (where == "") + auto whereOpt = ExecutablePath::load().findName(OS_STR("nix-env")); + if (!whereOpt) throw Error("couldn't figure out how Nix is installed, so I can't upgrade it"); + auto & where = *whereOpt; printInfo("found Nix in '%s'", where); - if (hasPrefix(where, "/run/current-system")) + if (hasPrefix(where.string(), "/run/current-system")) throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'"); - Path profileDir = dirOf(where); + Path profileDir = where.parent_path().string(); // Resolve profile to /nix/var/nix/profiles/ link. while (canonPath(profileDir).find("/profiles/") == std::string::npos && std::filesystem::is_symlink(profileDir)) @@ -128,7 +124,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand Path userEnv = canonPath(profileDir, true); - if (baseNameOf(where) != "bin" || + if (where.filename() != "bin" || !hasSuffix(userEnv, "user-environment")) throw Error("directory '%s' does not appear to be part of a Nix profile", where); @@ -147,7 +143,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand auto req = FileTransferRequest((std::string&) settings.upgradeNixStorePathUrl); auto res = getFileTransfer()->download(req); - auto state = std::make_unique(LookupPath{}, store); + auto state = std::make_unique(LookupPath{}, store, fetchSettings, evalSettings); auto v = state->allocValue(); state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); Bindings & bindings(*state->allocBindings(0)); diff --git a/src/nix/verify.cc b/src/nix/verify.cc index 2a0cbd19f..124a05bed 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -1,14 +1,14 @@ #include "command.hh" #include "shared.hh" #include "store-api.hh" -#include "sync.hh" #include "thread-pool.hh" -#include "references.hh" #include "signals.hh" #include "keys.hh" #include +#include "exit.hh" + using namespace nix; struct CmdVerify : StorePathsCommand diff --git a/src/nix/verify.md b/src/nix/verify.md index e1d55eab4..ae0b0acd6 100644 --- a/src/nix/verify.md +++ b/src/nix/verify.md @@ -46,4 +46,6 @@ The exit status of this command is the sum of the following values: * **4** if any path couldn't be verified for any other reason (such as an I/O error). +[Nix Archive]: @docroot@/store/file-system-object/content-address.md#serial-nix-archive + )"" diff --git a/src/perl/.version b/src/perl/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/src/perl/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/perl/.yath.rc.in b/src/perl/.yath.rc.in similarity index 100% rename from perl/.yath.rc.in rename to src/perl/.yath.rc.in diff --git a/perl/MANIFEST b/src/perl/MANIFEST similarity index 100% rename from perl/MANIFEST rename to src/perl/MANIFEST diff --git a/perl/lib/Nix/Config.pm.in b/src/perl/lib/Nix/Config.pm.in similarity index 92% rename from perl/lib/Nix/Config.pm.in rename to src/perl/lib/Nix/Config.pm.in index 508a15e15..ad51cff3b 100644 --- a/perl/lib/Nix/Config.pm.in +++ b/src/perl/lib/Nix/Config.pm.in @@ -5,7 +5,6 @@ use Nix::Store; $version = "@PACKAGE_VERSION@"; -$binDir = Nix::Store::getBinDir; $storeDir = Nix::Store::getStoreDir; %config = (); diff --git a/perl/lib/Nix/CopyClosure.pm b/src/perl/lib/Nix/CopyClosure.pm similarity index 100% rename from perl/lib/Nix/CopyClosure.pm rename to src/perl/lib/Nix/CopyClosure.pm diff --git a/perl/lib/Nix/Manifest.pm b/src/perl/lib/Nix/Manifest.pm similarity index 100% rename from perl/lib/Nix/Manifest.pm rename to src/perl/lib/Nix/Manifest.pm diff --git a/perl/lib/Nix/SSH.pm b/src/perl/lib/Nix/SSH.pm similarity index 100% rename from perl/lib/Nix/SSH.pm rename to src/perl/lib/Nix/SSH.pm diff --git a/perl/lib/Nix/Store.pm b/src/perl/lib/Nix/Store.pm similarity index 97% rename from perl/lib/Nix/Store.pm rename to src/perl/lib/Nix/Store.pm index 16f2e17c8..f2ae7e88f 100644 --- a/perl/lib/Nix/Store.pm +++ b/src/perl/lib/Nix/Store.pm @@ -24,7 +24,7 @@ our @EXPORT = qw( hashPath hashFile hashString convertHash signString checkSignature - getBinDir getStoreDir + getStoreDir setVerbosity ); diff --git a/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs similarity index 97% rename from perl/lib/Nix/Store.xs rename to src/perl/lib/Nix/Store.xs index ee211ef64..172c3500d 100644 --- a/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -1,4 +1,5 @@ -#include "nix/config.h" +#include "config-util.hh" +#include "config-store.hh" #include "EXTERN.h" #include "perl.h" @@ -258,7 +259,7 @@ hashPath(char * algo, int base32, char * path) try { Hash h = hashPath( PosixSourceAccessor::createAtRoot(path), - FileIngestionMethod::Recursive, parseHashAlgo(algo)); + FileIngestionMethod::NixArchive, parseHashAlgo(algo)).first; auto s = h.to_string(base32 ? HashFormat::Nix32 : HashFormat::Base16, false); XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0))); } catch (Error & e) { @@ -334,7 +335,7 @@ SV * StoreWrapper::addToStore(char * srcPath, int recursive, char * algo) PPCODE: try { - auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto method = recursive ? ContentAddressMethod::Raw::NixArchive : ContentAddressMethod::Raw::Flat; auto path = THIS->store->addToStore( std::string(baseNameOf(srcPath)), PosixSourceAccessor::createAtRoot(srcPath), @@ -350,7 +351,7 @@ StoreWrapper::makeFixedOutputPath(int recursive, char * algo, char * hash, char PPCODE: try { auto h = Hash::parseAny(hash, parseHashAlgo(algo)); - auto method = recursive ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + auto method = recursive ? FileIngestionMethod::NixArchive : FileIngestionMethod::Flat; auto path = THIS->store->makeFixedOutputPath(name, FixedOutputInfo { .method = method, .hash = h, @@ -423,11 +424,6 @@ StoreWrapper::addTempRoot(char * storePath) } -SV * getBinDir() - PPCODE: - XPUSHs(sv_2mortal(newSVpv(settings.nixBinDir.c_str(), 0))); - - SV * getStoreDir() PPCODE: XPUSHs(sv_2mortal(newSVpv(settings.nixStore.c_str(), 0))); diff --git a/perl/lib/Nix/Utils.pm b/src/perl/lib/Nix/Utils.pm similarity index 100% rename from perl/lib/Nix/Utils.pm rename to src/perl/lib/Nix/Utils.pm diff --git a/perl/lib/Nix/meson.build b/src/perl/lib/Nix/meson.build similarity index 96% rename from perl/lib/Nix/meson.build rename to src/perl/lib/Nix/meson.build index 9a79245cd..256e66096 100644 --- a/perl/lib/Nix/meson.build +++ b/src/perl/lib/Nix/meson.build @@ -43,6 +43,7 @@ nix_perl_store_lib = library( 'Store', sources : nix_perl_store_cc, name_prefix : '', + prelink : true, # For C++ static initializers install : true, install_mode : 'rwxr-xr-x', install_dir : join_paths(nix_perl_install_dir, 'auto', 'Nix', 'Store'), diff --git a/perl/meson.build b/src/perl/meson.build similarity index 95% rename from perl/meson.build rename to src/perl/meson.build index 350e5bd67..dcb6a68a4 100644 --- a/perl/meson.build +++ b/src/perl/meson.build @@ -7,34 +7,38 @@ project ( 'nix-perl', 'cpp', - meson_version : '>= 0.64.0', + version : files('.version'), + meson_version : '>= 1.1', license : 'LGPL-2.1-or-later', ) # setup env #------------------------------------------------- fs = import('fs') -nix_version = get_option('version') cpp = meson.get_compiler('cpp') nix_perl_conf = configuration_data() -nix_perl_conf.set('PACKAGE_VERSION', nix_version) +nix_perl_conf.set('PACKAGE_VERSION', meson.project_version()) # set error arguments #------------------------------------------------- error_args = [ - '-Wno-pedantic', - '-Wno-non-virtual-dtor', - '-Wno-unused-parameter', - '-Wno-variadic-macros', + '-Wdeprecated-copy', '-Wdeprecated-declarations', - '-Wno-missing-field-initializers', - '-Wno-unknown-warning-option', - '-Wno-unused-variable', - '-Wno-literal-suffix', - '-Wno-reserved-user-defined-literal', + '-Werror=suggest-override', + '-Werror=unused-result', + '-Wignored-qualifiers', '-Wno-duplicate-decl-specifier', + '-Wno-literal-suffix', + '-Wno-missing-field-initializers', + '-Wno-non-virtual-dtor', + '-Wno-pedantic', '-Wno-pointer-bool-conversion', + '-Wno-reserved-user-defined-literal', + '-Wno-unknown-warning-option', + '-Wno-unused-parameter', + '-Wno-unused-variable', + '-Wno-variadic-macros', ] add_project_arguments( @@ -64,7 +68,7 @@ yath = find_program('yath', required : false) bzip2_dep = dependency('bzip2') curl_dep = dependency('libcurl') libsodium_dep = dependency('libsodium') -# nix_util_dep = dependency('nix-util') + nix_store_dep = dependency('nix-store') diff --git a/perl/meson_options.txt b/src/perl/meson.options similarity index 88% rename from perl/meson_options.txt rename to src/perl/meson.options index 82ca52f37..9b5b6b1d9 100644 --- a/perl/meson_options.txt +++ b/src/perl/meson.options @@ -5,11 +5,6 @@ # compiler args #============================================================================ -option( - 'version', - type : 'string', - description : 'nix-perl version') - option( 'tests', type : 'feature', diff --git a/perl/default.nix b/src/perl/package.nix similarity index 51% rename from perl/default.nix rename to src/perl/package.nix index 45682381e..0b9343fba 100644 --- a/perl/default.nix +++ b/src/perl/package.nix @@ -1,52 +1,53 @@ { lib -, fileset , stdenv +, mkMesonDerivation , perl , perlPackages , meson , ninja , pkg-config -, nix +, nix-store +, darwin +, version , curl , bzip2 -, xz -, boost , libsodium -, darwin }: -perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: { - name = "nix-perl-${nix.version}"; +let + inherit (lib) fileset; +in - src = fileset.toSource { - root = ./.; - fileset = fileset.unions ([ - ./MANIFEST - ./lib - ./meson.build - ./meson_options.txt - ] ++ lib.optionals finalAttrs.doCheck [ - ./.yath.rc.in - ./t - ]); - }; +perl.pkgs.toPerlModule (mkMesonDerivation (finalAttrs: { + pname = "nix-perl"; + inherit version; + + workDir = ./.; + fileset = fileset.unions ([ + ./.version + ../../.version + ./MANIFEST + ./lib + ./meson.build + ./meson.options + ] ++ lib.optionals finalAttrs.doCheck [ + ./.yath.rc.in + ./t + ]); nativeBuildInputs = [ meson ninja pkg-config + perl + curl ]; buildInputs = [ - nix - curl + nix-store bzip2 - xz - perl - boost - ] - ++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium - ++ lib.optional stdenv.isDarwin darwin.apple_sdk.frameworks.Security; + libsodium + ]; # `perlPackages.Test2Harness` is marked broken for Darwin doCheck = !stdenv.isDarwin; @@ -55,8 +56,14 @@ perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: { perlPackages.Test2Harness ]; + preConfigure = + # "Inline" .version so its not a symlink, and includes the suffix + '' + chmod u+w .version + echo ${finalAttrs.version} > .version + ''; + mesonFlags = [ - (lib.mesonOption "version" (builtins.readFile ../.version)) (lib.mesonOption "dbi_path" "${perlPackages.DBI}/${perl.libPrefix}") (lib.mesonOption "dbd_sqlite_path" "${perlPackages.DBDSQLite}/${perl.libPrefix}") (lib.mesonEnable "tests" finalAttrs.doCheck) @@ -66,5 +73,5 @@ perl.pkgs.toPerlModule (stdenv.mkDerivation (finalAttrs: { "--print-errorlogs" ]; - enableParallelBuilding = true; + strictDeps = false; })) diff --git a/perl/t/init.t b/src/perl/t/init.t similarity index 100% rename from perl/t/init.t rename to src/perl/t/init.t diff --git a/perl/t/meson.build b/src/perl/t/meson.build similarity index 100% rename from perl/t/meson.build rename to src/perl/t/meson.build diff --git a/src/toml11/LICENSE b/src/toml11/LICENSE deleted file mode 100644 index f55c511d6..000000000 --- a/src/toml11/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2017 Toru Niina - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/src/toml11/README.md b/src/toml11/README.md deleted file mode 100644 index 62b586305..000000000 --- a/src/toml11/README.md +++ /dev/null @@ -1,1966 +0,0 @@ -toml11 -====== - -[![Build Status on GitHub Actions](https://github.com/ToruNiina/toml11/workflows/build/badge.svg)](https://github.com/ToruNiina/toml11/actions) -[![Build Status on TravisCI](https://travis-ci.org/ToruNiina/toml11.svg?branch=master)](https://travis-ci.org/ToruNiina/toml11) -[![Build status on Appveyor](https://ci.appveyor.com/api/projects/status/m2n08a926asvg5mg/branch/master?svg=true)](https://ci.appveyor.com/project/ToruNiina/toml11/branch/master) -[![Build status on CircleCI](https://circleci.com/gh/ToruNiina/toml11/tree/master.svg?style=svg)](https://circleci.com/gh/ToruNiina/toml11/tree/master) -[![Version](https://img.shields.io/github/release/ToruNiina/toml11.svg?style=flat)](https://github.com/ToruNiina/toml11/releases) -[![License](https://img.shields.io/github/license/ToruNiina/toml11.svg?style=flat)](LICENSE) -[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.1209136.svg)](https://doi.org/10.5281/zenodo.1209136) - -toml11 is a C++11 (or later) header-only toml parser/encoder depending only on C++ standard library. - -- It is compatible to the latest version of [TOML v1.0.0](https://toml.io/en/v1.0.0). -- It is one of the most TOML standard compliant libraries, tested with [the language agnostic test suite for TOML parsers by BurntSushi](https://github.com/BurntSushi/toml-test). -- It shows highly informative error messages. You can see the error messages about invalid files at [CircleCI](https://circleci.com/gh/ToruNiina/toml11). -- It has configurable container. You can use any random-access containers and key-value maps as backend containers. -- It optionally preserves comments without any overhead. -- It has configurable serializer that supports comments, inline tables, literal strings and multiline strings. -- It supports user-defined type conversion from/into toml values. -- It correctly handles UTF-8 sequences, with or without BOM, both on posix and Windows. - -## Example - -```cpp -#include -#include - -int main() -{ - // ```toml - // title = "an example toml file" - // nums = [3, 1, 4, 1, 5] - // ``` - auto data = toml::parse("example.toml"); - - // find a value with the specified type from a table - std::string title = toml::find(data, "title"); - - // convert the whole array into any container automatically - std::vector nums = toml::find>(data, "nums"); - - // access with STL-like manner - if(!data.contains("foo")) - { - data["foo"] = "bar"; - } - - // pass a fallback - std::string name = toml::find_or(data, "name", "not found"); - - // width-dependent formatting - std::cout << std::setw(80) << data << std::endl; - - return 0; -} -``` - -## Table of Contents - -- [Integration](#integration) -- [Decoding a toml file](#decoding-a-toml-file) - - [In the case of syntax error](#in-the-case-of-syntax-error) - - [Invalid UTF-8 Codepoints](#invalid-utf-8-codepoints) -- [Finding a toml value](#finding-a-toml-value) - - [Finding a value in a table](#finding-a-value-in-a-table) - - [In case of error](#in-case-of-error) - - [Dotted keys](#dotted-keys) -- [Casting a toml value](#casting-a-toml-value) -- [Checking value type](#checking-value-type) -- [More about conversion](#more-about-conversion) - - [Converting an array](#converting-an-array) - - [Converting a table](#converting-a-table) - - [Getting an array of tables](#getting-an-array-of-tables) - - [Cost of conversion](#cost-of-conversion) - - [Converting datetime and its variants](#converting-datetime-and-its-variants) -- [Getting with a fallback](#getting-with-a-fallback) -- [Expecting conversion](#expecting-conversion) -- [Visiting a toml::value](#visiting-a-tomlvalue) -- [Constructing a toml::value](#constructing-a-tomlvalue) -- [Preserving Comments](#preserving-comments) -- [Customizing containers](#customizing-containers) -- [TOML literal](#toml-literal) -- [Conversion between toml value and arbitrary types](#conversion-between-toml-value-and-arbitrary-types) -- [Formatting user-defined error messages](#formatting-user-defined-error-messages) -- [Obtaining location information](#obtaining-location-information) -- [Exceptions](#exceptions) -- [Colorize Error Messages](#colorize-error-messages) -- [Serializing TOML data](#serializing-toml-data) -- [Underlying types](#underlying-types) -- [Unreleased TOML features](#unreleased-toml-features) -- [Breaking Changes from v2](#breaking-changes-from-v2) -- [Running Tests](#running-tests) -- [Contributors](#contributors) -- [Licensing Terms](#licensing-terms) - -## Integration - -Just include the file after adding it to the include path. - -```cpp -#include // that's all! now you can use it. -#include - -int main() -{ - const auto data = toml::parse("example.toml"); - const auto title = toml::find(data, "title"); - std::cout << "the title is " << title << std::endl; - return 0; -} -``` - -The convenient way is to add this repository as a git-submodule or to install -it in your system by CMake. - -Note for MSVC: We recommend to set `/Zc:__cplusplus` to detect C++ version correctly. - -## Decoding a toml file - -To parse a toml file, the only thing you have to do is -to pass a filename to the `toml::parse` function. - -```cpp -const std::string fname("sample.toml"); -const toml::value data = toml::parse(fname); -``` - -As required by the TOML specification, the top-level value is always a table. -You can find a value inside it, cast it into a table explicitly, and insert it as a value into other `toml::value`. - -If it encounters an error while opening a file, it will throw `std::runtime_error`. - -You can also pass a `std::istream` to the `toml::parse` function. -To show a filename in an error message, however, it is recommended to pass the -filename with the stream. - -```cpp -std::ifstream ifs("sample.toml", std::ios_base::binary); -assert(ifs.good()); -const auto data = toml::parse(ifs, /*optional -> */ "sample.toml"); -``` - -**Note**: When you are **on Windows, open a file in binary mode**. -If a file is opened in text-mode, CRLF ("\r\n") will automatically be -converted to LF ("\n") and this causes inconsistency between file size -and the contents that would be read. This causes weird error. - -### In the case of syntax error - -If there is a syntax error in a toml file, `toml::parse` will throw -`toml::syntax_error` that inherits `std::exception`. - -toml11 has clean and informative error messages inspired by Rust and -it looks like the following. - -```console -terminate called after throwing an instance of 'toml::syntax_error' - what(): [error] toml::parse_table: invalid line format # error description - --> example.toml # file name - 3 | a = 42 = true # line num and content - | ^------ expected newline, but got '='. # error reason -``` - -If you (mistakenly) duplicate tables and got an error, it is helpful to see -where they are. toml11 shows both at the same time like the following. - -```console -terminate called after throwing an instance of 'toml::syntax_error' - what(): [error] toml::insert_value: table ("table") already exists. - --> duplicate-table.toml - 1 | [table] - | ~~~~~~~ table already exists here - ... - 3 | [table] - | ~~~~~~~ table defined twice -``` - -When toml11 encounters a malformed value, it tries to detect what type it is. -Then it shows hints to fix the format. An error message while reading one of -the malformed files in [the language agnostic test suite](https://github.com/BurntSushi/toml-test). -is shown below. - -```console -what(): [error] bad time: should be HH:MM:SS.subsec - --> ./datetime-malformed-no-secs.toml - 1 | no-secs = 1987-07-05T17:45Z - | ^------- HH:MM:SS.subsec - | -Hint: pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999 -Hint: fail: 1979-05-27T7:32:00, 1979-05-27 17:32 -``` - -You can find other examples in a job named `output_result` on -[CircleCI](https://circleci.com/gh/ToruNiina/toml11). - -Since the error message generation is generally a difficult task, the current -status is not ideal. If you encounter a weird error message, please let us know -and contribute to improve the quality! - -### Invalid UTF-8 codepoints - -It throws `syntax_error` if a value of an escape sequence -representing unicode character is not a valid UTF-8 codepoint. - -```console - what(): [error] toml::read_utf8_codepoint: input codepoint is too large. - --> utf8.toml - 1 | exceeds_unicode = "\U0011FFFF example" - | ^--------- should be in [0x00..0x10FFFF] -``` - -## Finding a toml value - -After parsing successfully, you can obtain the values from the result of -`toml::parse` using `toml::find` function. - -```toml -# sample.toml -answer = 42 -pi = 3.14 -numbers = [1,2,3] -time = 1979-05-27T07:32:00Z -``` - -``` cpp -const auto data = toml::parse("sample.toml"); -const auto answer = toml::find(data, "answer"); -const auto pi = toml::find(data, "pi"); -const auto numbers = toml::find>(data, "numbers"); -const auto timepoint = toml::find(data, "time"); -``` - -By default, `toml::find` returns a `toml::value`. - -```cpp -const toml::value& answer = toml::find(data, "answer"); -``` - -When you pass an exact TOML type that does not require type conversion, -`toml::find` returns a reference without copying the value. - -```cpp -const auto data = toml::parse("sample.toml"); -const auto& answer = toml::find(data, "answer"); -``` - -If the specified type requires conversion, you can't take a reference to the value. -See also [underlying types](#underlying-types). - -**NOTE**: For some technical reason, automatic conversion between `integer` and -`floating` is not supported. If you want to get a floating value even if a value -has integer value, you need to convert it manually after obtaining a value, -like the following. - -```cpp -const auto vx = toml::find(data, "x"); -double x = vx.is_floating() ? vx.as_floating(std::nothrow) : - static_cast(vx.as_integer()); // it throws if vx is neither - // floating nor integer. -``` - -### Finding a value in a table - -There are several way to get a value defined in a table. -First, you can get a table as a normal value and find a value from the table. - -```toml -[fruit] -name = "apple" -[fruit.physical] -color = "red" -shape = "round" -``` - -``` cpp -const auto data = toml::parse("fruit.toml"); -const auto& fruit = toml::find(data, "fruit"); -const auto name = toml::find(fruit, "name"); - -const auto& physical = toml::find(fruit, "physical"); -const auto color = toml::find(physical, "color"); -const auto shape = toml::find(physical, "shape"); -``` - -Here, variable `fruit` is a `toml::value` and can be used as the first argument -of `toml::find`. - -Second, you can pass as many arguments as the number of subtables to `toml::find`. - -```cpp -const auto data = toml::parse("fruit.toml"); -const auto color = toml::find(data, "fruit", "physical", "color"); -const auto shape = toml::find(data, "fruit", "physical", "shape"); -``` - -### Finding a value in an array - -You can find n-th value in an array by `toml::find`. - -```toml -values = ["foo", "bar", "baz"] -``` - -``` cpp -const auto data = toml::parse("sample.toml"); -const auto values = toml::find(data, "values"); -const auto bar = toml::find(values, 1); -``` - -`toml::find` can also search array recursively. - -```cpp -const auto data = toml::parse("fruit.toml"); -const auto bar = toml::find(data, "values", 1); -``` - -Before calling `toml::find`, you can check if a value corresponding to a key -exists. You can use both `bool toml::value::contains(const key&) const` and -`std::size_t toml::value::count(const key&) const`. Those behaves like the -`std::map::contains` and `std::map::count`. - -```cpp -const auto data = toml::parse("fruit.toml"); -if(data.contains("fruit") && data.at("fruit").count("physical") != 0) -{ - // ... -} -``` - -### In case of error - -If the value does not exist, `toml::find` throws `std::out_of_range` with the -location of the table. - -```console -terminate called after throwing an instance of 'std::out_of_range' - what(): [error] key "answer" not found - --> example.toml - 6 | [tab] - | ~~~~~ in this table -``` - ----- - -If the specified type differs from the actual value contained, it throws -`toml::type_error` that inherits `std::exception`. - -Similar to the case of syntax error, toml11 also displays clean error messages. -The error message when you choose `int` to get `string` value would be like this. - -```console -terminate called after throwing an instance of 'toml::type_error' - what(): [error] toml::value bad_cast to integer - --> example.toml - 3 | title = "TOML Example" - | ~~~~~~~~~~~~~~ the actual type is string -``` - -**NOTE**: In order to show this kind of error message, all the toml values have -a pointer to represent its range in a file. The entire contents of a file is -shared by `toml::value`s and remains on the heap memory. It is recommended to -destruct all the `toml::value` classes after configuring your application -if you have a large TOML file compared to the memory resource. - -### Dotted keys - -TOML v0.5.0 has a new feature named "dotted keys". -You can chain keys to represent the structure of the data. - -```toml -physical.color = "orange" -physical.shape = "round" -``` - -This is equivalent to the following. - -```toml -[physical] -color = "orange" -shape = "round" -``` - -You can get both of the above tables with the same c++ code. - -```cpp -const auto physical = toml::find(data, "physical"); -const auto color = toml::find(physical, "color"); -``` - -The following code does not work for the above toml file. - -```cpp -// XXX this does not work! -const auto color = toml::find(data, "physical.color"); -``` - -The above code works with the following toml file. - -```toml -"physical.color" = "orange" -# equivalent to {"physical.color": "orange"}, -# NOT {"physical": {"color": "orange"}}. -``` - - -## Casting a toml value - -### `toml::get` - -`toml::parse` returns `toml::value`. `toml::value` is a union type that can -contain one of the following types. - -- `toml::boolean` (`bool`) -- `toml::integer` (`std::int64_t`) -- `toml::floating` (`double`) -- `toml::string` (a type convertible to std::string) -- `toml::local_date` -- `toml::local_time` -- `toml::local_datetime` -- `toml::offset_datetime` -- `toml::array` (by default, `std::vector`) - - It depends. See [customizing containers](#customizing-containers) for detail. -- `toml::table` (by default, `std::unordered_map`) - - It depends. See [customizing containers](#customizing-containers) for detail. - -To get a value inside, you can use `toml::get()`. The usage is the same as -`toml::find` (actually, `toml::find` internally uses `toml::get` after casting -a value to `toml::table`). - -``` cpp -const toml::value data = toml::parse("sample.toml"); -const toml::value answer_ = toml::get(data).at("answer"); -const std::int64_t answer = toml::get(answer_); -``` - -When you pass an exact TOML type that does not require type conversion, -`toml::get` returns a reference through which you can modify the content -(if the `toml::value` is `const`, it returns `const` reference). - -```cpp -toml::value data = toml::parse("sample.toml"); -toml::value answer_ = toml::get(data).at("answer"); -toml::integer& answer = toml::get(answer_); -answer = 6 * 9; // write to data.answer. now `answer_` contains 54. -``` - -If the specified type requires conversion, you can't take a reference to the value. -See also [underlying types](#underlying-types). - -It also throws a `toml::type_error` if the type differs. - -### `as_xxx` - -You can also use a member function to cast a value. - -```cpp -const std::int64_t answer = data.as_table().at("answer").as_integer(); -``` - -It also throws a `toml::type_error` if the type differs. If you are sure that -the value `v` contains a value of the specified type, you can suppress checking -by passing `std::nothrow`. - -```cpp -const auto& answer = data.as_table().at("answer"); -if(answer.is_integer() && answer.as_integer(std::nothrow) == 42) -{ - std::cout << "value is 42" << std::endl; -} -``` - -If `std::nothrow` is passed, the functions are marked as noexcept. - -By casting a `toml::value` into an array or a table, you can iterate over the -elements. - -```cpp -const auto data = toml::parse("example.toml"); -std::cout << "keys in the top-level table are the following: \n"; -for(const auto& [k, v] : data.as_table()) -{ - std::cout << k << '\n'; -} - -const auto& fruits = toml::find(data, "fruits"); -for(const auto& v : fruits.as_array()) -{ - std::cout << toml::find(v, "name") << '\n'; -} -``` - -The full list of the functions is below. - -```cpp -namespace toml { -class value { - // ... - const boolean& as_boolean() const&; - const integer& as_integer() const&; - const floating& as_floating() const&; - const string& as_string() const&; - const offset_datetime& as_offset_datetime() const&; - const local_datetime& as_local_datetime() const&; - const local_date& as_local_date() const&; - const local_time& as_local_time() const&; - const array& as_array() const&; - const table& as_table() const&; - // -------------------------------------------------------- - // non-const version - boolean& as_boolean() &; - // ditto... - // -------------------------------------------------------- - // rvalue version - boolean&& as_boolean() &&; - // ditto... - - // -------------------------------------------------------- - // noexcept versions ... - const boolean& as_boolean(const std::nothrow_t&) const& noexcept; - boolean& as_boolean(const std::nothrow_t&) & noexcept; - boolean&& as_boolean(const std::nothrow_t&) && noexcept; - // ditto... -}; -} // toml -``` - -### `at()` - -You can access to the element of a table and an array by `toml::basic_value::at`. - -```cpp -const toml::value v{1,2,3,4,5}; -std::cout << v.at(2).as_integer() << std::endl; // 3 - -const toml::value v{{"foo", 42}, {"bar", 3.14}}; -std::cout << v.at("foo").as_integer() << std::endl; // 42 -``` - -If an invalid key (integer for a table, string for an array), it throws -`toml::type_error` for the conversion. If the provided key is out-of-range, -it throws `std::out_of_range`. - -Note that, although `std::string` has `at()` member function, `toml::value::at` -throws if the contained type is a string. Because `std::string` does not -contain `toml::value`. - -### `operator[]` - -You can also access to the element of a table and an array by -`toml::basic_value::operator[]`. - -```cpp -const toml::value v{1,2,3,4,5}; -std::cout << v[2].as_integer() << std::endl; // 3 - -const toml::value v{{"foo", 42}, {"bar", 3.14}}; -std::cout << v["foo"].as_integer() << std::endl; // 42 -``` - -When you access to a `toml::value` that is not initialized yet via -`operator[](const std::string&)`, the `toml::value` will be a table, -just like the `std::map`. - -```cpp -toml::value v; // not initialized as a table. -v["foo"] = 42; // OK. `v` will be a table. -``` - -Contrary, if you access to a `toml::value` that contains an array via `operator[]`, -it does not check anything. It converts `toml::value` without type check and then -access to the n-th element without boundary check, just like the `std::vector::operator[]`. - -```cpp -toml::value v; // not initialized as an array -v[2] = 42; // error! UB -``` - -Please make sure that the `toml::value` has an array inside when you access to -its element via `operator[]`. - -## Checking value type - -You can check the type of a value by `is_xxx` function. - -```cpp -const toml::value v = /* ... */; -if(v.is_integer()) -{ - std::cout << "value is an integer" << std::endl; -} -``` - -The complete list of the functions is below. - -```cpp -namespace toml { -class value { - // ... - bool is_boolean() const noexcept; - bool is_integer() const noexcept; - bool is_floating() const noexcept; - bool is_string() const noexcept; - bool is_offset_datetime() const noexcept; - bool is_local_datetime() const noexcept; - bool is_local_date() const noexcept; - bool is_local_time() const noexcept; - bool is_array() const noexcept; - bool is_table() const noexcept; - bool is_uninitialized() const noexcept; - // ... -}; -} // toml -``` - -Also, you can get `enum class value_t` from `toml::value::type()`. - -```cpp -switch(data.at("something").type()) -{ - case toml::value_t::integer: /*do some stuff*/ ; break; - case toml::value_t::floating: /*do some stuff*/ ; break; - case toml::value_t::string : /*do some stuff*/ ; break; - default : throw std::runtime_error( - "unexpected type : " + toml::stringize(data.at("something").type())); -} -``` - -The complete list of the `enum`s can be found in the section -[underlying types](#underlying-types). - -The `enum`s can be used as a parameter of `toml::value::is` function like the following. - -```cpp -toml::value v = /* ... */; -if(v.is(toml::value_t::boolean)) // ... -``` - -## More about conversion - -Since `toml::find` internally uses `toml::get`, all the following examples work -with both `toml::get` and `toml::find`. - -### Converting an array - -You can get any kind of `container` class from a `toml::array` -except for `map`-like classes. - -``` cpp -// # sample.toml -// numbers = [1,2,3] - -const auto numbers = toml::find(data, "numbers"); - -const auto vc = toml::get >(numbers); -const auto ls = toml::get >(numbers); -const auto dq = toml::get >(numbers); -const auto ar = toml::get>(numbers); -// if the size of data.at("numbers") is larger than that of std::array, -// it will throw toml::type_error because std::array is not resizable. -``` - -Surprisingly, you can convert `toml::array` into `std::pair` and `std::tuple`. - -```cpp -// numbers = [1,2,3] -const auto tp = toml::get>(numbers); -``` - -This functionality is helpful when you have a toml file like the following. - -```toml -array_of_arrays = [[1, 2, 3], ["foo", "bar", "baz"]] # toml allows this -``` - -What is the corresponding C++ type? -Obviously, it is a `std::pair` of `std::vector`s. - -```cpp -const auto array_of_arrays = toml::find(data, "array_of_arrays"); -const auto aofa = toml::get< - std::pair, std::vector> - >(array_of_arrays); -``` - -If you don't know the type of the elements, you can use `toml::array`, -which is a `std::vector` of `toml::value`, instead. - -```cpp -const auto a_of_a = toml::get(array_of_arrays); -const auto first = toml::get>(a_of_a.at(0)); -``` - -You can change the implementation of `toml::array` with `std::deque` or some -other array-like container. See [Customizing containers](#customizing-containers) -for detail. - -### Converting a table - -When all the values of the table have the same type, toml11 allows you to -convert a `toml::table` to a `map` that contains the convertible type. - -```toml -[tab] -key1 = "foo" # all the values are -key2 = "bar" # toml String -``` - -```cpp -const auto data = toml::parse("sample.toml"); -const auto tab = toml::find>(data, "tab"); -std::cout << tab["key1"] << std::endl; // foo -std::cout << tab["key2"] << std::endl; // bar -``` - -But since `toml::table` is just an alias of `std::unordered_map`, -normally you don't need to convert it because it has all the functionalities that -`std::unordered_map` has (e.g. `operator[]`, `count`, and `find`). In most cases -`toml::table` is sufficient. - -```cpp -toml::table tab = toml::get(data); -if(data.count("title") != 0) -{ - data["title"] = std::string("TOML example"); -} -``` - -You can change the implementation of `toml::table` with `std::map` or some -other map-like container. See [Customizing containers](#customizing-containers) -for detail. - -### Getting an array of tables - -An array of tables is just an array of tables. -You can get it in completely the same way as the other arrays and tables. - -```toml -# sample.toml -array_of_inline_tables = [{key = "value1"}, {key = "value2"}, {key = "value3"}] - -[[array_of_tables]] -key = "value4" -[[array_of_tables]] -key = "value5" -[[array_of_tables]] -key = "value6" -``` - -```cpp -const auto data = toml::parse("sample.toml"); -const auto aot1 = toml::find>(data, "array_of_inline_tables"); -const auto aot2 = toml::find>(data, "array_of_tables"); -``` - -### Cost of conversion - -Although conversion through `toml::(get|find)` is convenient, it has additional -copy-cost because it copies data contained in `toml::value` to the -user-specified type. Of course in some cases this overhead is not ignorable. - -```cpp -// the following code constructs a std::vector. -// it requires heap allocation for vector and element conversion. -const auto array = toml::find>(data, "foo"); -``` - -By passing the exact types, `toml::get` returns reference that has no overhead. - -``` cpp -const auto& tab = toml::find(data, "tab"); -const auto& numbers = toml::find(data, "numbers"); -``` - -Also, `as_xxx` are zero-overhead because they always return a reference. - -``` cpp -const auto& tab = toml::find(data, "tab" ).as_table(); -const auto& numbers = toml::find(data, "numbers").as_array(); -``` - -In this case you need to call `toml::get` each time you access to -the element of `toml::array` because `toml::array` is an array of `toml::value`. - -```cpp -const auto& num0 = toml::get(numbers.at(0)); -const auto& num1 = toml::get(numbers.at(1)); -const auto& num2 = toml::get(numbers.at(2)); -``` - -### Converting datetime and its variants - -TOML v0.5.0 has 4 different datetime objects, `local_date`, `local_time`, -`local_datetime`, and `offset_datetime`. - -Since `local_date`, `local_datetime`, and `offset_datetime` represent a time -point, you can convert them to `std::chrono::system_clock::time_point`. - -Contrary, `local_time` does not represents a time point because they lack a -date information, but it can be converted to `std::chrono::duration` that -represents a duration from the beginning of the day, `00:00:00.000`. - -```toml -# sample.toml -date = 2018-12-23 -time = 12:30:00 -l_dt = 2018-12-23T12:30:00 -o_dt = 2018-12-23T12:30:00+09:30 -``` - -```cpp -const auto data = toml::parse("sample.toml"); - -const auto date = toml::get(data.at("date")); -const auto l_dt = toml::get(data.at("l_dt")); -const auto o_dt = toml::get(data.at("o_dt")); - -const auto time = toml::get(data.at("time")); // 12 * 60 + 30 min -``` - -`local_date` and `local_datetime` are assumed to be in the local timezone when -they are converted into `time_point`. On the other hand, `offset_datetime` only -uses the offset part of the data and it does not take local timezone into account. - -To contain datetime data, toml11 defines its own datetime types. -For more detail, you can see the definitions in [toml/datetime.hpp](toml/datetime.hpp). - -## Getting with a fallback - -`toml::find_or` returns a default value if the value is not found or has a -different type. - -```cpp -const auto data = toml::parse("example.toml"); -const auto num = toml::find_or(data, "num", 42); -``` - -It works recursively if you pass several keys for subtables. -In that case, the last argument is considered to be the optional value. -All other arguments between `toml::value` and the optinoal value are considered as keys. - -```cpp -// [fruit.physical] -// color = "red" -auto data = toml::parse("fruit.toml"); -auto color = toml::find_or(data, "fruit", "physical", "color", "red"); -// ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ^^^^^ -// arguments optional value -``` - -Also, `toml::get_or` returns a default value if `toml::get` failed. - -```cpp -toml::value v("foo"); // v contains String -const int value = toml::get_or(v, 42); // conversion fails. it returns 42. -``` - -These functions automatically deduce what type you want to get -from the default value you passed. - -To get a reference through this function, take care about the default value. - -```cpp -toml::value v("foo"); // v contains String -toml::integer& i = toml::get_or(v, 42); // does not work because binding `42` - // to `integer&` is invalid -toml::integer opt = 42; -toml::integer& i = toml::get_or(v, opt); // this works. -``` - -## Expecting conversion - -By using `toml::expect`, you will get your expected value or an error message -without throwing `toml::type_error`. - -```cpp -const auto value = toml::expect(data.at("title")); -if(value.is_ok()) { - std::cout << value.unwrap() << std::endl; -} else { - std::cout << value.unwrap_err() << std::endl; -} -``` - -Also, you can pass a function object to modify the expected value. - -```cpp -const auto value = toml::expect(data.at("number")) - .map(// function that receives expected type (here, int) - [](const int number) -> double { - return number * 1.5 + 1.0; - }).unwrap_or(/*default value =*/ 3.14); -``` - -## Visiting a toml::value - -toml11 provides `toml::visit` to apply a function to `toml::value` in the -same way as `std::variant`. - -```cpp -const toml::value v(3.14); -toml::visit([](const auto& val) -> void { - std::cout << val << std::endl; - }, v); -``` - -The function object that would be passed to `toml::visit` must be able to -receive all the possible TOML types. Also, the result types should be the same -each other. - -## Constructing a toml::value - -`toml::value` can be constructed in various ways. - -```cpp -toml::value v(true); // boolean -toml::value v(42); // integer -toml::value v(3.14); // floating -toml::value v("foobar"); // string -toml::value v(toml::local_date(2019, toml::month_t::Apr, 1)); // date -toml::value v{1, 2, 3, 4, 5}; // array -toml::value v{{"foo", 42}, {"bar", 3.14}, {"baz", "qux"}}; // table -``` - -When constructing a string, you can choose to use either literal or basic string. -By default, it will be a basic string. - -```cpp -toml::value v("foobar", toml::string_t::basic ); -toml::value v("foobar", toml::string_t::literal); -``` - -Datetime objects can be constructed from `std::tm` and -`std::chrono::system_clock::time_point`. But you need to specify what type -you use to avoid ambiguity. - -```cpp -const auto now = std::chrono::system_clock::now(); -toml::value v(toml::local_date(now)); -toml::value v(toml::local_datetime(now)); -toml::value v(toml::offset_datetime(now)); -``` - -Since local time is not equivalent to a time point, because it lacks date -information, it will be constructed from `std::chrono::duration`. - -```cpp -toml::value v(toml::local_time(std::chrono::hours(10))); -``` - -You can construct an array object not only from `initializer_list`, but also -from STL containers. In that case, the element type must be convertible to -`toml::value`. - -```cpp -std::vector vec{1,2,3,4,5}; -toml::value v(vec); -``` - -When you construct an array value, all the elements of `initializer_list` -must be convertible into `toml::value`. - -If a `toml::value` has an array, you can `push_back` an element in it. - -```cpp -toml::value v{1,2,3,4,5}; -v.push_back(6); -``` - -`emplace_back` also works. - -## Preserving comments - -toml11 v3 or later allows you yo choose whether comments are preserved or not via template parameter - -```cpp -const auto data1 = toml::parse("example.toml"); -const auto data2 = toml::parse("example.toml"); -``` - -or macro definition. - -```cpp -#define TOML11_PRESERVE_COMMENTS_BY_DEFAULT -#include -``` - -This feature is controlled by template parameter in `toml::basic_value<...>`. -`toml::value` is an alias of `toml::basic_value<...>`. - -If template parameter is explicitly specified, the return value of `toml::parse` -will be `toml::basic_value`. -If the macro is defined, the alias `toml::value` will be -`toml::basic_value`. - -Comments related to a value can be obtained by `toml::value::comments()`. -The return value has the same interface as `std::vector`. - -```cpp -const auto& com = v.comments(); -for(const auto& c : com) -{ - std::cout << c << std::endl; -} -``` - -Comments just before and just after (within the same line) a value are kept in a value. - -```toml -# this is a comment for v1. -v1 = "foo" - -v2 = "bar" # this is a comment for v2. -# Note that this comment is NOT a comment for v2. - -# this comment is not related to any value -# because there are empty lines between v3. -# this comment will be ignored even if you set `preserve_comments`. - -# this is a comment for v3 -# this is also a comment for v3. -v3 = "baz" # ditto. -``` - -Each comment line becomes one element of a `std::vector`. - -Hash signs will be removed, but spaces after hash sign will not be removed. - -```cpp -v1.comments().at(0) == " this is a comment for v1."s; - -v2.comments().at(1) == " this is a comment for v1."s; - -v3.comments().at(0) == " this is a comment for v3."s; -v3.comments().at(1) == " this is also a comment for v3."s; -v3.comments().at(2) == " ditto."s; -``` - -Note that a comment just after an opening brace of an array will not be a -comment for the array. - -```toml -# this is a comment for a. -a = [ # this is not a comment for a. this will be ignored. - 1, 2, 3, - # this is a comment for `42`. - 42, # this is also a comment for `42`. - 5 -] # this is a comment for a. -``` - -You can also append and modify comments. -The interfaces are the same as `std::vector`. - -```cpp -toml::basic_value v(42); -v.comments().push_back(" add this comment."); -// # add this comment. -// i = 42 -``` - -Also, you can pass a `std::vector` when constructing a -`toml::basic_value`. - -```cpp -std::vector comments{"comment 1", "comment 2"}; -const toml::basic_value v1(42, std::move(comments)); -const toml::basic_value v2(42, {"comment 1", "comment 2"}); -``` - -When `toml::discard_comments` is chosen, comments will not be contained in a value. -`value::comments()` will always be kept empty. -All the modification on comments would be ignored. -All the element access in a `discard_comments` causes the same error as accessing -an element of an empty `std::vector`. - -The comments will also be serialized. If comments exist, those comments will be -added just before the values. - -__NOTE__: Result types from `toml::parse(...)` and -`toml::parse(...)` are different. - -## Customizing containers - -Actually, `toml::basic_value` has 3 template arguments. - -```cpp -template class Table = std::unordered_map, - template class Array = std::vector> -class basic_value; -``` - -This enables you to change the containers used inside. E.g. you can use -`std::map` to contain a table object instead of `std::unordered_map`. -And also can use `std::deque` as a array object instead of `std::vector`. - -You can set these parameters while calling `toml::parse` function. - -```cpp -const auto data = toml::parse< - toml::preserve_comments, std::map, std::deque - >("example.toml"); -``` - -Needless to say, the result types from `toml::parse(...)` and -`toml::parse(...)` are different (unless you specify the same -types as default). - -Note that, since `toml::table` and `toml::array` is an alias for a table and an -array of a default `toml::value`, so it is different from the types actually -contained in a `toml::basic_value` when you customize containers. -To get the actual type in a generic way, use -`typename toml::basic_type::table_type` and -`typename toml::basic_type::array_type`. - -## TOML literal - -toml11 supports `"..."_toml` literal. -It accept both a bare value and a file content. - -```cpp -using namespace toml::literals::toml_literals; - -// `_toml` can convert a bare value without key -const toml::value v = u8"0xDEADBEEF"_toml; -// v is an Integer value containing 0xDEADBEEF. - -// raw string literal (`R"(...)"` is useful for this purpose) -const toml::value t = u8R"( - title = "this is TOML literal" - [table] - key = "value" -)"_toml; -// the literal will be parsed and the result will be contained in t -``` - -The literal function is defined in the same way as the standard library literals -such as `std::literals::string_literals::operator""s`. - -```cpp -namespace toml -{ -inline namespace literals -{ -inline namespace toml_literals -{ -toml::value operator"" _toml(const char* str, std::size_t len); -} // toml_literals -} // literals -} // toml -``` - -Access to the operator can be gained with `using namespace toml::literals;`, -`using namespace toml::toml_literals`, and `using namespace toml::literals::toml_literals`. - -Note that a key that is composed only of digits is allowed in TOML. -And, unlike the file parser, toml-literal allows a bare value without a key. -Thus it is difficult to distinguish arrays having integers and definitions of -tables that are named as digits. -Currently, literal `[1]` becomes a table named "1". -To ensure a literal to be considered as an array with one element, you need to -add a comma after the first element (like `[1,]`). - -```cpp -"[1,2,3]"_toml; // This is an array -"[table]"_toml; // This is a table that has an empty table named "table" inside. -"[[1,2,3]]"_toml; // This is an array of arrays -"[[table]]"_toml; // This is a table that has an array of tables inside. - -"[[1]]"_toml; // This literal is ambiguous. - // Currently, it becomes a table that has array of table "1". -"1 = [{}]"_toml; // This is a table that has an array of table named 1. -"[[1,]]"_toml; // This is an array of arrays. -"[[1],]"_toml; // ditto. -``` - -NOTE: `_toml` literal returns a `toml::value` that does not have comments. - -## Conversion between toml value and arbitrary types - -You can also use `toml::get` and other related functions with the types -you defined after you implement a way to convert it. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; -}; -} // ext - -const auto data = toml::parse("example.toml"); - -// to do this -const foo f = toml::find(data, "foo"); -``` - -There are 3 ways to use `toml::get` with the types that you defined. - -The first one is to implement `from_toml(const toml::value&)` member function. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; - - void from_toml(const toml::value& v) - { - this->a = toml::find(v, "a"); - this->b = toml::find(v, "b"); - this->c = toml::find(v, "c"); - return; - } -}; -} // ext -``` - -In this way, because `toml::get` first constructs `foo` without arguments, -the type should be default-constructible. - -The second is to implement `constructor(const toml::value&)`. - -```cpp -namespace ext -{ -struct foo -{ - explicit foo(const toml::value& v) - : a(toml::find(v, "a")), b(toml::find(v, "b")), - c(toml::find(v, "c")) - {} - - int a; - double b; - std::string c; -}; -} // ext -``` - -Note that implicit default constructor declaration will be suppressed -when a constructor is defined. If you want to use the struct (here, `foo`) -in a container (e.g. `std::vector`), you may need to define default -constructor explicitly. - -The third is to implement specialization of `toml::from` for your type. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; -}; -} // ext - -namespace toml -{ -template<> -struct from -{ - static ext::foo from_toml(const value& v) - { - ext::foo f; - f.a = find(v, "a"); - f.b = find(v, "b"); - f.c = find(v, "c"); - return f; - } -}; -} // toml -``` - -In this way, since the conversion function is defined outside of the class, -you can add conversion between `toml::value` and classes defined in another library. - -In some cases, a class has a templatized constructor that takes a template, `T`. -It confuses `toml::get/find` because it makes the class "constructible" from -`toml::value`. To avoid this problem, `toml::from` and `from_toml` always -precede constructor. It makes easier to implement conversion between -`toml::value` and types defined in other libraries because it skips constructor. - -But, importantly, you cannot define `toml::from` and `T.from_toml` at the same -time because it causes ambiguity in the overload resolution of `toml::get` and `toml::find`. - -So the precedence is `toml::from` == `T.from_toml()` > `T(toml::value)`. - -If you want to convert any versions of `toml::basic_value`, -you need to templatize the conversion function as follows. - -```cpp -struct foo -{ - template class M, template class A> - void from_toml(const toml::basic_value& v) - { - this->a = toml::find(v, "a"); - this->b = toml::find(v, "b"); - this->c = toml::find(v, "c"); - return; - } -}; -// or -namespace toml -{ -template<> -struct from -{ - template class M, template class A> - static ext::foo from_toml(const basic_value& v) - { - ext::foo f; - f.a = find(v, "a"); - f.b = find(v, "b"); - f.c = find(v, "c"); - return f; - } -}; -} // toml -``` - ----- - -The opposite direction is also supported in a similar way. You can directly -pass your type to `toml::value`'s constructor by introducing `into_toml` or -`toml::into`. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; - - toml::value into_toml() const // you need to mark it const. - { - return toml::value{{"a", this->a}, {"b", this->b}, {"c", this->c}}; - } -}; -} // ext - -ext::foo f{42, 3.14, "foobar"}; -toml::value v(f); -``` - -The definition of `toml::into` is similar to `toml::from`. - -```cpp -namespace ext -{ -struct foo -{ - int a; - double b; - std::string c; -}; -} // ext - -namespace toml -{ -template<> -struct into -{ - static toml::value into_toml(const ext::foo& f) - { - return toml::value{{"a", f.a}, {"b", f.b}, {"c", f.c}}; - } -}; -} // toml - -ext::foo f{42, 3.14, "foobar"}; -toml::value v(f); -``` - -Any type that can be converted to `toml::value`, e.g. `int`, `toml::table` and -`toml::array` are okay to return from `into_toml`. - -You can also return a custom `toml::basic_value` from `toml::into`. - -```cpp -namespace toml -{ -template<> -struct into -{ - static toml::basic_value into_toml(const ext::foo& f) - { - toml::basic_value v{{"a", f.a}, {"b", f.b}, {"c", f.c}}; - v.comments().push_back(" comment"); - return v; - } -}; -} // toml -``` - -But note that, if this `basic_value` would be assigned into other `toml::value` -that discards `comments`, the comments would be dropped. - -### Macro to automatically define conversion functions - -There is a helper macro that automatically generates conversion functions `from` and `into` for a simple struct. - -```cpp -namespace foo -{ -struct Foo -{ - std::string s; - double d; - int i; -}; -} // foo - -TOML11_DEFINE_CONVERSION_NON_INTRUSIVE(foo::Foo, s, d, i) - -int main() -{ - const auto file = toml::parse("example.toml"); - auto f = toml::find(file, "foo"); -} -``` - -And then you can use `toml::find(file, "foo");` - -**Note** that, because of a slight difference in implementation of preprocessor between gcc/clang and MSVC, [you need to define `/Zc:preprocessor`](https://github.com/ToruNiina/toml11/issues/139#issuecomment-803683682) to use it in MSVC (Thank you @glebm !). - -## Formatting user-defined error messages - -When you encounter an error after you read the toml value, you may want to -show the error with the value. - -toml11 provides you a function that formats user-defined error message with -related values. With a code like the following, - -```cpp -const auto value = toml::find(data, "num"); -if(value < 0) -{ - std::cerr << toml::format_error("[error] value should be positive", - data.at("num"), "positive number required") - << std::endl; -} -``` - -you will get an error message like this. - -```console -[error] value should be positive - --> example.toml - 3 | num = -42 - | ~~~ positive number required -``` - -When you pass two values to `toml::format_error`, - -```cpp -const auto min = toml::find(range, "min"); -const auto max = toml::find(range, "max"); -if(max < min) -{ - std::cerr << toml::format_error("[error] max should be larger than min", - data.at("min"), "minimum number here", - data.at("max"), "maximum number here"); - << std::endl; -} -``` - -you will get an error message like this. - -```console -[error] max should be larger than min - --> example.toml - 3 | min = 54 - | ~~ minimum number here - ... - 4 | max = 42 - | ~~ maximum number here -``` - -You can print hints at the end of the message. - -```cpp -std::vector hints; -hints.push_back("positive number means n >= 0."); -hints.push_back("negative number is not positive."); -std::cerr << toml::format_error("[error] value should be positive", - data.at("num"), "positive number required", hints) - << std::endl; -``` - -```console -[error] value should be positive - --> example.toml - 2 | num = 42 - | ~~ positive number required - | -Hint: positive number means n >= 0. -Hint: negative number is not positive. -``` - -## Obtaining location information - -You can also format error messages in your own way by using `source_location`. - -```cpp -struct source_location -{ - std::uint_least32_t line() const noexcept; - std::uint_least32_t column() const noexcept; - std::uint_least32_t region() const noexcept; - std::string const& file_name() const noexcept; - std::string const& line_str() const noexcept; -}; -// +-- line() +--- length of the region (here, region() == 9) -// v .---+---. -// 12 | value = "foo bar" <- line_str() returns the line itself. -// ^-------- column() points here -``` - -You can get this by -```cpp -const toml::value v = /*...*/; -const toml::source_location loc = v.location(); -``` - -## Exceptions - -The following `exception` classes inherits `toml::exception` that inherits -`std::exception`. - -```cpp -namespace toml { -struct exception : public std::exception {/**/}; -struct syntax_error : public toml::exception {/**/}; -struct type_error : public toml::exception {/**/}; -struct internal_error : public toml::exception {/**/}; -} // toml -``` - -`toml::exception` has `toml::exception::location()` member function that returns -`toml::source_location`, in addition to `what()`. - -```cpp -namespace toml { -struct exception : public std::exception -{ - // ... - source_location const& location() const noexcept; -}; -} // toml -``` - -It represents where the error occurs. - -`syntax_error` will be thrown from `toml::parse` and `_toml` literal. -`type_error` will be thrown from `toml::get/find`, `toml::value::as_xxx()`, and -other functions that takes a content inside of `toml::value`. - -Note that, currently, from `toml::value::at()` and `toml::find(value, key)` -may throw an `std::out_of_range` that does not inherits `toml::exception`. - -Also, in some cases, most likely in the file open error, it will throw an -`std::runtime_error`. - -## Colorize Error Messages - -By defining `TOML11_COLORIZE_ERROR_MESSAGE`, the error messages from -`toml::parse` and `toml::find|get` will be colorized. By default, this feature -is turned off. - -With the following toml file taken from `toml-lang/toml/tests/hard_example.toml`, - -```toml -[error] -array = [ - "This might most likely happen in multiline arrays", - Like here, - "or here, - and here" - ] End of array comment, forgot the # -``` - -the error message would be like this. - -![error-message-1](https://github.com/ToruNiina/toml11/blob/misc/misc/toml11-err-msg-1.png) - -With the following, - -```toml -[error] -# array = [ -# "This might most likely happen in multiline arrays", -# Like here, -# "or here, -# and here" -# ] End of array comment, forgot the # -number = 3.14 pi <--again forgot the # -``` - -the error message would be like this. - -![error-message-2](https://github.com/ToruNiina/toml11/blob/misc/misc/toml11-err-msg-2.png) - -The message would be messy when it is written to a file, not a terminal because -it uses [ANSI escape code](https://en.wikipedia.org/wiki/ANSI_escape_code). - -Without `TOML11_COLORIZE_ERROR_MESSAGE`, you can still colorize user-defined -error message by passing `true` to the `toml::format_error` function. -If you define `TOML11_COLORIZE_ERROR_MESSAGE`, the value is `true` by default. -If not, the default value would be `false`. - -```cpp -std::cerr << toml::format_error("[error] value should be positive", - data.at("num"), "positive number required", - hints, /*colorize = */ true) << std::endl; -``` - -Note: It colorize `[error]` in red. That means that it detects `[error]` prefix -at the front of the error message. If there is no `[error]` prefix, -`format_error` adds it to the error message. - -## Serializing TOML data - -toml11 enables you to serialize data into toml format. - -```cpp -const toml::value data{{"foo", 42}, {"bar", "baz"}}; -std::cout << data << std::endl; -// bar = "baz" -// foo = 42 -``` - -toml11 automatically makes a small table and small array inline. -You can specify the width to make them inline by `std::setw` for streams. - -```cpp -const toml::value data{ - {"qux", {{"foo", 42}, {"bar", "baz"}}}, - {"quux", {"small", "array", "of", "strings"}}, - {"foobar", {"this", "array", "of", "strings", "is", "too", "long", - "to", "print", "into", "single", "line", "isn't", "it?"}}, -}; - -// the threshold becomes 80. -std::cout << std::setw(80) << data << std::endl; -// foobar = [ -// "this","array","of","strings","is","too","long","to","print","into", -// "single","line","isn't","it?", -// ] -// quux = ["small","array","of","strings"] -// qux = {bar="baz",foo=42} - - -// the width is 0. nothing become inline. -std::cout << std::setw(0) << data << std::endl; -// foobar = [ -// "this", -// ... (snip) -// "it?", -// ] -// quux = [ -// "small", -// "array", -// "of", -// "strings", -// ] -// [qux] -// bar = "baz" -// foo = 42 -``` - -It is recommended to set width before printing data. Some I/O functions changes -width to 0, and it makes all the stuff (including `toml::array`) multiline. -The resulting files becomes too long. - -To control the precision of floating point numbers, you need to pass -`std::setprecision` to stream. - -```cpp -const toml::value data{ - {"pi", 3.141592653589793}, - {"e", 2.718281828459045} -}; -std::cout << std::setprecision(17) << data << std::endl; -// e = 2.7182818284590451 -// pi = 3.1415926535897931 -std::cout << std::setprecision( 7) << data << std::endl; -// e = 2.718282 -// pi = 3.141593 -``` - -There is another way to format toml values, `toml::format()`. -It returns `std::string` that represents a value. - -```cpp -const toml::value v{{"a", 42}}; -const std::string fmt = toml::format(v); -// a = 42 -``` - -Note that since `toml::format` formats a value, the resulting string may lack -the key value. - -```cpp -const toml::value v{3.14}; -const std::string fmt = toml::format(v); -// 3.14 -``` - -To control the width and precision, `toml::format` receives optional second and -third arguments to set them. By default, the width is 80 and the precision is -`std::numeric_limits::max_digit10`. - -```cpp -const auto serial = toml::format(data, /*width = */ 0, /*prec = */ 17); -``` - -When you pass a comment-preserving-value, the comment will also be serialized. -An array or a table containing a value that has a comment would not be inlined. - -## Underlying types - -The toml types (can be used as `toml::*` in this library) and corresponding `enum` names are listed in the table below. - -| TOML type | underlying c++ type | enum class | -| -------------- | ---------------------------------- | -------------------------------- | -| Boolean | `bool` | `toml::value_t::boolean` | -| Integer | `std::int64_t` | `toml::value_t::integer` | -| Float | `double` | `toml::value_t::floating` | -| String | `toml::string` | `toml::value_t::string` | -| LocalDate | `toml::local_date` | `toml::value_t::local_date` | -| LocalTime | `toml::local_time` | `toml::value_t::local_time` | -| LocalDatetime | `toml::local_datetime` | `toml::value_t::local_datetime` | -| OffsetDatetime | `toml::offset_datetime` | `toml::value_t::offset_datetime` | -| Array | `array-like` | `toml::value_t::array` | -| Table | `map-like` | `toml::value_t::table` | - -`array-like` and `map-like` are the STL containers that works like a `std::vector` and -`std::unordered_map`, respectively. By default, `std::vector` and `std::unordered_map` -are used. See [Customizing containers](#customizing-containers) for detail. - -`toml::string` is effectively the same as `std::string` but has an additional -flag that represents a kind of a string, `string_t::basic` and `string_t::literal`. -Although `std::string` is not an exact toml type, still you can get a reference -that points to internal `std::string` by using `toml::get()` for convenience. -The most important difference between `std::string` and `toml::string` is that -`toml::string` will be formatted as a TOML string when outputted with `ostream`. -This feature is introduced to make it easy to write a custom serializer. - -`Datetime` variants are `struct` that are defined in this library. -Because `std::chrono::system_clock::time_point` is a __time point__, -not capable of representing a Local Time independent from a specific day. - -## Unreleased TOML features - -Since TOML v1.0.0-rc.1 has been released, those features are now activated by -default. We no longer need to define `TOML11_USE_UNRELEASED_FEATURES`. - -- Leading zeroes in exponent parts of floats are permitted. - - e.g. `1.0e+01`, `5e+05` - - [toml-lang/toml/PR/656](https://github.com/toml-lang/toml/pull/656) -- Allow raw tab characters in basic strings and multi-line basic strings. - - [toml-lang/toml/PR/627](https://github.com/toml-lang/toml/pull/627) -- Allow heterogeneous arrays - - [toml-lang/toml/PR/676](https://github.com/toml-lang/toml/pull/676) - -## Note about heterogeneous arrays - -Although `toml::parse` allows heterogeneous arrays, constructor of `toml::value` -does not. Here the reason is explained. - -```cpp -// this won't be compiled -toml::value v{ - "foo", 3.14, 42, {1,2,3,4,5}, {{"key", "value"}} -} -``` - -There is a workaround for this. By explicitly converting values into -`toml::value`, you can initialize `toml::value` with a heterogeneous array. -Also, you can first initialize a `toml::value` with an array and then -`push_back` into it. - -```cpp -// OK! -toml::value v{ - toml::value("foo"), toml::value(3.14), toml::value(42), - toml::value{1,2,3,4,5}, toml::value{{"key", "value"}} -} - -// OK! -toml::value v(toml::array{}); -v.push_back("foo"); -v.push_back(3.14); - -// OK! -toml::array a; -a.push_back("foo"); -a.push_back(3.14); -toml::value v(std::move(a)); -``` - -The reason why the first example is not allowed is the following. -Let's assume that you are initializing a `toml::value` with a table. - -```cpp - // # expecting TOML table. -toml::value v{ // [v] - {"answer", 42}, // answer = 42 - {"pi", 3.14}, // pi = 3.14 - {"foo", "bar"} // foo = "bar" -}; -``` - -This is indistinguishable from a (heterogeneous) TOML array definition. - -```toml -v = [ - ["answer", 42], - ["pi", 3.14], - ["foo", "bar"], -] -``` - -This means that the above C++ code makes constructor's overload resolution -ambiguous. So a constructor that allows both "table as an initializer-list" and -"heterogeneous array as an initializer-list" cannot be implemented. - -Thus, although it is painful, we need to explicitly cast values into -`toml::value` when you initialize heterogeneous array in a C++ code. - -```cpp -toml::value v{ - toml::value("foo"), toml::value(3.14), toml::value(42), - toml::value{1,2,3,4,5}, toml::value{{"key", "value"}} -}; -``` - -## Breaking Changes from v2 - -Although toml11 is relatively new library (it's three years old now), it had -some confusing and inconvenient user-interfaces because of historical reasons. - -Between v2 and v3, those interfaces are rearranged. - -- `toml::parse` now returns a `toml::value`, not `toml::table`. -- `toml::value` is now an alias of `toml::basic_value`. - - See [Customizing containers](#customizing-containers) for detail. -- The elements of `toml::value_t` are renamed as `snake_case`. - - See [Underlying types](#underlying-types) for detail. -- Supports for the CamelCaseNames are dropped. - - See [Underlying types](#underlying-types) for detail. -- `(is|as)_float` has been removed to make the function names consistent with others. - - Since `float` is a keyword, toml11 named a float type as `toml::floating`. - - Also a `value_t` corresponds to `toml::floating` is named `value_t::floating`. - - So `(is|as)_floating` is introduced and `is_float` has been removed. - - See [Casting a toml::value](#casting-a-tomlvalue) and [Checking value type](#checking-value-type) for detail. -- An overload of `toml::find` for `toml::table` has been dropped. Use `toml::value` version instead. - - Because type conversion between a table and a value causes ambiguity while overload resolution - - Since `toml::parse` now returns a `toml::value`, this feature becomes less important. - - Also because `toml::table` is a normal STL container, implementing utility function is easy. - - See [Finding a toml::value](#finding-a-toml-value) for detail. -- An overload of `operator<<` and `toml::format` for `toml::table`s are dropped. - - Use `toml::value` instead. - - See [Serializing TOML data](#serializing-toml-data) for detail. -- Interface around comments. - - See [Preserving Comments](#preserving-comments) for detail. -- An ancient `from_toml/into_toml` has been removed. Use arbitrary type conversion support. - - See [Conversion between toml value and arbitrary types](#conversion-between-toml-value-and-arbitrary-types) for detail. - -Such a big change will not happen in the coming years. - -## Running Tests - -After cloning this repository, run the following command (thank you @jwillikers -for automating test set fetching!). - -```sh -$ mkdir build -$ cd build -$ cmake .. -Dtoml11_BUILD_TEST=ON -$ make -$ make test -``` - -To run the language agnostic test suite, you need to compile -`tests/check_toml_test.cpp` and pass it to the tester. - -## Contributors - -I appreciate the help of the contributors who introduced the great feature to this library. - -- Guillaume Fraux (@Luthaf) - - Windows support and CI on Appvayor - - Intel Compiler support -- Quentin Khan (@xaxousis) - - Found & Fixed a bug around ODR - - Improved error messages for invalid keys to show the location where the parser fails -- Petr Beneš (@wbenny) - - Fixed warnings on MSVC -- Ivan Shynkarenka (@chronoxor) - - Fixed Visual Studio 2019 warnings -- @khoitd1997 - - Fixed warnings while type conversion -- @KerstinKeller - - Added installation script to CMake -- J.C. Moyer (@jcmoyer) - - Fixed an example code in the documentation -- Jt Freeman (@blockparty-sh) - - Fixed feature test macro around `localtime_s` - - Suppress warnings in Debug mode -- OGAWA Kenichi (@kenichiice) - - Suppress warnings on intel compiler -- Jordan Williams (@jwillikers) - - Fixed clang range-loop-analysis warnings - - Fixed feature test macro to suppress -Wundef - - Use cache variables in CMakeLists.txt - - Automate test set fetching, update and refactor CMakeLists.txt -- Scott McCaskill - - Parse 9 digits (nanoseconds) of fractional seconds in a `local_time` -- Shu Wang (@halfelf) - - fix "Finding a value in an array" example in README -- @maass-tv and @SeverinLeonhardt - - Fix MSVC warning C4866 -- OGAWA KenIchi (@kenichiice) - - Fix include path in README -- Mohammed Alyousef (@MoAlyousef) - - Made testing optional in CMake -- Ivan Shynkarenka (@chronoxor) - - Fix compilation error in `` with MinGW -- Alex Merry (@amerry) - - Add missing include files -- sneakypete81 (@sneakypete81) - - Fix typo in error message -- Oliver Kahrmann (@founderio) - - Fix missing filename in error message if parsed file is empty -- Karl Nilsson (@karl-nilsson) - - Fix many spelling errors -- ohdarling88 (@ohdarling) - - Fix a bug in a constructor of serializer -- estshorter (@estshorter) - - Fix MSVC warning C26478 -- Philip Top (@phlptp) - - Improve checking standard library feature availability check -- Louis Marascio (@marascio) - - Fix free-nonheap-object warning - - -## Licensing terms - -This product is licensed under the terms of the [MIT License](LICENSE). - -- Copyright (c) 2017-2021 Toru Niina - -All rights reserved. diff --git a/src/toml11/toml.hpp b/src/toml11/toml.hpp deleted file mode 100644 index f34cfccca..000000000 --- a/src/toml11/toml.hpp +++ /dev/null @@ -1,46 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) 2017 Toru Niina - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -#ifndef TOML_FOR_MODERN_CPP -#define TOML_FOR_MODERN_CPP - -#ifndef __cplusplus -# error "__cplusplus is not defined" -#endif - -#if __cplusplus < 201103L && _MSC_VER < 1900 -# error "toml11 requires C++11 or later." -#endif - -#define TOML11_VERSION_MAJOR 3 -#define TOML11_VERSION_MINOR 7 -#define TOML11_VERSION_PATCH 0 - -#include "toml/parser.hpp" -#include "toml/literal.hpp" -#include "toml/serializer.hpp" -#include "toml/get.hpp" -#include "toml/macros.hpp" - -#endif// TOML_FOR_MODERN_CPP diff --git a/src/toml11/toml/color.hpp b/src/toml11/toml/color.hpp deleted file mode 100644 index 4cb572cb0..000000000 --- a/src/toml11/toml/color.hpp +++ /dev/null @@ -1,64 +0,0 @@ -#ifndef TOML11_COLOR_HPP -#define TOML11_COLOR_HPP -#include -#include - -#ifdef TOML11_COLORIZE_ERROR_MESSAGE -#define TOML11_ERROR_MESSAGE_COLORIZED true -#else -#define TOML11_ERROR_MESSAGE_COLORIZED false -#endif - -namespace toml -{ - -// put ANSI escape sequence to ostream -namespace color_ansi -{ -namespace detail -{ -inline int colorize_index() -{ - static const int index = std::ios_base::xalloc(); - return index; -} -} // detail - -inline std::ostream& colorize(std::ostream& os) -{ - // by default, it is zero. - os.iword(detail::colorize_index()) = 1; - return os; -} -inline std::ostream& nocolorize(std::ostream& os) -{ - os.iword(detail::colorize_index()) = 0; - return os; -} -inline std::ostream& reset (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[00m";} return os;} -inline std::ostream& bold (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[01m";} return os;} -inline std::ostream& grey (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[30m";} return os;} -inline std::ostream& red (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[31m";} return os;} -inline std::ostream& green (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[32m";} return os;} -inline std::ostream& yellow (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[33m";} return os;} -inline std::ostream& blue (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[34m";} return os;} -inline std::ostream& magenta(std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[35m";} return os;} -inline std::ostream& cyan (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[36m";} return os;} -inline std::ostream& white (std::ostream& os) -{if(os.iword(detail::colorize_index()) == 1) {os << "\033[37m";} return os;} -} // color_ansi - -// ANSI escape sequence is the only and default colorization method currently -namespace color = color_ansi; - -} // toml -#endif// TOML11_COLOR_HPP diff --git a/src/toml11/toml/combinator.hpp b/src/toml11/toml/combinator.hpp deleted file mode 100644 index 33ecca1eb..000000000 --- a/src/toml11/toml/combinator.hpp +++ /dev/null @@ -1,306 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_COMBINATOR_HPP -#define TOML11_COMBINATOR_HPP -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "region.hpp" -#include "result.hpp" -#include "traits.hpp" -#include "utility.hpp" - -// they scans characters and returns region if it matches to the condition. -// when they fail, it does not change the location. -// in lexer.hpp, these are used. - -namespace toml -{ -namespace detail -{ - -// to output character as an error message. -inline std::string show_char(const char c) -{ - // It suppresses an error that occurs only in Debug mode of MSVC++ on Windows. - // I'm not completely sure but they check the value of char to be in the - // range [0, 256) and some of the COMPLETELY VALID utf-8 character sometimes - // has negative value (if char has sign). So here it re-interprets c as - // unsigned char through pointer. In general, converting pointer to a - // pointer that has different type cause UB, but `(signed|unsigned)?char` - // are one of the exceptions. Converting pointer only to char and std::byte - // (c++17) are valid. - if(std::isgraph(*reinterpret_cast(std::addressof(c)))) - { - return std::string(1, c); - } - else - { - std::array buf; - buf.fill('\0'); - const auto r = std::snprintf( - buf.data(), buf.size(), "0x%02x", static_cast(c) & 0xFF); - (void) r; // Unused variable warning - assert(r == static_cast(buf.size()) - 1); - return std::string(buf.data()); - } -} - -template -struct character -{ - static constexpr char target = C; - - static result - invoke(location& loc) - { - if(loc.iter() == loc.end()) {return none();} - const auto first = loc.iter(); - - const char c = *(loc.iter()); - if(c != target) - { - return none(); - } - loc.advance(); // update location - - return ok(region(loc, first, loc.iter())); - } -}; -template -constexpr char character::target; - -// closed interval [Low, Up]. both Low and Up are included. -template -struct in_range -{ - // assuming ascii part of UTF-8... - static_assert(Low <= Up, "lower bound should be less than upper bound."); - - static constexpr char upper = Up; - static constexpr char lower = Low; - - static result - invoke(location& loc) - { - if(loc.iter() == loc.end()) {return none();} - const auto first = loc.iter(); - - const char c = *(loc.iter()); - if(c < lower || upper < c) - { - return none(); - } - - loc.advance(); - return ok(region(loc, first, loc.iter())); - } -}; -template constexpr char in_range::upper; -template constexpr char in_range::lower; - -// keep iterator if `Combinator` matches. otherwise, increment `iter` by 1 char. -// for detecting invalid characters, like control sequences in toml string. -template -struct exclude -{ - static result - invoke(location& loc) - { - if(loc.iter() == loc.end()) {return none();} - auto first = loc.iter(); - - auto rslt = Combinator::invoke(loc); - if(rslt.is_ok()) - { - loc.reset(first); - return none(); - } - loc.reset(std::next(first)); // XXX maybe loc.advance() is okay but... - return ok(region(loc, first, loc.iter())); - } -}; - -// increment `iter`, if matches. otherwise, just return empty string. -template -struct maybe -{ - static result - invoke(location& loc) - { - const auto rslt = Combinator::invoke(loc); - if(rslt.is_ok()) - { - return rslt; - } - return ok(region(loc)); - } -}; - -template -struct sequence; - -template -struct sequence -{ - static result - invoke(location& loc) - { - const auto first = loc.iter(); - auto rslt = Head::invoke(loc); - if(rslt.is_err()) - { - loc.reset(first); - return none(); - } - return sequence::invoke(loc, std::move(rslt.unwrap()), first); - } - - // called from the above function only, recursively. - template - static result - invoke(location& loc, region reg, Iterator first) - { - const auto rslt = Head::invoke(loc); - if(rslt.is_err()) - { - loc.reset(first); - return none(); - } - reg += rslt.unwrap(); // concat regions - return sequence::invoke(loc, std::move(reg), first); - } -}; - -template -struct sequence -{ - // would be called from sequence::invoke only. - template - static result - invoke(location& loc, region reg, Iterator first) - { - const auto rslt = Head::invoke(loc); - if(rslt.is_err()) - { - loc.reset(first); - return none(); - } - reg += rslt.unwrap(); // concat regions - return ok(reg); - } -}; - -template -struct either; - -template -struct either -{ - static result - invoke(location& loc) - { - const auto rslt = Head::invoke(loc); - if(rslt.is_ok()) {return rslt;} - return either::invoke(loc); - } -}; -template -struct either -{ - static result - invoke(location& loc) - { - return Head::invoke(loc); - } -}; - -template -struct repeat; - -template struct exactly{}; -template struct at_least{}; -struct unlimited{}; - -template -struct repeat> -{ - static result - invoke(location& loc) - { - region retval(loc); - const auto first = loc.iter(); - for(std::size_t i=0; i -struct repeat> -{ - static result - invoke(location& loc) - { - region retval(loc); - - const auto first = loc.iter(); - for(std::size_t i=0; i -struct repeat -{ - static result - invoke(location& loc) - { - region retval(loc); - while(true) - { - auto rslt = T::invoke(loc); - if(rslt.is_err()) - { - return ok(std::move(retval)); - } - retval += rslt.unwrap(); - } - } -}; - -} // detail -} // toml -#endif// TOML11_COMBINATOR_HPP diff --git a/src/toml11/toml/comments.hpp b/src/toml11/toml/comments.hpp deleted file mode 100644 index ec2504117..000000000 --- a/src/toml11/toml/comments.hpp +++ /dev/null @@ -1,472 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_COMMENTS_HPP -#define TOML11_COMMENTS_HPP -#include -#include -#include -#include -#include -#include -#include - -#ifdef TOML11_PRESERVE_COMMENTS_BY_DEFAULT -# define TOML11_DEFAULT_COMMENT_STRATEGY ::toml::preserve_comments -#else -# define TOML11_DEFAULT_COMMENT_STRATEGY ::toml::discard_comments -#endif - -// This file provides mainly two classes, `preserve_comments` and `discard_comments`. -// Those two are a container that have the same interface as `std::vector` -// but bahaves in the opposite way. `preserve_comments` is just the same as -// `std::vector` and each `std::string` corresponds to a comment line. -// Conversely, `discard_comments` discards all the strings and ignores everything -// assigned in it. `discard_comments` is always empty and you will encounter an -// error whenever you access to the element. -namespace toml -{ -struct discard_comments; // forward decl - -// use it in the following way -// -// const toml::basic_value data = -// toml::parse("example.toml"); -// -// the interface is almost the same as std::vector. -struct preserve_comments -{ - // `container_type` is not provided in discard_comments. - // do not use this inner-type in a generic code. - using container_type = std::vector; - - using size_type = container_type::size_type; - using difference_type = container_type::difference_type; - using value_type = container_type::value_type; - using reference = container_type::reference; - using const_reference = container_type::const_reference; - using pointer = container_type::pointer; - using const_pointer = container_type::const_pointer; - using iterator = container_type::iterator; - using const_iterator = container_type::const_iterator; - using reverse_iterator = container_type::reverse_iterator; - using const_reverse_iterator = container_type::const_reverse_iterator; - - preserve_comments() = default; - ~preserve_comments() = default; - preserve_comments(preserve_comments const&) = default; - preserve_comments(preserve_comments &&) = default; - preserve_comments& operator=(preserve_comments const&) = default; - preserve_comments& operator=(preserve_comments &&) = default; - - explicit preserve_comments(const std::vector& c): comments(c){} - explicit preserve_comments(std::vector&& c) - : comments(std::move(c)) - {} - preserve_comments& operator=(const std::vector& c) - { - comments = c; - return *this; - } - preserve_comments& operator=(std::vector&& c) - { - comments = std::move(c); - return *this; - } - - explicit preserve_comments(const discard_comments&) {} - - explicit preserve_comments(size_type n): comments(n) {} - preserve_comments(size_type n, const std::string& x): comments(n, x) {} - preserve_comments(std::initializer_list x): comments(x) {} - template - preserve_comments(InputIterator first, InputIterator last) - : comments(first, last) - {} - - template - void assign(InputIterator first, InputIterator last) {comments.assign(first, last);} - void assign(std::initializer_list ini) {comments.assign(ini);} - void assign(size_type n, const std::string& val) {comments.assign(n, val);} - - // Related to the issue #97. - // - // It is known that `std::vector::insert` and `std::vector::erase` in - // the standard library implementation included in GCC 4.8.5 takes - // `std::vector::iterator` instead of `std::vector::const_iterator`. - // Because of the const-correctness, we cannot convert a `const_iterator` to - // an `iterator`. It causes compilation error in GCC 4.8.5. -#if defined(__GNUC__) && defined(__GNUC_MINOR__) && defined(__GNUC_PATCHLEVEL__) && !defined(__clang__) -# if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) <= 40805 -# define TOML11_WORKAROUND_GCC_4_8_X_STANDARD_LIBRARY_IMPLEMENTATION -# endif -#endif - -#ifdef TOML11_WORKAROUND_GCC_4_8_X_STANDARD_LIBRARY_IMPLEMENTATION - iterator insert(iterator p, const std::string& x) - { - return comments.insert(p, x); - } - iterator insert(iterator p, std::string&& x) - { - return comments.insert(p, std::move(x)); - } - void insert(iterator p, size_type n, const std::string& x) - { - return comments.insert(p, n, x); - } - template - void insert(iterator p, InputIterator first, InputIterator last) - { - return comments.insert(p, first, last); - } - void insert(iterator p, std::initializer_list ini) - { - return comments.insert(p, ini); - } - - template - iterator emplace(iterator p, Ts&& ... args) - { - return comments.emplace(p, std::forward(args)...); - } - - iterator erase(iterator pos) {return comments.erase(pos);} - iterator erase(iterator first, iterator last) - { - return comments.erase(first, last); - } -#else - iterator insert(const_iterator p, const std::string& x) - { - return comments.insert(p, x); - } - iterator insert(const_iterator p, std::string&& x) - { - return comments.insert(p, std::move(x)); - } - iterator insert(const_iterator p, size_type n, const std::string& x) - { - return comments.insert(p, n, x); - } - template - iterator insert(const_iterator p, InputIterator first, InputIterator last) - { - return comments.insert(p, first, last); - } - iterator insert(const_iterator p, std::initializer_list ini) - { - return comments.insert(p, ini); - } - - template - iterator emplace(const_iterator p, Ts&& ... args) - { - return comments.emplace(p, std::forward(args)...); - } - - iterator erase(const_iterator pos) {return comments.erase(pos);} - iterator erase(const_iterator first, const_iterator last) - { - return comments.erase(first, last); - } -#endif - - void swap(preserve_comments& other) {comments.swap(other.comments);} - - void push_back(const std::string& v) {comments.push_back(v);} - void push_back(std::string&& v) {comments.push_back(std::move(v));} - void pop_back() {comments.pop_back();} - - template - void emplace_back(Ts&& ... args) {comments.emplace_back(std::forward(args)...);} - - void clear() {comments.clear();} - - size_type size() const noexcept {return comments.size();} - size_type max_size() const noexcept {return comments.max_size();} - size_type capacity() const noexcept {return comments.capacity();} - bool empty() const noexcept {return comments.empty();} - - void reserve(size_type n) {comments.reserve(n);} - void resize(size_type n) {comments.resize(n);} - void resize(size_type n, const std::string& c) {comments.resize(n, c);} - void shrink_to_fit() {comments.shrink_to_fit();} - - reference operator[](const size_type n) noexcept {return comments[n];} - const_reference operator[](const size_type n) const noexcept {return comments[n];} - reference at(const size_type n) {return comments.at(n);} - const_reference at(const size_type n) const {return comments.at(n);} - reference front() noexcept {return comments.front();} - const_reference front() const noexcept {return comments.front();} - reference back() noexcept {return comments.back();} - const_reference back() const noexcept {return comments.back();} - - pointer data() noexcept {return comments.data();} - const_pointer data() const noexcept {return comments.data();} - - iterator begin() noexcept {return comments.begin();} - iterator end() noexcept {return comments.end();} - const_iterator begin() const noexcept {return comments.begin();} - const_iterator end() const noexcept {return comments.end();} - const_iterator cbegin() const noexcept {return comments.cbegin();} - const_iterator cend() const noexcept {return comments.cend();} - - reverse_iterator rbegin() noexcept {return comments.rbegin();} - reverse_iterator rend() noexcept {return comments.rend();} - const_reverse_iterator rbegin() const noexcept {return comments.rbegin();} - const_reverse_iterator rend() const noexcept {return comments.rend();} - const_reverse_iterator crbegin() const noexcept {return comments.crbegin();} - const_reverse_iterator crend() const noexcept {return comments.crend();} - - friend bool operator==(const preserve_comments&, const preserve_comments&); - friend bool operator!=(const preserve_comments&, const preserve_comments&); - friend bool operator< (const preserve_comments&, const preserve_comments&); - friend bool operator<=(const preserve_comments&, const preserve_comments&); - friend bool operator> (const preserve_comments&, const preserve_comments&); - friend bool operator>=(const preserve_comments&, const preserve_comments&); - - friend void swap(preserve_comments&, std::vector&); - friend void swap(std::vector&, preserve_comments&); - - private: - - container_type comments; -}; - -inline bool operator==(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments == rhs.comments;} -inline bool operator!=(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments != rhs.comments;} -inline bool operator< (const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments < rhs.comments;} -inline bool operator<=(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments <= rhs.comments;} -inline bool operator> (const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments > rhs.comments;} -inline bool operator>=(const preserve_comments& lhs, const preserve_comments& rhs) {return lhs.comments >= rhs.comments;} - -inline void swap(preserve_comments& lhs, preserve_comments& rhs) -{ - lhs.swap(rhs); - return; -} -inline void swap(preserve_comments& lhs, std::vector& rhs) -{ - lhs.comments.swap(rhs); - return; -} -inline void swap(std::vector& lhs, preserve_comments& rhs) -{ - lhs.swap(rhs.comments); - return; -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const preserve_comments& com) -{ - for(const auto& c : com) - { - os << '#' << c << '\n'; - } - return os; -} - -namespace detail -{ - -// To provide the same interface with `preserve_comments`, `discard_comments` -// should have an iterator. But it does not contain anything, so we need to -// add an iterator that points nothing. -// -// It always points null, so DO NOT unwrap this iterator. It always crashes -// your program. -template -struct empty_iterator -{ - using value_type = T; - using reference_type = typename std::conditional::type; - using pointer_type = typename std::conditional::type; - using difference_type = std::ptrdiff_t; - using iterator_category = std::random_access_iterator_tag; - - empty_iterator() = default; - ~empty_iterator() = default; - empty_iterator(empty_iterator const&) = default; - empty_iterator(empty_iterator &&) = default; - empty_iterator& operator=(empty_iterator const&) = default; - empty_iterator& operator=(empty_iterator &&) = default; - - // DO NOT call these operators. - reference_type operator*() const noexcept {std::terminate();} - pointer_type operator->() const noexcept {return nullptr;} - reference_type operator[](difference_type) const noexcept {return this->operator*();} - - // These operators do nothing. - empty_iterator& operator++() noexcept {return *this;} - empty_iterator operator++(int) noexcept {return *this;} - empty_iterator& operator--() noexcept {return *this;} - empty_iterator operator--(int) noexcept {return *this;} - - empty_iterator& operator+=(difference_type) noexcept {return *this;} - empty_iterator& operator-=(difference_type) noexcept {return *this;} - - empty_iterator operator+(difference_type) const noexcept {return *this;} - empty_iterator operator-(difference_type) const noexcept {return *this;} -}; - -template -bool operator==(const empty_iterator&, const empty_iterator&) noexcept {return true;} -template -bool operator!=(const empty_iterator&, const empty_iterator&) noexcept {return false;} -template -bool operator< (const empty_iterator&, const empty_iterator&) noexcept {return false;} -template -bool operator<=(const empty_iterator&, const empty_iterator&) noexcept {return true;} -template -bool operator> (const empty_iterator&, const empty_iterator&) noexcept {return false;} -template -bool operator>=(const empty_iterator&, const empty_iterator&) noexcept {return true;} - -template -typename empty_iterator::difference_type -operator-(const empty_iterator&, const empty_iterator&) noexcept {return 0;} - -template -empty_iterator -operator+(typename empty_iterator::difference_type, const empty_iterator& rhs) noexcept {return rhs;} -template -empty_iterator -operator+(const empty_iterator& lhs, typename empty_iterator::difference_type) noexcept {return lhs;} - -} // detail - -// The default comment type. It discards all the comments. It requires only one -// byte to contain, so the memory footprint is smaller than preserve_comments. -// -// It just ignores `push_back`, `insert`, `erase`, and any other modifications. -// IT always returns size() == 0, the iterator taken by `begin()` is always the -// same as that of `end()`, and accessing through `operator[]` or iterators -// always causes a segmentation fault. DO NOT access to the element of this. -// -// Why this is chose as the default type is because the last version (2.x.y) -// does not contain any comments in a value. To minimize the impact on the -// efficiency, this is chosen as a default. -// -// To reduce the memory footprint, later we can try empty base optimization (EBO). -struct discard_comments -{ - using size_type = std::size_t; - using difference_type = std::ptrdiff_t; - using value_type = std::string; - using reference = std::string&; - using const_reference = std::string const&; - using pointer = std::string*; - using const_pointer = std::string const*; - using iterator = detail::empty_iterator; - using const_iterator = detail::empty_iterator; - using reverse_iterator = detail::empty_iterator; - using const_reverse_iterator = detail::empty_iterator; - - discard_comments() = default; - ~discard_comments() = default; - discard_comments(discard_comments const&) = default; - discard_comments(discard_comments &&) = default; - discard_comments& operator=(discard_comments const&) = default; - discard_comments& operator=(discard_comments &&) = default; - - explicit discard_comments(const std::vector&) noexcept {} - explicit discard_comments(std::vector&&) noexcept {} - discard_comments& operator=(const std::vector&) noexcept {return *this;} - discard_comments& operator=(std::vector&&) noexcept {return *this;} - - explicit discard_comments(const preserve_comments&) noexcept {} - - explicit discard_comments(size_type) noexcept {} - discard_comments(size_type, const std::string&) noexcept {} - discard_comments(std::initializer_list) noexcept {} - template - discard_comments(InputIterator, InputIterator) noexcept {} - - template - void assign(InputIterator, InputIterator) noexcept {} - void assign(std::initializer_list) noexcept {} - void assign(size_type, const std::string&) noexcept {} - - iterator insert(const_iterator, const std::string&) {return iterator{};} - iterator insert(const_iterator, std::string&&) {return iterator{};} - iterator insert(const_iterator, size_type, const std::string&) {return iterator{};} - template - iterator insert(const_iterator, InputIterator, InputIterator) {return iterator{};} - iterator insert(const_iterator, std::initializer_list) {return iterator{};} - - template - iterator emplace(const_iterator, Ts&& ...) {return iterator{};} - iterator erase(const_iterator) {return iterator{};} - iterator erase(const_iterator, const_iterator) {return iterator{};} - - void swap(discard_comments&) {return;} - - void push_back(const std::string&) {return;} - void push_back(std::string&& ) {return;} - void pop_back() {return;} - - template - void emplace_back(Ts&& ...) {return;} - - void clear() {return;} - - size_type size() const noexcept {return 0;} - size_type max_size() const noexcept {return 0;} - size_type capacity() const noexcept {return 0;} - bool empty() const noexcept {return true;} - - void reserve(size_type) {return;} - void resize(size_type) {return;} - void resize(size_type, const std::string&) {return;} - void shrink_to_fit() {return;} - - // DO NOT access to the element of this container. This container is always - // empty, so accessing through operator[], front/back, data causes address - // error. - - reference operator[](const size_type) noexcept {return *data();} - const_reference operator[](const size_type) const noexcept {return *data();} - reference at(const size_type) {throw std::out_of_range("toml::discard_comment is always empty.");} - const_reference at(const size_type) const {throw std::out_of_range("toml::discard_comment is always empty.");} - reference front() noexcept {return *data();} - const_reference front() const noexcept {return *data();} - reference back() noexcept {return *data();} - const_reference back() const noexcept {return *data();} - - pointer data() noexcept {return nullptr;} - const_pointer data() const noexcept {return nullptr;} - - iterator begin() noexcept {return iterator{};} - iterator end() noexcept {return iterator{};} - const_iterator begin() const noexcept {return const_iterator{};} - const_iterator end() const noexcept {return const_iterator{};} - const_iterator cbegin() const noexcept {return const_iterator{};} - const_iterator cend() const noexcept {return const_iterator{};} - - reverse_iterator rbegin() noexcept {return iterator{};} - reverse_iterator rend() noexcept {return iterator{};} - const_reverse_iterator rbegin() const noexcept {return const_iterator{};} - const_reverse_iterator rend() const noexcept {return const_iterator{};} - const_reverse_iterator crbegin() const noexcept {return const_iterator{};} - const_reverse_iterator crend() const noexcept {return const_iterator{};} -}; - -inline bool operator==(const discard_comments&, const discard_comments&) noexcept {return true;} -inline bool operator!=(const discard_comments&, const discard_comments&) noexcept {return false;} -inline bool operator< (const discard_comments&, const discard_comments&) noexcept {return false;} -inline bool operator<=(const discard_comments&, const discard_comments&) noexcept {return true;} -inline bool operator> (const discard_comments&, const discard_comments&) noexcept {return false;} -inline bool operator>=(const discard_comments&, const discard_comments&) noexcept {return true;} - -inline void swap(const discard_comments&, const discard_comments&) noexcept {return;} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const discard_comments&) -{ - return os; -} - -} // toml11 -#endif// TOML11_COMMENTS_HPP diff --git a/src/toml11/toml/datetime.hpp b/src/toml11/toml/datetime.hpp deleted file mode 100644 index d8127c150..000000000 --- a/src/toml11/toml/datetime.hpp +++ /dev/null @@ -1,631 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_DATETIME_HPP -#define TOML11_DATETIME_HPP -#include -#include -#include - -#include -#include -#include -#include -#include - -namespace toml -{ - -// To avoid non-threadsafe std::localtime. In C11 (not C++11!), localtime_s is -// provided in the absolutely same purpose, but C++11 is actually not compatible -// with C11. We need to dispatch the function depending on the OS. -namespace detail -{ -// TODO: find more sophisticated way to handle this -#if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 1) || defined(_XOPEN_SOURCE) || defined(_BSD_SOURCE) || defined(_SVID_SOURCE) || defined(_POSIX_SOURCE) -inline std::tm localtime_s(const std::time_t* src) -{ - std::tm dst; - const auto result = ::localtime_r(src, &dst); - if (!result) { throw std::runtime_error("localtime_r failed."); } - return dst; -} -inline std::tm gmtime_s(const std::time_t* src) -{ - std::tm dst; - const auto result = ::gmtime_r(src, &dst); - if (!result) { throw std::runtime_error("gmtime_r failed."); } - return dst; -} -#elif defined(_MSC_VER) -inline std::tm localtime_s(const std::time_t* src) -{ - std::tm dst; - const auto result = ::localtime_s(&dst, src); - if (result) { throw std::runtime_error("localtime_s failed."); } - return dst; -} -inline std::tm gmtime_s(const std::time_t* src) -{ - std::tm dst; - const auto result = ::gmtime_s(&dst, src); - if (result) { throw std::runtime_error("gmtime_s failed."); } - return dst; -} -#else // fallback. not threadsafe -inline std::tm localtime_s(const std::time_t* src) -{ - const auto result = std::localtime(src); - if (!result) { throw std::runtime_error("localtime failed."); } - return *result; -} -inline std::tm gmtime_s(const std::time_t* src) -{ - const auto result = std::gmtime(src); - if (!result) { throw std::runtime_error("gmtime failed."); } - return *result; -} -#endif -} // detail - -enum class month_t : std::uint8_t -{ - Jan = 0, - Feb = 1, - Mar = 2, - Apr = 3, - May = 4, - Jun = 5, - Jul = 6, - Aug = 7, - Sep = 8, - Oct = 9, - Nov = 10, - Dec = 11 -}; - -struct local_date -{ - std::int16_t year; // A.D. (like, 2018) - std::uint8_t month; // [0, 11] - std::uint8_t day; // [1, 31] - - local_date(int y, month_t m, int d) - : year (static_cast(y)), - month(static_cast(m)), - day (static_cast(d)) - {} - - explicit local_date(const std::tm& t) - : year (static_cast(t.tm_year + 1900)), - month(static_cast(t.tm_mon)), - day (static_cast(t.tm_mday)) - {} - - explicit local_date(const std::chrono::system_clock::time_point& tp) - { - const auto t = std::chrono::system_clock::to_time_t(tp); - const auto time = detail::localtime_s(&t); - *this = local_date(time); - } - - explicit local_date(const std::time_t t) - : local_date(std::chrono::system_clock::from_time_t(t)) - {} - - operator std::chrono::system_clock::time_point() const - { - // std::mktime returns date as local time zone. no conversion needed - std::tm t; - t.tm_sec = 0; - t.tm_min = 0; - t.tm_hour = 0; - t.tm_mday = static_cast(this->day); - t.tm_mon = static_cast(this->month); - t.tm_year = static_cast(this->year) - 1900; - t.tm_wday = 0; // the value will be ignored - t.tm_yday = 0; // the value will be ignored - t.tm_isdst = -1; - return std::chrono::system_clock::from_time_t(std::mktime(&t)); - } - - operator std::time_t() const - { - return std::chrono::system_clock::to_time_t( - std::chrono::system_clock::time_point(*this)); - } - - local_date() = default; - ~local_date() = default; - local_date(local_date const&) = default; - local_date(local_date&&) = default; - local_date& operator=(local_date const&) = default; - local_date& operator=(local_date&&) = default; -}; - -inline bool operator==(const local_date& lhs, const local_date& rhs) -{ - return std::make_tuple(lhs.year, lhs.month, lhs.day) == - std::make_tuple(rhs.year, rhs.month, rhs.day); -} -inline bool operator!=(const local_date& lhs, const local_date& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const local_date& lhs, const local_date& rhs) -{ - return std::make_tuple(lhs.year, lhs.month, lhs.day) < - std::make_tuple(rhs.year, rhs.month, rhs.day); -} -inline bool operator<=(const local_date& lhs, const local_date& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const local_date& lhs, const local_date& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const local_date& lhs, const local_date& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const local_date& date) -{ - os << std::setfill('0') << std::setw(4) << static_cast(date.year ) << '-'; - os << std::setfill('0') << std::setw(2) << static_cast(date.month) + 1 << '-'; - os << std::setfill('0') << std::setw(2) << static_cast(date.day ) ; - return os; -} - -struct local_time -{ - std::uint8_t hour; // [0, 23] - std::uint8_t minute; // [0, 59] - std::uint8_t second; // [0, 60] - std::uint16_t millisecond; // [0, 999] - std::uint16_t microsecond; // [0, 999] - std::uint16_t nanosecond; // [0, 999] - - local_time(int h, int m, int s, - int ms = 0, int us = 0, int ns = 0) - : hour (static_cast(h)), - minute(static_cast(m)), - second(static_cast(s)), - millisecond(static_cast(ms)), - microsecond(static_cast(us)), - nanosecond (static_cast(ns)) - {} - - explicit local_time(const std::tm& t) - : hour (static_cast(t.tm_hour)), - minute(static_cast(t.tm_min)), - second(static_cast(t.tm_sec)), - millisecond(0), microsecond(0), nanosecond(0) - {} - - template - explicit local_time(const std::chrono::duration& t) - { - const auto h = std::chrono::duration_cast(t); - this->hour = static_cast(h.count()); - const auto t2 = t - h; - const auto m = std::chrono::duration_cast(t2); - this->minute = static_cast(m.count()); - const auto t3 = t2 - m; - const auto s = std::chrono::duration_cast(t3); - this->second = static_cast(s.count()); - const auto t4 = t3 - s; - const auto ms = std::chrono::duration_cast(t4); - this->millisecond = static_cast(ms.count()); - const auto t5 = t4 - ms; - const auto us = std::chrono::duration_cast(t5); - this->microsecond = static_cast(us.count()); - const auto t6 = t5 - us; - const auto ns = std::chrono::duration_cast(t6); - this->nanosecond = static_cast(ns.count()); - } - - operator std::chrono::nanoseconds() const - { - return std::chrono::nanoseconds (this->nanosecond) + - std::chrono::microseconds(this->microsecond) + - std::chrono::milliseconds(this->millisecond) + - std::chrono::seconds(this->second) + - std::chrono::minutes(this->minute) + - std::chrono::hours(this->hour); - } - - local_time() = default; - ~local_time() = default; - local_time(local_time const&) = default; - local_time(local_time&&) = default; - local_time& operator=(local_time const&) = default; - local_time& operator=(local_time&&) = default; -}; - -inline bool operator==(const local_time& lhs, const local_time& rhs) -{ - return std::make_tuple(lhs.hour, lhs.minute, lhs.second, lhs.millisecond, lhs.microsecond, lhs.nanosecond) == - std::make_tuple(rhs.hour, rhs.minute, rhs.second, rhs.millisecond, rhs.microsecond, rhs.nanosecond); -} -inline bool operator!=(const local_time& lhs, const local_time& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const local_time& lhs, const local_time& rhs) -{ - return std::make_tuple(lhs.hour, lhs.minute, lhs.second, lhs.millisecond, lhs.microsecond, lhs.nanosecond) < - std::make_tuple(rhs.hour, rhs.minute, rhs.second, rhs.millisecond, rhs.microsecond, rhs.nanosecond); -} -inline bool operator<=(const local_time& lhs, const local_time& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const local_time& lhs, const local_time& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const local_time& lhs, const local_time& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const local_time& time) -{ - os << std::setfill('0') << std::setw(2) << static_cast(time.hour ) << ':'; - os << std::setfill('0') << std::setw(2) << static_cast(time.minute) << ':'; - os << std::setfill('0') << std::setw(2) << static_cast(time.second); - if(time.millisecond != 0 || time.microsecond != 0 || time.nanosecond != 0) - { - os << '.'; - os << std::setfill('0') << std::setw(3) << static_cast(time.millisecond); - if(time.microsecond != 0 || time.nanosecond != 0) - { - os << std::setfill('0') << std::setw(3) << static_cast(time.microsecond); - if(time.nanosecond != 0) - { - os << std::setfill('0') << std::setw(3) << static_cast(time.nanosecond); - } - } - } - return os; -} - -struct time_offset -{ - std::int8_t hour; // [-12, 12] - std::int8_t minute; // [-59, 59] - - time_offset(int h, int m) - : hour (static_cast(h)), - minute(static_cast(m)) - {} - - operator std::chrono::minutes() const - { - return std::chrono::minutes(this->minute) + - std::chrono::hours(this->hour); - } - - time_offset() = default; - ~time_offset() = default; - time_offset(time_offset const&) = default; - time_offset(time_offset&&) = default; - time_offset& operator=(time_offset const&) = default; - time_offset& operator=(time_offset&&) = default; -}; - -inline bool operator==(const time_offset& lhs, const time_offset& rhs) -{ - return std::make_tuple(lhs.hour, lhs.minute) == - std::make_tuple(rhs.hour, rhs.minute); -} -inline bool operator!=(const time_offset& lhs, const time_offset& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const time_offset& lhs, const time_offset& rhs) -{ - return std::make_tuple(lhs.hour, lhs.minute) < - std::make_tuple(rhs.hour, rhs.minute); -} -inline bool operator<=(const time_offset& lhs, const time_offset& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const time_offset& lhs, const time_offset& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const time_offset& lhs, const time_offset& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const time_offset& offset) -{ - if(offset.hour == 0 && offset.minute == 0) - { - os << 'Z'; - return os; - } - int minute = static_cast(offset.hour) * 60 + offset.minute; - if(minute < 0){os << '-'; minute = std::abs(minute);} else {os << '+';} - os << std::setfill('0') << std::setw(2) << minute / 60 << ':'; - os << std::setfill('0') << std::setw(2) << minute % 60; - return os; -} - -struct local_datetime -{ - local_date date; - local_time time; - - local_datetime(local_date d, local_time t): date(d), time(t) {} - - explicit local_datetime(const std::tm& t): date(t), time(t){} - - explicit local_datetime(const std::chrono::system_clock::time_point& tp) - { - const auto t = std::chrono::system_clock::to_time_t(tp); - std::tm ltime = detail::localtime_s(&t); - - this->date = local_date(ltime); - this->time = local_time(ltime); - - // std::tm lacks subsecond information, so diff between tp and tm - // can be used to get millisecond & microsecond information. - const auto t_diff = tp - - std::chrono::system_clock::from_time_t(std::mktime(<ime)); - this->time.millisecond = static_cast( - std::chrono::duration_cast(t_diff).count()); - this->time.microsecond = static_cast( - std::chrono::duration_cast(t_diff).count()); - this->time.nanosecond = static_cast( - std::chrono::duration_cast(t_diff).count()); - } - - explicit local_datetime(const std::time_t t) - : local_datetime(std::chrono::system_clock::from_time_t(t)) - {} - - operator std::chrono::system_clock::time_point() const - { - using internal_duration = - typename std::chrono::system_clock::time_point::duration; - - // Normally DST begins at A.M. 3 or 4. If we re-use conversion operator - // of local_date and local_time independently, the conversion fails if - // it is the day when DST begins or ends. Since local_date considers the - // time is 00:00 A.M. and local_time does not consider DST because it - // does not have any date information. We need to consider both date and - // time information at the same time to convert it correctly. - - std::tm t; - t.tm_sec = static_cast(this->time.second); - t.tm_min = static_cast(this->time.minute); - t.tm_hour = static_cast(this->time.hour); - t.tm_mday = static_cast(this->date.day); - t.tm_mon = static_cast(this->date.month); - t.tm_year = static_cast(this->date.year) - 1900; - t.tm_wday = 0; // the value will be ignored - t.tm_yday = 0; // the value will be ignored - t.tm_isdst = -1; - - // std::mktime returns date as local time zone. no conversion needed - auto dt = std::chrono::system_clock::from_time_t(std::mktime(&t)); - dt += std::chrono::duration_cast( - std::chrono::milliseconds(this->time.millisecond) + - std::chrono::microseconds(this->time.microsecond) + - std::chrono::nanoseconds (this->time.nanosecond)); - return dt; - } - - operator std::time_t() const - { - return std::chrono::system_clock::to_time_t( - std::chrono::system_clock::time_point(*this)); - } - - local_datetime() = default; - ~local_datetime() = default; - local_datetime(local_datetime const&) = default; - local_datetime(local_datetime&&) = default; - local_datetime& operator=(local_datetime const&) = default; - local_datetime& operator=(local_datetime&&) = default; -}; - -inline bool operator==(const local_datetime& lhs, const local_datetime& rhs) -{ - return std::make_tuple(lhs.date, lhs.time) == - std::make_tuple(rhs.date, rhs.time); -} -inline bool operator!=(const local_datetime& lhs, const local_datetime& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const local_datetime& lhs, const local_datetime& rhs) -{ - return std::make_tuple(lhs.date, lhs.time) < - std::make_tuple(rhs.date, rhs.time); -} -inline bool operator<=(const local_datetime& lhs, const local_datetime& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const local_datetime& lhs, const local_datetime& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const local_datetime& lhs, const local_datetime& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const local_datetime& dt) -{ - os << dt.date << 'T' << dt.time; - return os; -} - -struct offset_datetime -{ - local_date date; - local_time time; - time_offset offset; - - offset_datetime(local_date d, local_time t, time_offset o) - : date(d), time(t), offset(o) - {} - offset_datetime(const local_datetime& dt, time_offset o) - : date(dt.date), time(dt.time), offset(o) - {} - explicit offset_datetime(const local_datetime& ld) - : date(ld.date), time(ld.time), offset(get_local_offset(nullptr)) - // use the current local timezone offset - {} - explicit offset_datetime(const std::chrono::system_clock::time_point& tp) - : offset(0, 0) // use gmtime - { - const auto timet = std::chrono::system_clock::to_time_t(tp); - const auto tm = detail::gmtime_s(&timet); - this->date = local_date(tm); - this->time = local_time(tm); - } - explicit offset_datetime(const std::time_t& t) - : offset(0, 0) // use gmtime - { - const auto tm = detail::gmtime_s(&t); - this->date = local_date(tm); - this->time = local_time(tm); - } - explicit offset_datetime(const std::tm& t) - : offset(0, 0) // assume gmtime - { - this->date = local_date(t); - this->time = local_time(t); - } - - operator std::chrono::system_clock::time_point() const - { - // get date-time - using internal_duration = - typename std::chrono::system_clock::time_point::duration; - - // first, convert it to local date-time information in the same way as - // local_datetime does. later we will use time_t to adjust time offset. - std::tm t; - t.tm_sec = static_cast(this->time.second); - t.tm_min = static_cast(this->time.minute); - t.tm_hour = static_cast(this->time.hour); - t.tm_mday = static_cast(this->date.day); - t.tm_mon = static_cast(this->date.month); - t.tm_year = static_cast(this->date.year) - 1900; - t.tm_wday = 0; // the value will be ignored - t.tm_yday = 0; // the value will be ignored - t.tm_isdst = -1; - const std::time_t tp_loc = std::mktime(std::addressof(t)); - - auto tp = std::chrono::system_clock::from_time_t(tp_loc); - tp += std::chrono::duration_cast( - std::chrono::milliseconds(this->time.millisecond) + - std::chrono::microseconds(this->time.microsecond) + - std::chrono::nanoseconds (this->time.nanosecond)); - - // Since mktime uses local time zone, it should be corrected. - // `12:00:00+09:00` means `03:00:00Z`. So mktime returns `03:00:00Z` if - // we are in `+09:00` timezone. To represent `12:00:00Z` there, we need - // to add `+09:00` to `03:00:00Z`. - // Here, it uses the time_t converted from date-time info to handle - // daylight saving time. - const auto ofs = get_local_offset(std::addressof(tp_loc)); - tp += std::chrono::hours (ofs.hour); - tp += std::chrono::minutes(ofs.minute); - - // We got `12:00:00Z` by correcting local timezone applied by mktime. - // Then we will apply the offset. Let's say `12:00:00-08:00` is given. - // And now, we have `12:00:00Z`. `12:00:00-08:00` means `20:00:00Z`. - // So we need to subtract the offset. - tp -= std::chrono::minutes(this->offset); - return tp; - } - - operator std::time_t() const - { - return std::chrono::system_clock::to_time_t( - std::chrono::system_clock::time_point(*this)); - } - - offset_datetime() = default; - ~offset_datetime() = default; - offset_datetime(offset_datetime const&) = default; - offset_datetime(offset_datetime&&) = default; - offset_datetime& operator=(offset_datetime const&) = default; - offset_datetime& operator=(offset_datetime&&) = default; - - private: - - static time_offset get_local_offset(const std::time_t* tp) - { - // get local timezone with the same date-time information as mktime - const auto t = detail::localtime_s(tp); - - std::array buf; - const auto result = std::strftime(buf.data(), 6, "%z", &t); // +hhmm\0 - if(result != 5) - { - throw std::runtime_error("toml::offset_datetime: cannot obtain " - "timezone information of current env"); - } - const int ofs = std::atoi(buf.data()); - const int ofs_h = ofs / 100; - const int ofs_m = ofs - (ofs_h * 100); - return time_offset(ofs_h, ofs_m); - } -}; - -inline bool operator==(const offset_datetime& lhs, const offset_datetime& rhs) -{ - return std::make_tuple(lhs.date, lhs.time, lhs.offset) == - std::make_tuple(rhs.date, rhs.time, rhs.offset); -} -inline bool operator!=(const offset_datetime& lhs, const offset_datetime& rhs) -{ - return !(lhs == rhs); -} -inline bool operator< (const offset_datetime& lhs, const offset_datetime& rhs) -{ - return std::make_tuple(lhs.date, lhs.time, lhs.offset) < - std::make_tuple(rhs.date, rhs.time, rhs.offset); -} -inline bool operator<=(const offset_datetime& lhs, const offset_datetime& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -inline bool operator> (const offset_datetime& lhs, const offset_datetime& rhs) -{ - return !(lhs <= rhs); -} -inline bool operator>=(const offset_datetime& lhs, const offset_datetime& rhs) -{ - return !(lhs < rhs); -} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const offset_datetime& dt) -{ - os << dt.date << 'T' << dt.time << dt.offset; - return os; -} - -}//toml -#endif// TOML11_DATETIME diff --git a/src/toml11/toml/exception.hpp b/src/toml11/toml/exception.hpp deleted file mode 100644 index c64651d0a..000000000 --- a/src/toml11/toml/exception.hpp +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_EXCEPTION_HPP -#define TOML11_EXCEPTION_HPP -#include -#include - -#include "source_location.hpp" - -namespace toml -{ - -struct exception : public std::exception -{ - public: - explicit exception(const source_location& loc): loc_(loc) {} - virtual ~exception() noexcept override = default; - virtual const char* what() const noexcept override {return "";} - virtual source_location const& location() const noexcept {return loc_;} - - protected: - source_location loc_; -}; - -struct syntax_error : public toml::exception -{ - public: - explicit syntax_error(const std::string& what_arg, const source_location& loc) - : exception(loc), what_(what_arg) - {} - virtual ~syntax_error() noexcept override = default; - virtual const char* what() const noexcept override {return what_.c_str();} - - protected: - std::string what_; -}; - -struct type_error : public toml::exception -{ - public: - explicit type_error(const std::string& what_arg, const source_location& loc) - : exception(loc), what_(what_arg) - {} - virtual ~type_error() noexcept override = default; - virtual const char* what() const noexcept override {return what_.c_str();} - - protected: - std::string what_; -}; - -struct internal_error : public toml::exception -{ - public: - explicit internal_error(const std::string& what_arg, const source_location& loc) - : exception(loc), what_(what_arg) - {} - virtual ~internal_error() noexcept override = default; - virtual const char* what() const noexcept override {return what_.c_str();} - - protected: - std::string what_; -}; - -} // toml -#endif // TOML_EXCEPTION diff --git a/src/toml11/toml/from.hpp b/src/toml11/toml/from.hpp deleted file mode 100644 index 10815caf5..000000000 --- a/src/toml11/toml/from.hpp +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_FROM_HPP -#define TOML11_FROM_HPP - -namespace toml -{ - -template -struct from; -// { -// static T from_toml(const toml::value& v) -// { -// // User-defined conversions ... -// } -// }; - -} // toml -#endif // TOML11_FROM_HPP diff --git a/src/toml11/toml/get.hpp b/src/toml11/toml/get.hpp deleted file mode 100644 index d7fdf553b..000000000 --- a/src/toml11/toml/get.hpp +++ /dev/null @@ -1,1117 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_GET_HPP -#define TOML11_GET_HPP -#include - -#include "from.hpp" -#include "result.hpp" -#include "value.hpp" - -namespace toml -{ - -// ============================================================================ -// exact toml::* type - -template class M, template class V> -detail::enable_if_t>::value, T> & -get(basic_value& v) -{ - return v.template cast>::value>(); -} - -template class M, template class V> -detail::enable_if_t>::value, T> const& -get(const basic_value& v) -{ - return v.template cast>::value>(); -} - -template class M, template class V> -detail::enable_if_t>::value, T> -get(basic_value&& v) -{ - return T(std::move(v).template cast>::value>()); -} - -// ============================================================================ -// T == toml::value; identity transformation. - -template class M, template class V> -inline detail::enable_if_t>::value, T>& -get(basic_value& v) -{ - return v; -} - -template class M, template class V> -inline detail::enable_if_t>::value, T> const& -get(const basic_value& v) -{ - return v; -} - -template class M, template class V> -inline detail::enable_if_t>::value, T> -get(basic_value&& v) -{ - return basic_value(std::move(v)); -} - -// ============================================================================ -// T == toml::basic_value; basic_value -> basic_value - -template class M, template class V> -inline detail::enable_if_t, - detail::negation>> - >::value, T> -get(const basic_value& v) -{ - return T(v); -} - -// ============================================================================ -// integer convertible from toml::Integer - -template class M, template class V> -inline detail::enable_if_t, // T is integral - detail::negation>, // but not bool - detail::negation< // but not toml::integer - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - return static_cast(v.as_integer()); -} - -// ============================================================================ -// floating point convertible from toml::Float - -template class M, template class V> -inline detail::enable_if_t, // T is floating_point - detail::negation< // but not toml::floating - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - return static_cast(v.as_floating()); -} - -// ============================================================================ -// std::string; toml uses its own toml::string, but it should be convertible to -// std::string seamlessly - -template class M, template class V> -inline detail::enable_if_t::value, std::string>& -get(basic_value& v) -{ - return v.as_string().str; -} - -template class M, template class V> -inline detail::enable_if_t::value, std::string> const& -get(const basic_value& v) -{ - return v.as_string().str; -} - -template class M, template class V> -inline detail::enable_if_t::value, std::string> -get(basic_value&& v) -{ - return std::string(std::move(v.as_string().str)); -} - -// ============================================================================ -// std::string_view - -#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 -template class M, template class V> -inline detail::enable_if_t::value, std::string_view> -get(const basic_value& v) -{ - return std::string_view(v.as_string().str); -} -#endif - -// ============================================================================ -// std::chrono::duration from toml::local_time. - -template class M, template class V> -inline detail::enable_if_t::value, T> -get(const basic_value& v) -{ - return std::chrono::duration_cast( - std::chrono::nanoseconds(v.as_local_time())); -} - -// ============================================================================ -// std::chrono::system_clock::time_point from toml::datetime variants - -template class M, template class V> -inline detail::enable_if_t< - std::is_same::value, T> -get(const basic_value& v) -{ - switch(v.type()) - { - case value_t::local_date: - { - return std::chrono::system_clock::time_point(v.as_local_date()); - } - case value_t::local_datetime: - { - return std::chrono::system_clock::time_point(v.as_local_datetime()); - } - case value_t::offset_datetime: - { - return std::chrono::system_clock::time_point(v.as_offset_datetime()); - } - default: - { - throw type_error(detail::format_underline("toml::value: " - "bad_cast to std::chrono::system_clock::time_point", { - {v.location(), concat_to_string("the actual type is ", v.type())} - }), v.location()); - } - } -} - -// ============================================================================ -// forward declaration to use this recursively. ignore this and go ahead. - -// array-like type with push_back(value) method -template class M, template class V> -detail::enable_if_t, // T is a container - detail::has_push_back_method, // T::push_back(value) works - detail::negation< // but not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value&); - -// array-like type without push_back(value) method -template class M, template class V> -detail::enable_if_t, // T is a container - detail::negation>, // w/o push_back(...) - detail::negation< // not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value&); - -// std::pair -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value&); - -// std::tuple -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value&); - -// map-like classes -template class M, template class V> -detail::enable_if_t, // T is map - detail::negation< // but not toml::table - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value&); - -// T.from_toml(v) -template class M, template class V> -detail::enable_if_t>>, - detail::has_from_toml_method, // but has from_toml(toml::value) - std::is_default_constructible // and default constructible - >::value, T> -get(const basic_value&); - -// toml::from::from_toml(v) -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value&); - -// T(const toml::value&) and T is not toml::basic_value, -// and it does not have `from` nor `from_toml`. -template class M, template class V> -detail::enable_if_t>, - std::is_constructible&>, - detail::negation>, - detail::negation> - >::value, T> -get(const basic_value&); - -// ============================================================================ -// array-like types; most likely STL container, like std::vector, etc. - -template class M, template class V> -detail::enable_if_t, // T is a container - detail::has_push_back_method, // container.push_back(elem) works - detail::negation< // but not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - using value_type = typename T::value_type; - const auto& ary = v.as_array(); - - T container; - try_reserve(container, ary.size()); - - for(const auto& elem : ary) - { - container.push_back(get(elem)); - } - return container; -} - -// ============================================================================ -// std::forward_list does not have push_back, insert, or emplace. -// It has insert_after, emplace_after, push_front. - -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value& v) -{ - using value_type = typename T::value_type; - T container; - for(const auto& elem : v.as_array()) - { - container.push_front(get(elem)); - } - container.reverse(); - return container; -} - -// ============================================================================ -// array-like types, without push_back(). most likely [std|boost]::array. - -template class M, template class V> -detail::enable_if_t, // T is a container - detail::negation>, // w/o push_back - detail::negation< // T is not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - using value_type = typename T::value_type; - const auto& ar = v.as_array(); - - T container; - if(ar.size() != container.size()) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "toml::get: specified container size is ", container.size(), - " but there are ", ar.size(), " elements in toml array."), { - {v.location(), "here"} - })); - } - for(std::size_t i=0; i(ar[i]); - } - return container; -} - -// ============================================================================ -// std::pair. - -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value& v) -{ - using first_type = typename T::first_type; - using second_type = typename T::second_type; - - const auto& ar = v.as_array(); - if(ar.size() != 2) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "toml::get: specified std::pair but there are ", ar.size(), - " elements in toml array."), {{v.location(), "here"}})); - } - return std::make_pair(::toml::get(ar.at(0)), - ::toml::get(ar.at(1))); -} - -// ============================================================================ -// std::tuple. - -namespace detail -{ -template -T get_tuple_impl(const Array& a, index_sequence) -{ - return std::make_tuple( - ::toml::get::type>(a.at(I))...); -} -} // detail - -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value& v) -{ - const auto& ar = v.as_array(); - if(ar.size() != std::tuple_size::value) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "toml::get: specified std::tuple with ", - std::tuple_size::value, " elements, but there are ", ar.size(), - " elements in toml array."), {{v.location(), "here"}})); - } - return detail::get_tuple_impl(ar, - detail::make_index_sequence::value>{}); -} - -// ============================================================================ -// map-like types; most likely STL map, like std::map or std::unordered_map. - -template class M, template class V> -detail::enable_if_t, // T is map - detail::negation< // but not toml::array - detail::is_exact_toml_type>> - >::value, T> -get(const basic_value& v) -{ - using key_type = typename T::key_type; - using mapped_type = typename T::mapped_type; - static_assert(std::is_convertible::value, - "toml::get only supports map type of which key_type is " - "convertible from std::string."); - T map; - for(const auto& kv : v.as_table()) - { - map.emplace(key_type(kv.first), get(kv.second)); - } - return map; -} - -// ============================================================================ -// user-defined, but compatible types. - -template class M, template class V> -detail::enable_if_t>>, - detail::has_from_toml_method, // but has from_toml(toml::value) memfn - std::is_default_constructible // and default constructible - >::value, T> -get(const basic_value& v) -{ - T ud; - ud.from_toml(v); - return ud; -} -template class M, template class V> -detail::enable_if_t::value, T> -get(const basic_value& v) -{ - return ::toml::from::from_toml(v); -} - -template class M, template class V> -detail::enable_if_t>, // T is not a toml::value - std::is_constructible&>, // T is constructible from toml::value - detail::negation>, // and T does not have T.from_toml(v); - detail::negation> // and T does not have toml::from{}; - >::value, T> -get(const basic_value& v) -{ - return T(v); -} - -// ============================================================================ -// find - -// ---------------------------------------------------------------------------- -// these overloads do not require to set T. and returns value itself. -template class M, template class V> -basic_value const& find(const basic_value& v, const key& ky) -{ - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return tab.at(ky); -} -template class M, template class V> -basic_value& find(basic_value& v, const key& ky) -{ - auto& tab = v.as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return tab.at(ky); -} -template class M, template class V> -basic_value find(basic_value&& v, const key& ky) -{ - typename basic_value::table_type tab = std::move(v).as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return basic_value(std::move(tab.at(ky))); -} - -// ---------------------------------------------------------------------------- -// find(value, idx) -template class M, template class V> -basic_value const& -find(const basic_value& v, const std::size_t idx) -{ - const auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ary.at(idx); -} -template class M, template class V> -basic_value& find(basic_value& v, const std::size_t idx) -{ - auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ary.at(idx); -} -template class M, template class V> -basic_value find(basic_value&& v, const std::size_t idx) -{ - auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return basic_value(std::move(ary.at(idx))); -} - -// ---------------------------------------------------------------------------- -// find(value, key); - -template class M, template class V> -decltype(::toml::get(std::declval const&>())) -find(const basic_value& v, const key& ky) -{ - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return ::toml::get(tab.at(ky)); -} - -template class M, template class V> -decltype(::toml::get(std::declval&>())) -find(basic_value& v, const key& ky) -{ - auto& tab = v.as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return ::toml::get(tab.at(ky)); -} - -template class M, template class V> -decltype(::toml::get(std::declval&&>())) -find(basic_value&& v, const key& ky) -{ - typename basic_value::table_type tab = std::move(v).as_table(); - if(tab.count(ky) == 0) - { - detail::throw_key_not_found_error(v, ky); - } - return ::toml::get(std::move(tab.at(ky))); -} - -// ---------------------------------------------------------------------------- -// find(value, idx) -template class M, template class V> -decltype(::toml::get(std::declval const&>())) -find(const basic_value& v, const std::size_t idx) -{ - const auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ::toml::get(ary.at(idx)); -} -template class M, template class V> -decltype(::toml::get(std::declval&>())) -find(basic_value& v, const std::size_t idx) -{ - auto& ary = v.as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ::toml::get(ary.at(idx)); -} -template class M, template class V> -decltype(::toml::get(std::declval&&>())) -find(basic_value&& v, const std::size_t idx) -{ - typename basic_value::array_type ary = std::move(v).as_array(); - if(ary.size() <= idx) - { - throw std::out_of_range(detail::format_underline(concat_to_string( - "index ", idx, " is out of range"), {{v.location(), "in this array"}})); - } - return ::toml::get(std::move(ary.at(idx))); -} - -// -------------------------------------------------------------------------- -// toml::find(toml::value, toml::key, Ts&& ... keys) - -namespace detail -{ -// It suppresses warnings by -Wsign-conversion. Let's say we have the following -// code. -// ```cpp -// const auto x = toml::find(data, "array", 0); -// ``` -// Here, the type of literal number `0` is `int`. `int` is a signed integer. -// `toml::find` takes `std::size_t` as an index. So it causes implicit sign -// conversion and `-Wsign-conversion` warns about it. Using `0u` instead of `0` -// suppresses the warning, but it makes user code messy. -// To suppress this warning, we need to be aware of type conversion caused -// by `toml::find(v, key1, key2, ... keys)`. But the thing is that the types of -// keys can be any combination of {string-like, size_t-like}. Of course we can't -// write down all the combinations. Thus we need to use some function that -// recognize the type of argument and cast it into `std::string` or -// `std::size_t` depending on the context. -// `key_cast` does the job. It has 2 overloads. One is invoked when the -// argument type is an integer and cast the argument into `std::size_t`. The -// other is invoked when the argument type is not an integer, possibly one of -// std::string, const char[N] or const char*, and construct std::string from -// the argument. -// `toml::find(v, k1, k2, ... ks)` uses `key_cast` before passing `ks` to -// `toml::find(v, k)` to suppress -Wsign-conversion. - -template -enable_if_t>, - negation, bool>>>::value, std::size_t> -key_cast(T&& v) noexcept -{ - return std::size_t(v); -} -template -enable_if_t>, - negation, bool>>>>::value, std::string> -key_cast(T&& v) noexcept -{ - return std::string(std::forward(v)); -} -} // detail - -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -const basic_value& -find(const basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(v, detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -basic_value& -find(basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(v, detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -basic_value -find(basic_value&& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(std::move(v), std::forward(k1)), - detail::key_cast(k2), std::forward(keys)...); -} - -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -decltype(::toml::get(std::declval&>())) -find(const basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(v, detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -decltype(::toml::get(std::declval&>())) -find(basic_value& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(v, detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} -template class M, template class V, - typename Key1, typename Key2, typename ... Keys> -decltype(::toml::get(std::declval&&>())) -find(basic_value&& v, Key1&& k1, Key2&& k2, Keys&& ... keys) -{ - return ::toml::find(::toml::find(std::move(v), detail::key_cast(k1)), - detail::key_cast(k2), std::forward(keys)...); -} - -// ============================================================================ -// get_or(value, fallback) - -template class M, template class V> -basic_value const& -get_or(const basic_value& v, const basic_value&) -{ - return v; -} -template class M, template class V> -basic_value& -get_or(basic_value& v, basic_value&) -{ - return v; -} -template class M, template class V> -basic_value -get_or(basic_value&& v, basic_value&&) -{ - return v; -} - -// ---------------------------------------------------------------------------- -// specialization for the exact toml types (return type becomes lvalue ref) - -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, T> const& -get_or(const basic_value& v, const T& opt) -{ - try - { - return get>(v); - } - catch(...) - { - return opt; - } -} -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, T>& -get_or(basic_value& v, T& opt) -{ - try - { - return get>(v); - } - catch(...) - { - return opt; - } -} -template class M, template class V> -detail::enable_if_t, - basic_value>::value, detail::remove_cvref_t> -get_or(basic_value&& v, T&& opt) -{ - try - { - return get>(std::move(v)); - } - catch(...) - { - return detail::remove_cvref_t(std::forward(opt)); - } -} - -// ---------------------------------------------------------------------------- -// specialization for std::string (return type becomes lvalue ref) - -template class M, template class V> -detail::enable_if_t, std::string>::value, - std::string> const& -get_or(const basic_value& v, const T& opt) -{ - try - { - return v.as_string().str; - } - catch(...) - { - return opt; - } -} -template class M, template class V> -detail::enable_if_t::value, std::string>& -get_or(basic_value& v, T& opt) -{ - try - { - return v.as_string().str; - } - catch(...) - { - return opt; - } -} -template class M, template class V> -detail::enable_if_t< - std::is_same, std::string>::value, std::string> -get_or(basic_value&& v, T&& opt) -{ - try - { - return std::move(v.as_string().str); - } - catch(...) - { - return std::string(std::forward(opt)); - } -} - -// ---------------------------------------------------------------------------- -// specialization for string literal - -template class M, template class V> -detail::enable_if_t::type>::value, std::string> -get_or(const basic_value& v, T&& opt) -{ - try - { - return std::move(v.as_string().str); - } - catch(...) - { - return std::string(std::forward(opt)); - } -} - -// ---------------------------------------------------------------------------- -// others (require type conversion and return type cannot be lvalue reference) - -template class M, template class V> -detail::enable_if_t, - basic_value>>, - detail::negation>>, - detail::negation::type>> - >::value, detail::remove_cvref_t> -get_or(const basic_value& v, T&& opt) -{ - try - { - return get>(v); - } - catch(...) - { - return detail::remove_cvref_t(std::forward(opt)); - } -} - -// =========================================================================== -// find_or(value, key, fallback) - -template class M, template class V> -basic_value const& -find_or(const basic_value& v, const key& ky, - const basic_value& opt) -{ - if(!v.is_table()) {return opt;} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return tab.at(ky); -} - -template class M, template class V> -basic_value& -find_or(basic_value& v, const toml::key& ky, basic_value& opt) -{ - if(!v.is_table()) {return opt;} - auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return tab.at(ky); -} - -template class M, template class V> -basic_value -find_or(basic_value&& v, const toml::key& ky, basic_value&& opt) -{ - if(!v.is_table()) {return opt;} - auto tab = std::move(v).as_table(); - if(tab.count(ky) == 0) {return opt;} - return basic_value(std::move(tab.at(ky))); -} - -// --------------------------------------------------------------------------- -// exact types (return type can be a reference) -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, T> const& -find_or(const basic_value& v, const key& ky, const T& opt) -{ - if(!v.is_table()) {return opt;} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return get_or(tab.at(ky), opt); -} - -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, T>& -find_or(basic_value& v, const toml::key& ky, T& opt) -{ - if(!v.is_table()) {return opt;} - auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return get_or(tab.at(ky), opt); -} - -template class M, template class V> -detail::enable_if_t< - detail::is_exact_toml_type>::value, - detail::remove_cvref_t> -find_or(basic_value&& v, const toml::key& ky, T&& opt) -{ - if(!v.is_table()) {return std::forward(opt);} - auto tab = std::move(v).as_table(); - if(tab.count(ky) == 0) {return std::forward(opt);} - return get_or(std::move(tab.at(ky)), std::forward(opt)); -} - -// --------------------------------------------------------------------------- -// std::string (return type can be a reference) - -template class M, template class V> -detail::enable_if_t::value, std::string> const& -find_or(const basic_value& v, const key& ky, const T& opt) -{ - if(!v.is_table()) {return opt;} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return get_or(tab.at(ky), opt); -} -template class M, template class V> -detail::enable_if_t::value, std::string>& -find_or(basic_value& v, const toml::key& ky, T& opt) -{ - if(!v.is_table()) {return opt;} - auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return opt;} - return get_or(tab.at(ky), opt); -} -template class M, template class V> -detail::enable_if_t::value, std::string> -find_or(basic_value&& v, const toml::key& ky, T&& opt) -{ - if(!v.is_table()) {return std::forward(opt);} - auto tab = std::move(v).as_table(); - if(tab.count(ky) == 0) {return std::forward(opt);} - return get_or(std::move(tab.at(ky)), std::forward(opt)); -} - -// --------------------------------------------------------------------------- -// string literal (deduced as std::string) -template class M, template class V> -detail::enable_if_t< - detail::is_string_literal::type>::value, - std::string> -find_or(const basic_value& v, const toml::key& ky, T&& opt) -{ - if(!v.is_table()) {return std::string(opt);} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return std::string(opt);} - return get_or(tab.at(ky), std::forward(opt)); -} - -// --------------------------------------------------------------------------- -// others (require type conversion and return type cannot be lvalue reference) -template class M, template class V> -detail::enable_if_t, basic_value>>, - // T is not std::string - detail::negation>>, - // T is not a string literal - detail::negation::type>> - >::value, detail::remove_cvref_t> -find_or(const basic_value& v, const toml::key& ky, T&& opt) -{ - if(!v.is_table()) {return std::forward(opt);} - const auto& tab = v.as_table(); - if(tab.count(ky) == 0) {return std::forward(opt);} - return get_or(tab.at(ky), std::forward(opt)); -} - -// --------------------------------------------------------------------------- -// recursive find-or with type deduction (find_or(value, keys, opt)) - -template 1), std::nullptr_t> = nullptr> - // here we need to add SFINAE in the template parameter to avoid - // infinite recursion in type deduction on gcc -auto find_or(Value&& v, const toml::key& ky, Ks&& ... keys) - -> decltype(find_or(std::forward(v), ky, detail::last_one(std::forward(keys)...))) -{ - if(!v.is_table()) - { - return detail::last_one(std::forward(keys)...); - } - auto&& tab = std::forward(v).as_table(); - if(tab.count(ky) == 0) - { - return detail::last_one(std::forward(keys)...); - } - return find_or(std::forward(tab).at(ky), std::forward(keys)...); -} - -// --------------------------------------------------------------------------- -// recursive find_or with explicit type specialization, find_or(value, keys...) - -template 1), std::nullptr_t> = nullptr> - // here we need to add SFINAE in the template parameter to avoid - // infinite recursion in type deduction on gcc -auto find_or(Value&& v, const toml::key& ky, Ks&& ... keys) - -> decltype(find_or(std::forward(v), ky, detail::last_one(std::forward(keys)...))) -{ - if(!v.is_table()) - { - return detail::last_one(std::forward(keys)...); - } - auto&& tab = std::forward(v).as_table(); - if(tab.count(ky) == 0) - { - return detail::last_one(std::forward(keys)...); - } - return find_or(std::forward(tab).at(ky), std::forward(keys)...); -} - -// ============================================================================ -// expect - -template class M, template class V> -result expect(const basic_value& v) noexcept -{ - try - { - return ok(get(v)); - } - catch(const std::exception& e) - { - return err(e.what()); - } -} -template class M, template class V> -result -expect(const basic_value& v, const toml::key& k) noexcept -{ - try - { - return ok(find(v, k)); - } - catch(const std::exception& e) - { - return err(e.what()); - } -} - -} // toml -#endif// TOML11_GET diff --git a/src/toml11/toml/into.hpp b/src/toml11/toml/into.hpp deleted file mode 100644 index 74495560e..000000000 --- a/src/toml11/toml/into.hpp +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_INTO_HPP -#define TOML11_INTO_HPP - -namespace toml -{ - -template -struct into; -// { -// static toml::value into_toml(const T& user_defined_type) -// { -// // User-defined conversions ... -// } -// }; - -} // toml -#endif // TOML11_INTO_HPP diff --git a/src/toml11/toml/lexer.hpp b/src/toml11/toml/lexer.hpp deleted file mode 100644 index ea5050b8d..000000000 --- a/src/toml11/toml/lexer.hpp +++ /dev/null @@ -1,293 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_LEXER_HPP -#define TOML11_LEXER_HPP -#include -#include -#include -#include - -#include "combinator.hpp" - -namespace toml -{ -namespace detail -{ - -// these scans contents from current location in a container of char -// and extract a region that matches their own pattern. -// to see the implementation of each component, see combinator.hpp. - -using lex_wschar = either, character<'\t'>>; -using lex_ws = repeat>; -using lex_newline = either, - sequence, character<'\n'>>>; -using lex_lower = in_range<'a', 'z'>; -using lex_upper = in_range<'A', 'Z'>; -using lex_alpha = either; -using lex_digit = in_range<'0', '9'>; -using lex_nonzero = in_range<'1', '9'>; -using lex_oct_dig = in_range<'0', '7'>; -using lex_bin_dig = in_range<'0', '1'>; -using lex_hex_dig = either, in_range<'a', 'f'>>; - -using lex_hex_prefix = sequence, character<'x'>>; -using lex_oct_prefix = sequence, character<'o'>>; -using lex_bin_prefix = sequence, character<'b'>>; -using lex_underscore = character<'_'>; -using lex_plus = character<'+'>; -using lex_minus = character<'-'>; -using lex_sign = either; - -// digit | nonzero 1*(digit | _ digit) -using lex_unsigned_dec_int = either>, at_least<1>>>, - lex_digit>; -// (+|-)? unsigned_dec_int -using lex_dec_int = sequence, lex_unsigned_dec_int>; - -// hex_prefix hex_dig *(hex_dig | _ hex_dig) -using lex_hex_int = sequence>, unlimited>>>; -// oct_prefix oct_dig *(oct_dig | _ oct_dig) -using lex_oct_int = sequence>, unlimited>>>; -// bin_prefix bin_dig *(bin_dig | _ bin_dig) -using lex_bin_int = sequence>, unlimited>>>; - -// (dec_int | hex_int | oct_int | bin_int) -using lex_integer = either; - -// =========================================================================== - -using lex_inf = sequence, character<'n'>, character<'f'>>; -using lex_nan = sequence, character<'a'>, character<'n'>>; -using lex_special_float = sequence, either>; - -using lex_zero_prefixable_int = sequence>, unlimited>>; - -using lex_fractional_part = sequence, lex_zero_prefixable_int>; - -using lex_exponent_part = sequence, character<'E'>>, - maybe, lex_zero_prefixable_int>; - -using lex_float = either>>>>; - -// =========================================================================== - -using lex_true = sequence, character<'r'>, - character<'u'>, character<'e'>>; -using lex_false = sequence, character<'a'>, character<'l'>, - character<'s'>, character<'e'>>; -using lex_boolean = either; - -// =========================================================================== - -using lex_date_fullyear = repeat>; -using lex_date_month = repeat>; -using lex_date_mday = repeat>; -using lex_time_delim = either, character<'t'>, character<' '>>; -using lex_time_hour = repeat>; -using lex_time_minute = repeat>; -using lex_time_second = repeat>; -using lex_time_secfrac = sequence, - repeat>>; - -using lex_time_numoffset = sequence, character<'-'>>, - sequence, - lex_time_minute>>; -using lex_time_offset = either, character<'z'>, - lex_time_numoffset>; - -using lex_partial_time = sequence, - lex_time_minute, character<':'>, - lex_time_second, maybe>; -using lex_full_date = sequence, - lex_date_month, character<'-'>, - lex_date_mday>; -using lex_full_time = sequence; - -using lex_offset_date_time = sequence; -using lex_local_date_time = sequence; -using lex_local_date = lex_full_date; -using lex_local_time = lex_partial_time; - -// =========================================================================== - -using lex_quotation_mark = character<'"'>; -using lex_basic_unescaped = exclude, // 0x09 (tab) is allowed - in_range<0x0A, 0x1F>, - character<0x22>, character<0x5C>, - character<0x7F>>>; - -using lex_escape = character<'\\'>; -using lex_escape_unicode_short = sequence, - repeat>>; -using lex_escape_unicode_long = sequence, - repeat>>; -using lex_escape_seq_char = either, character<'\\'>, - character<'b'>, character<'f'>, - character<'n'>, character<'r'>, - character<'t'>, - lex_escape_unicode_short, - lex_escape_unicode_long - >; -using lex_escaped = sequence; -using lex_basic_char = either; -using lex_basic_string = sequence, - lex_quotation_mark>; - -// After toml post-v0.5.0, it is explicitly clarified how quotes in ml-strings -// are allowed to be used. -// After this, the following strings are *explicitly* allowed. -// - One or two `"`s in a multi-line basic string is allowed wherever it is. -// - Three consecutive `"`s in a multi-line basic string is considered as a delimiter. -// - One or two `"`s can appear just before or after the delimiter. -// ```toml -// str4 = """Here are two quotation marks: "". Simple enough.""" -// str5 = """Here are three quotation marks: ""\".""" -// str6 = """Here are fifteen quotation marks: ""\"""\"""\"""\"""\".""" -// str7 = """"This," she said, "is just a pointless statement."""" -// ``` -// In the current implementation (v3.3.0), it is difficult to parse `str7` in -// the above example. It is difficult to recognize `"` at the end of string body -// collectly. It will be misunderstood as a `"""` delimiter and an additional, -// invalid `"`. Like this: -// ```console -// what(): [error] toml::parse_table: invalid line format -// --> hoge.toml -// | -// 13 | str7 = """"This," she said, "is just a pointless statement."""" -// | ^- expected newline, but got '"'. -// ``` -// As a quick workaround for this problem, `lex_ml_basic_string_delim` was -// split into two, `lex_ml_basic_string_open` and `lex_ml_basic_string_close`. -// `lex_ml_basic_string_open` allows only `"""`. `_close` allows 3-5 `"`s. -// In parse_ml_basic_string() function, the trailing `"`s will be attached to -// the string body. -// -using lex_ml_basic_string_delim = repeat>; -using lex_ml_basic_string_open = lex_ml_basic_string_delim; -using lex_ml_basic_string_close = sequence< - repeat>, - maybe, maybe - >; - -using lex_ml_basic_unescaped = exclude, // 0x09 is tab - in_range<0x0A, 0x1F>, - character<0x5C>, // backslash - character<0x7F>, // DEL - lex_ml_basic_string_delim>>; - -using lex_ml_basic_escaped_newline = sequence< - lex_escape, maybe, lex_newline, - repeat, unlimited>>; - -using lex_ml_basic_char = either; -using lex_ml_basic_body = repeat, - unlimited>; -using lex_ml_basic_string = sequence; - -using lex_literal_char = exclude, in_range<0x0A, 0x1F>, - character<0x7F>, character<0x27>>>; -using lex_apostrophe = character<'\''>; -using lex_literal_string = sequence, - lex_apostrophe>; - -// the same reason as above. -using lex_ml_literal_string_delim = repeat>; -using lex_ml_literal_string_open = lex_ml_literal_string_delim; -using lex_ml_literal_string_close = sequence< - repeat>, - maybe, maybe - >; - -using lex_ml_literal_char = exclude, - in_range<0x0A, 0x1F>, - character<0x7F>, - lex_ml_literal_string_delim>>; -using lex_ml_literal_body = repeat, - unlimited>; -using lex_ml_literal_string = sequence; - -using lex_string = either; - -// =========================================================================== -using lex_dot_sep = sequence, character<'.'>, maybe>; - -using lex_unquoted_key = repeat, character<'_'>>, - at_least<1>>; -using lex_quoted_key = either; -using lex_simple_key = either; -using lex_dotted_key = sequence, - at_least<1> - > - >; -using lex_key = either; - -using lex_keyval_sep = sequence, - character<'='>, - maybe>; - -using lex_std_table_open = character<'['>; -using lex_std_table_close = character<']'>; -using lex_std_table = sequence, - lex_key, - maybe, - lex_std_table_close>; - -using lex_array_table_open = sequence; -using lex_array_table_close = sequence; -using lex_array_table = sequence, - lex_key, - maybe, - lex_array_table_close>; - -using lex_utf8_1byte = in_range<0x00, 0x7F>; -using lex_utf8_2byte = sequence< - in_range(0xC2), static_cast(0xDF)>, - in_range(0x80), static_cast(0xBF)> - >; -using lex_utf8_3byte = sequence(0xE0)>, in_range(0xA0), static_cast(0xBF)>>, - sequence(0xE1), static_cast(0xEC)>, in_range(0x80), static_cast(0xBF)>>, - sequence(0xED)>, in_range(0x80), static_cast(0x9F)>>, - sequence(0xEE), static_cast(0xEF)>, in_range(0x80), static_cast(0xBF)>> - >, in_range(0x80), static_cast(0xBF)>>; -using lex_utf8_4byte = sequence(0xF0)>, in_range(0x90), static_cast(0xBF)>>, - sequence(0xF1), static_cast(0xF3)>, in_range(0x80), static_cast(0xBF)>>, - sequence(0xF4)>, in_range(0x80), static_cast(0x8F)>> - >, in_range(0x80), static_cast(0xBF)>, - in_range(0x80), static_cast(0xBF)>>; -using lex_utf8_code = either< - lex_utf8_1byte, - lex_utf8_2byte, - lex_utf8_3byte, - lex_utf8_4byte - >; - -using lex_comment_start_symbol = character<'#'>; -using lex_non_eol_ascii = either, in_range<0x20, 0x7E>>; -using lex_comment = sequence, unlimited>>; - -} // detail -} // toml -#endif // TOML_LEXER_HPP diff --git a/src/toml11/toml/literal.hpp b/src/toml11/toml/literal.hpp deleted file mode 100644 index 04fbbc13e..000000000 --- a/src/toml11/toml/literal.hpp +++ /dev/null @@ -1,113 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_LITERAL_HPP -#define TOML11_LITERAL_HPP -#include "parser.hpp" - -namespace toml -{ -inline namespace literals -{ -inline namespace toml_literals -{ - -// implementation -inline ::toml::basic_value -literal_internal_impl(::toml::detail::location loc) -{ - using value_type = ::toml::basic_value< - TOML11_DEFAULT_COMMENT_STRATEGY, std::unordered_map, std::vector>; - // if there are some comments or empty lines, skip them. - using skip_line = ::toml::detail::repeat, - ::toml::detail::maybe<::toml::detail::lex_comment>, - ::toml::detail::lex_newline - >, ::toml::detail::at_least<1>>; - skip_line::invoke(loc); - - // if there are some whitespaces before a value, skip them. - using skip_ws = ::toml::detail::repeat< - ::toml::detail::lex_ws, ::toml::detail::at_least<1>>; - skip_ws::invoke(loc); - - // to distinguish arrays and tables, first check it is a table or not. - // - // "[1,2,3]"_toml; // this is an array - // "[table]"_toml; // a table that has an empty table named "table" inside. - // "[[1,2,3]]"_toml; // this is an array of arrays - // "[[table]]"_toml; // this is a table that has an array of tables inside. - // - // "[[1]]"_toml; // this can be both... (currently it becomes a table) - // "1 = [{}]"_toml; // this is a table that has an array of table named 1. - // "[[1,]]"_toml; // this is an array of arrays. - // "[[1],]"_toml; // this also. - - const auto the_front = loc.iter(); - - const bool is_table_key = ::toml::detail::lex_std_table::invoke(loc); - loc.reset(the_front); - - const bool is_aots_key = ::toml::detail::lex_array_table::invoke(loc); - loc.reset(the_front); - - // If it is neither a table-key or a array-of-table-key, it may be a value. - if(!is_table_key && !is_aots_key) - { - if(auto data = ::toml::detail::parse_value(loc)) - { - return data.unwrap(); - } - } - - // Note that still it can be a table, because the literal might be something - // like the following. - // ```cpp - // R"( // c++11 raw string literals - // key = "value" - // int = 42 - // )"_toml; - // ``` - // It is a valid toml file. - // It should be parsed as if we parse a file with this content. - - if(auto data = ::toml::detail::parse_toml_file(loc)) - { - return data.unwrap(); - } - else // none of them. - { - throw ::toml::syntax_error(data.unwrap_err(), source_location(loc)); - } - -} - -inline ::toml::basic_value -operator"" _toml(const char* str, std::size_t len) -{ - ::toml::detail::location loc( - std::string("TOML literal encoded in a C++ code"), - std::vector(str, str + len)); - // literal length does not include the null character at the end. - return literal_internal_impl(std::move(loc)); -} - -// value of __cplusplus in C++2a/20 mode is not fixed yet along compilers. -// So here we use the feature test macro for `char8_t` itself. -#if defined(__cpp_char8_t) && __cpp_char8_t >= 201811L -// value of u8"" literal has been changed from char to char8_t and char8_t is -// NOT compatible to char -inline ::toml::basic_value -operator"" _toml(const char8_t* str, std::size_t len) -{ - ::toml::detail::location loc( - std::string("TOML literal encoded in a C++ code"), - std::vector(reinterpret_cast(str), - reinterpret_cast(str) + len)); - return literal_internal_impl(std::move(loc)); -} -#endif - -} // toml_literals -} // literals -} // toml -#endif//TOML11_LITERAL_HPP diff --git a/src/toml11/toml/macros.hpp b/src/toml11/toml/macros.hpp deleted file mode 100644 index e8f91aecd..000000000 --- a/src/toml11/toml/macros.hpp +++ /dev/null @@ -1,121 +0,0 @@ -#ifndef TOML11_MACROS_HPP -#define TOML11_MACROS_HPP - -#define TOML11_STRINGIZE_AUX(x) #x -#define TOML11_STRINGIZE(x) TOML11_STRINGIZE_AUX(x) - -#define TOML11_CONCATENATE_AUX(x, y) x##y -#define TOML11_CONCATENATE(x, y) TOML11_CONCATENATE_AUX(x, y) - -// ============================================================================ -// TOML11_DEFINE_CONVERSION_NON_INTRUSIVE - -#ifndef TOML11_WITHOUT_DEFINE_NON_INTRUSIVE - -// ---------------------------------------------------------------------------- -// TOML11_ARGS_SIZE - -#define TOML11_INDEX_RSEQ() \ - 32, 31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, \ - 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0 -#define TOML11_ARGS_SIZE_IMPL(\ - ARG1, ARG2, ARG3, ARG4, ARG5, ARG6, ARG7, ARG8, ARG9, ARG10, \ - ARG11, ARG12, ARG13, ARG14, ARG15, ARG16, ARG17, ARG18, ARG19, ARG20, \ - ARG21, ARG22, ARG23, ARG24, ARG25, ARG26, ARG27, ARG28, ARG29, ARG30, \ - ARG31, ARG32, N, ...) N -#define TOML11_ARGS_SIZE_AUX(...) TOML11_ARGS_SIZE_IMPL(__VA_ARGS__) -#define TOML11_ARGS_SIZE(...) TOML11_ARGS_SIZE_AUX(__VA_ARGS__, TOML11_INDEX_RSEQ()) - -// ---------------------------------------------------------------------------- -// TOML11_FOR_EACH_VA_ARGS - -#define TOML11_FOR_EACH_VA_ARGS_AUX_1( FUNCTOR, ARG1 ) FUNCTOR(ARG1) -#define TOML11_FOR_EACH_VA_ARGS_AUX_2( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_1( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_3( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_2( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_4( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_3( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_5( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_4( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_6( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_5( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_7( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_6( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_8( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_7( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_9( FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_8( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_10(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_9( FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_11(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_10(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_12(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_11(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_13(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_12(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_14(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_13(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_15(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_14(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_16(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_15(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_17(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_16(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_18(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_17(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_19(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_18(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_20(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_19(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_21(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_20(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_22(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_21(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_23(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_22(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_24(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_23(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_25(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_24(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_26(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_25(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_27(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_26(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_28(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_27(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_29(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_28(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_30(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_29(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_31(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_30(FUNCTOR, __VA_ARGS__) -#define TOML11_FOR_EACH_VA_ARGS_AUX_32(FUNCTOR, ARG1, ...) FUNCTOR(ARG1) TOML11_FOR_EACH_VA_ARGS_AUX_31(FUNCTOR, __VA_ARGS__) - -#define TOML11_FOR_EACH_VA_ARGS(FUNCTOR, ...)\ - TOML11_CONCATENATE(TOML11_FOR_EACH_VA_ARGS_AUX_, TOML11_ARGS_SIZE(__VA_ARGS__))(FUNCTOR, __VA_ARGS__) - -// ---------------------------------------------------------------------------- -// TOML11_DEFINE_CONVERSION_NON_INTRUSIVE - -// use it in the following way. -// ```cpp -// namespace foo -// { -// struct Foo -// { -// std::string s; -// double d; -// int i; -// }; -// } // foo -// -// TOML11_DEFINE_CONVERSION_NON_INTRUSIVE(foo::Foo, s, d, i) -// ``` -// And then you can use `toml::find(file, "foo");` -// -#define TOML11_FIND_MEMBER_VARIABLE_FROM_VALUE(VAR_NAME)\ - obj.VAR_NAME = toml::find(v, TOML11_STRINGIZE(VAR_NAME)); - -#define TOML11_ASSIGN_MEMBER_VARIABLE_TO_VALUE(VAR_NAME)\ - v[TOML11_STRINGIZE(VAR_NAME)] = obj.VAR_NAME; - -#define TOML11_DEFINE_CONVERSION_NON_INTRUSIVE(NAME, ...)\ - namespace toml { \ - template<> \ - struct from \ - { \ - template class T, \ - template class A> \ - static NAME from_toml(const basic_value& v) \ - { \ - NAME obj; \ - TOML11_FOR_EACH_VA_ARGS(TOML11_FIND_MEMBER_VARIABLE_FROM_VALUE, __VA_ARGS__) \ - return obj; \ - } \ - }; \ - template<> \ - struct into \ - { \ - static value into_toml(const NAME& obj) \ - { \ - ::toml::value v = ::toml::table{}; \ - TOML11_FOR_EACH_VA_ARGS(TOML11_ASSIGN_MEMBER_VARIABLE_TO_VALUE, __VA_ARGS__) \ - return v; \ - } \ - }; \ - } /* toml */ - -#endif// TOML11_WITHOUT_DEFINE_NON_INTRUSIVE - -#endif// TOML11_MACROS_HPP diff --git a/src/toml11/toml/parser.hpp b/src/toml11/toml/parser.hpp deleted file mode 100644 index e31179918..000000000 --- a/src/toml11/toml/parser.hpp +++ /dev/null @@ -1,2364 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_PARSER_HPP -#define TOML11_PARSER_HPP -#include -#include -#include - -#include "combinator.hpp" -#include "lexer.hpp" -#include "region.hpp" -#include "result.hpp" -#include "types.hpp" -#include "value.hpp" - -#ifndef TOML11_DISABLE_STD_FILESYSTEM -#ifdef __cpp_lib_filesystem -#if __has_include() -#define TOML11_HAS_STD_FILESYSTEM -#include -#endif // has_include() -#endif // __cpp_lib_filesystem -#endif // TOML11_DISABLE_STD_FILESYSTEM - -namespace toml -{ -namespace detail -{ - -inline result, std::string> -parse_boolean(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_boolean::invoke(loc)) - { - const auto reg = token.unwrap(); - if (reg.str() == "true") {return ok(std::make_pair(true, reg));} - else if(reg.str() == "false") {return ok(std::make_pair(false, reg));} - else // internal error. - { - throw internal_error(format_underline( - "toml::parse_boolean: internal error", - {{source_location(reg), "invalid token"}}), - source_location(reg)); - } - } - loc.reset(first); //rollback - return err(format_underline("toml::parse_boolean: ", - {{source_location(loc), "the next token is not a boolean"}})); -} - -inline result, std::string> -parse_binary_integer(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_bin_int::invoke(loc)) - { - auto str = token.unwrap().str(); - assert(str.size() > 2); // minimum -> 0b1 - integer retval(0), base(1); - for(auto i(str.rbegin()), e(str.rend() - 2); i!=e; ++i) - { - if (*i == '1'){retval += base; base *= 2;} - else if(*i == '0'){base *= 2;} - else if(*i == '_'){/* do nothing. */} - else // internal error. - { - throw internal_error(format_underline( - "toml::parse_integer: internal error", - {{source_location(token.unwrap()), "invalid token"}}), - source_location(loc)); - } - } - return ok(std::make_pair(retval, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_binary_integer:", - {{source_location(loc), "the next token is not an integer"}})); -} - -inline result, std::string> -parse_octal_integer(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_oct_int::invoke(loc)) - { - auto str = token.unwrap().str(); - str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); - str.erase(str.begin()); str.erase(str.begin()); // remove `0o` prefix - - std::istringstream iss(str); - integer retval(0); - iss >> std::oct >> retval; - return ok(std::make_pair(retval, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_octal_integer:", - {{source_location(loc), "the next token is not an integer"}})); -} - -inline result, std::string> -parse_hexadecimal_integer(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_hex_int::invoke(loc)) - { - auto str = token.unwrap().str(); - str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); - str.erase(str.begin()); str.erase(str.begin()); // remove `0x` prefix - - std::istringstream iss(str); - integer retval(0); - iss >> std::hex >> retval; - return ok(std::make_pair(retval, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_hexadecimal_integer", - {{source_location(loc), "the next token is not an integer"}})); -} - -inline result, std::string> -parse_integer(location& loc) -{ - const auto first = loc.iter(); - if(first != loc.end() && *first == '0') - { - const auto second = std::next(first); - if(second == loc.end()) // the token is just zero. - { - loc.advance(); - return ok(std::make_pair(0, region(loc, first, second))); - } - - if(*second == 'b') {return parse_binary_integer (loc);} // 0b1100 - if(*second == 'o') {return parse_octal_integer (loc);} // 0o775 - if(*second == 'x') {return parse_hexadecimal_integer(loc);} // 0xC0FFEE - - if(std::isdigit(*second)) - { - return err(format_underline("toml::parse_integer: " - "leading zero in an Integer is not allowed.", - {{source_location(loc), "leading zero"}})); - } - else if(std::isalpha(*second)) - { - return err(format_underline("toml::parse_integer: " - "unknown integer prefix appeared.", - {{source_location(loc), "none of 0x, 0o, 0b"}})); - } - } - - if(const auto token = lex_dec_int::invoke(loc)) - { - auto str = token.unwrap().str(); - str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); - - std::istringstream iss(str); - integer retval(0); - iss >> retval; - return ok(std::make_pair(retval, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_integer: ", - {{source_location(loc), "the next token is not an integer"}})); -} - -inline result, std::string> -parse_floating(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_float::invoke(loc)) - { - auto str = token.unwrap().str(); - if(str == "inf" || str == "+inf") - { - if(std::numeric_limits::has_infinity) - { - return ok(std::make_pair( - std::numeric_limits::infinity(), token.unwrap())); - } - else - { - throw std::domain_error("toml::parse_floating: inf value found" - " but the current environment does not support inf. Please" - " make sure that the floating-point implementation conforms" - " IEEE 754/ISO 60559 international standard."); - } - } - else if(str == "-inf") - { - if(std::numeric_limits::has_infinity) - { - return ok(std::make_pair( - -std::numeric_limits::infinity(), token.unwrap())); - } - else - { - throw std::domain_error("toml::parse_floating: inf value found" - " but the current environment does not support inf. Please" - " make sure that the floating-point implementation conforms" - " IEEE 754/ISO 60559 international standard."); - } - } - else if(str == "nan" || str == "+nan") - { - if(std::numeric_limits::has_quiet_NaN) - { - return ok(std::make_pair( - std::numeric_limits::quiet_NaN(), token.unwrap())); - } - else if(std::numeric_limits::has_signaling_NaN) - { - return ok(std::make_pair( - std::numeric_limits::signaling_NaN(), token.unwrap())); - } - else - { - throw std::domain_error("toml::parse_floating: NaN value found" - " but the current environment does not support NaN. Please" - " make sure that the floating-point implementation conforms" - " IEEE 754/ISO 60559 international standard."); - } - } - else if(str == "-nan") - { - if(std::numeric_limits::has_quiet_NaN) - { - return ok(std::make_pair( - -std::numeric_limits::quiet_NaN(), token.unwrap())); - } - else if(std::numeric_limits::has_signaling_NaN) - { - return ok(std::make_pair( - -std::numeric_limits::signaling_NaN(), token.unwrap())); - } - else - { - throw std::domain_error("toml::parse_floating: NaN value found" - " but the current environment does not support NaN. Please" - " make sure that the floating-point implementation conforms" - " IEEE 754/ISO 60559 international standard."); - } - } - str.erase(std::remove(str.begin(), str.end(), '_'), str.end()); - std::istringstream iss(str); - floating v(0.0); - iss >> v; - return ok(std::make_pair(v, token.unwrap())); - } - loc.reset(first); - return err(format_underline("toml::parse_floating: ", - {{source_location(loc), "the next token is not a float"}})); -} - -inline std::string read_utf8_codepoint(const region& reg, const location& loc) -{ - const auto str = reg.str().substr(1); - std::uint_least32_t codepoint; - std::istringstream iss(str); - iss >> std::hex >> codepoint; - - const auto to_char = [](const std::uint_least32_t i) noexcept -> char { - const auto uc = static_cast(i); - return *reinterpret_cast(std::addressof(uc)); - }; - - std::string character; - if(codepoint < 0x80) // U+0000 ... U+0079 ; just an ASCII. - { - character += static_cast(codepoint); - } - else if(codepoint < 0x800) //U+0080 ... U+07FF - { - // 110yyyyx 10xxxxxx; 0x3f == 0b0011'1111 - character += to_char(0xC0| codepoint >> 6); - character += to_char(0x80|(codepoint & 0x3F)); - } - else if(codepoint < 0x10000) // U+0800...U+FFFF - { - if(0xD800 <= codepoint && codepoint <= 0xDFFF) - { - throw syntax_error(format_underline( - "toml::read_utf8_codepoint: codepoints in the range " - "[0xD800, 0xDFFF] are not valid UTF-8.", {{ - source_location(loc), "not a valid UTF-8 codepoint" - }}), source_location(loc)); - } - assert(codepoint < 0xD800 || 0xDFFF < codepoint); - // 1110yyyy 10yxxxxx 10xxxxxx - character += to_char(0xE0| codepoint >> 12); - character += to_char(0x80|(codepoint >> 6 & 0x3F)); - character += to_char(0x80|(codepoint & 0x3F)); - } - else if(codepoint < 0x110000) // U+010000 ... U+10FFFF - { - // 11110yyy 10yyxxxx 10xxxxxx 10xxxxxx - character += to_char(0xF0| codepoint >> 18); - character += to_char(0x80|(codepoint >> 12 & 0x3F)); - character += to_char(0x80|(codepoint >> 6 & 0x3F)); - character += to_char(0x80|(codepoint & 0x3F)); - } - else // out of UTF-8 region - { - throw syntax_error(format_underline("toml::read_utf8_codepoint:" - " input codepoint is too large.", - {{source_location(loc), "should be in [0x00..0x10FFFF]"}}), - source_location(loc)); - } - return character; -} - -inline result parse_escape_sequence(location& loc) -{ - const auto first = loc.iter(); - if(first == loc.end() || *first != '\\') - { - return err(format_underline("toml::parse_escape_sequence: ", {{ - source_location(loc), "the next token is not a backslash \"\\\""}})); - } - loc.advance(); - switch(*loc.iter()) - { - case '\\':{loc.advance(); return ok(std::string("\\"));} - case '"' :{loc.advance(); return ok(std::string("\""));} - case 'b' :{loc.advance(); return ok(std::string("\b"));} - case 't' :{loc.advance(); return ok(std::string("\t"));} - case 'n' :{loc.advance(); return ok(std::string("\n"));} - case 'f' :{loc.advance(); return ok(std::string("\f"));} - case 'r' :{loc.advance(); return ok(std::string("\r"));} - case 'u' : - { - if(const auto token = lex_escape_unicode_short::invoke(loc)) - { - return ok(read_utf8_codepoint(token.unwrap(), loc)); - } - else - { - return err(format_underline("parse_escape_sequence: " - "invalid token found in UTF-8 codepoint uXXXX.", - {{source_location(loc), "here"}})); - } - } - case 'U': - { - if(const auto token = lex_escape_unicode_long::invoke(loc)) - { - return ok(read_utf8_codepoint(token.unwrap(), loc)); - } - else - { - return err(format_underline("parse_escape_sequence: " - "invalid token found in UTF-8 codepoint Uxxxxxxxx", - {{source_location(loc), "here"}})); - } - } - } - - const auto msg = format_underline("parse_escape_sequence: " - "unknown escape sequence appeared.", {{source_location(loc), - "escape sequence is one of \\, \", b, t, n, f, r, uxxxx, Uxxxxxxxx"}}, - /* Hints = */{"if you want to write backslash as just one backslash, " - "use literal string like: regex = '<\\i\\c*\\s*>'"}); - loc.reset(first); - return err(msg); -} - -inline std::ptrdiff_t check_utf8_validity(const std::string& reg) -{ - location loc("tmp", reg); - const auto u8 = repeat::invoke(loc); - if(!u8 || loc.iter() != loc.end()) - { - const auto error_location = std::distance(loc.begin(), loc.iter()); - assert(0 <= error_location); - return error_location; - } - return -1; -} - -inline result, std::string> -parse_ml_basic_string(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_ml_basic_string::invoke(loc)) - { - auto inner_loc = loc; - inner_loc.reset(first); - - std::string retval; - retval.reserve(token.unwrap().size()); - - auto delim = lex_ml_basic_string_open::invoke(inner_loc); - if(!delim) - { - throw internal_error(format_underline( - "parse_ml_basic_string: invalid token", - {{source_location(inner_loc), "should be \"\"\""}}), - source_location(inner_loc)); - } - // immediate newline is ignored (if exists) - /* discard return value */ lex_newline::invoke(inner_loc); - - delim = none(); - while(!delim) - { - using lex_unescaped_seq = repeat< - either, unlimited>; - if(auto unescaped = lex_unescaped_seq::invoke(inner_loc)) - { - retval += unescaped.unwrap().str(); - } - if(auto escaped = parse_escape_sequence(inner_loc)) - { - retval += escaped.unwrap(); - } - if(auto esc_nl = lex_ml_basic_escaped_newline::invoke(inner_loc)) - { - // ignore newline after escape until next non-ws char - } - if(inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "parse_ml_basic_string: unexpected end of region", - {{source_location(inner_loc), "not sufficient token"}}), - source_location(inner_loc)); - } - delim = lex_ml_basic_string_close::invoke(inner_loc); - } - // `lex_ml_basic_string_close` allows 3 to 5 `"`s to allow 1 or 2 `"`s - // at just before the delimiter. Here, we need to attach `"`s at the - // end of the string body, if it exists. - // For detail, see the definition of `lex_ml_basic_string_close`. - assert(std::all_of(delim.unwrap().first(), delim.unwrap().last(), - [](const char c) noexcept {return c == '\"';})); - switch(delim.unwrap().size()) - { - case 3: {break;} - case 4: {retval += "\""; break;} - case 5: {retval += "\"\""; break;} - default: - { - throw internal_error(format_underline( - "parse_ml_basic_string: closing delimiter has invalid length", - {{source_location(inner_loc), "end of this"}}), - source_location(inner_loc)); - } - } - - const auto err_loc = check_utf8_validity(token.unwrap().str()); - if(err_loc == -1) - { - return ok(std::make_pair(toml::string(retval), token.unwrap())); - } - else - { - inner_loc.reset(first); - inner_loc.advance(err_loc); - throw syntax_error(format_underline( - "parse_ml_basic_string: invalid utf8 sequence found", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_ml_basic_string: " - "the next token is not a valid multiline string", - {{source_location(loc), "here"}})); - } -} - -inline result, std::string> -parse_basic_string(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_basic_string::invoke(loc)) - { - auto inner_loc = loc; - inner_loc.reset(first); - - auto quot = lex_quotation_mark::invoke(inner_loc); - if(!quot) - { - throw internal_error(format_underline("parse_basic_string: " - "invalid token", {{source_location(inner_loc), "should be \""}}), - source_location(inner_loc)); - } - - std::string retval; - retval.reserve(token.unwrap().size()); - - quot = none(); - while(!quot) - { - using lex_unescaped_seq = repeat; - if(auto unescaped = lex_unescaped_seq::invoke(inner_loc)) - { - retval += unescaped.unwrap().str(); - } - if(auto escaped = parse_escape_sequence(inner_loc)) - { - retval += escaped.unwrap(); - } - if(inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "parse_basic_string: unexpected end of region", - {{source_location(inner_loc), "not sufficient token"}}), - source_location(inner_loc)); - } - quot = lex_quotation_mark::invoke(inner_loc); - } - - const auto err_loc = check_utf8_validity(token.unwrap().str()); - if(err_loc == -1) - { - return ok(std::make_pair(toml::string(retval), token.unwrap())); - } - else - { - inner_loc.reset(first); - inner_loc.advance(err_loc); - throw syntax_error(format_underline( - "parse_ml_basic_string: invalid utf8 sequence found", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - else - { - loc.reset(first); // rollback - return err(format_underline("toml::parse_basic_string: " - "the next token is not a valid string", - {{source_location(loc), "here"}})); - } -} - -inline result, std::string> -parse_ml_literal_string(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_ml_literal_string::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto open = lex_ml_literal_string_open::invoke(inner_loc); - if(!open) - { - throw internal_error(format_underline( - "parse_ml_literal_string: invalid token", - {{source_location(inner_loc), "should be '''"}}), - source_location(inner_loc)); - } - // immediate newline is ignored (if exists) - /* discard return value */ lex_newline::invoke(inner_loc); - - const auto body = lex_ml_literal_body::invoke(inner_loc); - - const auto close = lex_ml_literal_string_close::invoke(inner_loc); - if(!close) - { - throw internal_error(format_underline( - "parse_ml_literal_string: invalid token", - {{source_location(inner_loc), "should be '''"}}), - source_location(inner_loc)); - } - // `lex_ml_literal_string_close` allows 3 to 5 `'`s to allow 1 or 2 `'`s - // at just before the delimiter. Here, we need to attach `'`s at the - // end of the string body, if it exists. - // For detail, see the definition of `lex_ml_basic_string_close`. - - std::string retval = body.unwrap().str(); - assert(std::all_of(close.unwrap().first(), close.unwrap().last(), - [](const char c) noexcept {return c == '\'';})); - switch(close.unwrap().size()) - { - case 3: {break;} - case 4: {retval += "'"; break;} - case 5: {retval += "''"; break;} - default: - { - throw internal_error(format_underline( - "parse_ml_literal_string: closing delimiter has invalid length", - {{source_location(inner_loc), "end of this"}}), - source_location(inner_loc)); - } - } - - const auto err_loc = check_utf8_validity(token.unwrap().str()); - if(err_loc == -1) - { - return ok(std::make_pair(toml::string(retval, toml::string_t::literal), - token.unwrap())); - } - else - { - inner_loc.reset(first); - inner_loc.advance(err_loc); - throw syntax_error(format_underline( - "parse_ml_basic_string: invalid utf8 sequence found", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - else - { - loc.reset(first); // rollback - return err(format_underline("toml::parse_ml_literal_string: " - "the next token is not a valid multiline literal string", - {{source_location(loc), "here"}})); - } -} - -inline result, std::string> -parse_literal_string(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_literal_string::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto open = lex_apostrophe::invoke(inner_loc); - if(!open) - { - throw internal_error(format_underline( - "parse_literal_string: invalid token", - {{source_location(inner_loc), "should be '"}}), - source_location(inner_loc)); - } - - const auto body = repeat::invoke(inner_loc); - - const auto close = lex_apostrophe::invoke(inner_loc); - if(!close) - { - throw internal_error(format_underline( - "parse_literal_string: invalid token", - {{source_location(inner_loc), "should be '"}}), - source_location(inner_loc)); - } - - const auto err_loc = check_utf8_validity(token.unwrap().str()); - if(err_loc == -1) - { - return ok(std::make_pair( - toml::string(body.unwrap().str(), toml::string_t::literal), - token.unwrap())); - } - else - { - inner_loc.reset(first); - inner_loc.advance(err_loc); - throw syntax_error(format_underline( - "parse_ml_basic_string: invalid utf8 sequence found", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - else - { - loc.reset(first); // rollback - return err(format_underline("toml::parse_literal_string: " - "the next token is not a valid literal string", - {{source_location(loc), "here"}})); - } -} - -inline result, std::string> -parse_string(location& loc) -{ - if(loc.iter() != loc.end() && *(loc.iter()) == '"') - { - if(loc.iter() + 1 != loc.end() && *(loc.iter() + 1) == '"' && - loc.iter() + 2 != loc.end() && *(loc.iter() + 2) == '"') - { - return parse_ml_basic_string(loc); - } - else - { - return parse_basic_string(loc); - } - } - else if(loc.iter() != loc.end() && *(loc.iter()) == '\'') - { - if(loc.iter() + 1 != loc.end() && *(loc.iter() + 1) == '\'' && - loc.iter() + 2 != loc.end() && *(loc.iter() + 2) == '\'') - { - return parse_ml_literal_string(loc); - } - else - { - return parse_literal_string(loc); - } - } - return err(format_underline("toml::parse_string: ", - {{source_location(loc), "the next token is not a string"}})); -} - -inline result, std::string> -parse_local_date(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_local_date::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto y = lex_date_fullyear::invoke(inner_loc); - if(!y || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != '-') - { - throw internal_error(format_underline( - "toml::parse_inner_local_date: invalid year format", - {{source_location(inner_loc), "should be `-`"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto m = lex_date_month::invoke(inner_loc); - if(!m || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != '-') - { - throw internal_error(format_underline( - "toml::parse_local_date: invalid month format", - {{source_location(inner_loc), "should be `-`"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto d = lex_date_mday::invoke(inner_loc); - if(!d) - { - throw internal_error(format_underline( - "toml::parse_local_date: invalid day format", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - - const auto year = static_cast(from_string(y.unwrap().str(), 0)); - const auto month = static_cast(from_string(m.unwrap().str(), 0)); - const auto day = static_cast(from_string(d.unwrap().str(), 0)); - - // We briefly check whether the input date is valid or not. But here, we - // only check if the RFC3339 compliance. - // Actually there are several special date that does not exist, - // because of historical reasons, such as 1582/10/5-1582/10/14 (only in - // several countries). But here, we do not care about such a complicated - // rule. It makes the code complicated and there is only low probability - // that such a specific date is needed in practice. If someone need to - // validate date accurately, that means that the one need a specialized - // library for their purpose in a different layer. - { - const bool is_leap = (year % 4 == 0) && ((year % 100 != 0) || (year % 400 == 0)); - const auto max_day = (month == 2) ? (is_leap ? 29 : 28) : - ((month == 4 || month == 6 || month == 9 || month == 11) ? 30 : 31); - - if((month < 1 || 12 < month) || (day < 1 || max_day < day)) - { - throw syntax_error(format_underline("toml::parse_date: " - "invalid date: it does not conform RFC3339.", {{ - source_location(loc), "month should be 01-12, day should be" - " 01-28,29,30,31, depending on month/year." - }}), source_location(inner_loc)); - } - } - return ok(std::make_pair(local_date(year, static_cast(month - 1), day), - token.unwrap())); - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_local_date: ", - {{source_location(loc), "the next token is not a local_date"}})); - } -} - -inline result, std::string> -parse_local_time(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_local_time::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto h = lex_time_hour::invoke(inner_loc); - if(!h || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != ':') - { - throw internal_error(format_underline( - "toml::parse_local_time: invalid year format", - {{source_location(inner_loc), "should be `:`"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto m = lex_time_minute::invoke(inner_loc); - if(!m || inner_loc.iter() == inner_loc.end() || *inner_loc.iter() != ':') - { - throw internal_error(format_underline( - "toml::parse_local_time: invalid month format", - {{source_location(inner_loc), "should be `:`"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto s = lex_time_second::invoke(inner_loc); - if(!s) - { - throw internal_error(format_underline( - "toml::parse_local_time: invalid second format", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - - const int hour = from_string(h.unwrap().str(), 0); - const int minute = from_string(m.unwrap().str(), 0); - const int second = from_string(s.unwrap().str(), 0); - - if((hour < 0 || 23 < hour) || (minute < 0 || 59 < minute) || - (second < 0 || 60 < second)) // it may be leap second - { - throw syntax_error(format_underline("toml::parse_time: " - "invalid time: it does not conform RFC3339.", {{ - source_location(loc), "hour should be 00-23, minute should be" - " 00-59, second should be 00-60 (depending on the leap" - " second rules.)"}}), source_location(inner_loc)); - } - - local_time time(hour, minute, second, 0, 0); - - const auto before_secfrac = inner_loc.iter(); - if(const auto secfrac = lex_time_secfrac::invoke(inner_loc)) - { - auto sf = secfrac.unwrap().str(); - sf.erase(sf.begin()); // sf.front() == '.' - switch(sf.size() % 3) - { - case 2: sf += '0'; break; - case 1: sf += "00"; break; - case 0: break; - default: break; - } - if(sf.size() >= 9) - { - time.millisecond = from_string(sf.substr(0, 3), 0u); - time.microsecond = from_string(sf.substr(3, 3), 0u); - time.nanosecond = from_string(sf.substr(6, 3), 0u); - } - else if(sf.size() >= 6) - { - time.millisecond = from_string(sf.substr(0, 3), 0u); - time.microsecond = from_string(sf.substr(3, 3), 0u); - } - else if(sf.size() >= 3) - { - time.millisecond = from_string(sf, 0u); - time.microsecond = 0u; - } - } - else - { - if(before_secfrac != inner_loc.iter()) - { - throw internal_error(format_underline( - "toml::parse_local_time: invalid subsecond format", - {{source_location(inner_loc), "here"}}), - source_location(inner_loc)); - } - } - return ok(std::make_pair(time, token.unwrap())); - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_local_time: ", - {{source_location(loc), "the next token is not a local_time"}})); - } -} - -inline result, std::string> -parse_local_datetime(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_local_date_time::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - const auto date = parse_local_date(inner_loc); - if(!date || inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "toml::parse_local_datetime: invalid datetime format", - {{source_location(inner_loc), "date, not datetime"}}), - source_location(inner_loc)); - } - const char delim = *(inner_loc.iter()); - if(delim != 'T' && delim != 't' && delim != ' ') - { - throw internal_error(format_underline( - "toml::parse_local_datetime: invalid datetime format", - {{source_location(inner_loc), "should be `T` or ` ` (space)"}}), - source_location(inner_loc)); - } - inner_loc.advance(); - const auto time = parse_local_time(inner_loc); - if(!time) - { - throw internal_error(format_underline( - "toml::parse_local_datetime: invalid datetime format", - {{source_location(inner_loc), "invalid time format"}}), - source_location(inner_loc)); - } - return ok(std::make_pair( - local_datetime(date.unwrap().first, time.unwrap().first), - token.unwrap())); - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_local_datetime: ", - {{source_location(loc), "the next token is not a local_datetime"}})); - } -} - -inline result, std::string> -parse_offset_datetime(location& loc) -{ - const auto first = loc.iter(); - if(const auto token = lex_offset_date_time::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - const auto datetime = parse_local_datetime(inner_loc); - if(!datetime || inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "toml::parse_offset_datetime: invalid datetime format", - {{source_location(inner_loc), "date, not datetime"}}), - source_location(inner_loc)); - } - time_offset offset(0, 0); - if(const auto ofs = lex_time_numoffset::invoke(inner_loc)) - { - const auto str = ofs.unwrap().str(); - - const auto hour = from_string(str.substr(1,2), 0); - const auto minute = from_string(str.substr(4,2), 0); - - if((hour < 0 || 23 < hour) || (minute < 0 || 59 < minute)) - { - throw syntax_error(format_underline("toml::parse_offset_datetime: " - "invalid offset: it does not conform RFC3339.", {{ - source_location(loc), "month should be 01-12, day should be" - " 01-28,29,30,31, depending on month/year." - }}), source_location(inner_loc)); - } - - if(str.front() == '+') - { - offset = time_offset(hour, minute); - } - else - { - offset = time_offset(-hour, -minute); - } - } - else if(*inner_loc.iter() != 'Z' && *inner_loc.iter() != 'z') - { - throw internal_error(format_underline( - "toml::parse_offset_datetime: invalid datetime format", - {{source_location(inner_loc), "should be `Z` or `+HH:MM`"}}), - source_location(inner_loc)); - } - return ok(std::make_pair(offset_datetime(datetime.unwrap().first, offset), - token.unwrap())); - } - else - { - loc.reset(first); - return err(format_underline("toml::parse_offset_datetime: ", - {{source_location(loc), "the next token is not a offset_datetime"}})); - } -} - -inline result, std::string> -parse_simple_key(location& loc) -{ - if(const auto bstr = parse_basic_string(loc)) - { - return ok(std::make_pair(bstr.unwrap().first.str, bstr.unwrap().second)); - } - if(const auto lstr = parse_literal_string(loc)) - { - return ok(std::make_pair(lstr.unwrap().first.str, lstr.unwrap().second)); - } - if(const auto bare = lex_unquoted_key::invoke(loc)) - { - const auto reg = bare.unwrap(); - return ok(std::make_pair(reg.str(), reg)); - } - return err(format_underline("toml::parse_simple_key: ", - {{source_location(loc), "the next token is not a simple key"}})); -} - -// dotted key become vector of keys -inline result, region>, std::string> -parse_key(location& loc) -{ - const auto first = loc.iter(); - // dotted key -> `foo.bar.baz` where several single keys are chained by - // dots. Whitespaces between keys and dots are allowed. - if(const auto token = lex_dotted_key::invoke(loc)) - { - const auto reg = token.unwrap(); - location inner_loc(loc.name(), reg.str()); - std::vector keys; - - while(inner_loc.iter() != inner_loc.end()) - { - lex_ws::invoke(inner_loc); - if(const auto k = parse_simple_key(inner_loc)) - { - keys.push_back(k.unwrap().first); - } - else - { - throw internal_error(format_underline( - "toml::detail::parse_key: dotted key contains invalid key", - {{source_location(inner_loc), k.unwrap_err()}}), - source_location(inner_loc)); - } - - lex_ws::invoke(inner_loc); - if(inner_loc.iter() == inner_loc.end()) - { - break; - } - else if(*inner_loc.iter() == '.') - { - inner_loc.advance(); // to skip `.` - } - else - { - throw internal_error(format_underline("toml::parse_key: " - "dotted key contains invalid key ", - {{source_location(inner_loc), "should be `.`"}}), - source_location(inner_loc)); - } - } - return ok(std::make_pair(keys, reg)); - } - loc.reset(first); - - // simple_key: a single (basic_string|literal_string|bare key) - if(const auto smpl = parse_simple_key(loc)) - { - return ok(std::make_pair(std::vector(1, smpl.unwrap().first), - smpl.unwrap().second)); - } - return err(format_underline("toml::parse_key: an invalid key appeared.", - {{source_location(loc), "is not a valid key"}}, { - "bare keys : non-empty strings composed only of [A-Za-z0-9_-].", - "quoted keys: same as \"basic strings\" or 'literal strings'.", - "dotted keys: sequence of bare or quoted keys joined with a dot." - })); -} - -// forward-decl to implement parse_array and parse_table -template -result parse_value(location&); - -template -result, std::string> -parse_array(location& loc) -{ - using value_type = Value; - using array_type = typename value_type::array_type; - - const auto first = loc.iter(); - if(loc.iter() == loc.end()) - { - return err("toml::parse_array: input is empty"); - } - if(*loc.iter() != '[') - { - return err("toml::parse_array: token is not an array"); - } - loc.advance(); - - using lex_ws_comment_newline = repeat< - either, unlimited>; - - array_type retval; - while(loc.iter() != loc.end()) - { - lex_ws_comment_newline::invoke(loc); // skip - - if(loc.iter() != loc.end() && *loc.iter() == ']') - { - loc.advance(); // skip ']' - return ok(std::make_pair(retval, - region(loc, first, loc.iter()))); - } - - if(auto val = parse_value(loc)) - { - // After TOML v1.0.0-rc.1, array becomes to be able to have values - // with different types. So here we will omit this by default. - // - // But some of the test-suite checks if the parser accepts a hetero- - // geneous arrays, so we keep this for a while. -#ifdef TOML11_DISALLOW_HETEROGENEOUS_ARRAYS - if(!retval.empty() && retval.front().type() != val.as_ok().type()) - { - auto array_start_loc = loc; - array_start_loc.reset(first); - - throw syntax_error(format_underline("toml::parse_array: " - "type of elements should be the same each other.", { - {source_location(array_start_loc), "array starts here"}, - { - retval.front().location(), - "value has type " + stringize(retval.front().type()) - }, - { - val.unwrap().location(), - "value has different type, " + stringize(val.unwrap().type()) - } - }), source_location(loc)); - } -#endif - retval.push_back(std::move(val.unwrap())); - } - else - { - auto array_start_loc = loc; - array_start_loc.reset(first); - - throw syntax_error(format_underline("toml::parse_array: " - "value having invalid format appeared in an array", { - {source_location(array_start_loc), "array starts here"}, - {source_location(loc), "it is not a valid value."} - }), source_location(loc)); - } - - using lex_array_separator = sequence, character<','>>; - const auto sp = lex_array_separator::invoke(loc); - if(!sp) - { - lex_ws_comment_newline::invoke(loc); - if(loc.iter() != loc.end() && *loc.iter() == ']') - { - loc.advance(); // skip ']' - return ok(std::make_pair(retval, - region(loc, first, loc.iter()))); - } - else - { - auto array_start_loc = loc; - array_start_loc.reset(first); - - throw syntax_error(format_underline("toml::parse_array:" - " missing array separator `,` after a value", { - {source_location(array_start_loc), "array starts here"}, - {source_location(loc), "should be `,`"} - }), source_location(loc)); - } - } - } - loc.reset(first); - throw syntax_error(format_underline("toml::parse_array: " - "array did not closed by `]`", - {{source_location(loc), "should be closed"}}), - source_location(loc)); -} - -template -result, region>, Value>, std::string> -parse_key_value_pair(location& loc) -{ - using value_type = Value; - - const auto first = loc.iter(); - auto key_reg = parse_key(loc); - if(!key_reg) - { - std::string msg = std::move(key_reg.unwrap_err()); - // if the next token is keyvalue-separator, it means that there are no - // key. then we need to show error as "empty key is not allowed". - if(const auto keyval_sep = lex_keyval_sep::invoke(loc)) - { - loc.reset(first); - msg = format_underline("toml::parse_key_value_pair: " - "empty key is not allowed.", - {{source_location(loc), "key expected before '='"}}); - } - return err(std::move(msg)); - } - - const auto kvsp = lex_keyval_sep::invoke(loc); - if(!kvsp) - { - std::string msg; - // if the line contains '=' after the invalid sequence, possibly the - // error is in the key (like, invalid character in bare key). - const auto line_end = std::find(loc.iter(), loc.end(), '\n'); - if(std::find(loc.iter(), line_end, '=') != line_end) - { - msg = format_underline("toml::parse_key_value_pair: " - "invalid format for key", - {{source_location(loc), "invalid character in key"}}, - {"Did you forget '.' to separate dotted-key?", - "Allowed characters for bare key are [0-9a-zA-Z_-]."}); - } - else // if not, the error is lack of key-value separator. - { - msg = format_underline("toml::parse_key_value_pair: " - "missing key-value separator `=`", - {{source_location(loc), "should be `=`"}}); - } - loc.reset(first); - return err(std::move(msg)); - } - - const auto after_kvsp = loc.iter(); // err msg - auto val = parse_value(loc); - if(!val) - { - std::string msg; - loc.reset(after_kvsp); - // check there is something not a comment/whitespace after `=` - if(sequence, maybe, lex_newline>::invoke(loc)) - { - loc.reset(after_kvsp); - msg = format_underline("toml::parse_key_value_pair: " - "missing value after key-value separator '='", - {{source_location(loc), "expected value, but got nothing"}}); - } - else // there is something not a comment/whitespace, so invalid format. - { - msg = std::move(val.unwrap_err()); - } - loc.reset(first); - return err(msg); - } - return ok(std::make_pair(std::move(key_reg.unwrap()), - std::move(val.unwrap()))); -} - -// for error messages. -template -std::string format_dotted_keys(InputIterator first, const InputIterator last) -{ - static_assert(std::is_same::value_type>::value,""); - - std::string retval(*first++); - for(; first != last; ++first) - { - retval += '.'; - retval += *first; - } - return retval; -} - -// forward decl for is_valid_forward_table_definition -result, region>, std::string> -parse_table_key(location& loc); -template -result, std::string> -parse_inline_table(location& loc); - -// The following toml file is allowed. -// ```toml -// [a.b.c] # here, table `a` has element `b`. -// foo = "bar" -// [a] # merge a = {baz = "qux"} to a = {b = {...}} -// baz = "qux" -// ``` -// But the following is not allowed. -// ```toml -// [a] -// b.c.foo = "bar" -// [a] # error! the same table [a] defined! -// baz = "qux" -// ``` -// The following is neither allowed. -// ```toml -// a = { b.c.foo = "bar"} -// [a] # error! the same table [a] defined! -// baz = "qux" -// ``` -// Here, it parses region of `tab->at(k)` as a table key and check the depth -// of the key. If the key region points deeper node, it would be allowed. -// Otherwise, the key points the same node. It would be rejected. -template -bool is_valid_forward_table_definition(const Value& fwd, const Value& inserting, - Iterator key_first, Iterator key_curr, Iterator key_last) -{ - // ------------------------------------------------------------------------ - // check type of the value to be inserted/merged - - std::string inserting_reg = ""; - if(const auto ptr = detail::get_region(inserting)) - { - inserting_reg = ptr->str(); - } - location inserting_def("internal", std::move(inserting_reg)); - if(const auto inlinetable = parse_inline_table(inserting_def)) - { - // check if we are overwriting existing table. - // ```toml - // # NG - // a.b = 42 - // a = {d = 3.14} - // ``` - // Inserting an inline table to a existing super-table is not allowed in - // any case. If we found it, we can reject it without further checking. - return false; - } - - // ------------------------------------------------------------------------ - // check table defined before - - std::string internal = ""; - if(const auto ptr = detail::get_region(fwd)) - { - internal = ptr->str(); - } - location def("internal", std::move(internal)); - if(const auto tabkeys = parse_table_key(def)) // [table.key] - { - // table keys always contains all the nodes from the root. - const auto& tks = tabkeys.unwrap().first; - if(std::size_t(std::distance(key_first, key_last)) == tks.size() && - std::equal(tks.begin(), tks.end(), key_first)) - { - // the keys are equivalent. it is not allowed. - return false; - } - // the keys are not equivalent. it is allowed. - return true; - } - if(const auto dotkeys = parse_key(def)) - { - // consider the following case. - // [a] - // b.c = {d = 42} - // [a.b.c] - // e = 2.71 - // this defines the table [a.b.c] twice. no? - - // a dotted key starts from the node representing a table in which the - // dotted key belongs to. - const auto& dks = dotkeys.unwrap().first; - if(std::size_t(std::distance(key_curr, key_last)) == dks.size() && - std::equal(dks.begin(), dks.end(), key_curr)) - { - // the keys are equivalent. it is not allowed. - return false; - } - // the keys are not equivalent. it is allowed. - return true; - } - return false; -} - -template -result -insert_nested_key(typename Value::table_type& root, const Value& v, - InputIterator iter, const InputIterator last, - region key_reg, - const bool is_array_of_table = false) -{ - static_assert(std::is_same::value_type>::value,""); - - using value_type = Value; - using table_type = typename value_type::table_type; - using array_type = typename value_type::array_type; - - const auto first = iter; - assert(iter != last); - - table_type* tab = std::addressof(root); - for(; iter != last; ++iter) // search recursively - { - const key& k = *iter; - if(std::next(iter) == last) // k is the last key - { - // XXX if the value is array-of-tables, there can be several - // tables that are in the same array. in that case, we need to - // find the last element and insert it to there. - if(is_array_of_table) - { - if(tab->count(k) == 1) // there is already an array of table - { - if(tab->at(k).is_table()) - { - // show special err msg for conflicting table - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of table (\"", - format_dotted_keys(first, last), - "\") cannot be defined"), { - {tab->at(k).location(), "table already defined"}, - {v.location(), "this conflicts with the previous table"} - }), v.location()); - } - else if(!(tab->at(k).is_array())) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of table (\"", - format_dotted_keys(first, last), "\") collides with" - " existing value"), { - {tab->at(k).location(), - concat_to_string("this ", tab->at(k).type(), - " value already exists")}, - {v.location(), - "while inserting this array-of-tables"} - }), v.location()); - } - // the above if-else-if checks tab->at(k) is an array - auto& a = tab->at(k).as_array(); - // If table element is defined as [[array_of_tables]], it - // cannot be an empty array. If an array of tables is - // defined as `aot = []`, it cannot be appended. - if(a.empty() || !(a.front().is_table())) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of table (\"", - format_dotted_keys(first, last), "\") collides with" - " existing value"), { - {tab->at(k).location(), - concat_to_string("this ", tab->at(k).type(), - " value already exists")}, - {v.location(), - "while inserting this array-of-tables"} - }), v.location()); - } - // avoid conflicting array of table like the following. - // ```toml - // a = [{b = 42}] # define a as an array of *inline* tables - // [[a]] # a is an array of *multi-line* tables - // b = 54 - // ``` - // Here, from the type information, these cannot be detected - // because inline table is also a table. - // But toml v0.5.0 explicitly says it is invalid. The above - // array-of-tables has a static size and appending to the - // array is invalid. - // In this library, multi-line table value has a region - // that points to the key of the table (e.g. [[a]]). By - // comparing the first two letters in key, we can detect - // the array-of-table is inline or multiline. - if(const auto ptr = detail::get_region(a.front())) - { - if(ptr->str().substr(0,2) != "[[") - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of table (\"", - format_dotted_keys(first, last), "\") collides " - "with existing array-of-tables"), { - {tab->at(k).location(), - concat_to_string("this ", tab->at(k).type(), - " value has static size")}, - {v.location(), - "appending it to the statically sized array"} - }), v.location()); - } - } - a.push_back(v); - return ok(true); - } - else // if not, we need to create the array of table - { - // XXX: Consider the following array of tables. - // ```toml - // # This is a comment. - // [[aot]] - // foo = "bar" - // ``` - // Here, the comment is for `aot`. But here, actually two - // values are defined. An array that contains tables, named - // `aot`, and the 0th element of the `aot`, `{foo = "bar"}`. - // Those two are different from each other. But both of them - // points to the same portion of the TOML file, `[[aot]]`, - // so `key_reg.comments()` returns `# This is a comment`. - // If it is assigned as a comment of `aot` defined here, the - // comment will be duplicated. Both the `aot` itself and - // the 0-th element will have the same comment. This causes - // "duplication of the same comments" bug when the data is - // serialized. - // Next, consider the following. - // ```toml - // # comment 1 - // aot = [ - // # comment 2 - // {foo = "bar"}, - // ] - // ``` - // In this case, we can distinguish those two comments. So - // here we need to add "comment 1" to the `aot` and - // "comment 2" to the 0th element of that. - // To distinguish those two, we check the key region. - std::vector comments{/* empty by default */}; - if(key_reg.str().substr(0, 2) != "[[") - { - comments = key_reg.comments(); - } - value_type aot(array_type(1, v), key_reg, std::move(comments)); - tab->insert(std::make_pair(k, aot)); - return ok(true); - } - } // end if(array of table) - - if(tab->count(k) == 1) - { - if(tab->at(k).is_table() && v.is_table()) - { - if(!is_valid_forward_table_definition( - tab->at(k), v, first, iter, last)) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: table (\"", - format_dotted_keys(first, last), - "\") already exists."), { - {tab->at(k).location(), "table already exists here"}, - {v.location(), "table defined twice"} - }), v.location()); - } - // to allow the following toml file. - // [a.b.c] - // d = 42 - // [a] - // e = 2.71 - auto& t = tab->at(k).as_table(); - for(const auto& kv : v.as_table()) - { - if(tab->at(k).contains(kv.first)) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: value (\"", - format_dotted_keys(first, last), - "\") already exists."), { - {t.at(kv.first).location(), "already exists here"}, - {v.location(), "this defined twice"} - }), v.location()); - } - t[kv.first] = kv.second; - } - detail::change_region(tab->at(k), key_reg); - return ok(true); - } - else if(v.is_table() && - tab->at(k).is_array() && - tab->at(k).as_array().size() > 0 && - tab->at(k).as_array().front().is_table()) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: array of tables (\"", - format_dotted_keys(first, last), "\") already exists."), { - {tab->at(k).location(), "array of tables defined here"}, - {v.location(), "table conflicts with the previous array of table"} - }), v.location()); - } - else - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: value (\"", - format_dotted_keys(first, last), "\") already exists."), { - {tab->at(k).location(), "value already exists here"}, - {v.location(), "value defined twice"} - }), v.location()); - } - } - tab->insert(std::make_pair(k, v)); - return ok(true); - } - else // k is not the last one, we should insert recursively - { - // if there is no corresponding value, insert it first. - // related: you don't need to write - // # [x] - // # [x.y] - // to write - // [x.y.z] - if(tab->count(k) == 0) - { - // a table that is defined implicitly doesn't have any comments. - (*tab)[k] = value_type(table_type{}, key_reg, {/*no comment*/}); - } - - // type checking... - if(tab->at(k).is_table()) - { - // According to toml-lang/toml:36d3091b3 "Clarify that inline - // tables are immutable", check if it adds key-value pair to an - // inline table. - if(const auto* ptr = get_region(tab->at(k))) - { - // here, if the value is a (multi-line) table, the region - // should be something like `[table-name]`. - if(ptr->front() == '{') - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: inserting to an inline table (", - format_dotted_keys(first, std::next(iter)), - ") but inline tables are immutable"), { - {tab->at(k).location(), "inline tables are immutable"}, - {v.location(), "inserting this"} - }), v.location()); - } - } - tab = std::addressof((*tab)[k].as_table()); - } - else if(tab->at(k).is_array()) // inserting to array-of-tables? - { - auto& a = (*tab)[k].as_array(); - if(!a.back().is_table()) - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: target (", - format_dotted_keys(first, std::next(iter)), - ") is neither table nor an array of tables"), { - {a.back().location(), concat_to_string( - "actual type is ", a.back().type())}, - {v.location(), "inserting this"} - }), v.location()); - } - tab = std::addressof(a.back().as_table()); - } - else - { - throw syntax_error(format_underline(concat_to_string( - "toml::insert_value: target (", - format_dotted_keys(first, std::next(iter)), - ") is neither table nor an array of tables"), { - {tab->at(k).location(), concat_to_string( - "actual type is ", tab->at(k).type())}, - {v.location(), "inserting this"} - }), v.location()); - } - } - } - return err(std::string("toml::detail::insert_nested_key: never reach here")); -} - -template -result, std::string> -parse_inline_table(location& loc) -{ - using value_type = Value; - using table_type = typename value_type::table_type; - - const auto first = loc.iter(); - table_type retval; - if(!(loc.iter() != loc.end() && *loc.iter() == '{')) - { - return err(format_underline("toml::parse_inline_table: ", - {{source_location(loc), "the next token is not an inline table"}})); - } - loc.advance(); - - // check if the inline table is an empty table = { } - maybe::invoke(loc); - if(loc.iter() != loc.end() && *loc.iter() == '}') - { - loc.advance(); // skip `}` - return ok(std::make_pair(retval, region(loc, first, loc.iter()))); - } - - // it starts from "{". it should be formatted as inline-table - while(loc.iter() != loc.end()) - { - const auto kv_r = parse_key_value_pair(loc); - if(!kv_r) - { - return err(kv_r.unwrap_err()); - } - - const auto& kvpair = kv_r.unwrap(); - const std::vector& keys = kvpair.first.first; - const auto& key_reg = kvpair.first.second; - const value_type& val = kvpair.second; - - const auto inserted = - insert_nested_key(retval, val, keys.begin(), keys.end(), key_reg); - if(!inserted) - { - throw internal_error("toml::parse_inline_table: " - "failed to insert value into table: " + inserted.unwrap_err(), - source_location(loc)); - } - - using lex_table_separator = sequence, character<','>>; - const auto sp = lex_table_separator::invoke(loc); - - if(!sp) - { - maybe::invoke(loc); - - if(loc.iter() == loc.end()) - { - throw syntax_error(format_underline( - "toml::parse_inline_table: missing table separator `}` ", - {{source_location(loc), "should be `}`"}}), - source_location(loc)); - } - else if(*loc.iter() == '}') - { - loc.advance(); // skip `}` - return ok(std::make_pair( - retval, region(loc, first, loc.iter()))); - } - else if(*loc.iter() == '#' || *loc.iter() == '\r' || *loc.iter() == '\n') - { - throw syntax_error(format_underline( - "toml::parse_inline_table: missing curly brace `}`", - {{source_location(loc), "should be `}`"}}), - source_location(loc)); - } - else - { - throw syntax_error(format_underline( - "toml::parse_inline_table: missing table separator `,` ", - {{source_location(loc), "should be `,`"}}), - source_location(loc)); - } - } - else // `,` is found - { - maybe::invoke(loc); - if(loc.iter() != loc.end() && *loc.iter() == '}') - { - throw syntax_error(format_underline( - "toml::parse_inline_table: trailing comma is not allowed in" - " an inline table", - {{source_location(loc), "should be `}`"}}), - source_location(loc)); - } - } - } - loc.reset(first); - throw syntax_error(format_underline("toml::parse_inline_table: " - "inline table did not closed by `}`", - {{source_location(loc), "should be closed"}}), - source_location(loc)); -} - -inline result guess_number_type(const location& l) -{ - // This function tries to find some (common) mistakes by checking characters - // that follows the last character of a value. But it is often difficult - // because some non-newline characters can appear after a value. E.g. - // spaces, tabs, commas (in an array or inline table), closing brackets - // (of an array or inline table), comment-sign (#). Since this function - // does not parse further, those characters are always allowed to be there. - location loc = l; - - if(lex_offset_date_time::invoke(loc)) {return ok(value_t::offset_datetime);} - loc.reset(l.iter()); - - if(lex_local_date_time::invoke(loc)) - { - // bad offset may appear after this. - if(loc.iter() != loc.end() && (*loc.iter() == '+' || *loc.iter() == '-' - || *loc.iter() == 'Z' || *loc.iter() == 'z')) - { - return err(format_underline("bad offset: should be [+-]HH:MM or Z", - {{source_location(loc), "[+-]HH:MM or Z"}}, - {"pass: +09:00, -05:30", "fail: +9:00, -5:30"})); - } - return ok(value_t::local_datetime); - } - loc.reset(l.iter()); - - if(lex_local_date::invoke(loc)) - { - // bad time may appear after this. - // A space is allowed as a delimiter between local time. But there are - // both cases in which a space becomes valid or invalid. - // - invalid: 2019-06-16 7:00:00 - // - valid : 2019-06-16 07:00:00 - if(loc.iter() != loc.end()) - { - const auto c = *loc.iter(); - if(c == 'T' || c == 't') - { - return err(format_underline("bad time: should be HH:MM:SS.subsec", - {{source_location(loc), "HH:MM:SS.subsec"}}, - {"pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999", - "fail: 1979-05-27T7:32:00, 1979-05-27 17:32"})); - } - if('0' <= c && c <= '9') - { - return err(format_underline("bad time: missing T", - {{source_location(loc), "T or space required here"}}, - {"pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999", - "fail: 1979-05-27T7:32:00, 1979-05-27 7:32"})); - } - if(c == ' ' && std::next(loc.iter()) != loc.end() && - ('0' <= *std::next(loc.iter()) && *std::next(loc.iter())<= '9')) - { - loc.advance(); - return err(format_underline("bad time: should be HH:MM:SS.subsec", - {{source_location(loc), "HH:MM:SS.subsec"}}, - {"pass: 1979-05-27T07:32:00, 1979-05-27 07:32:00.999999", - "fail: 1979-05-27T7:32:00, 1979-05-27 7:32"})); - } - } - return ok(value_t::local_date); - } - loc.reset(l.iter()); - - if(lex_local_time::invoke(loc)) {return ok(value_t::local_time);} - loc.reset(l.iter()); - - if(lex_float::invoke(loc)) - { - if(loc.iter() != loc.end() && *loc.iter() == '_') - { - return err(format_underline("bad float: `_` should be surrounded by digits", - {{source_location(loc), "here"}}, - {"pass: +1.0, -2e-2, 3.141_592_653_589, inf, nan", - "fail: .0, 1., _1.0, 1.0_, 1_.0, 1.0__0"})); - } - return ok(value_t::floating); - } - loc.reset(l.iter()); - - if(lex_integer::invoke(loc)) - { - if(loc.iter() != loc.end()) - { - const auto c = *loc.iter(); - if(c == '_') - { - return err(format_underline("bad integer: `_` should be surrounded by digits", - {{source_location(loc), "here"}}, - {"pass: -42, 1_000, 1_2_3_4_5, 0xC0FFEE, 0b0010, 0o755", - "fail: 1__000, 0123"})); - } - if('0' <= c && c <= '9') - { - // leading zero. point '0' - loc.retrace(); - return err(format_underline("bad integer: leading zero", - {{source_location(loc), "here"}}, - {"pass: -42, 1_000, 1_2_3_4_5, 0xC0FFEE, 0b0010, 0o755", - "fail: 1__000, 0123"})); - } - if(c == ':' || c == '-') - { - return err(format_underline("bad datetime: invalid format", - {{source_location(loc), "here"}}, - {"pass: 1979-05-27T07:32:00-07:00, 1979-05-27 07:32:00.999999Z", - "fail: 1979-05-27T7:32:00-7:00, 1979-05-27 7:32-00:30"})); - } - if(c == '.' || c == 'e' || c == 'E') - { - return err(format_underline("bad float: invalid format", - {{source_location(loc), "here"}}, - {"pass: +1.0, -2e-2, 3.141_592_653_589, inf, nan", - "fail: .0, 1., _1.0, 1.0_, 1_.0, 1.0__0"})); - } - } - return ok(value_t::integer); - } - if(loc.iter() != loc.end() && *loc.iter() == '.') - { - return err(format_underline("bad float: invalid format", - {{source_location(loc), "integer part required before this"}}, - {"pass: +1.0, -2e-2, 3.141_592_653_589, inf, nan", - "fail: .0, 1., _1.0, 1.0_, 1_.0, 1.0__0"})); - } - if(loc.iter() != loc.end() && *loc.iter() == '_') - { - return err(format_underline("bad number: `_` should be surrounded by digits", - {{source_location(loc), "`_` is not surrounded by digits"}}, - {"pass: -42, 1_000, 1_2_3_4_5, 0xC0FFEE, 0b0010, 0o755", - "fail: 1__000, 0123"})); - } - return err(format_underline("bad format: unknown value appeared", - {{source_location(loc), "here"}})); -} - -inline result guess_value_type(const location& loc) -{ - switch(*loc.iter()) - { - case '"' : {return ok(value_t::string); } - case '\'': {return ok(value_t::string); } - case 't' : {return ok(value_t::boolean); } - case 'f' : {return ok(value_t::boolean); } - case '[' : {return ok(value_t::array); } - case '{' : {return ok(value_t::table); } - case 'i' : {return ok(value_t::floating);} // inf. - case 'n' : {return ok(value_t::floating);} // nan. - default : {return guess_number_type(loc);} - } -} - -template -result -parse_value_helper(result, std::string> rslt) -{ - if(rslt.is_ok()) - { - auto comments = rslt.as_ok().second.comments(); - return ok(Value(std::move(rslt.as_ok()), std::move(comments))); - } - else - { - return err(std::move(rslt.as_err())); - } -} - -template -result parse_value(location& loc) -{ - const auto first = loc.iter(); - if(first == loc.end()) - { - return err(format_underline("toml::parse_value: input is empty", - {{source_location(loc), ""}})); - } - - const auto type = guess_value_type(loc); - if(!type) - { - return err(type.unwrap_err()); - } - - switch(type.unwrap()) - { - case value_t::boolean : {return parse_value_helper(parse_boolean(loc) );} - case value_t::integer : {return parse_value_helper(parse_integer(loc) );} - case value_t::floating : {return parse_value_helper(parse_floating(loc) );} - case value_t::string : {return parse_value_helper(parse_string(loc) );} - case value_t::offset_datetime: {return parse_value_helper(parse_offset_datetime(loc) );} - case value_t::local_datetime : {return parse_value_helper(parse_local_datetime(loc) );} - case value_t::local_date : {return parse_value_helper(parse_local_date(loc) );} - case value_t::local_time : {return parse_value_helper(parse_local_time(loc) );} - case value_t::array : {return parse_value_helper(parse_array(loc) );} - case value_t::table : {return parse_value_helper(parse_inline_table(loc));} - default: - { - const auto msg = format_underline("toml::parse_value: " - "unknown token appeared", {{source_location(loc), "unknown"}}); - loc.reset(first); - return err(msg); - } - } -} - -inline result, region>, std::string> -parse_table_key(location& loc) -{ - if(auto token = lex_std_table::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto open = lex_std_table_open::invoke(inner_loc); - if(!open || inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "toml::parse_table_key: no `[`", - {{source_location(inner_loc), "should be `[`"}}), - source_location(inner_loc)); - } - // to skip [ a . b . c ] - // ^----------- this whitespace - lex_ws::invoke(inner_loc); - const auto keys = parse_key(inner_loc); - if(!keys) - { - throw internal_error(format_underline( - "toml::parse_table_key: invalid key", - {{source_location(inner_loc), "not key"}}), - source_location(inner_loc)); - } - // to skip [ a . b . c ] - // ^-- this whitespace - lex_ws::invoke(inner_loc); - const auto close = lex_std_table_close::invoke(inner_loc); - if(!close) - { - throw internal_error(format_underline( - "toml::parse_table_key: no `]`", - {{source_location(inner_loc), "should be `]`"}}), - source_location(inner_loc)); - } - - // after [table.key], newline or EOF(empty table) required. - if(loc.iter() != loc.end()) - { - using lex_newline_after_table_key = - sequence, maybe, lex_newline>; - const auto nl = lex_newline_after_table_key::invoke(loc); - if(!nl) - { - throw syntax_error(format_underline( - "toml::parse_table_key: newline required after [table.key]", - {{source_location(loc), "expected newline"}}), - source_location(loc)); - } - } - return ok(std::make_pair(keys.unwrap().first, token.unwrap())); - } - else - { - return err(format_underline("toml::parse_table_key: " - "not a valid table key", {{source_location(loc), "here"}})); - } -} - -inline result, region>, std::string> -parse_array_table_key(location& loc) -{ - if(auto token = lex_array_table::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - - const auto open = lex_array_table_open::invoke(inner_loc); - if(!open || inner_loc.iter() == inner_loc.end()) - { - throw internal_error(format_underline( - "toml::parse_array_table_key: no `[[`", - {{source_location(inner_loc), "should be `[[`"}}), - source_location(inner_loc)); - } - lex_ws::invoke(inner_loc); - const auto keys = parse_key(inner_loc); - if(!keys) - { - throw internal_error(format_underline( - "toml::parse_array_table_key: invalid key", - {{source_location(inner_loc), "not a key"}}), - source_location(inner_loc)); - } - lex_ws::invoke(inner_loc); - const auto close = lex_array_table_close::invoke(inner_loc); - if(!close) - { - throw internal_error(format_underline( - "toml::parse_table_key: no `]]`", - {{source_location(inner_loc), "should be `]]`"}}), - source_location(inner_loc)); - } - - // after [[table.key]], newline or EOF(empty table) required. - if(loc.iter() != loc.end()) - { - using lex_newline_after_table_key = - sequence, maybe, lex_newline>; - const auto nl = lex_newline_after_table_key::invoke(loc); - if(!nl) - { - throw syntax_error(format_underline("toml::" - "parse_array_table_key: newline required after [[table.key]]", - {{source_location(loc), "expected newline"}}), - source_location(loc)); - } - } - return ok(std::make_pair(keys.unwrap().first, token.unwrap())); - } - else - { - return err(format_underline("toml::parse_array_table_key: " - "not a valid table key", {{source_location(loc), "here"}})); - } -} - -// parse table body (key-value pairs until the iter hits the next [tablekey]) -template -result -parse_ml_table(location& loc) -{ - using value_type = Value; - using table_type = typename value_type::table_type; - - const auto first = loc.iter(); - if(first == loc.end()) - { - return ok(table_type{}); - } - - // XXX at lest one newline is needed. - using skip_line = repeat< - sequence, maybe, lex_newline>, at_least<1>>; - skip_line::invoke(loc); - lex_ws::invoke(loc); - - table_type tab; - while(loc.iter() != loc.end()) - { - lex_ws::invoke(loc); - const auto before = loc.iter(); - if(const auto tmp = parse_array_table_key(loc)) // next table found - { - loc.reset(before); - return ok(tab); - } - if(const auto tmp = parse_table_key(loc)) // next table found - { - loc.reset(before); - return ok(tab); - } - - if(const auto kv = parse_key_value_pair(loc)) - { - const auto& kvpair = kv.unwrap(); - const std::vector& keys = kvpair.first.first; - const auto& key_reg = kvpair.first.second; - const value_type& val = kvpair.second; - const auto inserted = - insert_nested_key(tab, val, keys.begin(), keys.end(), key_reg); - if(!inserted) - { - return err(inserted.unwrap_err()); - } - } - else - { - return err(kv.unwrap_err()); - } - - // comment lines are skipped by the above function call. - // However, since the `skip_line` requires at least 1 newline, it fails - // if the file ends with ws and/or comment without newline. - // `skip_line` matches `ws? + comment? + newline`, not `ws` or `comment` - // itself. To skip the last ws and/or comment, call lexers. - // It does not matter if these fails, so the return value is discarded. - lex_ws::invoke(loc); - lex_comment::invoke(loc); - - // skip_line is (whitespace? comment? newline)_{1,}. multiple empty lines - // and comments after the last key-value pairs are allowed. - const auto newline = skip_line::invoke(loc); - if(!newline && loc.iter() != loc.end()) - { - const auto before2 = loc.iter(); - lex_ws::invoke(loc); // skip whitespace - const auto msg = format_underline("toml::parse_table: " - "invalid line format", {{source_location(loc), concat_to_string( - "expected newline, but got '", show_char(*loc.iter()), "'.")}}); - loc.reset(before2); - return err(msg); - } - - // the skip_lines only matches with lines that includes newline. - // to skip the last line that includes comment and/or whitespace - // but no newline, call them one more time. - lex_ws::invoke(loc); - lex_comment::invoke(loc); - } - return ok(tab); -} - -template -result parse_toml_file(location& loc) -{ - using value_type = Value; - using table_type = typename value_type::table_type; - - const auto first = loc.iter(); - if(first == loc.end()) - { - // For empty files, return an empty table with an empty region (zero-length). - // Without the region, error messages would miss the filename. - return ok(value_type(table_type{}, region(loc, first, first), {})); - } - - // put the first line as a region of a file - // Here first != loc.end(), so taking std::next is okay - const region file(loc, first, std::next(loc.iter())); - - // The first successive comments that are separated from the first value - // by an empty line are for a file itself. - // ```toml - // # this is a comment for a file. - // - // key = "the first value" - // ``` - // ```toml - // # this is a comment for "the first value". - // key = "the first value" - // ``` - std::vector comments; - using lex_first_comments = sequence< - repeat, lex_comment, lex_newline>, at_least<1>>, - sequence, lex_newline> - >; - if(const auto token = lex_first_comments::invoke(loc)) - { - location inner_loc(loc.name(), token.unwrap().str()); - while(inner_loc.iter() != inner_loc.end()) - { - maybe::invoke(inner_loc); // remove ws if exists - if(lex_newline::invoke(inner_loc)) - { - assert(inner_loc.iter() == inner_loc.end()); - break; // empty line found. - } - auto com = lex_comment::invoke(inner_loc).unwrap().str(); - com.erase(com.begin()); // remove # sign - comments.push_back(std::move(com)); - lex_newline::invoke(inner_loc); - } - } - - table_type data; - // root object is also a table, but without [tablename] - if(const auto tab = parse_ml_table(loc)) - { - data = std::move(tab.unwrap()); - } - else // failed (empty table is regarded as success in parse_ml_table) - { - return err(tab.unwrap_err()); - } - while(loc.iter() != loc.end()) - { - // here, the region of [table] is regarded as the table-key because - // the table body is normally too big and it is not so informative - // if the first key-value pair of the table is shown in the error - // message. - if(const auto tabkey = parse_array_table_key(loc)) - { - const auto tab = parse_ml_table(loc); - if(!tab){return err(tab.unwrap_err());} - - const auto& tk = tabkey.unwrap(); - const auto& keys = tk.first; - const auto& reg = tk.second; - - const auto inserted = insert_nested_key(data, - value_type(tab.unwrap(), reg, reg.comments()), - keys.begin(), keys.end(), reg, - /*is_array_of_table=*/ true); - if(!inserted) {return err(inserted.unwrap_err());} - - continue; - } - if(const auto tabkey = parse_table_key(loc)) - { - const auto tab = parse_ml_table(loc); - if(!tab){return err(tab.unwrap_err());} - - const auto& tk = tabkey.unwrap(); - const auto& keys = tk.first; - const auto& reg = tk.second; - - const auto inserted = insert_nested_key(data, - value_type(tab.unwrap(), reg, reg.comments()), - keys.begin(), keys.end(), reg); - if(!inserted) {return err(inserted.unwrap_err());} - - continue; - } - return err(format_underline("toml::parse_toml_file: " - "unknown line appeared", {{source_location(loc), "unknown format"}})); - } - - return ok(Value(std::move(data), file, comments)); -} - -} // detail - -template class Table = std::unordered_map, - template class Array = std::vector> -basic_value -parse(std::istream& is, const std::string& fname = "unknown file") -{ - using value_type = basic_value; - - const auto beg = is.tellg(); - is.seekg(0, std::ios::end); - const auto end = is.tellg(); - const auto fsize = end - beg; - is.seekg(beg); - - // read whole file as a sequence of char - assert(fsize >= 0); - std::vector letters(static_cast(fsize)); - is.read(letters.data(), fsize); - - // append LF. - // Although TOML does not require LF at the EOF, to make parsing logic - // simpler, we "normalize" the content by adding LF if it does not exist. - // It also checks if the last char is CR, to avoid changing the meaning. - // This is not the *best* way to deal with the last character, but is a - // simple and quick fix. - if(!letters.empty() && letters.back() != '\n' && letters.back() != '\r') - { - letters.push_back('\n'); - } - - detail::location loc(std::move(fname), std::move(letters)); - - // skip BOM if exists. - // XXX component of BOM (like 0xEF) exceeds the representable range of - // signed char, so on some (actually, most) of the environment, these cannot - // be compared to char. However, since we are always out of luck, we need to - // check our chars are equivalent to BOM. To do this, first we need to - // convert char to unsigned char to guarantee the comparability. - if(loc.source()->size() >= 3) - { - std::array BOM; - std::memcpy(BOM.data(), loc.source()->data(), 3); - if(BOM[0] == 0xEF && BOM[1] == 0xBB && BOM[2] == 0xBF) - { - loc.advance(3); // BOM found. skip. - } - } - - const auto data = detail::parse_toml_file(loc); - if(!data) - { - throw syntax_error(data.unwrap_err(), source_location(loc)); - } - return data.unwrap(); -} - -template class Table = std::unordered_map, - template class Array = std::vector> -basic_value parse(const std::string& fname) -{ - std::ifstream ifs(fname.c_str(), std::ios_base::binary); - if(!ifs.good()) - { - throw std::runtime_error("toml::parse: file open error -> " + fname); - } - return parse(ifs, fname); -} - -#ifdef TOML11_HAS_STD_FILESYSTEM -// This function just forwards `parse("filename.toml")` to std::string version -// to avoid the ambiguity in overload resolution. -// -// Both std::string and std::filesystem::path are convertible from const char*. -// Without this, both parse(std::string) and parse(std::filesystem::path) -// matches to parse("filename.toml"). This breaks the existing code. -// -// This function exactly matches to the invocation with c-string. -// So this function is preferred than others and the ambiguity disappears. -template class Table = std::unordered_map, - template class Array = std::vector> -basic_value parse(const char* fname) -{ - return parse(std::string(fname)); -} - -template class Table = std::unordered_map, - template class Array = std::vector> -basic_value parse(const std::filesystem::path& fpath) -{ - std::ifstream ifs(fpath, std::ios_base::binary); - if(!ifs.good()) - { - throw std::runtime_error("toml::parse: file open error -> " + - fpath.string()); - } - return parse(ifs, fpath.string()); -} -#endif // TOML11_HAS_STD_FILESYSTEM - -} // toml -#endif// TOML11_PARSER_HPP diff --git a/src/toml11/toml/region.hpp b/src/toml11/toml/region.hpp deleted file mode 100644 index 2e01e51d0..000000000 --- a/src/toml11/toml/region.hpp +++ /dev/null @@ -1,417 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_REGION_HPP -#define TOML11_REGION_HPP -#include -#include -#include -#include -#include -#include -#include -#include "color.hpp" - -namespace toml -{ -namespace detail -{ - -// helper function to avoid std::string(0, 'c') or std::string(iter, iter) -template -std::string make_string(Iterator first, Iterator last) -{ - if(first == last) {return "";} - return std::string(first, last); -} -inline std::string make_string(std::size_t len, char c) -{ - if(len == 0) {return "";} - return std::string(len, c); -} - -// region_base is a base class of location and region that are defined below. -// it will be used to generate better error messages. -struct region_base -{ - region_base() = default; - virtual ~region_base() = default; - region_base(const region_base&) = default; - region_base(region_base&& ) = default; - region_base& operator=(const region_base&) = default; - region_base& operator=(region_base&& ) = default; - - virtual bool is_ok() const noexcept {return false;} - virtual char front() const noexcept {return '\0';} - - virtual std::string str() const {return std::string("unknown region");} - virtual std::string name() const {return std::string("unknown file");} - virtual std::string line() const {return std::string("unknown line");} - virtual std::string line_num() const {return std::string("?");} - - // length of the region - virtual std::size_t size() const noexcept {return 0;} - // number of characters in the line before the region - virtual std::size_t before() const noexcept {return 0;} - // number of characters in the line after the region - virtual std::size_t after() const noexcept {return 0;} - - virtual std::vector comments() const {return {};} - // ```toml - // # comment_before - // key = "value" # comment_inline - // ``` -}; - -// location represents a position in a container, which contains a file content. -// it can be considered as a region that contains only one character. -// -// it contains pointer to the file content and iterator that points the current -// location. -struct location final : public region_base -{ - using const_iterator = typename std::vector::const_iterator; - using difference_type = typename const_iterator::difference_type; - using source_ptr = std::shared_ptr>; - - location(std::string source_name, std::vector cont) - : source_(std::make_shared>(std::move(cont))), - line_number_(1), source_name_(std::move(source_name)), iter_(source_->cbegin()) - {} - location(std::string source_name, const std::string& cont) - : source_(std::make_shared>(cont.begin(), cont.end())), - line_number_(1), source_name_(std::move(source_name)), iter_(source_->cbegin()) - {} - - location(const location&) = default; - location(location&&) = default; - location& operator=(const location&) = default; - location& operator=(location&&) = default; - ~location() = default; - - bool is_ok() const noexcept override {return static_cast(source_);} - char front() const noexcept override {return *iter_;} - - // this const prohibits codes like `++(loc.iter())`. - const const_iterator iter() const noexcept {return iter_;} - - const_iterator begin() const noexcept {return source_->cbegin();} - const_iterator end() const noexcept {return source_->cend();} - - // XXX `location::line_num()` used to be implemented using `std::count` to - // count a number of '\n'. But with a long toml file (typically, 10k lines), - // it becomes intolerably slow because each time it generates error messages, - // it counts '\n' from thousands of characters. To workaround it, I decided - // to introduce `location::line_number_` member variable and synchronize it - // to the location changes the point to look. So an overload of `iter()` - // which returns mutable reference is removed and `advance()`, `retrace()` - // and `reset()` is added. - void advance(difference_type n = 1) noexcept - { - this->line_number_ += static_cast( - std::count(this->iter_, std::next(this->iter_, n), '\n')); - this->iter_ += n; - return; - } - void retrace(difference_type n = 1) noexcept - { - this->line_number_ -= static_cast( - std::count(std::prev(this->iter_, n), this->iter_, '\n')); - this->iter_ -= n; - return; - } - void reset(const_iterator rollback) noexcept - { - // since c++11, std::distance works in both ways for random-access - // iterators and returns a negative value if `first > last`. - if(0 <= std::distance(rollback, this->iter_)) // rollback < iter - { - this->line_number_ -= static_cast( - std::count(rollback, this->iter_, '\n')); - } - else // iter < rollback [[unlikely]] - { - this->line_number_ += static_cast( - std::count(this->iter_, rollback, '\n')); - } - this->iter_ = rollback; - return; - } - - std::string str() const override {return make_string(1, *this->iter());} - std::string name() const override {return source_name_;} - - std::string line_num() const override - { - return std::to_string(this->line_number_); - } - - std::string line() const override - { - return make_string(this->line_begin(), this->line_end()); - } - - const_iterator line_begin() const noexcept - { - using reverse_iterator = std::reverse_iterator; - return std::find(reverse_iterator(this->iter()), - reverse_iterator(this->begin()), '\n').base(); - } - const_iterator line_end() const noexcept - { - return std::find(this->iter(), this->end(), '\n'); - } - - // location is always points a character. so the size is 1. - std::size_t size() const noexcept override - { - return 1u; - } - std::size_t before() const noexcept override - { - const auto sz = std::distance(this->line_begin(), this->iter()); - assert(sz >= 0); - return static_cast(sz); - } - std::size_t after() const noexcept override - { - const auto sz = std::distance(this->iter(), this->line_end()); - assert(sz >= 0); - return static_cast(sz); - } - - source_ptr const& source() const& noexcept {return source_;} - source_ptr&& source() && noexcept {return std::move(source_);} - - private: - - source_ptr source_; - std::size_t line_number_; - std::string source_name_; - const_iterator iter_; -}; - -// region represents a range in a container, which contains a file content. -// -// it contains pointer to the file content and iterator that points the first -// and last location. -struct region final : public region_base -{ - using const_iterator = typename std::vector::const_iterator; - using source_ptr = std::shared_ptr>; - - // delete default constructor. source_ never be null. - region() = delete; - - explicit region(const location& loc) - : source_(loc.source()), source_name_(loc.name()), - first_(loc.iter()), last_(loc.iter()) - {} - explicit region(location&& loc) - : source_(loc.source()), source_name_(loc.name()), - first_(loc.iter()), last_(loc.iter()) - {} - - region(const location& loc, const_iterator f, const_iterator l) - : source_(loc.source()), source_name_(loc.name()), first_(f), last_(l) - {} - region(location&& loc, const_iterator f, const_iterator l) - : source_(loc.source()), source_name_(loc.name()), first_(f), last_(l) - {} - - region(const region&) = default; - region(region&&) = default; - region& operator=(const region&) = default; - region& operator=(region&&) = default; - ~region() = default; - - region& operator+=(const region& other) - { - // different regions cannot be concatenated - assert(this->begin() == other.begin() && this->end() == other.end() && - this->last_ == other.first_); - - this->last_ = other.last_; - return *this; - } - - bool is_ok() const noexcept override {return static_cast(source_);} - char front() const noexcept override {return *first_;} - - std::string str() const override {return make_string(first_, last_);} - std::string line() const override - { - if(this->contain_newline()) - { - return make_string(this->line_begin(), - std::find(this->line_begin(), this->last(), '\n')); - } - return make_string(this->line_begin(), this->line_end()); - } - std::string line_num() const override - { - return std::to_string(1 + std::count(this->begin(), this->first(), '\n')); - } - - std::size_t size() const noexcept override - { - const auto sz = std::distance(first_, last_); - assert(sz >= 0); - return static_cast(sz); - } - std::size_t before() const noexcept override - { - const auto sz = std::distance(this->line_begin(), this->first()); - assert(sz >= 0); - return static_cast(sz); - } - std::size_t after() const noexcept override - { - const auto sz = std::distance(this->last(), this->line_end()); - assert(sz >= 0); - return static_cast(sz); - } - - bool contain_newline() const noexcept - { - return std::find(this->first(), this->last(), '\n') != this->last(); - } - - const_iterator line_begin() const noexcept - { - using reverse_iterator = std::reverse_iterator; - return std::find(reverse_iterator(this->first()), - reverse_iterator(this->begin()), '\n').base(); - } - const_iterator line_end() const noexcept - { - return std::find(this->last(), this->end(), '\n'); - } - - const_iterator begin() const noexcept {return source_->cbegin();} - const_iterator end() const noexcept {return source_->cend();} - const_iterator first() const noexcept {return first_;} - const_iterator last() const noexcept {return last_;} - - source_ptr const& source() const& noexcept {return source_;} - source_ptr&& source() && noexcept {return std::move(source_);} - - std::string name() const override {return source_name_;} - - std::vector comments() const override - { - // assuming the current region (`*this`) points a value. - // ```toml - // a = "value" - // ^^^^^^^- this region - // ``` - using rev_iter = std::reverse_iterator; - - std::vector com{}; - { - // find comments just before the current region. - // ```toml - // # this should be collected. - // # this also. - // a = value # not this. - // ``` - - // # this is a comment for `a`, not array elements. - // a = [1, 2, 3, 4, 5] - if(this->first() == std::find_if(this->line_begin(), this->first(), - [](const char c) noexcept -> bool {return c == '[' || c == '{';})) - { - auto iter = this->line_begin(); // points the first character - while(iter != this->begin()) - { - iter = std::prev(iter); - - // range [line_start, iter) represents the previous line - const auto line_start = std::find( - rev_iter(iter), rev_iter(this->begin()), '\n').base(); - const auto comment_found = std::find(line_start, iter, '#'); - if(comment_found == iter) - { - break; // comment not found. - } - - // exclude the following case. - // > a = "foo" # comment // <-- this is not a comment for b but a. - // > b = "current value" - if(std::all_of(line_start, comment_found, - [](const char c) noexcept -> bool { - return c == ' ' || c == '\t'; - })) - { - // unwrap the first '#' by std::next. - auto s = make_string(std::next(comment_found), iter); - if(!s.empty() && s.back() == '\r') {s.pop_back();} - com.push_back(std::move(s)); - } - else - { - break; - } - iter = line_start; - } - } - } - - if(com.size() > 1) - { - std::reverse(com.begin(), com.end()); - } - - { - // find comments just after the current region. - // ```toml - // # not this. - // a = value # this one. - // a = [ # not this (technically difficult) - // - // ] # and this. - // ``` - // The reason why it's difficult is that it requires parsing in the - // following case. - // ```toml - // a = [ 10 # this comment is for `10`. not for `a` but `a[0]`. - // # ... - // ] # this is apparently a comment for a. - // - // b = [ - // 3.14 ] # there is no way to add a comment to `3.14` currently. - // - // c = [ - // 3.14 # do this if you need a comment here. - // ] - // ``` - const auto comment_found = - std::find(this->last(), this->line_end(), '#'); - if(comment_found != this->line_end()) // '#' found - { - // table = {key = "value"} # what is this for? - // the above comment is not for "value", but {key="value"}. - if(comment_found == std::find_if(this->last(), comment_found, - [](const char c) noexcept -> bool { - return !(c == ' ' || c == '\t' || c == ','); - })) - { - // unwrap the first '#' by std::next. - auto s = make_string(std::next(comment_found), this->line_end()); - if(!s.empty() && s.back() == '\r') {s.pop_back();} - com.push_back(std::move(s)); - } - } - } - return com; - } - - private: - - source_ptr source_; - std::string source_name_; - const_iterator first_, last_; -}; - -} // detail -} // toml -#endif// TOML11_REGION_H diff --git a/src/toml11/toml/result.hpp b/src/toml11/toml/result.hpp deleted file mode 100644 index 77cd46c64..000000000 --- a/src/toml11/toml/result.hpp +++ /dev/null @@ -1,717 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_RESULT_HPP -#define TOML11_RESULT_HPP -#include "traits.hpp" -#include -#include -#include -#include -#include -#include -#include - -namespace toml -{ - -template -struct success -{ - using value_type = T; - value_type value; - - explicit success(const value_type& v) - noexcept(std::is_nothrow_copy_constructible::value) - : value(v) - {} - explicit success(value_type&& v) - noexcept(std::is_nothrow_move_constructible::value) - : value(std::move(v)) - {} - - template - explicit success(U&& v): value(std::forward(v)) {} - - template - explicit success(const success& v): value(v.value) {} - template - explicit success(success&& v): value(std::move(v.value)) {} - - ~success() = default; - success(const success&) = default; - success(success&&) = default; - success& operator=(const success&) = default; - success& operator=(success&&) = default; -}; - -template -struct failure -{ - using value_type = T; - value_type value; - - explicit failure(const value_type& v) - noexcept(std::is_nothrow_copy_constructible::value) - : value(v) - {} - explicit failure(value_type&& v) - noexcept(std::is_nothrow_move_constructible::value) - : value(std::move(v)) - {} - - template - explicit failure(U&& v): value(std::forward(v)) {} - - template - explicit failure(const failure& v): value(v.value) {} - template - explicit failure(failure&& v): value(std::move(v.value)) {} - - ~failure() = default; - failure(const failure&) = default; - failure(failure&&) = default; - failure& operator=(const failure&) = default; - failure& operator=(failure&&) = default; -}; - -template -success::type>::type> -ok(T&& v) -{ - return success< - typename std::remove_cv::type>::type - >(std::forward(v)); -} -template -failure::type>::type> -err(T&& v) -{ - return failure< - typename std::remove_cv::type>::type - >(std::forward(v)); -} - -inline success ok(const char* literal) -{ - return success(std::string(literal)); -} -inline failure err(const char* literal) -{ - return failure(std::string(literal)); -} - - -template -struct result -{ - using value_type = T; - using error_type = E; - using success_type = success; - using failure_type = failure; - - result(const success_type& s): is_ok_(true) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(s); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - result(const failure_type& f): is_ok_(false) - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(f); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - result(success_type&& s): is_ok_(true) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s)); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - result(failure_type&& f): is_ok_(false) - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f)); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - - template - result(const success& s): is_ok_(true) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(s.value); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - template - result(const failure& f): is_ok_(false) - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(f.value); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - template - result(success&& s): is_ok_(true) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s.value)); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - template - result(failure&& f): is_ok_(false) - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f.value)); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - - result& operator=(const success_type& s) - { - this->cleanup(); - this->is_ok_ = true; - auto tmp = ::new(std::addressof(this->succ)) success_type(s); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - return *this; - } - result& operator=(const failure_type& f) - { - this->cleanup(); - this->is_ok_ = false; - auto tmp = ::new(std::addressof(this->fail)) failure_type(f); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - return *this; - } - result& operator=(success_type&& s) - { - this->cleanup(); - this->is_ok_ = true; - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s)); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - return *this; - } - result& operator=(failure_type&& f) - { - this->cleanup(); - this->is_ok_ = false; - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f)); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - return *this; - } - - template - result& operator=(const success& s) - { - this->cleanup(); - this->is_ok_ = true; - auto tmp = ::new(std::addressof(this->succ)) success_type(s.value); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - return *this; - } - template - result& operator=(const failure& f) - { - this->cleanup(); - this->is_ok_ = false; - auto tmp = ::new(std::addressof(this->fail)) failure_type(f.value); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - return *this; - } - template - result& operator=(success&& s) - { - this->cleanup(); - this->is_ok_ = true; - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(s.value)); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - return *this; - } - template - result& operator=(failure&& f) - { - this->cleanup(); - this->is_ok_ = false; - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(f.value)); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - return *this; - } - - ~result() noexcept {this->cleanup();} - - result(const result& other): is_ok_(other.is_ok()) - { - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - } - result(result&& other): is_ok_(other.is_ok()) - { - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - } - - template - result(const result& other): is_ok_(other.is_ok()) - { - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - } - template - result(result&& other): is_ok_(other.is_ok()) - { - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - } - - result& operator=(const result& other) - { - this->cleanup(); - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - is_ok_ = other.is_ok(); - return *this; - } - result& operator=(result&& other) - { - this->cleanup(); - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - is_ok_ = other.is_ok(); - return *this; - } - - template - result& operator=(const result& other) - { - this->cleanup(); - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(other.as_ok()); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(other.as_err()); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - is_ok_ = other.is_ok(); - return *this; - } - template - result& operator=(result&& other) - { - this->cleanup(); - if(other.is_ok()) - { - auto tmp = ::new(std::addressof(this->succ)) success_type(std::move(other.as_ok())); - assert(tmp == std::addressof(this->succ)); - (void)tmp; - } - else - { - auto tmp = ::new(std::addressof(this->fail)) failure_type(std::move(other.as_err())); - assert(tmp == std::addressof(this->fail)); - (void)tmp; - } - is_ok_ = other.is_ok(); - return *this; - } - - bool is_ok() const noexcept {return is_ok_;} - bool is_err() const noexcept {return !is_ok_;} - - operator bool() const noexcept {return is_ok_;} - - value_type& unwrap() & - { - if(is_err()) - { - throw std::runtime_error("toml::result: bad unwrap: " + - format_error(this->as_err())); - } - return this->succ.value; - } - value_type const& unwrap() const& - { - if(is_err()) - { - throw std::runtime_error("toml::result: bad unwrap: " + - format_error(this->as_err())); - } - return this->succ.value; - } - value_type&& unwrap() && - { - if(is_err()) - { - throw std::runtime_error("toml::result: bad unwrap: " + - format_error(this->as_err())); - } - return std::move(this->succ.value); - } - - value_type& unwrap_or(value_type& opt) & - { - if(is_err()) {return opt;} - return this->succ.value; - } - value_type const& unwrap_or(value_type const& opt) const& - { - if(is_err()) {return opt;} - return this->succ.value; - } - value_type unwrap_or(value_type opt) && - { - if(is_err()) {return opt;} - return this->succ.value; - } - - error_type& unwrap_err() & - { - if(is_ok()) {throw std::runtime_error("toml::result: bad unwrap_err");} - return this->fail.value; - } - error_type const& unwrap_err() const& - { - if(is_ok()) {throw std::runtime_error("toml::result: bad unwrap_err");} - return this->fail.value; - } - error_type&& unwrap_err() && - { - if(is_ok()) {throw std::runtime_error("toml::result: bad unwrap_err");} - return std::move(this->fail.value); - } - - value_type& as_ok() & noexcept {return this->succ.value;} - value_type const& as_ok() const& noexcept {return this->succ.value;} - value_type&& as_ok() && noexcept {return std::move(this->succ.value);} - - error_type& as_err() & noexcept {return this->fail.value;} - error_type const& as_err() const& noexcept {return this->fail.value;} - error_type&& as_err() && noexcept {return std::move(this->fail.value);} - - - // prerequisities - // F: T -> U - // retval: result - template - result, error_type> - map(F&& f) & - { - if(this->is_ok()){return ok(f(this->as_ok()));} - return err(this->as_err()); - } - template - result, error_type> - map(F&& f) const& - { - if(this->is_ok()){return ok(f(this->as_ok()));} - return err(this->as_err()); - } - template - result, error_type> - map(F&& f) && - { - if(this->is_ok()){return ok(f(std::move(this->as_ok())));} - return err(std::move(this->as_err())); - } - - // prerequisities - // F: E -> F - // retval: result - template - result> - map_err(F&& f) & - { - if(this->is_err()){return err(f(this->as_err()));} - return ok(this->as_ok()); - } - template - result> - map_err(F&& f) const& - { - if(this->is_err()){return err(f(this->as_err()));} - return ok(this->as_ok()); - } - template - result> - map_err(F&& f) && - { - if(this->is_err()){return err(f(std::move(this->as_err())));} - return ok(std::move(this->as_ok())); - } - - // prerequisities - // F: T -> U - // retval: U - template - detail::return_type_of_t - map_or_else(F&& f, U&& opt) & - { - if(this->is_err()){return std::forward(opt);} - return f(this->as_ok()); - } - template - detail::return_type_of_t - map_or_else(F&& f, U&& opt) const& - { - if(this->is_err()){return std::forward(opt);} - return f(this->as_ok()); - } - template - detail::return_type_of_t - map_or_else(F&& f, U&& opt) && - { - if(this->is_err()){return std::forward(opt);} - return f(std::move(this->as_ok())); - } - - // prerequisities - // F: E -> U - // retval: U - template - detail::return_type_of_t - map_err_or_else(F&& f, U&& opt) & - { - if(this->is_ok()){return std::forward(opt);} - return f(this->as_err()); - } - template - detail::return_type_of_t - map_err_or_else(F&& f, U&& opt) const& - { - if(this->is_ok()){return std::forward(opt);} - return f(this->as_err()); - } - template - detail::return_type_of_t - map_err_or_else(F&& f, U&& opt) && - { - if(this->is_ok()){return std::forward(opt);} - return f(std::move(this->as_err())); - } - - // prerequisities: - // F: func T -> U - // toml::err(error_type) should be convertible to U. - // normally, type U is another result and E is convertible to F - template - detail::return_type_of_t - and_then(F&& f) & - { - if(this->is_ok()){return f(this->as_ok());} - return err(this->as_err()); - } - template - detail::return_type_of_t - and_then(F&& f) const& - { - if(this->is_ok()){return f(this->as_ok());} - return err(this->as_err()); - } - template - detail::return_type_of_t - and_then(F&& f) && - { - if(this->is_ok()){return f(std::move(this->as_ok()));} - return err(std::move(this->as_err())); - } - - // prerequisities: - // F: func E -> U - // toml::ok(value_type) should be convertible to U. - // normally, type U is another result and T is convertible to S - template - detail::return_type_of_t - or_else(F&& f) & - { - if(this->is_err()){return f(this->as_err());} - return ok(this->as_ok()); - } - template - detail::return_type_of_t - or_else(F&& f) const& - { - if(this->is_err()){return f(this->as_err());} - return ok(this->as_ok()); - } - template - detail::return_type_of_t - or_else(F&& f) && - { - if(this->is_err()){return f(std::move(this->as_err()));} - return ok(std::move(this->as_ok())); - } - - // if *this is error, returns *this. otherwise, returns other. - result and_other(const result& other) const& - { - return this->is_err() ? *this : other; - } - result and_other(result&& other) && - { - return this->is_err() ? std::move(*this) : std::move(other); - } - - // if *this is okay, returns *this. otherwise, returns other. - result or_other(const result& other) const& - { - return this->is_ok() ? *this : other; - } - result or_other(result&& other) && - { - return this->is_ok() ? std::move(*this) : std::move(other); - } - - void swap(result& other) - { - result tmp(std::move(*this)); - *this = std::move(other); - other = std::move(tmp); - return ; - } - - private: - - static std::string format_error(std::exception const& excpt) - { - return std::string(excpt.what()); - } - template::value, std::nullptr_t>::type = nullptr> - static std::string format_error(U const& others) - { - std::ostringstream oss; oss << others; - return oss.str(); - } - - void cleanup() noexcept - { - if(this->is_ok_) {this->succ.~success_type();} - else {this->fail.~failure_type();} - return; - } - - private: - - bool is_ok_; - union - { - success_type succ; - failure_type fail; - }; -}; - -template -void swap(result& lhs, result& rhs) -{ - lhs.swap(rhs); - return; -} - -// this might be confusing because it eagerly evaluated, while in the other -// cases operator && and || are short-circuited. -// -// template -// inline result -// operator&&(const result& lhs, const result& rhs) noexcept -// { -// return lhs.is_ok() ? rhs : lhs; -// } -// -// template -// inline result -// operator||(const result& lhs, const result& rhs) noexcept -// { -// return lhs.is_ok() ? lhs : rhs; -// } - -// ---------------------------------------------------------------------------- -// re-use result as a optional with none_t - -namespace detail -{ -struct none_t {}; -inline bool operator==(const none_t&, const none_t&) noexcept {return true;} -inline bool operator!=(const none_t&, const none_t&) noexcept {return false;} -inline bool operator< (const none_t&, const none_t&) noexcept {return false;} -inline bool operator<=(const none_t&, const none_t&) noexcept {return true;} -inline bool operator> (const none_t&, const none_t&) noexcept {return false;} -inline bool operator>=(const none_t&, const none_t&) noexcept {return true;} -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const none_t&) -{ - os << "none"; - return os; -} -inline failure none() noexcept {return failure{none_t{}};} -} // detail -} // toml11 -#endif// TOML11_RESULT_H diff --git a/src/toml11/toml/serializer.hpp b/src/toml11/toml/serializer.hpp deleted file mode 100644 index 88ae775a8..000000000 --- a/src/toml11/toml/serializer.hpp +++ /dev/null @@ -1,922 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_SERIALIZER_HPP -#define TOML11_SERIALIZER_HPP -#include -#include - -#include - -#include "lexer.hpp" -#include "value.hpp" - -namespace toml -{ - -// This function serialize a key. It checks a string is a bare key and -// escapes special characters if the string is not compatible to a bare key. -// ```cpp -// std::string k("non.bare.key"); // the key itself includes `.`s. -// std::string formatted = toml::format_key(k); -// assert(formatted == "\"non.bare.key\""); -// ``` -// -// This function is exposed to make it easy to write a user-defined serializer. -// Since toml restricts characters available in a bare key, generally a string -// should be escaped. But checking whether a string needs to be surrounded by -// a `"` and escaping some special character is boring. -template -std::basic_string -format_key(const std::basic_string& k) -{ - if(k.empty()) - { - return std::string("\"\""); - } - - // check the key can be a bare (unquoted) key - detail::location loc(k, std::vector(k.begin(), k.end())); - detail::lex_unquoted_key::invoke(loc); - if(loc.iter() == loc.end()) - { - return k; // all the tokens are consumed. the key is unquoted-key. - } - - //if it includes special characters, then format it in a "quoted" key. - std::basic_string serialized("\""); - for(const char c : k) - { - switch(c) - { - case '\\': {serialized += "\\\\"; break;} - case '\"': {serialized += "\\\""; break;} - case '\b': {serialized += "\\b"; break;} - case '\t': {serialized += "\\t"; break;} - case '\f': {serialized += "\\f"; break;} - case '\n': {serialized += "\\n"; break;} - case '\r': {serialized += "\\r"; break;} - default : {serialized += c; break;} - } - } - serialized += "\""; - return serialized; -} - -template -std::basic_string -format_keys(const std::vector>& keys) -{ - if(keys.empty()) - { - return std::string("\"\""); - } - - std::basic_string serialized; - for(const auto& ky : keys) - { - serialized += format_key(ky); - serialized += charT('.'); - } - serialized.pop_back(); // remove the last dot '.' - return serialized; -} - -template -struct serializer -{ - static_assert(detail::is_basic_value::value, - "toml::serializer is for toml::value and its variants, " - "toml::basic_value<...>."); - - using value_type = Value; - using key_type = typename value_type::key_type ; - using comment_type = typename value_type::comment_type ; - using boolean_type = typename value_type::boolean_type ; - using integer_type = typename value_type::integer_type ; - using floating_type = typename value_type::floating_type ; - using string_type = typename value_type::string_type ; - using local_time_type = typename value_type::local_time_type ; - using local_date_type = typename value_type::local_date_type ; - using local_datetime_type = typename value_type::local_datetime_type ; - using offset_datetime_type = typename value_type::offset_datetime_type; - using array_type = typename value_type::array_type ; - using table_type = typename value_type::table_type ; - - serializer(const std::size_t w = 80u, - const int float_prec = std::numeric_limits::max_digits10, - const bool can_be_inlined = false, - const bool no_comment = false, - std::vector ks = {}, - const bool value_has_comment = false) - : can_be_inlined_(can_be_inlined), no_comment_(no_comment), - value_has_comment_(value_has_comment && !no_comment), - float_prec_(float_prec), width_(w), keys_(std::move(ks)) - {} - ~serializer() = default; - - std::string operator()(const boolean_type& b) const - { - return b ? "true" : "false"; - } - std::string operator()(const integer_type i) const - { - return std::to_string(i); - } - std::string operator()(const floating_type f) const - { - if(std::isnan(f)) - { - if(std::signbit(f)) - { - return std::string("-nan"); - } - else - { - return std::string("nan"); - } - } - else if(!std::isfinite(f)) - { - if(std::signbit(f)) - { - return std::string("-inf"); - } - else - { - return std::string("inf"); - } - } - - const auto fmt = "%.*g"; - const auto bsz = std::snprintf(nullptr, 0, fmt, this->float_prec_, f); - // +1 for null character(\0) - std::vector buf(static_cast(bsz + 1), '\0'); - std::snprintf(buf.data(), buf.size(), fmt, this->float_prec_, f); - - std::string token(buf.begin(), std::prev(buf.end())); - if(!token.empty() && token.back() == '.') // 1. => 1.0 - { - token += '0'; - } - - const auto e = std::find_if( - token.cbegin(), token.cend(), [](const char c) noexcept -> bool { - return c == 'e' || c == 'E'; - }); - const auto has_exponent = (token.cend() != e); - const auto has_fraction = (token.cend() != std::find( - token.cbegin(), token.cend(), '.')); - - if(!has_exponent && !has_fraction) - { - // the resulting value does not have any float specific part! - token += ".0"; - } - return token; - } - std::string operator()(const string_type& s) const - { - if(s.kind == string_t::basic) - { - if((std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || - std::find(s.str.cbegin(), s.str.cend(), '\"') != s.str.cend()) && - this->width_ != (std::numeric_limits::max)()) - { - // if linefeed or double-quote is contained, - // make it multiline basic string. - const auto escaped = this->escape_ml_basic_string(s.str); - std::string open("\"\"\""); - std::string close("\"\"\""); - if(escaped.find('\n') != std::string::npos || - this->width_ < escaped.size() + 6) - { - // if the string body contains newline or is enough long, - // add newlines after and before delimiters. - open += "\n"; - close = std::string("\\\n") + close; - } - return open + escaped + close; - } - - // no linefeed. try to make it oneline-string. - std::string oneline = this->escape_basic_string(s.str); - if(oneline.size() + 2 < width_ || width_ < 2) - { - const std::string quote("\""); - return quote + oneline + quote; - } - - // the line is too long compared to the specified width. - // split it into multiple lines. - std::string token("\"\"\"\n"); - while(!oneline.empty()) - { - if(oneline.size() < width_) - { - token += oneline; - oneline.clear(); - } - else if(oneline.at(width_-2) == '\\') - { - token += oneline.substr(0, width_-2); - token += "\\\n"; - oneline.erase(0, width_-2); - } - else - { - token += oneline.substr(0, width_-1); - token += "\\\n"; - oneline.erase(0, width_-1); - } - } - return token + std::string("\\\n\"\"\""); - } - else // the string `s` is literal-string. - { - if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || - std::find(s.str.cbegin(), s.str.cend(), '\'') != s.str.cend() ) - { - std::string open("'''"); - if(this->width_ + 6 < s.str.size()) - { - open += '\n'; // the first newline is ignored by TOML spec - } - const std::string close("'''"); - return open + s.str + close; - } - else - { - const std::string quote("'"); - return quote + s.str + quote; - } - } - } - - std::string operator()(const local_date_type& d) const - { - std::ostringstream oss; - oss << d; - return oss.str(); - } - std::string operator()(const local_time_type& t) const - { - std::ostringstream oss; - oss << t; - return oss.str(); - } - std::string operator()(const local_datetime_type& dt) const - { - std::ostringstream oss; - oss << dt; - return oss.str(); - } - std::string operator()(const offset_datetime_type& odt) const - { - std::ostringstream oss; - oss << odt; - return oss.str(); - } - - std::string operator()(const array_type& v) const - { - if(v.empty()) - { - return std::string("[]"); - } - if(this->is_array_of_tables(v)) - { - return make_array_of_tables(v); - } - - // not an array of tables. normal array. - // first, try to make it inline if none of the elements have a comment. - if( ! this->has_comment_inside(v)) - { - const auto inl = this->make_inline_array(v); - if(inl.size() < this->width_ && - std::find(inl.cbegin(), inl.cend(), '\n') == inl.cend()) - { - return inl; - } - } - - // if the length exceeds this->width_, print multiline array. - // key = [ - // # ... - // 42, - // ... - // ] - std::string token; - std::string current_line; - token += "[\n"; - for(const auto& item : v) - { - if( ! item.comments().empty() && !no_comment_) - { - // if comment exists, the element must be the only element in the line. - // e.g. the following is not allowed. - // ```toml - // array = [ - // # comment for what? - // 1, 2, 3, 4, 5 - // ] - // ``` - if(!current_line.empty()) - { - if(current_line.back() != '\n') - { - current_line += '\n'; - } - token += current_line; - current_line.clear(); - } - for(const auto& c : item.comments()) - { - token += '#'; - token += c; - token += '\n'; - } - token += toml::visit(*this, item); - if(!token.empty() && token.back() == '\n') {token.pop_back();} - token += ",\n"; - continue; - } - std::string next_elem; - if(item.is_table()) - { - serializer ser(*this); - ser.can_be_inlined_ = true; - ser.width_ = (std::numeric_limits::max)(); - next_elem += toml::visit(ser, item); - } - else - { - next_elem += toml::visit(*this, item); - } - - // comma before newline. - if(!next_elem.empty() && next_elem.back() == '\n') {next_elem.pop_back();} - - // if current line does not exceeds the width limit, continue. - if(current_line.size() + next_elem.size() + 1 < this->width_) - { - current_line += next_elem; - current_line += ','; - } - else if(current_line.empty()) - { - // if current line was empty, force put the next_elem because - // next_elem is not splittable - token += next_elem; - token += ",\n"; - // current_line is kept empty - } - else // reset current_line - { - assert(current_line.back() == ','); - token += current_line; - token += '\n'; - current_line = next_elem; - current_line += ','; - } - } - if(!current_line.empty()) - { - if(!current_line.empty() && current_line.back() != '\n') - { - current_line += '\n'; - } - token += current_line; - } - token += "]\n"; - return token; - } - - // templatize for any table-like container - std::string operator()(const table_type& v) const - { - // if an element has a comment, then it can't be inlined. - // table = {# how can we write a comment for this? key = "value"} - if(this->can_be_inlined_ && !(this->has_comment_inside(v))) - { - std::string token; - if(!this->keys_.empty()) - { - token += format_key(this->keys_.back()); - token += " = "; - } - token += this->make_inline_table(v); - if(token.size() < this->width_ && - token.end() == std::find(token.begin(), token.end(), '\n')) - { - return token; - } - } - - std::string token; - if(!keys_.empty()) - { - token += '['; - token += format_keys(keys_); - token += "]\n"; - } - token += this->make_multiline_table(v); - return token; - } - - private: - - std::string escape_basic_string(const std::string& s) const - { - //XXX assuming `s` is a valid utf-8 sequence. - std::string retval; - for(const char c : s) - { - switch(c) - { - case '\\': {retval += "\\\\"; break;} - case '\"': {retval += "\\\""; break;} - case '\b': {retval += "\\b"; break;} - case '\t': {retval += "\\t"; break;} - case '\f': {retval += "\\f"; break;} - case '\n': {retval += "\\n"; break;} - case '\r': {retval += "\\r"; break;} - default : - { - if((0x00 <= c && c <= 0x08) || (0x0A <= c && c <= 0x1F) || c == 0x7F) - { - retval += "\\u00"; - retval += char(48 + (c / 16)); - retval += char((c % 16 < 10 ? 48 : 55) + (c % 16)); - } - else - { - retval += c; - } - } - } - } - return retval; - } - - std::string escape_ml_basic_string(const std::string& s) const - { - std::string retval; - for(auto i=s.cbegin(), e=s.cend(); i!=e; ++i) - { - switch(*i) - { - case '\\': {retval += "\\\\"; break;} - // One or two consecutive "s are allowed. - // Later we will check there are no three consecutive "s. - // case '\"': {retval += "\\\""; break;} - case '\b': {retval += "\\b"; break;} - case '\t': {retval += "\\t"; break;} - case '\f': {retval += "\\f"; break;} - case '\n': {retval += "\n"; break;} - case '\r': - { - if(std::next(i) != e && *std::next(i) == '\n') - { - retval += "\r\n"; - ++i; - } - else - { - retval += "\\r"; - } - break; - } - default : - { - const auto c = *i; - if((0x00 <= c && c <= 0x08) || (0x0A <= c && c <= 0x1F) || c == 0x7F) - { - retval += "\\u00"; - retval += char(48 + (c / 16)); - retval += char((c % 16 < 10 ? 48 : 55) + (c % 16)); - } - else - { - retval += c; - } - } - - } - } - // Only 1 or 2 consecutive `"`s are allowed in multiline basic string. - // 3 consecutive `"`s are considered as a closing delimiter. - // We need to check if there are 3 or more consecutive `"`s and insert - // backslash to break them down into several short `"`s like the `str6` - // in the following example. - // ```toml - // str4 = """Here are two quotation marks: "". Simple enough.""" - // # str5 = """Here are three quotation marks: """.""" # INVALID - // str5 = """Here are three quotation marks: ""\".""" - // str6 = """Here are fifteen quotation marks: ""\"""\"""\"""\"""\".""" - // ``` - auto found_3_quotes = retval.find("\"\"\""); - while(found_3_quotes != std::string::npos) - { - retval.replace(found_3_quotes, 3, "\"\"\\\""); - found_3_quotes = retval.find("\"\"\""); - } - return retval; - } - - // if an element of a table or an array has a comment, it cannot be inlined. - bool has_comment_inside(const array_type& a) const noexcept - { - // if no_comment is set, comments would not be written. - if(this->no_comment_) {return false;} - - for(const auto& v : a) - { - if(!v.comments().empty()) {return true;} - } - return false; - } - bool has_comment_inside(const table_type& t) const noexcept - { - // if no_comment is set, comments would not be written. - if(this->no_comment_) {return false;} - - for(const auto& kv : t) - { - if(!kv.second.comments().empty()) {return true;} - } - return false; - } - - std::string make_inline_array(const array_type& v) const - { - assert(!has_comment_inside(v)); - std::string token; - token += '['; - bool is_first = true; - for(const auto& item : v) - { - if(is_first) {is_first = false;} else {token += ',';} - token += visit(serializer( - (std::numeric_limits::max)(), this->float_prec_, - /* inlined */ true, /*no comment*/ false, /*keys*/ {}, - /*has_comment*/ !item.comments().empty()), item); - } - token += ']'; - return token; - } - - std::string make_inline_table(const table_type& v) const - { - assert(!has_comment_inside(v)); - assert(this->can_be_inlined_); - std::string token; - token += '{'; - bool is_first = true; - for(const auto& kv : v) - { - // in inline tables, trailing comma is not allowed (toml-lang #569). - if(is_first) {is_first = false;} else {token += ',';} - token += format_key(kv.first); - token += '='; - token += visit(serializer( - (std::numeric_limits::max)(), this->float_prec_, - /* inlined */ true, /*no comment*/ false, /*keys*/ {}, - /*has_comment*/ !kv.second.comments().empty()), kv.second); - } - token += '}'; - return token; - } - - std::string make_multiline_table(const table_type& v) const - { - std::string token; - - // print non-table elements first. - // ```toml - // [foo] # a table we're writing now here - // key = "value" # <- non-table element, "key" - // # ... - // [foo.bar] # <- table element, "bar" - // ``` - // because after printing [foo.bar], the remaining non-table values will - // be assigned into [foo.bar], not [foo]. Those values should be printed - // earlier. - for(const auto& kv : v) - { - if(kv.second.is_table() || is_array_of_tables(kv.second)) - { - continue; - } - - token += write_comments(kv.second); - - const auto key_and_sep = format_key(kv.first) + " = "; - const auto residual_width = (this->width_ > key_and_sep.size()) ? - this->width_ - key_and_sep.size() : 0; - token += key_and_sep; - token += visit(serializer(residual_width, this->float_prec_, - /*can be inlined*/ true, /*no comment*/ false, /*keys*/ {}, - /*has_comment*/ !kv.second.comments().empty()), kv.second); - - if(token.back() != '\n') - { - token += '\n'; - } - } - - // normal tables / array of tables - - // after multiline table appeared, the other tables cannot be inline - // because the table would be assigned into the table. - // [foo] - // ... - // bar = {...} # <- bar will be a member of [foo]. - bool multiline_table_printed = false; - for(const auto& kv : v) - { - if(!kv.second.is_table() && !is_array_of_tables(kv.second)) - { - continue; // other stuff are already serialized. skip them. - } - - std::vector ks(this->keys_); - ks.push_back(kv.first); - - auto tmp = visit(serializer(this->width_, this->float_prec_, - !multiline_table_printed, this->no_comment_, ks, - /*has_comment*/ !kv.second.comments().empty()), kv.second); - - // If it is the first time to print a multi-line table, it would be - // helpful to separate normal key-value pair and subtables by a - // newline. - // (this checks if the current key-value pair contains newlines. - // but it is not perfect because multi-line string can also contain - // a newline. in such a case, an empty line will be written) TODO - if((!multiline_table_printed) && - std::find(tmp.cbegin(), tmp.cend(), '\n') != tmp.cend()) - { - multiline_table_printed = true; - token += '\n'; // separate key-value pairs and subtables - - token += write_comments(kv.second); - token += tmp; - - // care about recursive tables (all tables in each level prints - // newline and there will be a full of newlines) - if(tmp.substr(tmp.size() - 2, 2) != "\n\n" && - tmp.substr(tmp.size() - 4, 4) != "\r\n\r\n" ) - { - token += '\n'; - } - } - else - { - token += write_comments(kv.second); - token += tmp; - token += '\n'; - } - } - return token; - } - - std::string make_array_of_tables(const array_type& v) const - { - // if it's not inlined, we need to add `[[table.key]]`. - // but if it can be inlined, we can format it as the following. - // ``` - // table.key = [ - // {...}, - // # comment - // {...}, - // ] - // ``` - // This function checks if inlinization is possible or not, and then - // format the array-of-tables in a proper way. - // - // Note about comments: - // - // If the array itself has a comment (value_has_comment_ == true), we - // should try to make it inline. - // ```toml - // # comment about array - // array = [ - // # comment about table element - // {of = "table"} - // ] - // ``` - // If it is formatted as a multiline table, the two comments becomes - // indistinguishable. - // ```toml - // # comment about array - // # comment about table element - // [[array]] - // of = "table" - // ``` - // So we need to try to make it inline, and it force-inlines regardless - // of the line width limit. - // It may fail if the element of a table has comment. In that case, - // the array-of-tables will be formatted as a multiline table. - if(this->can_be_inlined_ || this->value_has_comment_) - { - std::string token; - if(!keys_.empty()) - { - token += format_key(keys_.back()); - token += " = "; - } - - bool failed = false; - token += "[\n"; - for(const auto& item : v) - { - // if an element of the table has a comment, the table - // cannot be inlined. - if(this->has_comment_inside(item.as_table())) - { - failed = true; - break; - } - // write comments for the table itself - token += write_comments(item); - - const auto t = this->make_inline_table(item.as_table()); - - if(t.size() + 1 > width_ || // +1 for the last comma {...}, - std::find(t.cbegin(), t.cend(), '\n') != t.cend()) - { - // if the value itself has a comment, ignore the line width limit - if( ! this->value_has_comment_) - { - failed = true; - break; - } - } - token += t; - token += ",\n"; - } - - if( ! failed) - { - token += "]\n"; - return token; - } - // if failed, serialize them as [[array.of.tables]]. - } - - std::string token; - for(const auto& item : v) - { - token += write_comments(item); - token += "[["; - token += format_keys(keys_); - token += "]]\n"; - token += this->make_multiline_table(item.as_table()); - } - return token; - } - - std::string write_comments(const value_type& v) const - { - std::string retval; - if(this->no_comment_) {return retval;} - - for(const auto& c : v.comments()) - { - retval += '#'; - retval += c; - retval += '\n'; - } - return retval; - } - - bool is_array_of_tables(const value_type& v) const - { - if(!v.is_array() || v.as_array().empty()) {return false;} - return is_array_of_tables(v.as_array()); - } - bool is_array_of_tables(const array_type& v) const - { - // Since TOML v0.5.0, heterogeneous arrays are allowed. So we need to - // check all the element in an array to check if the array is an array - // of tables. - return std::all_of(v.begin(), v.end(), [](const value_type& elem) { - return elem.is_table(); - }); - } - - private: - - bool can_be_inlined_; - bool no_comment_; - bool value_has_comment_; - int float_prec_; - std::size_t width_; - std::vector keys_; -}; - -template class M, template class V> -std::string -format(const basic_value& v, std::size_t w = 80u, - int fprec = std::numeric_limits::max_digits10, - bool no_comment = false, bool force_inline = false) -{ - using value_type = basic_value; - // if value is a table, it is considered to be a root object. - // the root object can't be an inline table. - if(v.is_table()) - { - std::ostringstream oss; - if(!v.comments().empty()) - { - oss << v.comments(); - oss << '\n'; // to split the file comment from the first element - } - const auto serialized = visit(serializer(w, fprec, false, no_comment), v); - oss << serialized; - return oss.str(); - } - return visit(serializer(w, fprec, force_inline), v); -} - -namespace detail -{ -template -int comment_index(std::basic_ostream&) -{ - static const int index = std::ios_base::xalloc(); - return index; -} -} // detail - -template -std::basic_ostream& -nocomment(std::basic_ostream& os) -{ - // by default, it is zero. and by default, it shows comments. - os.iword(detail::comment_index(os)) = 1; - return os; -} - -template -std::basic_ostream& -showcomment(std::basic_ostream& os) -{ - // by default, it is zero. and by default, it shows comments. - os.iword(detail::comment_index(os)) = 0; - return os; -} - -template class M, template class V> -std::basic_ostream& -operator<<(std::basic_ostream& os, const basic_value& v) -{ - using value_type = basic_value; - - // get status of std::setw(). - const auto w = static_cast(os.width()); - const int fprec = static_cast(os.precision()); - os.width(0); - - // by default, iword is initialized by 0. And by default, toml11 outputs - // comments. So `0` means showcomment. 1 means nocommnet. - const bool no_comment = (1 == os.iword(detail::comment_index(os))); - - if(!no_comment && v.is_table() && !v.comments().empty()) - { - os << v.comments(); - os << '\n'; // to split the file comment from the first element - } - // the root object can't be an inline table. so pass `false`. - const auto serialized = visit(serializer(w, fprec, no_comment, false), v); - os << serialized; - - // if v is a non-table value, and has only one comment, then - // put a comment just after a value. in the following way. - // - // ```toml - // key = "value" # comment. - // ``` - // - // Since the top-level toml object is a table, one who want to put a - // non-table toml value must use this in a following way. - // - // ```cpp - // toml::value v; - // std::cout << "user-defined-key = " << v << std::endl; - // ``` - // - // In this case, it is impossible to put comments before key-value pair. - // The only way to preserve comments is to put all of them after a value. - if(!no_comment && !v.is_table() && !v.comments().empty()) - { - os << " #"; - for(const auto& c : v.comments()) {os << c;} - } - return os; -} - -} // toml -#endif// TOML11_SERIALIZER_HPP diff --git a/src/toml11/toml/source_location.hpp b/src/toml11/toml/source_location.hpp deleted file mode 100644 index fa175b5b4..000000000 --- a/src/toml11/toml/source_location.hpp +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright Toru Niina 2019. -// Distributed under the MIT License. -#ifndef TOML11_SOURCE_LOCATION_HPP -#define TOML11_SOURCE_LOCATION_HPP -#include -#include - -#include "region.hpp" - -namespace toml -{ - -// A struct to contain location in a toml file. -// The interface imitates std::experimental::source_location, -// but not completely the same. -// -// It would be constructed by toml::value. It can be used to generate -// user-defined error messages. -// -// - std::uint_least32_t line() const noexcept -// - returns the line number where the region is on. -// - std::uint_least32_t column() const noexcept -// - returns the column number where the region starts. -// - std::uint_least32_t region() const noexcept -// - returns the size of the region. -// -// +-- line() +-- region of interest (region() == 9) -// v .---+---. -// 12 | value = "foo bar" -// ^ -// +-- column() -// -// - std::string const& file_name() const noexcept; -// - name of the file. -// - std::string const& line_str() const noexcept; -// - the whole line that contains the region of interest. -// -struct source_location -{ - public: - - source_location() - : line_num_(1), column_num_(1), region_size_(1), - file_name_("unknown file"), line_str_("") - {} - - explicit source_location(const detail::region_base* reg) - : line_num_(1), column_num_(1), region_size_(1), - file_name_("unknown file"), line_str_("") - { - if(reg) - { - if(reg->line_num() != detail::region_base().line_num()) - { - line_num_ = static_cast( - std::stoul(reg->line_num())); - } - column_num_ = static_cast(reg->before() + 1); - region_size_ = static_cast(reg->size()); - file_name_ = reg->name(); - line_str_ = reg->line(); - } - } - - explicit source_location(const detail::region& reg) - : line_num_(static_cast(std::stoul(reg.line_num()))), - column_num_(static_cast(reg.before() + 1)), - region_size_(static_cast(reg.size())), - file_name_(reg.name()), - line_str_ (reg.line()) - {} - explicit source_location(const detail::location& loc) - : line_num_(static_cast(std::stoul(loc.line_num()))), - column_num_(static_cast(loc.before() + 1)), - region_size_(static_cast(loc.size())), - file_name_(loc.name()), - line_str_ (loc.line()) - {} - - ~source_location() = default; - source_location(source_location const&) = default; - source_location(source_location &&) = default; - source_location& operator=(source_location const&) = default; - source_location& operator=(source_location &&) = default; - - std::uint_least32_t line() const noexcept {return line_num_;} - std::uint_least32_t column() const noexcept {return column_num_;} - std::uint_least32_t region() const noexcept {return region_size_;} - - std::string const& file_name() const noexcept {return file_name_;} - std::string const& line_str() const noexcept {return line_str_;} - - private: - - std::uint_least32_t line_num_; - std::uint_least32_t column_num_; - std::uint_least32_t region_size_; - std::string file_name_; - std::string line_str_; -}; - -namespace detail -{ - -// internal error message generation. -inline std::string format_underline(const std::string& message, - const std::vector>& loc_com, - const std::vector& helps = {}, - const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) -{ - std::size_t line_num_width = 0; - for(const auto& lc : loc_com) - { - std::uint_least32_t line = lc.first.line(); - std::size_t digit = 0; - while(line != 0) - { - line /= 10; - digit += 1; - } - line_num_width = (std::max)(line_num_width, digit); - } - // 1 is the minimum width - line_num_width = std::max(line_num_width, 1); - - std::ostringstream retval; - - if(colorize) - { - retval << color::colorize; // turn on ANSI color - } - - // XXX - // Here, before `colorize` support, it does not output `[error]` prefix - // automatically. So some user may output it manually and this change may - // duplicate the prefix. To avoid it, check the first 7 characters and - // if it is "[error]", it removes that part from the message shown. - if(message.size() > 7 && message.substr(0, 7) == "[error]") - { - retval << color::bold << color::red << "[error]" << color::reset - << color::bold << message.substr(7) << color::reset << '\n'; - } - else - { - retval << color::bold << color::red << "[error] " << color::reset - << color::bold << message << color::reset << '\n'; - } - - const auto format_one_location = [line_num_width] - (std::ostringstream& oss, - const source_location& loc, const std::string& comment) -> void - { - oss << ' ' << color::bold << color::blue - << std::setw(static_cast(line_num_width)) - << std::right << loc.line() << " | " << color::reset - << loc.line_str() << '\n'; - - oss << make_string(line_num_width + 1, ' ') - << color::bold << color::blue << " | " << color::reset - << make_string(loc.column()-1 /*1-origin*/, ' '); - - if(loc.region() == 1) - { - // invalid - // ^------ - oss << color::bold << color::red << "^---" << color::reset; - } - else - { - // invalid - // ~~~~~~~ - const auto underline_len = (std::min)( - static_cast(loc.region()), loc.line_str().size()); - oss << color::bold << color::red - << make_string(underline_len, '~') << color::reset; - } - oss << ' '; - oss << comment; - return; - }; - - assert(!loc_com.empty()); - - // --> example.toml - // | - retval << color::bold << color::blue << " --> " << color::reset - << loc_com.front().first.file_name() << '\n'; - retval << make_string(line_num_width + 1, ' ') - << color::bold << color::blue << " |\n" << color::reset; - // 1 | key value - // | ^--- missing = - format_one_location(retval, loc_com.front().first, loc_com.front().second); - - // process the rest of the locations - for(std::size_t i=1; i filename.toml" again - { - retval << color::bold << color::blue << " --> " << color::reset - << curr.first.file_name() << '\n'; - retval << make_string(line_num_width + 1, ' ') - << color::bold << color::blue << " |\n" << color::reset; - } - - format_one_location(retval, curr.first, curr.second); - } - - if(!helps.empty()) - { - retval << '\n'; - retval << make_string(line_num_width + 1, ' '); - retval << color::bold << color::blue << " |" << color::reset; - for(const auto& help : helps) - { - retval << color::bold << "\nHint: " << color::reset; - retval << help; - } - } - return retval.str(); -} - -} // detail -} // toml -#endif// TOML11_SOURCE_LOCATION_HPP diff --git a/src/toml11/toml/storage.hpp b/src/toml11/toml/storage.hpp deleted file mode 100644 index 202f9035f..000000000 --- a/src/toml11/toml/storage.hpp +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_STORAGE_HPP -#define TOML11_STORAGE_HPP -#include "utility.hpp" - -namespace toml -{ -namespace detail -{ - -// this contains pointer and deep-copy the content if copied. -// to avoid recursive pointer. -template -struct storage -{ - using value_type = T; - - explicit storage(value_type const& v): ptr(toml::make_unique(v)) {} - explicit storage(value_type&& v): ptr(toml::make_unique(std::move(v))) {} - ~storage() = default; - storage(const storage& rhs): ptr(toml::make_unique(*rhs.ptr)) {} - storage& operator=(const storage& rhs) - { - this->ptr = toml::make_unique(*rhs.ptr); - return *this; - } - storage(storage&&) = default; - storage& operator=(storage&&) = default; - - bool is_ok() const noexcept {return static_cast(ptr);} - - value_type& value() & noexcept {return *ptr;} - value_type const& value() const& noexcept {return *ptr;} - value_type&& value() && noexcept {return std::move(*ptr);} - - private: - std::unique_ptr ptr; -}; - -} // detail -} // toml -#endif// TOML11_STORAGE_HPP diff --git a/src/toml11/toml/string.hpp b/src/toml11/toml/string.hpp deleted file mode 100644 index 5136d8c56..000000000 --- a/src/toml11/toml/string.hpp +++ /dev/null @@ -1,225 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_STRING_HPP -#define TOML11_STRING_HPP -#include - -#include -#include - -#if __cplusplus >= 201703L -#if __has_include() -#define TOML11_USING_STRING_VIEW 1 -#include -#endif -#endif - -namespace toml -{ - -enum class string_t : std::uint8_t -{ - basic = 0, - literal = 1, -}; - -struct string -{ - string() = default; - ~string() = default; - string(const string& s) = default; - string(string&& s) = default; - string& operator=(const string& s) = default; - string& operator=(string&& s) = default; - - string(const std::string& s): kind(string_t::basic), str(s){} - string(const std::string& s, string_t k): kind(k), str(s){} - string(const char* s): kind(string_t::basic), str(s){} - string(const char* s, string_t k): kind(k), str(s){} - - string(std::string&& s): kind(string_t::basic), str(std::move(s)){} - string(std::string&& s, string_t k): kind(k), str(std::move(s)){} - - string& operator=(const std::string& s) - {kind = string_t::basic; str = s; return *this;} - string& operator=(std::string&& s) - {kind = string_t::basic; str = std::move(s); return *this;} - - operator std::string& () & noexcept {return str;} - operator std::string const& () const& noexcept {return str;} - operator std::string&& () && noexcept {return std::move(str);} - - string& operator+=(const char* rhs) {str += rhs; return *this;} - string& operator+=(const char rhs) {str += rhs; return *this;} - string& operator+=(const std::string& rhs) {str += rhs; return *this;} - string& operator+=(const string& rhs) {str += rhs.str; return *this;} - -#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 - explicit string(std::string_view s): kind(string_t::basic), str(s){} - string(std::string_view s, string_t k): kind(k), str(s){} - - string& operator=(std::string_view s) - {kind = string_t::basic; str = s; return *this;} - - explicit operator std::string_view() const noexcept - {return std::string_view(str);} - - string& operator+=(const std::string_view& rhs) {str += rhs; return *this;} -#endif - - string_t kind; - std::string str; -}; - -inline bool operator==(const string& lhs, const string& rhs) -{ - return lhs.kind == rhs.kind && lhs.str == rhs.str; -} -inline bool operator!=(const string& lhs, const string& rhs) -{ - return !(lhs == rhs); -} -inline bool operator<(const string& lhs, const string& rhs) -{ - return (lhs.kind == rhs.kind) ? (lhs.str < rhs.str) : (lhs.kind < rhs.kind); -} -inline bool operator>(const string& lhs, const string& rhs) -{ - return rhs < lhs; -} -inline bool operator<=(const string& lhs, const string& rhs) -{ - return !(rhs < lhs); -} -inline bool operator>=(const string& lhs, const string& rhs) -{ - return !(lhs < rhs); -} - -inline bool -operator==(const string& lhs, const std::string& rhs) {return lhs.str == rhs;} -inline bool -operator!=(const string& lhs, const std::string& rhs) {return lhs.str != rhs;} -inline bool -operator< (const string& lhs, const std::string& rhs) {return lhs.str < rhs;} -inline bool -operator> (const string& lhs, const std::string& rhs) {return lhs.str > rhs;} -inline bool -operator<=(const string& lhs, const std::string& rhs) {return lhs.str <= rhs;} -inline bool -operator>=(const string& lhs, const std::string& rhs) {return lhs.str >= rhs;} - -inline bool -operator==(const std::string& lhs, const string& rhs) {return lhs == rhs.str;} -inline bool -operator!=(const std::string& lhs, const string& rhs) {return lhs != rhs.str;} -inline bool -operator< (const std::string& lhs, const string& rhs) {return lhs < rhs.str;} -inline bool -operator> (const std::string& lhs, const string& rhs) {return lhs > rhs.str;} -inline bool -operator<=(const std::string& lhs, const string& rhs) {return lhs <= rhs.str;} -inline bool -operator>=(const std::string& lhs, const string& rhs) {return lhs >= rhs.str;} - -inline bool -operator==(const string& lhs, const char* rhs) {return lhs.str == std::string(rhs);} -inline bool -operator!=(const string& lhs, const char* rhs) {return lhs.str != std::string(rhs);} -inline bool -operator< (const string& lhs, const char* rhs) {return lhs.str < std::string(rhs);} -inline bool -operator> (const string& lhs, const char* rhs) {return lhs.str > std::string(rhs);} -inline bool -operator<=(const string& lhs, const char* rhs) {return lhs.str <= std::string(rhs);} -inline bool -operator>=(const string& lhs, const char* rhs) {return lhs.str >= std::string(rhs);} - -inline bool -operator==(const char* lhs, const string& rhs) {return std::string(lhs) == rhs.str;} -inline bool -operator!=(const char* lhs, const string& rhs) {return std::string(lhs) != rhs.str;} -inline bool -operator< (const char* lhs, const string& rhs) {return std::string(lhs) < rhs.str;} -inline bool -operator> (const char* lhs, const string& rhs) {return std::string(lhs) > rhs.str;} -inline bool -operator<=(const char* lhs, const string& rhs) {return std::string(lhs) <= rhs.str;} -inline bool -operator>=(const char* lhs, const string& rhs) {return std::string(lhs) >= rhs.str;} - -template -std::basic_ostream& -operator<<(std::basic_ostream& os, const string& s) -{ - if(s.kind == string_t::basic) - { - if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend()) - { - // it contains newline. make it multiline string. - os << "\"\"\"\n"; - for(auto i=s.str.cbegin(), e=s.str.cend(); i!=e; ++i) - { - switch(*i) - { - case '\\': {os << "\\\\"; break;} - case '\"': {os << "\\\""; break;} - case '\b': {os << "\\b"; break;} - case '\t': {os << "\\t"; break;} - case '\f': {os << "\\f"; break;} - case '\n': {os << '\n'; break;} - case '\r': - { - // since it is a multiline string, - // CRLF is not needed to be escaped. - if(std::next(i) != e && *std::next(i) == '\n') - { - os << "\r\n"; - ++i; - } - else - { - os << "\\r"; - } - break; - } - default: {os << *i; break;} - } - } - os << "\\\n\"\"\""; - return os; - } - // no newline. make it inline. - os << "\""; - for(const auto c : s.str) - { - switch(c) - { - case '\\': {os << "\\\\"; break;} - case '\"': {os << "\\\""; break;} - case '\b': {os << "\\b"; break;} - case '\t': {os << "\\t"; break;} - case '\f': {os << "\\f"; break;} - case '\n': {os << "\\n"; break;} - case '\r': {os << "\\r"; break;} - default : {os << c; break;} - } - } - os << "\""; - return os; - } - // the string `s` is literal-string. - if(std::find(s.str.cbegin(), s.str.cend(), '\n') != s.str.cend() || - std::find(s.str.cbegin(), s.str.cend(), '\'') != s.str.cend() ) - { - // contains newline or single quote. make it multiline. - os << "'''\n" << s.str << "'''"; - return os; - } - // normal literal string - os << '\'' << s.str << '\''; - return os; -} - -} // toml -#endif// TOML11_STRING_H diff --git a/src/toml11/toml/traits.hpp b/src/toml11/toml/traits.hpp deleted file mode 100644 index 5495c93b2..000000000 --- a/src/toml11/toml/traits.hpp +++ /dev/null @@ -1,327 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_TRAITS_HPP -#define TOML11_TRAITS_HPP - -#include "from.hpp" -#include "into.hpp" - -#include -#include -#include -#include -#include -#include - -#if __cplusplus >= 201703L -#if __has_include() -#include -#endif // has_include() -#endif // cplusplus >= C++17 - -namespace toml -{ -template class T, template class A> -class basic_value; - -namespace detail -{ -// --------------------------------------------------------------------------- -// check whether type T is a kind of container/map class - -struct has_iterator_impl -{ - template static std::true_type check(typename T::iterator*); - template static std::false_type check(...); -}; -struct has_value_type_impl -{ - template static std::true_type check(typename T::value_type*); - template static std::false_type check(...); -}; -struct has_key_type_impl -{ - template static std::true_type check(typename T::key_type*); - template static std::false_type check(...); -}; -struct has_mapped_type_impl -{ - template static std::true_type check(typename T::mapped_type*); - template static std::false_type check(...); -}; -struct has_reserve_method_impl -{ - template static std::false_type check(...); - template static std::true_type check( - decltype(std::declval().reserve(std::declval()))*); -}; -struct has_push_back_method_impl -{ - template static std::false_type check(...); - template static std::true_type check( - decltype(std::declval().push_back(std::declval()))*); -}; -struct is_comparable_impl -{ - template static std::false_type check(...); - template static std::true_type check( - decltype(std::declval() < std::declval())*); -}; - -struct has_from_toml_method_impl -{ - template class Tb, template class A> - static std::true_type check( - decltype(std::declval().from_toml( - std::declval<::toml::basic_value>()))*); - - template class Tb, template class A> - static std::false_type check(...); -}; -struct has_into_toml_method_impl -{ - template - static std::true_type check(decltype(std::declval().into_toml())*); - template - static std::false_type check(...); -}; - -struct has_specialized_from_impl -{ - template - static std::false_type check(...); - template)> - static std::true_type check(::toml::from*); -}; -struct has_specialized_into_impl -{ - template - static std::false_type check(...); - template)> - static std::true_type check(::toml::from*); -}; - - -/// Intel C++ compiler can not use decltype in parent class declaration, here -/// is a hack to work around it. https://stackoverflow.com/a/23953090/4692076 -#ifdef __INTEL_COMPILER -#define decltype(...) std::enable_if::type -#endif - -template -struct has_iterator : decltype(has_iterator_impl::check(nullptr)){}; -template -struct has_value_type : decltype(has_value_type_impl::check(nullptr)){}; -template -struct has_key_type : decltype(has_key_type_impl::check(nullptr)){}; -template -struct has_mapped_type : decltype(has_mapped_type_impl::check(nullptr)){}; -template -struct has_reserve_method : decltype(has_reserve_method_impl::check(nullptr)){}; -template -struct has_push_back_method : decltype(has_push_back_method_impl::check(nullptr)){}; -template -struct is_comparable : decltype(is_comparable_impl::check(nullptr)){}; - -template class Tb, template class A> -struct has_from_toml_method -: decltype(has_from_toml_method_impl::check(nullptr)){}; - -template -struct has_into_toml_method -: decltype(has_into_toml_method_impl::check(nullptr)){}; - -template -struct has_specialized_from : decltype(has_specialized_from_impl::check(nullptr)){}; -template -struct has_specialized_into : decltype(has_specialized_into_impl::check(nullptr)){}; - -#ifdef __INTEL_COMPILER -#undef decltype -#endif - -// --------------------------------------------------------------------------- -// C++17 and/or/not - -#if __cplusplus >= 201703L - -using std::conjunction; -using std::disjunction; -using std::negation; - -#else - -template struct conjunction : std::true_type{}; -template struct conjunction : T{}; -template -struct conjunction : - std::conditional(T::value), conjunction, T>::type -{}; - -template struct disjunction : std::false_type{}; -template struct disjunction : T {}; -template -struct disjunction : - std::conditional(T::value), T, disjunction>::type -{}; - -template -struct negation : std::integral_constant(T::value)>{}; - -#endif - -// --------------------------------------------------------------------------- -// type checkers - -template struct is_std_pair : std::false_type{}; -template -struct is_std_pair> : std::true_type{}; - -template struct is_std_tuple : std::false_type{}; -template -struct is_std_tuple> : std::true_type{}; - -template struct is_std_forward_list : std::false_type{}; -template -struct is_std_forward_list> : std::true_type{}; - -template struct is_chrono_duration: std::false_type{}; -template -struct is_chrono_duration>: std::true_type{}; - -template -struct is_map : conjunction< // map satisfies all the following conditions - has_iterator, // has T::iterator - has_value_type, // has T::value_type - has_key_type, // has T::key_type - has_mapped_type // has T::mapped_type - >{}; -template struct is_map : is_map{}; -template struct is_map : is_map{}; -template struct is_map : is_map{}; -template struct is_map : is_map{}; - -template -struct is_container : conjunction< - negation>, // not a map - negation>, // not a std::string -#if __cplusplus >= 201703L -#if __has_include() - negation>, // not a std::string_view -#endif // has_include() -#endif - has_iterator, // has T::iterator - has_value_type // has T::value_type - >{}; -template struct is_container : is_container{}; -template struct is_container : is_container{}; -template struct is_container : is_container{}; -template struct is_container : is_container{}; - -template -struct is_basic_value: std::false_type{}; -template struct is_basic_value : is_basic_value{}; -template struct is_basic_value : is_basic_value{}; -template struct is_basic_value : is_basic_value{}; -template struct is_basic_value : is_basic_value{}; -template class M, template class V> -struct is_basic_value<::toml::basic_value>: std::true_type{}; - -// --------------------------------------------------------------------------- -// C++14 index_sequence - -#if __cplusplus >= 201402L - -using std::index_sequence; -using std::make_index_sequence; - -#else - -template struct index_sequence{}; - -template struct push_back_index_sequence{}; -template -struct push_back_index_sequence, N> -{ - typedef index_sequence type; -}; - -template -struct index_sequence_maker -{ - typedef typename push_back_index_sequence< - typename index_sequence_maker::type, N>::type type; -}; -template<> -struct index_sequence_maker<0> -{ - typedef index_sequence<0> type; -}; -template -using make_index_sequence = typename index_sequence_maker::type; - -#endif // __cplusplus >= 2014 - -// --------------------------------------------------------------------------- -// C++14 enable_if_t - -#if __cplusplus >= 201402L - -using std::enable_if_t; - -#else - -template -using enable_if_t = typename std::enable_if::type; - -#endif // __cplusplus >= 2014 - -// --------------------------------------------------------------------------- -// return_type_of_t - -#if __cplusplus >= 201703L && defined(__cpp_lib_is_invocable) && __cpp_lib_is_invocable>=201703 - -template -using return_type_of_t = std::invoke_result_t; - -#else -// result_of is deprecated after C++17 -template -using return_type_of_t = typename std::result_of::type; - -#endif - -// --------------------------------------------------------------------------- -// is_string_literal -// -// to use this, pass `typename remove_reference::type` to T. - -template -struct is_string_literal: -disjunction< - std::is_same, - conjunction< - std::is_array, - std::is_same::type> - > - >{}; - -// --------------------------------------------------------------------------- -// C++20 remove_cvref_t - -template -struct remove_cvref -{ - using type = typename std::remove_cv< - typename std::remove_reference::type>::type; -}; - -template -using remove_cvref_t = typename remove_cvref::type; - -}// detail -}//toml -#endif // TOML_TRAITS diff --git a/src/toml11/toml/types.hpp b/src/toml11/toml/types.hpp deleted file mode 100644 index 1e420e7fd..000000000 --- a/src/toml11/toml/types.hpp +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_TYPES_HPP -#define TOML11_TYPES_HPP -#include -#include - -#include "comments.hpp" -#include "datetime.hpp" -#include "string.hpp" -#include "traits.hpp" - -namespace toml -{ - -template class Table, // map-like class - template class Array> // vector-like class -class basic_value; - -using character = char; -using key = std::string; - -#if !defined(__clang__) && defined(__GNUC__) && __GNUC__ <= 4 -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wshadow" -#endif - -using boolean = bool; -using integer = std::int64_t; -using floating = double; // "float" is a keyword, cannot use it here. -// the following stuffs are structs defined here, so aliases are not needed. -// - string -// - offset_datetime -// - offset_datetime -// - local_datetime -// - local_date -// - local_time - -#if defined(__GNUC__) && !defined(__clang__) -# pragma GCC diagnostic pop -#endif - -// default toml::value and default array/table. these are defined after defining -// basic_value itself. -// using value = basic_value; -// using array = typename value::array_type; -// using table = typename value::table_type; - -// to avoid warnings about `value_t::integer` is "shadowing" toml::integer in -// GCC -Wshadow=global. -#if defined(__GNUC__) && !defined(__clang__) -# pragma GCC diagnostic push -# if 7 <= __GNUC__ -# pragma GCC diagnostic ignored "-Wshadow=global" -# else // gcc-6 or older -# pragma GCC diagnostic ignored "-Wshadow" -# endif -#endif -enum class value_t : std::uint8_t -{ - empty = 0, - boolean = 1, - integer = 2, - floating = 3, - string = 4, - offset_datetime = 5, - local_datetime = 6, - local_date = 7, - local_time = 8, - array = 9, - table = 10, -}; -#if defined(__GNUC__) && !defined(__clang__) -# pragma GCC diagnostic pop -#endif - -template -inline std::basic_ostream& -operator<<(std::basic_ostream& os, value_t t) -{ - switch(t) - { - case value_t::boolean : os << "boolean"; return os; - case value_t::integer : os << "integer"; return os; - case value_t::floating : os << "floating"; return os; - case value_t::string : os << "string"; return os; - case value_t::offset_datetime : os << "offset_datetime"; return os; - case value_t::local_datetime : os << "local_datetime"; return os; - case value_t::local_date : os << "local_date"; return os; - case value_t::local_time : os << "local_time"; return os; - case value_t::array : os << "array"; return os; - case value_t::table : os << "table"; return os; - case value_t::empty : os << "empty"; return os; - default : os << "unknown"; return os; - } -} - -template, - typename alloc = std::allocator> -inline std::basic_string stringize(value_t t) -{ - std::basic_ostringstream oss; - oss << t; - return oss.str(); -} - -namespace detail -{ - -// helper to define a type that represents a value_t value. -template -using value_t_constant = std::integral_constant; - -// meta-function that convertes from value_t to the exact toml type that corresponds to. -// It takes toml::basic_value type because array and table types depend on it. -template struct enum_to_type {using type = void ;}; -template struct enum_to_type{using type = void ;}; -template struct enum_to_type{using type = boolean ;}; -template struct enum_to_type{using type = integer ;}; -template struct enum_to_type{using type = floating ;}; -template struct enum_to_type{using type = string ;}; -template struct enum_to_type{using type = offset_datetime ;}; -template struct enum_to_type{using type = local_datetime ;}; -template struct enum_to_type{using type = local_date ;}; -template struct enum_to_type{using type = local_time ;}; -template struct enum_to_type{using type = typename Value::array_type;}; -template struct enum_to_type{using type = typename Value::table_type;}; - -// meta-function that converts from an exact toml type to the enum that corresponds to. -template -struct type_to_enum : std::conditional< - std::is_same::value, // if T == array_type, - value_t_constant, // then value_t::array - typename std::conditional< // else... - std::is_same::value, // if T == table_type - value_t_constant, // then value_t::table - value_t_constant // else value_t::empty - >::type - >::type {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; -template struct type_to_enum: value_t_constant {}; - -// meta-function that checks the type T is the same as one of the toml::* types. -template -struct is_exact_toml_type : disjunction< - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same, - std::is_same - >{}; -template struct is_exact_toml_type : is_exact_toml_type{}; -template struct is_exact_toml_type : is_exact_toml_type{}; -template struct is_exact_toml_type : is_exact_toml_type{}; -template struct is_exact_toml_type: is_exact_toml_type{}; - -} // detail -} // toml - -#endif// TOML11_TYPES_H diff --git a/src/toml11/toml/utility.hpp b/src/toml11/toml/utility.hpp deleted file mode 100644 index 4a6b4309d..000000000 --- a/src/toml11/toml/utility.hpp +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_UTILITY_HPP -#define TOML11_UTILITY_HPP -#include -#include -#include - -#include "traits.hpp" - -#if __cplusplus >= 201402L -# define TOML11_MARK_AS_DEPRECATED(msg) [[deprecated(msg)]] -#elif defined(__GNUC__) -# define TOML11_MARK_AS_DEPRECATED(msg) __attribute__((deprecated(msg))) -#elif defined(_MSC_VER) -# define TOML11_MARK_AS_DEPRECATED(msg) __declspec(deprecated(msg)) -#else -# define TOML11_MARK_AS_DEPRECATED -#endif - -namespace toml -{ - -#if __cplusplus >= 201402L - -using std::make_unique; - -#else - -template -inline std::unique_ptr make_unique(Ts&& ... args) -{ - return std::unique_ptr(new T(std::forward(args)...)); -} - -#endif // __cplusplus >= 2014 - -namespace detail -{ -template -void try_reserve_impl(Container& container, std::size_t N, std::true_type) -{ - container.reserve(N); - return; -} -template -void try_reserve_impl(Container&, std::size_t, std::false_type) noexcept -{ - return; -} -} // detail - -template -void try_reserve(Container& container, std::size_t N) -{ - if(N <= container.size()) {return;} - detail::try_reserve_impl(container, N, detail::has_reserve_method{}); - return; -} - -namespace detail -{ -inline std::string concat_to_string_impl(std::ostringstream& oss) -{ - return oss.str(); -} -template -std::string concat_to_string_impl(std::ostringstream& oss, T&& head, Ts&& ... tail) -{ - oss << std::forward(head); - return concat_to_string_impl(oss, std::forward(tail) ... ); -} -} // detail - -template -std::string concat_to_string(Ts&& ... args) -{ - std::ostringstream oss; - oss << std::boolalpha << std::fixed; - return detail::concat_to_string_impl(oss, std::forward(args) ...); -} - -template -T from_string(const std::string& str, T opt) -{ - T v(opt); - std::istringstream iss(str); - iss >> v; - return v; -} - -namespace detail -{ -#if __cplusplus >= 201402L -template -decltype(auto) last_one(T&& tail) noexcept -{ - return std::forward(tail); -} - -template -decltype(auto) last_one(T&& /*head*/, Ts&& ... tail) noexcept -{ - return last_one(std::forward(tail)...); -} -#else // C++11 -// The following code -// ```cpp -// 1 | template -// 2 | auto last_one(T&& /*head*/, Ts&& ... tail) -// 3 | -> decltype(last_one(std::forward(tail)...)) -// 4 | { -// 5 | return last_one(std::forward(tail)...); -// 6 | } -// ``` -// does not work because the function `last_one(...)` is not yet defined at -// line #3, so `decltype()` cannot deduce the type returned from `last_one`. -// So we need to determine return type in a different way, like a meta func. - -template -struct last_one_in_pack -{ - using type = typename last_one_in_pack::type; -}; -template -struct last_one_in_pack -{ - using type = T; -}; -template -using last_one_in_pack_t = typename last_one_in_pack::type; - -template -T&& last_one(T&& tail) noexcept -{ - return std::forward(tail); -} -template -enable_if_t<(sizeof...(Ts) > 0), last_one_in_pack_t> -last_one(T&& /*head*/, Ts&& ... tail) -{ - return last_one(std::forward(tail)...); -} - -#endif -} // detail - -}// toml -#endif // TOML11_UTILITY diff --git a/src/toml11/toml/value.hpp b/src/toml11/toml/value.hpp deleted file mode 100644 index 1b43db8d4..000000000 --- a/src/toml11/toml/value.hpp +++ /dev/null @@ -1,2035 +0,0 @@ -// Copyright Toru Niina 2017. -// Distributed under the MIT License. -#ifndef TOML11_VALUE_HPP -#define TOML11_VALUE_HPP -#include - -#include "comments.hpp" -#include "exception.hpp" -#include "into.hpp" -#include "region.hpp" -#include "source_location.hpp" -#include "storage.hpp" -#include "traits.hpp" -#include "types.hpp" -#include "utility.hpp" - -namespace toml -{ - -namespace detail -{ - -// to show error messages. not recommended for users. -template -inline region_base const* get_region(const Value& v) -{ - return v.region_info_.get(); -} - -template -void change_region(Value& v, region reg) -{ - v.region_info_ = std::make_shared(std::move(reg)); - return; -} - -template -[[noreturn]] inline void -throw_bad_cast(const std::string& funcname, value_t actual, const Value& v) -{ - throw type_error(detail::format_underline( - concat_to_string(funcname, "bad_cast to ", Expected), { - {v.location(), concat_to_string("the actual type is ", actual)} - }), v.location()); -} - -// Throw `out_of_range` from `toml::value::at()` and `toml::find()` -// after generating an error message. -// -// The implementation is a bit complicated and there are many edge-cases. -// If you are not interested in the error message generation, just skip this. -template -[[noreturn]] void -throw_key_not_found_error(const Value& v, const key& ky) -{ - // The top-level table has its region at the first character of the file. - // That means that, in the case when a key is not found in the top-level - // table, the error message points to the first character. If the file has - // its first table at the first line, the error message would be like this. - // ```console - // [error] key "a" not found - // --> example.toml - // | - // 1 | [table] - // | ^------ in this table - // ``` - // It actually points to the top-level table at the first character, - // not `[table]`. But it is too confusing. To avoid the confusion, the error - // message should explicitly say "key not found in the top-level table", - // or "the parsed file is empty" if there is no content at all (0 bytes in file). - const auto loc = v.location(); - if(loc.line() == 1 && loc.region() == 0) - { - // First line with a zero-length region means "empty file". - // The region will be generated at `parse_toml_file` function - // if the file contains no bytes. - throw std::out_of_range(format_underline(concat_to_string( - "key \"", ky, "\" not found in the top-level table"), { - {loc, "the parsed file is empty"} - })); - } - else if(loc.line() == 1 && loc.region() == 1) - { - // Here it assumes that top-level table starts at the first character. - // The region corresponds to the top-level table will be generated at - // `parse_toml_file` function. - // It also assumes that the top-level table size is just one and - // the line number is `1`. It is always satisfied. And those conditions - // are satisfied only if the table is the top-level table. - // - // 1. one-character dot-key at the first line - // ```toml - // a.b = "c" - // ``` - // toml11 counts whole key as the table key. Here, `a.b` is the region - // of the table "a". It could be counter intuitive, but it works. - // The size of the region is 3, not 1. The above example is the shortest - // dot-key example. The size cannot be 1. - // - // 2. one-character inline-table at the first line - // ```toml - // a = {b = "c"} - // ``` - // toml11 considers the inline table body as the table region. Here, - // `{b = "c"}` is the region of the table "a". The size of the region - // is 9, not 1. The shotest inline table still has two characters, `{` - // and `}`. The size cannot be 1. - // - // 3. one-character table declaration at the first line - // ```toml - // [a] - // ``` - // toml11 considers the whole table key as the table region. Here, - // `[a]` is the table region. The size is 3, not 1. - // - throw std::out_of_range(format_underline(concat_to_string( - "key \"", ky, "\" not found in the top-level table"), { - {loc, "the top-level table starts here"} - })); - } - else - { - // normal table. - throw std::out_of_range(format_underline(concat_to_string( - "key \"", ky, "\" not found"), { {loc, "in this table"} })); - } -} - -// switch by `value_t` at the compile time. -template -struct switch_cast {}; -#define TOML11_GENERATE_SWITCH_CASTER(TYPE) \ - template<> \ - struct switch_cast \ - { \ - template \ - static typename Value::TYPE##_type& invoke(Value& v) \ - { \ - return v.as_##TYPE(); \ - } \ - template \ - static typename Value::TYPE##_type const& invoke(const Value& v) \ - { \ - return v.as_##TYPE(); \ - } \ - template \ - static typename Value::TYPE##_type&& invoke(Value&& v) \ - { \ - return std::move(v).as_##TYPE(); \ - } \ - }; \ - /**/ -TOML11_GENERATE_SWITCH_CASTER(boolean) -TOML11_GENERATE_SWITCH_CASTER(integer) -TOML11_GENERATE_SWITCH_CASTER(floating) -TOML11_GENERATE_SWITCH_CASTER(string) -TOML11_GENERATE_SWITCH_CASTER(offset_datetime) -TOML11_GENERATE_SWITCH_CASTER(local_datetime) -TOML11_GENERATE_SWITCH_CASTER(local_date) -TOML11_GENERATE_SWITCH_CASTER(local_time) -TOML11_GENERATE_SWITCH_CASTER(array) -TOML11_GENERATE_SWITCH_CASTER(table) - -#undef TOML11_GENERATE_SWITCH_CASTER - -}// detail - -template class Table = std::unordered_map, - template class Array = std::vector> -class basic_value -{ - template - static void assigner(T& dst, U&& v) - { - const auto tmp = ::new(std::addressof(dst)) T(std::forward(v)); - assert(tmp == std::addressof(dst)); - (void)tmp; - } - - using region_base = detail::region_base; - - template class T, - template class A> - friend class basic_value; - - public: - - using comment_type = Comment; - using key_type = ::toml::key; - using value_type = basic_value; - using boolean_type = ::toml::boolean; - using integer_type = ::toml::integer; - using floating_type = ::toml::floating; - using string_type = ::toml::string; - using local_time_type = ::toml::local_time; - using local_date_type = ::toml::local_date; - using local_datetime_type = ::toml::local_datetime; - using offset_datetime_type = ::toml::offset_datetime; - using array_type = Array; - using table_type = Table; - - public: - - basic_value() noexcept - : type_(value_t::empty), - region_info_(std::make_shared(region_base{})) - {} - ~basic_value() noexcept {this->cleanup();} - - basic_value(const basic_value& v) - : type_(v.type()), region_info_(v.region_info_), comments_(v.comments_) - { - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : assigner(array_ , v.array_ ); break; - case value_t::table : assigner(table_ , v.table_ ); break; - default: break; - } - } - basic_value(basic_value&& v) - : type_(v.type()), region_info_(std::move(v.region_info_)), - comments_(std::move(v.comments_)) - { - switch(this->type_) // here this->type_ is already initialized - { - case value_t::boolean : assigner(boolean_ , std::move(v.boolean_ )); break; - case value_t::integer : assigner(integer_ , std::move(v.integer_ )); break; - case value_t::floating : assigner(floating_ , std::move(v.floating_ )); break; - case value_t::string : assigner(string_ , std::move(v.string_ )); break; - case value_t::offset_datetime: assigner(offset_datetime_, std::move(v.offset_datetime_)); break; - case value_t::local_datetime : assigner(local_datetime_ , std::move(v.local_datetime_ )); break; - case value_t::local_date : assigner(local_date_ , std::move(v.local_date_ )); break; - case value_t::local_time : assigner(local_time_ , std::move(v.local_time_ )); break; - case value_t::array : assigner(array_ , std::move(v.array_ )); break; - case value_t::table : assigner(table_ , std::move(v.table_ )); break; - default: break; - } - } - basic_value& operator=(const basic_value& v) - { - this->cleanup(); - this->region_info_ = v.region_info_; - this->comments_ = v.comments_; - this->type_ = v.type(); - switch(this->type_) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : assigner(array_ , v.array_ ); break; - case value_t::table : assigner(table_ , v.table_ ); break; - default: break; - } - return *this; - } - basic_value& operator=(basic_value&& v) - { - this->cleanup(); - this->region_info_ = std::move(v.region_info_); - this->comments_ = std::move(v.comments_); - this->type_ = v.type(); - switch(this->type_) - { - case value_t::boolean : assigner(boolean_ , std::move(v.boolean_ )); break; - case value_t::integer : assigner(integer_ , std::move(v.integer_ )); break; - case value_t::floating : assigner(floating_ , std::move(v.floating_ )); break; - case value_t::string : assigner(string_ , std::move(v.string_ )); break; - case value_t::offset_datetime: assigner(offset_datetime_, std::move(v.offset_datetime_)); break; - case value_t::local_datetime : assigner(local_datetime_ , std::move(v.local_datetime_ )); break; - case value_t::local_date : assigner(local_date_ , std::move(v.local_date_ )); break; - case value_t::local_time : assigner(local_time_ , std::move(v.local_time_ )); break; - case value_t::array : assigner(array_ , std::move(v.array_ )); break; - case value_t::table : assigner(table_ , std::move(v.table_ )); break; - default: break; - } - return *this; - } - - // overwrite comments ---------------------------------------------------- - - basic_value(const basic_value& v, std::vector com) - : type_(v.type()), region_info_(v.region_info_), - comments_(std::move(com)) - { - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : assigner(array_ , v.array_ ); break; - case value_t::table : assigner(table_ , v.table_ ); break; - default: break; - } - } - - basic_value(basic_value&& v, std::vector com) - : type_(v.type()), region_info_(std::move(v.region_info_)), - comments_(std::move(com)) - { - switch(this->type_) // here this->type_ is already initialized - { - case value_t::boolean : assigner(boolean_ , std::move(v.boolean_ )); break; - case value_t::integer : assigner(integer_ , std::move(v.integer_ )); break; - case value_t::floating : assigner(floating_ , std::move(v.floating_ )); break; - case value_t::string : assigner(string_ , std::move(v.string_ )); break; - case value_t::offset_datetime: assigner(offset_datetime_, std::move(v.offset_datetime_)); break; - case value_t::local_datetime : assigner(local_datetime_ , std::move(v.local_datetime_ )); break; - case value_t::local_date : assigner(local_date_ , std::move(v.local_date_ )); break; - case value_t::local_time : assigner(local_time_ , std::move(v.local_time_ )); break; - case value_t::array : assigner(array_ , std::move(v.array_ )); break; - case value_t::table : assigner(table_ , std::move(v.table_ )); break; - default: break; - } - } - - // ----------------------------------------------------------------------- - // conversion between different basic_values. - template class T, - template class A> - basic_value(const basic_value& v) - : type_(v.type()), region_info_(v.region_info_), comments_(v.comments()) - { - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : - { - array_type tmp(v.as_array(std::nothrow).begin(), - v.as_array(std::nothrow).end()); - assigner(array_, std::move(tmp)); - break; - } - case value_t::table : - { - table_type tmp(v.as_table(std::nothrow).begin(), - v.as_table(std::nothrow).end()); - assigner(table_, std::move(tmp)); - break; - } - default: break; - } - } - template class T, - template class A> - basic_value(const basic_value& v, std::vector com) - : type_(v.type()), region_info_(v.region_info_), - comments_(std::move(com)) - { - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : - { - array_type tmp(v.as_array(std::nothrow).begin(), - v.as_array(std::nothrow).end()); - assigner(array_, std::move(tmp)); - break; - } - case value_t::table : - { - table_type tmp(v.as_table(std::nothrow).begin(), - v.as_table(std::nothrow).end()); - assigner(table_, std::move(tmp)); - break; - } - default: break; - } - } - template class T, - template class A> - basic_value& operator=(const basic_value& v) - { - this->region_info_ = v.region_info_; - this->comments_ = comment_type(v.comments()); - this->type_ = v.type(); - switch(v.type()) - { - case value_t::boolean : assigner(boolean_ , v.boolean_ ); break; - case value_t::integer : assigner(integer_ , v.integer_ ); break; - case value_t::floating : assigner(floating_ , v.floating_ ); break; - case value_t::string : assigner(string_ , v.string_ ); break; - case value_t::offset_datetime: assigner(offset_datetime_, v.offset_datetime_); break; - case value_t::local_datetime : assigner(local_datetime_ , v.local_datetime_ ); break; - case value_t::local_date : assigner(local_date_ , v.local_date_ ); break; - case value_t::local_time : assigner(local_time_ , v.local_time_ ); break; - case value_t::array : - { - array_type tmp(v.as_array(std::nothrow).begin(), - v.as_array(std::nothrow).end()); - assigner(array_, std::move(tmp)); - break; - } - case value_t::table : - { - table_type tmp(v.as_table(std::nothrow).begin(), - v.as_table(std::nothrow).end()); - assigner(table_, std::move(tmp)); - break; - } - default: break; - } - return *this; - } - - // boolean ============================================================== - - basic_value(boolean b) - : type_(value_t::boolean), - region_info_(std::make_shared(region_base{})) - { - assigner(this->boolean_, b); - } - basic_value& operator=(boolean b) - { - this->cleanup(); - this->type_ = value_t::boolean; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->boolean_, b); - return *this; - } - basic_value(boolean b, std::vector com) - : type_(value_t::boolean), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->boolean_, b); - } - - // integer ============================================================== - - template, detail::negation>>::value, - std::nullptr_t>::type = nullptr> - basic_value(T i) - : type_(value_t::integer), - region_info_(std::make_shared(region_base{})) - { - assigner(this->integer_, static_cast(i)); - } - - template, detail::negation>>::value, - std::nullptr_t>::type = nullptr> - basic_value& operator=(T i) - { - this->cleanup(); - this->type_ = value_t::integer; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->integer_, static_cast(i)); - return *this; - } - - template, detail::negation>>::value, - std::nullptr_t>::type = nullptr> - basic_value(T i, std::vector com) - : type_(value_t::integer), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->integer_, static_cast(i)); - } - - // floating ============================================================= - - template::value, std::nullptr_t>::type = nullptr> - basic_value(T f) - : type_(value_t::floating), - region_info_(std::make_shared(region_base{})) - { - assigner(this->floating_, static_cast(f)); - } - - - template::value, std::nullptr_t>::type = nullptr> - basic_value& operator=(T f) - { - this->cleanup(); - this->type_ = value_t::floating; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->floating_, static_cast(f)); - return *this; - } - - template::value, std::nullptr_t>::type = nullptr> - basic_value(T f, std::vector com) - : type_(value_t::floating), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->floating_, f); - } - - // string =============================================================== - - basic_value(toml::string s) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, std::move(s)); - } - basic_value& operator=(toml::string s) - { - this->cleanup(); - this->type_ = value_t::string ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->string_, s); - return *this; - } - basic_value(toml::string s, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, std::move(s)); - } - - basic_value(std::string s) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(std::move(s))); - } - basic_value& operator=(std::string s) - { - this->cleanup(); - this->type_ = value_t::string ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->string_, toml::string(std::move(s))); - return *this; - } - basic_value(std::string s, string_t kind) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(std::move(s), kind)); - } - basic_value(std::string s, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(std::move(s))); - } - basic_value(std::string s, string_t kind, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(std::move(s), kind)); - } - - basic_value(const char* s) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(std::string(s))); - } - basic_value& operator=(const char* s) - { - this->cleanup(); - this->type_ = value_t::string ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->string_, toml::string(std::string(s))); - return *this; - } - basic_value(const char* s, string_t kind) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(std::string(s), kind)); - } - basic_value(const char* s, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(std::string(s))); - } - basic_value(const char* s, string_t kind, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(std::string(s), kind)); - } - -#if defined(TOML11_USING_STRING_VIEW) && TOML11_USING_STRING_VIEW>0 - basic_value(std::string_view s) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(s)); - } - basic_value& operator=(std::string_view s) - { - this->cleanup(); - this->type_ = value_t::string ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->string_, toml::string(s)); - return *this; - } - basic_value(std::string_view s, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(s)); - } - basic_value(std::string_view s, string_t kind) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})) - { - assigner(this->string_, toml::string(s, kind)); - } - basic_value(std::string_view s, string_t kind, std::vector com) - : type_(value_t::string), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->string_, toml::string(s, kind)); - } -#endif - - // local date =========================================================== - - basic_value(const local_date& ld) - : type_(value_t::local_date), - region_info_(std::make_shared(region_base{})) - { - assigner(this->local_date_, ld); - } - basic_value& operator=(const local_date& ld) - { - this->cleanup(); - this->type_ = value_t::local_date; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->local_date_, ld); - return *this; - } - basic_value(const local_date& ld, std::vector com) - : type_(value_t::local_date), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->local_date_, ld); - } - - // local time =========================================================== - - basic_value(const local_time& lt) - : type_(value_t::local_time), - region_info_(std::make_shared(region_base{})) - { - assigner(this->local_time_, lt); - } - basic_value(const local_time& lt, std::vector com) - : type_(value_t::local_time), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->local_time_, lt); - } - basic_value& operator=(const local_time& lt) - { - this->cleanup(); - this->type_ = value_t::local_time; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->local_time_, lt); - return *this; - } - - template - basic_value(const std::chrono::duration& dur) - : type_(value_t::local_time), - region_info_(std::make_shared(region_base{})) - { - assigner(this->local_time_, local_time(dur)); - } - template - basic_value(const std::chrono::duration& dur, - std::vector com) - : type_(value_t::local_time), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->local_time_, local_time(dur)); - } - template - basic_value& operator=(const std::chrono::duration& dur) - { - this->cleanup(); - this->type_ = value_t::local_time; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->local_time_, local_time(dur)); - return *this; - } - - // local datetime ======================================================= - - basic_value(const local_datetime& ldt) - : type_(value_t::local_datetime), - region_info_(std::make_shared(region_base{})) - { - assigner(this->local_datetime_, ldt); - } - basic_value(const local_datetime& ldt, std::vector com) - : type_(value_t::local_datetime), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->local_datetime_, ldt); - } - basic_value& operator=(const local_datetime& ldt) - { - this->cleanup(); - this->type_ = value_t::local_datetime; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->local_datetime_, ldt); - return *this; - } - - // offset datetime ====================================================== - - basic_value(const offset_datetime& odt) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(region_base{})) - { - assigner(this->offset_datetime_, odt); - } - basic_value(const offset_datetime& odt, std::vector com) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->offset_datetime_, odt); - } - basic_value& operator=(const offset_datetime& odt) - { - this->cleanup(); - this->type_ = value_t::offset_datetime; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->offset_datetime_, odt); - return *this; - } - basic_value(const std::chrono::system_clock::time_point& tp) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(region_base{})) - { - assigner(this->offset_datetime_, offset_datetime(tp)); - } - basic_value(const std::chrono::system_clock::time_point& tp, - std::vector com) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->offset_datetime_, offset_datetime(tp)); - } - basic_value& operator=(const std::chrono::system_clock::time_point& tp) - { - this->cleanup(); - this->type_ = value_t::offset_datetime; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->offset_datetime_, offset_datetime(tp)); - return *this; - } - - // array ================================================================ - - basic_value(const array_type& ary) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})) - { - assigner(this->array_, ary); - } - basic_value(const array_type& ary, std::vector com) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->array_, ary); - } - basic_value& operator=(const array_type& ary) - { - this->cleanup(); - this->type_ = value_t::array ; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->array_, ary); - return *this; - } - - // array (initializer_list) ---------------------------------------------- - - template::value, - std::nullptr_t>::type = nullptr> - basic_value(std::initializer_list list) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})) - { - array_type ary(list.begin(), list.end()); - assigner(this->array_, std::move(ary)); - } - template::value, - std::nullptr_t>::type = nullptr> - basic_value(std::initializer_list list, std::vector com) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - array_type ary(list.begin(), list.end()); - assigner(this->array_, std::move(ary)); - } - template::value, - std::nullptr_t>::type = nullptr> - basic_value& operator=(std::initializer_list list) - { - this->cleanup(); - this->type_ = value_t::array; - this->region_info_ = std::make_shared(region_base{}); - - array_type ary(list.begin(), list.end()); - assigner(this->array_, std::move(ary)); - return *this; - } - - // array (STL Containers) ------------------------------------------------ - - template>, - detail::is_container - >::value, std::nullptr_t>::type = nullptr> - basic_value(const T& list) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})) - { - static_assert(std::is_convertible::value, - "elements of a container should be convertible to toml::value"); - - array_type ary(list.size()); - std::copy(list.begin(), list.end(), ary.begin()); - assigner(this->array_, std::move(ary)); - } - template>, - detail::is_container - >::value, std::nullptr_t>::type = nullptr> - basic_value(const T& list, std::vector com) - : type_(value_t::array), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - static_assert(std::is_convertible::value, - "elements of a container should be convertible to toml::value"); - - array_type ary(list.size()); - std::copy(list.begin(), list.end(), ary.begin()); - assigner(this->array_, std::move(ary)); - } - template>, - detail::is_container - >::value, std::nullptr_t>::type = nullptr> - basic_value& operator=(const T& list) - { - static_assert(std::is_convertible::value, - "elements of a container should be convertible to toml::value"); - - this->cleanup(); - this->type_ = value_t::array; - this->region_info_ = std::make_shared(region_base{}); - - array_type ary(list.size()); - std::copy(list.begin(), list.end(), ary.begin()); - assigner(this->array_, std::move(ary)); - return *this; - } - - // table ================================================================ - - basic_value(const table_type& tab) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})) - { - assigner(this->table_, tab); - } - basic_value(const table_type& tab, std::vector com) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - assigner(this->table_, tab); - } - basic_value& operator=(const table_type& tab) - { - this->cleanup(); - this->type_ = value_t::table; - this->region_info_ = std::make_shared(region_base{}); - assigner(this->table_, tab); - return *this; - } - - // initializer-list ------------------------------------------------------ - - basic_value(std::initializer_list> list) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})) - { - table_type tab; - for(const auto& elem : list) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - } - - basic_value(std::initializer_list> list, - std::vector com) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - table_type tab; - for(const auto& elem : list) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - } - basic_value& operator=(std::initializer_list> list) - { - this->cleanup(); - this->type_ = value_t::table; - this->region_info_ = std::make_shared(region_base{}); - - table_type tab; - for(const auto& elem : list) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - return *this; - } - - // other table-like ----------------------------------------------------- - - template>, - detail::is_map - >::value, std::nullptr_t>::type = nullptr> - basic_value(const Map& mp) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})) - { - table_type tab; - for(const auto& elem : mp) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - } - template>, - detail::is_map - >::value, std::nullptr_t>::type = nullptr> - basic_value(const Map& mp, std::vector com) - : type_(value_t::table), - region_info_(std::make_shared(region_base{})), - comments_(std::move(com)) - { - table_type tab; - for(const auto& elem : mp) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - } - template>, - detail::is_map - >::value, std::nullptr_t>::type = nullptr> - basic_value& operator=(const Map& mp) - { - this->cleanup(); - this->type_ = value_t::table; - this->region_info_ = std::make_shared(region_base{}); - - table_type tab; - for(const auto& elem : mp) {tab[elem.first] = elem.second;} - assigner(this->table_, std::move(tab)); - return *this; - } - - // user-defined ========================================================= - - // convert using into_toml() method ------------------------------------- - - template::value, std::nullptr_t>::type = nullptr> - basic_value(const T& ud): basic_value(ud.into_toml()) {} - - template::value, std::nullptr_t>::type = nullptr> - basic_value(const T& ud, std::vector com) - : basic_value(ud.into_toml(), std::move(com)) - {} - template::value, std::nullptr_t>::type = nullptr> - basic_value& operator=(const T& ud) - { - *this = ud.into_toml(); - return *this; - } - - // convert using into struct ----------------------------------------- - - template)> - basic_value(const T& ud): basic_value(::toml::into::into_toml(ud)) {} - template)> - basic_value(const T& ud, std::vector com) - : basic_value(::toml::into::into_toml(ud), std::move(com)) - {} - template)> - basic_value& operator=(const T& ud) - { - *this = ::toml::into::into_toml(ud); - return *this; - } - - // for internal use ------------------------------------------------------ - // - // Those constructors take detail::region that contains parse result. - - basic_value(boolean b, detail::region reg, std::vector cm) - : type_(value_t::boolean), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->boolean_, b); - } - template, detail::negation> - >::value, std::nullptr_t>::type = nullptr> - basic_value(T i, detail::region reg, std::vector cm) - : type_(value_t::integer), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->integer_, static_cast(i)); - } - template::value, std::nullptr_t>::type = nullptr> - basic_value(T f, detail::region reg, std::vector cm) - : type_(value_t::floating), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->floating_, static_cast(f)); - } - basic_value(toml::string s, detail::region reg, - std::vector cm) - : type_(value_t::string), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->string_, std::move(s)); - } - basic_value(const local_date& ld, detail::region reg, - std::vector cm) - : type_(value_t::local_date), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->local_date_, ld); - } - basic_value(const local_time& lt, detail::region reg, - std::vector cm) - : type_(value_t::local_time), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->local_time_, lt); - } - basic_value(const local_datetime& ldt, detail::region reg, - std::vector cm) - : type_(value_t::local_datetime), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->local_datetime_, ldt); - } - basic_value(const offset_datetime& odt, detail::region reg, - std::vector cm) - : type_(value_t::offset_datetime), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->offset_datetime_, odt); - } - basic_value(const array_type& ary, detail::region reg, - std::vector cm) - : type_(value_t::array), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->array_, ary); - } - basic_value(const table_type& tab, detail::region reg, - std::vector cm) - : type_(value_t::table), - region_info_(std::make_shared(std::move(reg))), - comments_(std::move(cm)) - { - assigner(this->table_, tab); - } - - template::value, - std::nullptr_t>::type = nullptr> - basic_value(std::pair parse_result, std::vector com) - : basic_value(std::move(parse_result.first), - std::move(parse_result.second), - std::move(com)) - {} - - // type checking and casting ============================================ - - template::value, - std::nullptr_t>::type = nullptr> - bool is() const noexcept - { - return detail::type_to_enum::value == this->type_; - } - bool is(value_t t) const noexcept {return t == this->type_;} - - bool is_uninitialized() const noexcept {return this->is(value_t::empty );} - bool is_boolean() const noexcept {return this->is(value_t::boolean );} - bool is_integer() const noexcept {return this->is(value_t::integer );} - bool is_floating() const noexcept {return this->is(value_t::floating );} - bool is_string() const noexcept {return this->is(value_t::string );} - bool is_offset_datetime() const noexcept {return this->is(value_t::offset_datetime);} - bool is_local_datetime() const noexcept {return this->is(value_t::local_datetime );} - bool is_local_date() const noexcept {return this->is(value_t::local_date );} - bool is_local_time() const noexcept {return this->is(value_t::local_time );} - bool is_array() const noexcept {return this->is(value_t::array );} - bool is_table() const noexcept {return this->is(value_t::table );} - - value_t type() const noexcept {return type_;} - - template - typename detail::enum_to_type::type& cast() & - { - if(this->type_ != T) - { - detail::throw_bad_cast("toml::value::cast: ", this->type_, *this); - } - return detail::switch_cast::invoke(*this); - } - template - typename detail::enum_to_type::type const& cast() const& - { - if(this->type_ != T) - { - detail::throw_bad_cast("toml::value::cast: ", this->type_, *this); - } - return detail::switch_cast::invoke(*this); - } - template - typename detail::enum_to_type::type&& cast() && - { - if(this->type_ != T) - { - detail::throw_bad_cast("toml::value::cast: ", this->type_, *this); - } - return detail::switch_cast::invoke(std::move(*this)); - } - - // ------------------------------------------------------------------------ - // nothrow version - - boolean const& as_boolean (const std::nothrow_t&) const& noexcept {return this->boolean_;} - integer const& as_integer (const std::nothrow_t&) const& noexcept {return this->integer_;} - floating const& as_floating (const std::nothrow_t&) const& noexcept {return this->floating_;} - string const& as_string (const std::nothrow_t&) const& noexcept {return this->string_;} - offset_datetime const& as_offset_datetime(const std::nothrow_t&) const& noexcept {return this->offset_datetime_;} - local_datetime const& as_local_datetime (const std::nothrow_t&) const& noexcept {return this->local_datetime_;} - local_date const& as_local_date (const std::nothrow_t&) const& noexcept {return this->local_date_;} - local_time const& as_local_time (const std::nothrow_t&) const& noexcept {return this->local_time_;} - array_type const& as_array (const std::nothrow_t&) const& noexcept {return this->array_.value();} - table_type const& as_table (const std::nothrow_t&) const& noexcept {return this->table_.value();} - - boolean & as_boolean (const std::nothrow_t&) & noexcept {return this->boolean_;} - integer & as_integer (const std::nothrow_t&) & noexcept {return this->integer_;} - floating & as_floating (const std::nothrow_t&) & noexcept {return this->floating_;} - string & as_string (const std::nothrow_t&) & noexcept {return this->string_;} - offset_datetime& as_offset_datetime(const std::nothrow_t&) & noexcept {return this->offset_datetime_;} - local_datetime & as_local_datetime (const std::nothrow_t&) & noexcept {return this->local_datetime_;} - local_date & as_local_date (const std::nothrow_t&) & noexcept {return this->local_date_;} - local_time & as_local_time (const std::nothrow_t&) & noexcept {return this->local_time_;} - array_type & as_array (const std::nothrow_t&) & noexcept {return this->array_.value();} - table_type & as_table (const std::nothrow_t&) & noexcept {return this->table_.value();} - - boolean && as_boolean (const std::nothrow_t&) && noexcept {return std::move(this->boolean_);} - integer && as_integer (const std::nothrow_t&) && noexcept {return std::move(this->integer_);} - floating && as_floating (const std::nothrow_t&) && noexcept {return std::move(this->floating_);} - string && as_string (const std::nothrow_t&) && noexcept {return std::move(this->string_);} - offset_datetime&& as_offset_datetime(const std::nothrow_t&) && noexcept {return std::move(this->offset_datetime_);} - local_datetime && as_local_datetime (const std::nothrow_t&) && noexcept {return std::move(this->local_datetime_);} - local_date && as_local_date (const std::nothrow_t&) && noexcept {return std::move(this->local_date_);} - local_time && as_local_time (const std::nothrow_t&) && noexcept {return std::move(this->local_time_);} - array_type && as_array (const std::nothrow_t&) && noexcept {return std::move(this->array_.value());} - table_type && as_table (const std::nothrow_t&) && noexcept {return std::move(this->table_.value());} - - // ======================================================================== - // throw version - // ------------------------------------------------------------------------ - // const reference {{{ - - boolean const& as_boolean() const& - { - if(this->type_ != value_t::boolean) - { - detail::throw_bad_cast( - "toml::value::as_boolean(): ", this->type_, *this); - } - return this->boolean_; - } - integer const& as_integer() const& - { - if(this->type_ != value_t::integer) - { - detail::throw_bad_cast( - "toml::value::as_integer(): ", this->type_, *this); - } - return this->integer_; - } - floating const& as_floating() const& - { - if(this->type_ != value_t::floating) - { - detail::throw_bad_cast( - "toml::value::as_floating(): ", this->type_, *this); - } - return this->floating_; - } - string const& as_string() const& - { - if(this->type_ != value_t::string) - { - detail::throw_bad_cast( - "toml::value::as_string(): ", this->type_, *this); - } - return this->string_; - } - offset_datetime const& as_offset_datetime() const& - { - if(this->type_ != value_t::offset_datetime) - { - detail::throw_bad_cast( - "toml::value::as_offset_datetime(): ", this->type_, *this); - } - return this->offset_datetime_; - } - local_datetime const& as_local_datetime() const& - { - if(this->type_ != value_t::local_datetime) - { - detail::throw_bad_cast( - "toml::value::as_local_datetime(): ", this->type_, *this); - } - return this->local_datetime_; - } - local_date const& as_local_date() const& - { - if(this->type_ != value_t::local_date) - { - detail::throw_bad_cast( - "toml::value::as_local_date(): ", this->type_, *this); - } - return this->local_date_; - } - local_time const& as_local_time() const& - { - if(this->type_ != value_t::local_time) - { - detail::throw_bad_cast( - "toml::value::as_local_time(): ", this->type_, *this); - } - return this->local_time_; - } - array_type const& as_array() const& - { - if(this->type_ != value_t::array) - { - detail::throw_bad_cast( - "toml::value::as_array(): ", this->type_, *this); - } - return this->array_.value(); - } - table_type const& as_table() const& - { - if(this->type_ != value_t::table) - { - detail::throw_bad_cast( - "toml::value::as_table(): ", this->type_, *this); - } - return this->table_.value(); - } - // }}} - // ------------------------------------------------------------------------ - // nonconst reference {{{ - - boolean & as_boolean() & - { - if(this->type_ != value_t::boolean) - { - detail::throw_bad_cast( - "toml::value::as_boolean(): ", this->type_, *this); - } - return this->boolean_; - } - integer & as_integer() & - { - if(this->type_ != value_t::integer) - { - detail::throw_bad_cast( - "toml::value::as_integer(): ", this->type_, *this); - } - return this->integer_; - } - floating & as_floating() & - { - if(this->type_ != value_t::floating) - { - detail::throw_bad_cast( - "toml::value::as_floating(): ", this->type_, *this); - } - return this->floating_; - } - string & as_string() & - { - if(this->type_ != value_t::string) - { - detail::throw_bad_cast( - "toml::value::as_string(): ", this->type_, *this); - } - return this->string_; - } - offset_datetime & as_offset_datetime() & - { - if(this->type_ != value_t::offset_datetime) - { - detail::throw_bad_cast( - "toml::value::as_offset_datetime(): ", this->type_, *this); - } - return this->offset_datetime_; - } - local_datetime & as_local_datetime() & - { - if(this->type_ != value_t::local_datetime) - { - detail::throw_bad_cast( - "toml::value::as_local_datetime(): ", this->type_, *this); - } - return this->local_datetime_; - } - local_date & as_local_date() & - { - if(this->type_ != value_t::local_date) - { - detail::throw_bad_cast( - "toml::value::as_local_date(): ", this->type_, *this); - } - return this->local_date_; - } - local_time & as_local_time() & - { - if(this->type_ != value_t::local_time) - { - detail::throw_bad_cast( - "toml::value::as_local_time(): ", this->type_, *this); - } - return this->local_time_; - } - array_type & as_array() & - { - if(this->type_ != value_t::array) - { - detail::throw_bad_cast( - "toml::value::as_array(): ", this->type_, *this); - } - return this->array_.value(); - } - table_type & as_table() & - { - if(this->type_ != value_t::table) - { - detail::throw_bad_cast( - "toml::value::as_table(): ", this->type_, *this); - } - return this->table_.value(); - } - - // }}} - // ------------------------------------------------------------------------ - // rvalue reference {{{ - - boolean && as_boolean() && - { - if(this->type_ != value_t::boolean) - { - detail::throw_bad_cast( - "toml::value::as_boolean(): ", this->type_, *this); - } - return std::move(this->boolean_); - } - integer && as_integer() && - { - if(this->type_ != value_t::integer) - { - detail::throw_bad_cast( - "toml::value::as_integer(): ", this->type_, *this); - } - return std::move(this->integer_); - } - floating && as_floating() && - { - if(this->type_ != value_t::floating) - { - detail::throw_bad_cast( - "toml::value::as_floating(): ", this->type_, *this); - } - return std::move(this->floating_); - } - string && as_string() && - { - if(this->type_ != value_t::string) - { - detail::throw_bad_cast( - "toml::value::as_string(): ", this->type_, *this); - } - return std::move(this->string_); - } - offset_datetime && as_offset_datetime() && - { - if(this->type_ != value_t::offset_datetime) - { - detail::throw_bad_cast( - "toml::value::as_offset_datetime(): ", this->type_, *this); - } - return std::move(this->offset_datetime_); - } - local_datetime && as_local_datetime() && - { - if(this->type_ != value_t::local_datetime) - { - detail::throw_bad_cast( - "toml::value::as_local_datetime(): ", this->type_, *this); - } - return std::move(this->local_datetime_); - } - local_date && as_local_date() && - { - if(this->type_ != value_t::local_date) - { - detail::throw_bad_cast( - "toml::value::as_local_date(): ", this->type_, *this); - } - return std::move(this->local_date_); - } - local_time && as_local_time() && - { - if(this->type_ != value_t::local_time) - { - detail::throw_bad_cast( - "toml::value::as_local_time(): ", this->type_, *this); - } - return std::move(this->local_time_); - } - array_type && as_array() && - { - if(this->type_ != value_t::array) - { - detail::throw_bad_cast( - "toml::value::as_array(): ", this->type_, *this); - } - return std::move(this->array_.value()); - } - table_type && as_table() && - { - if(this->type_ != value_t::table) - { - detail::throw_bad_cast( - "toml::value::as_table(): ", this->type_, *this); - } - return std::move(this->table_.value()); - } - // }}} - - // accessors ============================================================= - // - // may throw type_error or out_of_range - // - value_type& at(const key& k) - { - if(!this->is_table()) - { - detail::throw_bad_cast( - "toml::value::at(key): ", this->type_, *this); - } - if(this->as_table(std::nothrow).count(k) == 0) - { - detail::throw_key_not_found_error(*this, k); - } - return this->as_table(std::nothrow).at(k); - } - value_type const& at(const key& k) const - { - if(!this->is_table()) - { - detail::throw_bad_cast( - "toml::value::at(key): ", this->type_, *this); - } - if(this->as_table(std::nothrow).count(k) == 0) - { - detail::throw_key_not_found_error(*this, k); - } - return this->as_table(std::nothrow).at(k); - } - value_type& operator[](const key& k) - { - if(this->is_uninitialized()) - { - *this = table_type{}; - } - else if(!this->is_table()) // initialized, but not a table - { - detail::throw_bad_cast( - "toml::value::operator[](key): ", this->type_, *this); - } - return this->as_table(std::nothrow)[k]; - } - - value_type& at(const std::size_t idx) - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::at(idx): ", this->type_, *this); - } - if(this->as_array(std::nothrow).size() <= idx) - { - throw std::out_of_range(detail::format_underline( - "toml::value::at(idx): no element corresponding to the index", { - {this->location(), concat_to_string("the length is ", - this->as_array(std::nothrow).size(), - ", and the specified index is ", idx)} - })); - } - return this->as_array().at(idx); - } - value_type const& at(const std::size_t idx) const - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::at(idx): ", this->type_, *this); - } - if(this->as_array(std::nothrow).size() <= idx) - { - throw std::out_of_range(detail::format_underline( - "toml::value::at(idx): no element corresponding to the index", { - {this->location(), concat_to_string("the length is ", - this->as_array(std::nothrow).size(), - ", and the specified index is ", idx)} - })); - } - return this->as_array(std::nothrow).at(idx); - } - - value_type& operator[](const std::size_t idx) noexcept - { - // no check... - return this->as_array(std::nothrow)[idx]; - } - value_type const& operator[](const std::size_t idx) const noexcept - { - // no check... - return this->as_array(std::nothrow)[idx]; - } - - void push_back(const value_type& x) - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::push_back(value): ", this->type_, *this); - } - this->as_array(std::nothrow).push_back(x); - return; - } - void push_back(value_type&& x) - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::push_back(value): ", this->type_, *this); - } - this->as_array(std::nothrow).push_back(std::move(x)); - return; - } - - template - value_type& emplace_back(Ts&& ... args) - { - if(!this->is_array()) - { - detail::throw_bad_cast( - "toml::value::emplace_back(...): ", this->type_, *this); - } - this->as_array(std::nothrow).emplace_back(std::forward(args) ...); - return this->as_array(std::nothrow).back(); - } - - std::size_t size() const - { - switch(this->type_) - { - case value_t::array: - { - return this->as_array(std::nothrow).size(); - } - case value_t::table: - { - return this->as_table(std::nothrow).size(); - } - case value_t::string: - { - return this->as_string(std::nothrow).str.size(); - } - default: - { - throw type_error(detail::format_underline( - "toml::value::size(): bad_cast to container types", { - {this->location(), - concat_to_string("the actual type is ", this->type_)} - }), this->location()); - } - } - } - - std::size_t count(const key_type& k) const - { - if(!this->is_table()) - { - detail::throw_bad_cast( - "toml::value::count(key): ", this->type_, *this); - } - return this->as_table(std::nothrow).count(k); - } - - bool contains(const key_type& k) const - { - if(!this->is_table()) - { - detail::throw_bad_cast( - "toml::value::contains(key): ", this->type_, *this); - } - return (this->as_table(std::nothrow).count(k) != 0); - } - - source_location location() const - { - return source_location(this->region_info_.get()); - } - - comment_type const& comments() const noexcept {return this->comments_;} - comment_type& comments() noexcept {return this->comments_;} - - private: - - void cleanup() noexcept - { - switch(this->type_) - { - case value_t::string : {string_.~string(); return;} - case value_t::array : {array_.~array_storage(); return;} - case value_t::table : {table_.~table_storage(); return;} - default : return; - } - } - - // for error messages - template - friend region_base const* detail::get_region(const Value& v); - - template - friend void detail::change_region(Value& v, detail::region reg); - - private: - - using array_storage = detail::storage; - using table_storage = detail::storage; - - value_t type_; - union - { - boolean boolean_; - integer integer_; - floating floating_; - string string_; - offset_datetime offset_datetime_; - local_datetime local_datetime_; - local_date local_date_; - local_time local_time_; - array_storage array_; - table_storage table_; - }; - std::shared_ptr region_info_; - comment_type comments_; -}; - -// default toml::value and default array/table. -// TOML11_DEFAULT_COMMENT_STRATEGY is defined in comments.hpp -using value = basic_value; -using array = typename value::array_type; -using table = typename value::table_type; - -template class T, template class A> -inline bool -operator==(const basic_value& lhs, const basic_value& rhs) -{ - if(lhs.type() != rhs.type()) {return false;} - if(lhs.comments() != rhs.comments()) {return false;} - - switch(lhs.type()) - { - case value_t::boolean : - { - return lhs.as_boolean() == rhs.as_boolean(); - } - case value_t::integer : - { - return lhs.as_integer() == rhs.as_integer(); - } - case value_t::floating : - { - return lhs.as_floating() == rhs.as_floating(); - } - case value_t::string : - { - return lhs.as_string() == rhs.as_string(); - } - case value_t::offset_datetime: - { - return lhs.as_offset_datetime() == rhs.as_offset_datetime(); - } - case value_t::local_datetime: - { - return lhs.as_local_datetime() == rhs.as_local_datetime(); - } - case value_t::local_date: - { - return lhs.as_local_date() == rhs.as_local_date(); - } - case value_t::local_time: - { - return lhs.as_local_time() == rhs.as_local_time(); - } - case value_t::array : - { - return lhs.as_array() == rhs.as_array(); - } - case value_t::table : - { - return lhs.as_table() == rhs.as_table(); - } - case value_t::empty : {return true; } - default: {return false;} - } -} - -template class T, template class A> -inline bool operator!=(const basic_value& lhs, const basic_value& rhs) -{ - return !(lhs == rhs); -} - -template class T, template class A> -typename std::enable_if::array_type>, - detail::is_comparable::table_type> - >::value, bool>::type -operator<(const basic_value& lhs, const basic_value& rhs) -{ - if(lhs.type() != rhs.type()){return (lhs.type() < rhs.type());} - switch(lhs.type()) - { - case value_t::boolean : - { - return lhs.as_boolean() < rhs.as_boolean() || - (lhs.as_boolean() == rhs.as_boolean() && - lhs.comments() < rhs.comments()); - } - case value_t::integer : - { - return lhs.as_integer() < rhs.as_integer() || - (lhs.as_integer() == rhs.as_integer() && - lhs.comments() < rhs.comments()); - } - case value_t::floating : - { - return lhs.as_floating() < rhs.as_floating() || - (lhs.as_floating() == rhs.as_floating() && - lhs.comments() < rhs.comments()); - } - case value_t::string : - { - return lhs.as_string() < rhs.as_string() || - (lhs.as_string() == rhs.as_string() && - lhs.comments() < rhs.comments()); - } - case value_t::offset_datetime: - { - return lhs.as_offset_datetime() < rhs.as_offset_datetime() || - (lhs.as_offset_datetime() == rhs.as_offset_datetime() && - lhs.comments() < rhs.comments()); - } - case value_t::local_datetime: - { - return lhs.as_local_datetime() < rhs.as_local_datetime() || - (lhs.as_local_datetime() == rhs.as_local_datetime() && - lhs.comments() < rhs.comments()); - } - case value_t::local_date: - { - return lhs.as_local_date() < rhs.as_local_date() || - (lhs.as_local_date() == rhs.as_local_date() && - lhs.comments() < rhs.comments()); - } - case value_t::local_time: - { - return lhs.as_local_time() < rhs.as_local_time() || - (lhs.as_local_time() == rhs.as_local_time() && - lhs.comments() < rhs.comments()); - } - case value_t::array : - { - return lhs.as_array() < rhs.as_array() || - (lhs.as_array() == rhs.as_array() && - lhs.comments() < rhs.comments()); - } - case value_t::table : - { - return lhs.as_table() < rhs.as_table() || - (lhs.as_table() == rhs.as_table() && - lhs.comments() < rhs.comments()); - } - case value_t::empty : - { - return lhs.comments() < rhs.comments(); - } - default: - { - return lhs.comments() < rhs.comments(); - } - } -} - -template class T, template class A> -typename std::enable_if::array_type>, - detail::is_comparable::table_type> - >::value, bool>::type -operator<=(const basic_value& lhs, const basic_value& rhs) -{ - return (lhs < rhs) || (lhs == rhs); -} -template class T, template class A> -typename std::enable_if::array_type>, - detail::is_comparable::table_type> - >::value, bool>::type -operator>(const basic_value& lhs, const basic_value& rhs) -{ - return !(lhs <= rhs); -} -template class T, template class A> -typename std::enable_if::array_type>, - detail::is_comparable::table_type> - >::value, bool>::type -operator>=(const basic_value& lhs, const basic_value& rhs) -{ - return !(lhs < rhs); -} - -template class T, template class A> -inline std::string format_error(const std::string& err_msg, - const basic_value& v, const std::string& comment, - std::vector hints = {}, - const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) -{ - return detail::format_underline(err_msg, {{v.location(), comment}}, - std::move(hints), colorize); -} - -template class T, template class A> -inline std::string format_error(const std::string& err_msg, - const toml::basic_value& v1, const std::string& comment1, - const toml::basic_value& v2, const std::string& comment2, - std::vector hints = {}, - const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) -{ - return detail::format_underline(err_msg, { - {v1.location(), comment1}, {v2.location(), comment2} - }, std::move(hints), colorize); -} - -template class T, template class A> -inline std::string format_error(const std::string& err_msg, - const toml::basic_value& v1, const std::string& comment1, - const toml::basic_value& v2, const std::string& comment2, - const toml::basic_value& v3, const std::string& comment3, - std::vector hints = {}, - const bool colorize = TOML11_ERROR_MESSAGE_COLORIZED) -{ - return detail::format_underline(err_msg, {{v1.location(), comment1}, - {v2.location(), comment2}, {v3.location(), comment3} - }, std::move(hints), colorize); -} - -template class T, template class A> -detail::return_type_of_t -visit(Visitor&& visitor, const toml::basic_value& v) -{ - switch(v.type()) - { - case value_t::boolean : {return visitor(v.as_boolean ());} - case value_t::integer : {return visitor(v.as_integer ());} - case value_t::floating : {return visitor(v.as_floating ());} - case value_t::string : {return visitor(v.as_string ());} - case value_t::offset_datetime: {return visitor(v.as_offset_datetime());} - case value_t::local_datetime : {return visitor(v.as_local_datetime ());} - case value_t::local_date : {return visitor(v.as_local_date ());} - case value_t::local_time : {return visitor(v.as_local_time ());} - case value_t::array : {return visitor(v.as_array ());} - case value_t::table : {return visitor(v.as_table ());} - case value_t::empty : break; - default: break; - } - throw std::runtime_error(format_error("[error] toml::visit: toml::basic_value " - "does not have any valid basic_value.", v, "here")); -} - -template class T, template class A> -detail::return_type_of_t -visit(Visitor&& visitor, toml::basic_value& v) -{ - switch(v.type()) - { - case value_t::boolean : {return visitor(v.as_boolean ());} - case value_t::integer : {return visitor(v.as_integer ());} - case value_t::floating : {return visitor(v.as_floating ());} - case value_t::string : {return visitor(v.as_string ());} - case value_t::offset_datetime: {return visitor(v.as_offset_datetime());} - case value_t::local_datetime : {return visitor(v.as_local_datetime ());} - case value_t::local_date : {return visitor(v.as_local_date ());} - case value_t::local_time : {return visitor(v.as_local_time ());} - case value_t::array : {return visitor(v.as_array ());} - case value_t::table : {return visitor(v.as_table ());} - case value_t::empty : break; - default: break; - } - throw std::runtime_error(format_error("[error] toml::visit: toml::basic_value " - "does not have any valid basic_value.", v, "here")); -} - -template class T, template class A> -detail::return_type_of_t -visit(Visitor&& visitor, toml::basic_value&& v) -{ - switch(v.type()) - { - case value_t::boolean : {return visitor(std::move(v.as_boolean ()));} - case value_t::integer : {return visitor(std::move(v.as_integer ()));} - case value_t::floating : {return visitor(std::move(v.as_floating ()));} - case value_t::string : {return visitor(std::move(v.as_string ()));} - case value_t::offset_datetime: {return visitor(std::move(v.as_offset_datetime()));} - case value_t::local_datetime : {return visitor(std::move(v.as_local_datetime ()));} - case value_t::local_date : {return visitor(std::move(v.as_local_date ()));} - case value_t::local_time : {return visitor(std::move(v.as_local_time ()));} - case value_t::array : {return visitor(std::move(v.as_array ()));} - case value_t::table : {return visitor(std::move(v.as_table ()));} - case value_t::empty : break; - default: break; - } - throw std::runtime_error(format_error("[error] toml::visit: toml::basic_value " - "does not have any valid basic_value.", v, "here")); -} - -}// toml -#endif// TOML11_VALUE diff --git a/subprojects b/subprojects new file mode 120000 index 000000000..e8310385c --- /dev/null +++ b/subprojects @@ -0,0 +1 @@ +src \ No newline at end of file diff --git a/tests/functional/.version b/tests/functional/.version new file mode 120000 index 000000000..b7badcd0c --- /dev/null +++ b/tests/functional/.version @@ -0,0 +1 @@ +../../.version \ No newline at end of file diff --git a/tests/functional/add.sh b/tests/functional/add.sh old mode 100644 new mode 100755 index a4bb0e225..3b37ee7d4 --- a/tests/functional/add.sh +++ b/tests/functional/add.sh @@ -1,10 +1,12 @@ +#!/usr/bin/env bash + source common.sh path1=$(nix-store --add ./dummy) -echo $path1 +echo "$path1" path2=$(nix-store --add-fixed sha256 --recursive ./dummy) -echo $path2 +echo "$path2" if test "$path1" != "$path2"; then echo "nix-store --add and --add-fixed mismatch" @@ -12,24 +14,24 @@ if test "$path1" != "$path2"; then fi path3=$(nix-store --add-fixed sha256 ./dummy) -echo $path3 +echo "$path3" test "$path1" != "$path3" || exit 1 path4=$(nix-store --add-fixed sha1 --recursive ./dummy) -echo $path4 +echo "$path4" test "$path1" != "$path4" || exit 1 -hash1=$(nix-store -q --hash $path1) -echo $hash1 +hash1=$(nix-store -q --hash "$path1") +echo "$hash1" hash2=$(nix-hash --type sha256 --base32 ./dummy) -echo $hash2 +echo "$hash2" test "$hash1" = "sha256:$hash2" #### New style commands -clearStore +clearStoreIfPossible ( path1=$(nix store add ./dummy) diff --git a/tests/functional/bash-profile.sh b/tests/functional/bash-profile.sh old mode 100644 new mode 100755 index 3faeaaba1..4228d4a20 --- a/tests/functional/bash-profile.sh +++ b/tests/functional/bash-profile.sh @@ -1,9 +1,11 @@ +#!/usr/bin/env bash + source common.sh -sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../../scripts/nix-profile.sh.in > $TEST_ROOT/nix-profile.sh +sed -e "s|@localstatedir@|$TEST_ROOT/profile-var|g" -e "s|@coreutils@|$coreutils|g" < ../../scripts/nix-profile.sh.in > "$TEST_ROOT"/nix-profile.sh user=$(whoami) -rm -rf $TEST_HOME $TEST_ROOT/profile-var -mkdir -p $TEST_HOME +rm -rf "$TEST_HOME" "$TEST_ROOT/profile-var" +mkdir -p "$TEST_HOME" USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh; set" USER=$user $SHELL -e -c ". $TEST_ROOT/nix-profile.sh" # test idempotency diff --git a/tests/functional/binary-cache-build-remote.sh b/tests/functional/binary-cache-build-remote.sh old mode 100644 new mode 100755 index 81cd21a4a..5046d0064 --- a/tests/functional/binary-cache-build-remote.sh +++ b/tests/functional/binary-cache-build-remote.sh @@ -1,6 +1,10 @@ +#!/usr/bin/env bash + source common.sh -clearStore +TODO_NixOS + +clearStoreIfPossible clearCacheCache # Fails without remote builders @@ -10,7 +14,7 @@ clearCacheCache outPath=$(nix-build --store "file://$cacheDir" --builders 'auto - - 1 1' -j0 dependencies.nix) # Test that the path exactly exists in the destination store. -nix path-info --store "file://$cacheDir" $outPath +nix path-info --store "file://$cacheDir" "$outPath" # Succeeds without any build capability because no-op nix-build --store "file://$cacheDir" -j0 dependencies.nix diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh old mode 100644 new mode 100755 index 2a8d5ccdb..6a177b657 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + needLocalStore "'--no-require-sigs' can’t be used with the daemon" # We can produce drvs directly into the binary cache @@ -12,9 +16,9 @@ clearStore clearCache outPath=$(nix-build dependencies.nix --no-out-link) -nix copy --to file://$cacheDir $outPath +nix copy --to "file://$cacheDir" "$outPath" -readarray -t paths < <(nix path-info --all --json --store file://$cacheDir | jq 'keys|sort|.[]' -r) +readarray -t paths < <(nix path-info --all --json --store "file://$cacheDir" | jq 'keys|sort|.[]' -r) [[ "${#paths[@]}" -eq 3 ]] for path in "${paths[@]}"; do [[ "$path" =~ -dependencies-input-0$ ]] \ @@ -23,16 +27,16 @@ for path in "${paths[@]}"; do done # Test copying build logs to the binary cache. -expect 1 nix log --store file://$cacheDir $outPath 2>&1 | grep 'is not available' -nix store copy-log --to file://$cacheDir $outPath -nix log --store file://$cacheDir $outPath | grep FOO -rm -rf $TEST_ROOT/var/log/nix -expect 1 nix log $outPath 2>&1 | grep 'is not available' -nix log --substituters file://$cacheDir $outPath | grep FOO +expect 1 nix log --store "file://$cacheDir" "$outPath" 2>&1 | grep 'is not available' +nix store copy-log --to "file://$cacheDir" "$outPath" +nix log --store "file://$cacheDir" "$outPath" | grep FOO +rm -rf "$TEST_ROOT/var/log/nix" +expect 1 nix log "$outPath" 2>&1 | grep 'is not available' +nix log --substituters "file://$cacheDir" "$outPath" | grep FOO # Test copying build logs from the binary cache. -nix store copy-log --from file://$cacheDir $(nix-store -qd $outPath)^'*' -nix log $outPath | grep FOO +nix store copy-log --from "file://$cacheDir" "$(nix-store -qd "$outPath")"^'*' +nix log "$outPath" | grep FOO basicDownloadTests() { # No uploading tests bcause upload with force HTTP doesn't work. @@ -44,15 +48,15 @@ basicDownloadTests() { nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "---" - nix-store --substituters "file://$cacheDir" --no-require-sigs -r $outPath + nix-store --substituters "file://$cacheDir" --no-require-sigs -r "$outPath" - [ -x $outPath/program ] + [ -x "$outPath/program" ] # But with the right configuration, "nix-env -qas" should also work. clearStore clearCacheCache - echo "WantMassQuery: 1" >> $cacheDir/nix-cache-info + echo "WantMassQuery: 1" >> "$cacheDir/nix-cache-info" nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" nix-env --substituters "file://$cacheDir" -f dependencies.nix -qas \* | grep -- "--S" @@ -60,12 +64,12 @@ basicDownloadTests() { x=$(nix-env -f dependencies.nix -qas \* --prebuilt-only) [ -z "$x" ] - nix-store --substituters "file://$cacheDir" --no-require-sigs -r $outPath + nix-store --substituters "file://$cacheDir" --no-require-sigs -r "$outPath" - nix-store --check-validity $outPath - nix-store -qR $outPath | grep input-2 + nix-store --check-validity "$outPath" + nix-store -qR "$outPath" | grep input-2 - echo "WantMassQuery: 0" >> $cacheDir/nix-cache-info + echo "WantMassQuery: 0" >> "$cacheDir/nix-cache-info" } @@ -81,22 +85,22 @@ basicDownloadTests # Test whether Nix notices if the NAR doesn't match the hash in the NAR info. clearStore -nar=$(ls $cacheDir/nar/*.nar.xz | head -n1) -mv $nar $nar.good -mkdir -p $TEST_ROOT/empty -nix-store --dump $TEST_ROOT/empty | xz > $nar +nar=$(find "$cacheDir/nar/" -type f -name "*.nar.xz" | head -n1) +mv "$nar" "$nar".good +mkdir -p "$TEST_ROOT/empty" +nix-store --dump "$TEST_ROOT/empty" | xz > "$nar" -expect 1 nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grepQuiet "hash mismatch" $TEST_ROOT/log +expect 1 nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" 2>&1 | tee "$TEST_ROOT/log" +grepQuiet "hash mismatch" "$TEST_ROOT/log" -mv $nar.good $nar +mv "$nar".good "$nar" # Test whether this unsigned cache is rejected if the user requires signed caches. clearStore clearCacheCache -if nix-store --substituters "file://$cacheDir" -r $outPath; then +if nix-store --substituters "file://$cacheDir" -r "$outPath"; then echo "unsigned binary cache incorrectly accepted" exit 1 fi @@ -105,131 +109,134 @@ fi # Test whether fallback works if a NAR has disappeared. This does not require --fallback. clearStore -mv $cacheDir/nar $cacheDir/nar2 +mv "$cacheDir/nar" "$cacheDir/nar2" -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" -mv $cacheDir/nar2 $cacheDir/nar +mv "$cacheDir/nar2" "$cacheDir/nar" # Test whether fallback works if a NAR is corrupted. This does require --fallback. clearStore -mv $cacheDir/nar $cacheDir/nar2 -mkdir $cacheDir/nar -for i in $(cd $cacheDir/nar2 && echo *); do touch $cacheDir/nar/$i; done +mv "$cacheDir/nar" "$cacheDir/nar2" +mkdir "$cacheDir/nar" +for i in $(cd "$cacheDir/nar2" && echo *); do touch "$cacheDir"/nar/"$i"; done -(! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result) +(! nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result") -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result --fallback +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" --fallback -rm -rf $cacheDir/nar -mv $cacheDir/nar2 $cacheDir/nar +rm -rf "$cacheDir/nar" +mv "$cacheDir/nar2" "$cacheDir/nar" # Test whether building works if the binary cache contains an # incomplete closure. clearStore -rm -v $(grep -l "StorePath:.*dependencies-input-2" $cacheDir/*.narinfo) +rm -v "$(grep -l "StorePath:.*dependencies-input-2" "$cacheDir"/*.narinfo)" -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grepQuiet "copying path.*input-0" $TEST_ROOT/log -grepQuiet "copying path.*input-2" $TEST_ROOT/log -grepQuiet "copying path.*top" $TEST_ROOT/log +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" 2>&1 | tee "$TEST_ROOT/log" +grepQuiet "copying path.*input-0" "$TEST_ROOT/log" +grepQuiet "copying path.*input-2" "$TEST_ROOT/log" +grepQuiet "copying path.*top" "$TEST_ROOT/log" # Idem, but without cached .narinfo. clearStore clearCacheCache -nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o $TEST_ROOT/result 2>&1 | tee $TEST_ROOT/log -grepQuiet "don't know how to build" $TEST_ROOT/log -grepQuiet "building.*input-1" $TEST_ROOT/log -grepQuiet "building.*input-2" $TEST_ROOT/log -grepQuiet "copying path.*input-0" $TEST_ROOT/log -grepQuiet "copying path.*top" $TEST_ROOT/log +nix-build --substituters "file://$cacheDir" --no-require-sigs dependencies.nix -o "$TEST_ROOT/result" 2>&1 | tee "$TEST_ROOT/log" +grepQuiet "don't know how to build" "$TEST_ROOT/log" +grepQuiet "building.*input-1" "$TEST_ROOT/log" +grepQuiet "building.*input-2" "$TEST_ROOT/log" +grepQuiet "copying path.*input-0" "$TEST_ROOT/log" +grepQuiet "copying path.*top" "$TEST_ROOT/log" # Create a signed binary cache. clearCache clearCacheCache -nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk1 -publicKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk1) +nix key generate-secret --key-name test.nixos.org-1 > "$TEST_ROOT/sk1" +publicKey=$(nix key convert-secret-to-public < "$TEST_ROOT/sk1") -nix key generate-secret --key-name test.nixos.org-1 > $TEST_ROOT/sk2 -badKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk2) +nix key generate-secret --key-name test.nixos.org-1 > "$TEST_ROOT/sk2" +badKey=$(nix key convert-secret-to-public < "$TEST_ROOT/sk2") -nix key generate-secret --key-name foo.nixos.org-1 > $TEST_ROOT/sk3 -otherKey=$(nix key convert-secret-to-public < $TEST_ROOT/sk3) +nix key generate-secret --key-name foo.nixos.org-1 > "$TEST_ROOT/sk3" +otherKey=$(nix key convert-secret-to-public < "$TEST_ROOT/sk3") -_NIX_FORCE_HTTP= nix copy --to file://$cacheDir?secret-key=$TEST_ROOT/sk1 $outPath +_NIX_FORCE_HTTP='' nix copy --to "file://$cacheDir"?secret-key="$TEST_ROOT"/sk1 "$outPath" # Downloading should fail if we don't provide a key. clearStore clearCacheCache -(! nix-store -r $outPath --substituters "file://$cacheDir") +(! nix-store -r "$outPath" --substituters "file://$cacheDir") # And it should fail if we provide an incorrect key. clearStore clearCacheCache -(! nix-store -r $outPath --substituters "file://$cacheDir" --trusted-public-keys "$badKey") +(! nix-store -r "$outPath" --substituters "file://$cacheDir" --trusted-public-keys "$badKey") # It should succeed if we provide the correct key. -nix-store -r $outPath --substituters "file://$cacheDir" --trusted-public-keys "$otherKey $publicKey" +nix-store -r "$outPath" --substituters "file://$cacheDir" --trusted-public-keys "$otherKey $publicKey" # It should fail if we corrupt the .narinfo. clearStore cacheDir2=$TEST_ROOT/binary-cache-2 -rm -rf $cacheDir2 -cp -r $cacheDir $cacheDir2 +rm -rf "$cacheDir2" +cp -r "$cacheDir" "$cacheDir2" -for i in $cacheDir2/*.narinfo; do - grep -v References $i > $i.tmp - mv $i.tmp $i +for i in "$cacheDir2"/*.narinfo; do + grep -v References "$i" > "$i".tmp + mv "$i".tmp "$i" done clearCacheCache -(! nix-store -r $outPath --substituters "file://$cacheDir2" --trusted-public-keys "$publicKey") +(! nix-store -r "$outPath" --substituters "file://$cacheDir2" --trusted-public-keys "$publicKey") # If we provide a bad and a good binary cache, it should succeed. -nix-store -r $outPath --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" +nix-store -r "$outPath" --substituters "file://$cacheDir2 file://$cacheDir" --trusted-public-keys "$publicKey" unset _NIX_FORCE_HTTP # Test 'nix verify --all' on a binary cache. -nix store verify -vvvvv --all --store file://$cacheDir --no-trust +nix store verify -vvvvv --all --store "file://$cacheDir" --no-trust # Test local NAR caching. narCache=$TEST_ROOT/nar-cache -rm -rf $narCache -mkdir $narCache +rm -rf "$narCache" +mkdir "$narCache" -[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] +[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" "$outPath/foobar") = FOOBAR ]] rm -rfv "$cacheDir/nar" -[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" $outPath/foobar) = FOOBAR ]] +[[ $(nix store cat --store "file://$cacheDir?local-nar-cache=$narCache" "$outPath/foobar") = FOOBAR ]] -(! nix store cat --store file://$cacheDir $outPath/foobar) +(! nix store cat --store "file://$cacheDir" "$outPath/foobar") # Test NAR listing generation. clearCache + +# preserve quotes variables in the single-quoted string +# shellcheck disable=SC2016 outPath=$(nix-build --no-out-link -E ' with import ./config.nix; mkDerivation { @@ -238,16 +245,18 @@ outPath=$(nix-build --no-out-link -E ' } ') -nix copy --to file://$cacheDir?write-nar-listing=1 $outPath +nix copy --to "file://$cacheDir"?write-nar-listing=1 "$outPath" diff -u \ - <(jq -S < $cacheDir/$(basename $outPath | cut -c1-32).ls) \ + <(jq -S < "$cacheDir/$(basename "$outPath" | cut -c1-32).ls") \ <(echo '{"version":1,"root":{"type":"directory","entries":{"bar":{"type":"regular","size":4,"narOffset":232},"link":{"type":"symlink","target":"xyzzy"}}}}' | jq -S) # Test debug info index generation. clearCache +# preserve quotes variables in the single-quoted string +# shellcheck disable=SC2016 outPath=$(nix-build --no-out-link -E ' with import ./config.nix; mkDerivation { @@ -256,14 +265,16 @@ outPath=$(nix-build --no-out-link -E ' } ') -nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" $outPath +nix copy --to "file://$cacheDir?index-debug-info=1&compression=none" "$outPath" diff -u \ - <(cat $cacheDir/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug | jq -S) \ + <(jq -S < "$cacheDir"/debuginfo/02623eda209c26a59b1a8638ff7752f6b945c26b.debug) \ <(echo '{"archive":"../nar/100vxs724qr46phz8m24iswmg9p3785hsyagz0kchf6q6gf06sw6.nar","member":"lib/debug/.build-id/02/623eda209c26a59b1a8638ff7752f6b945c26b.debug"}' | jq -S) # Test against issue https://github.com/NixOS/nix/issues/3964 -# + +# preserve quotes variables in the single-quoted string +# shellcheck disable=SC2016 expr=' with import ./config.nix; mkDerivation { @@ -273,22 +284,22 @@ expr=' } ' outPath=$(nix-build --no-out-link -E "$expr") -docPath=$(nix-store -q --references $outPath) +docPath=$(nix-store -q --references "$outPath") # $ nix-store -q --tree $outPath # ...-multi-output # +---...-multi-output-doc -nix copy --to "file://$cacheDir" $outPath +nix copy --to "file://$cacheDir" "$outPath" hashpart() { basename "$1" | cut -c1-32 } # break the closure of out by removing doc -rm $cacheDir/$(hashpart $docPath).narinfo +rm "$cacheDir/$(hashpart "$docPath")".narinfo -nix-store --delete $outPath $docPath +nix-store --delete "$outPath" "$docPath" # -vvv is the level that logs during the loop timeout 60 nix-build --no-out-link -E "$expr" --option substituters "file://$cacheDir" \ --option trusted-binary-caches "file://$cacheDir" --no-require-sigs diff --git a/tests/functional/brotli.sh b/tests/functional/brotli.sh old mode 100644 new mode 100755 index dc9bbdb66..327eab4a5 --- a/tests/functional/brotli.sh +++ b/tests/functional/brotli.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore clearCache @@ -7,15 +11,15 @@ cacheURI="file://$cacheDir?compression=br" outPath=$(nix-build dependencies.nix --no-out-link) -nix copy --to $cacheURI $outPath +nix copy --to "$cacheURI" "$outPath" -HASH=$(nix hash path $outPath) +HASH=$(nix hash path "$outPath") clearStore clearCacheCache -nix copy --from $cacheURI $outPath --no-check-sigs +nix copy --from "$cacheURI" "$outPath" --no-check-sigs -HASH2=$(nix hash path $outPath) +HASH2=$(nix hash path "$outPath") -[[ $HASH = $HASH2 ]] +[[ $HASH == "$HASH2" ]] diff --git a/tests/functional/build-delete.sh b/tests/functional/build-delete.sh old mode 100644 new mode 100755 index 9c56b00e8..18841509d --- a/tests/functional/build-delete.sh +++ b/tests/functional/build-delete.sh @@ -1,28 +1,30 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible # https://github.com/NixOS/nix/issues/6572 issue_6572_independent_outputs() { - nix build -f multiple-outputs.nix --json independent --no-link > $TEST_ROOT/independent.json + nix build -f multiple-outputs.nix --json independent --no-link > "$TEST_ROOT"/independent.json # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation. p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths) nix-store --delete "$p" # Clean up for next test # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. - nix-store --delete "$(jq -r <$TEST_ROOT/independent.json .[0].outputs.first)" + nix-store --delete "$(jq -r <"$TEST_ROOT"/independent.json .[0].outputs.first)" p=$(nix build -f multiple-outputs.nix use-independent --no-link --print-out-paths) - cmp $p < $TEST_ROOT/a.json + nix build -f multiple-outputs.nix --json a --no-link > "$TEST_ROOT"/a.json # # Make sure that 'nix build' can build a derivation that depends on both outputs of another derivation. p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths) nix-store --delete "$p" # Clean up for next test # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. - nix-store --delete "$(jq -r <$TEST_ROOT/a.json .[0].outputs.second)" + nix-store --delete "$(jq -r <"$TEST_ROOT"/a.json .[0].outputs.second)" p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths) - cmp $p < $TEST_ROOT/post-build-hook.sh + cat < "$TEST_ROOT/post-build-hook.sh" #!/bin/sh echo "Post hook ran successfully" # Add an empty line to a counter file, just to check that this hook ran properly echo "" >> $TEST_ROOT/post-hook-counter EOF - chmod +x $TEST_ROOT/post-build-hook.sh - rm -f $TEST_ROOT/post-hook-counter + chmod +x "$TEST_ROOT/post-build-hook.sh" + rm -f "$TEST_ROOT/post-hook-counter" - echo "post-build-hook = $TEST_ROOT/post-build-hook.sh" >> $NIX_CONF_DIR/nix.conf + echo "post-build-hook = $TEST_ROOT/post-build-hook.sh" >> "$test_nix_conf" } registerBuildHook @@ -30,4 +32,4 @@ source build-remote.sh # `build-hook.nix` has four derivations to build, and the hook runs twice for # each derivation (once on the builder and once on the host), so the counter # should contain eight lines now -[[ $(cat $TEST_ROOT/post-hook-counter | wc -l) -eq 8 ]] +[[ $(wc -l < "$TEST_ROOT/post-hook-counter") -eq 8 ]] diff --git a/tests/functional/build-remote-trustless-after.sh b/tests/functional/build-remote-trustless-after.sh index 19f59e6ae..2fcdbf10a 100644 --- a/tests/functional/build-remote-trustless-after.sh +++ b/tests/functional/build-remote-trustless-after.sh @@ -1,2 +1,7 @@ -outPath=$(readlink -f $TEST_ROOT/result) -grep 'FOO BAR BAZ' ${remoteDir}/${outPath} +# shellcheck shell=bash + +# Variables must be defined by caller, so +# shellcheck disable=SC2154 + +outPath=$(readlink -f "$TEST_ROOT/result") +grep 'FOO BAR BAZ' "${remoteDir}/${outPath}" diff --git a/tests/functional/build-remote-trustless-should-fail-0.sh b/tests/functional/build-remote-trustless-should-fail-0.sh old mode 100644 new mode 100755 index 3d4a4b097..4eccb73e0 --- a/tests/functional/build-remote-trustless-should-fail-0.sh +++ b/tests/functional/build-remote-trustless-should-fail-0.sh @@ -1,7 +1,10 @@ +#!/usr/bin/env bash + source common.sh enableFeatures "daemon-trust-override" +TODO_NixOS restartDaemon requireSandboxSupport @@ -22,8 +25,12 @@ nix-build build-hook.nix -A passthru.input2 \ # copy our already-build `input2` to the remote store. That store object # is input-addressed, so this will fail. +# For script below +# shellcheck disable=SC2034 file=build-hook.nix +# shellcheck disable=SC2034 prog=$(readlink -e ./nix-daemon-untrusting.sh) +# shellcheck disable=SC2034 proto=ssh-ng expectStderr 1 source build-remote-trustless.sh \ diff --git a/tests/functional/build-remote-trustless-should-pass-0.sh b/tests/functional/build-remote-trustless-should-pass-0.sh old mode 100644 new mode 100755 index 2a7ebd8c6..b81060907 --- a/tests/functional/build-remote-trustless-should-pass-0.sh +++ b/tests/functional/build-remote-trustless-should-pass-0.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh # Remote trusts us diff --git a/tests/functional/build-remote-trustless-should-pass-1.sh b/tests/functional/build-remote-trustless-should-pass-1.sh old mode 100644 new mode 100755 index 516bdf092..b8dc038bf --- a/tests/functional/build-remote-trustless-should-pass-1.sh +++ b/tests/functional/build-remote-trustless-should-pass-1.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh # Remote trusts us diff --git a/tests/functional/build-remote-trustless-should-pass-2.sh b/tests/functional/build-remote-trustless-should-pass-2.sh old mode 100644 new mode 100755 index b769a88f0..34ce7fbe4 --- a/tests/functional/build-remote-trustless-should-pass-2.sh +++ b/tests/functional/build-remote-trustless-should-pass-2.sh @@ -1,7 +1,11 @@ +#!/usr/bin/env bash + source common.sh enableFeatures "daemon-trust-override" +TODO_NixOS + restartDaemon # Remote doesn't trust us diff --git a/tests/functional/build-remote-trustless-should-pass-3.sh b/tests/functional/build-remote-trustless-should-pass-3.sh old mode 100644 new mode 100755 index 40f81da5a..d01d79191 --- a/tests/functional/build-remote-trustless-should-pass-3.sh +++ b/tests/functional/build-remote-trustless-should-pass-3.sh @@ -1,7 +1,10 @@ +#!/usr/bin/env bash + source common.sh enableFeatures "daemon-trust-override" +TODO_NixOS restartDaemon # Remote doesn't trusts us, but this is fine because we are only diff --git a/tests/functional/build-remote-trustless.sh b/tests/functional/build-remote-trustless.sh index 81e5253bf..c498d46c3 100644 --- a/tests/functional/build-remote-trustless.sh +++ b/tests/functional/build-remote-trustless.sh @@ -1,5 +1,11 @@ +# shellcheck shell=bash + +# All variables should be defined externally by the scripts that source +# this, `set -u` will catch any that are forgotten. +# shellcheck disable=SC2154 + requireSandboxSupport -[[ $busybox =~ busybox ]] || skipTest "no busybox" +[[ "$busybox" =~ busybox ]] || skipTest "no busybox" unset NIX_STORE_DIR unset NIX_STATE_DIR @@ -8,7 +14,7 @@ remoteDir=$TEST_ROOT/remote # Note: ssh{-ng}://localhost bypasses ssh. See tests/functional/build-remote.sh for # more details. -nix-build $file -o $TEST_ROOT/result --max-jobs 0 \ - --arg busybox $busybox \ - --store $TEST_ROOT/local \ +nix-build "$file" -o "$TEST_ROOT/result" --max-jobs 0 \ + --arg busybox "$busybox" \ + --store "$TEST_ROOT/local" \ --builders "$proto://localhost?remote-program=$prog&remote-store=${remoteDir}%3Fsystem-features=foo%20bar%20baz - - 1 1 foo,bar,baz" diff --git a/tests/functional/build-remote-with-mounted-ssh-ng.sh b/tests/functional/build-remote-with-mounted-ssh-ng.sh old mode 100644 new mode 100755 index 443acb6ca..e2627af39 --- a/tests/functional/build-remote-with-mounted-ssh-ng.sh +++ b/tests/functional/build-remote-with-mounted-ssh-ng.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh requireSandboxSupport @@ -6,17 +8,17 @@ requireSandboxSupport enableFeatures mounted-ssh-store nix build -Lvf simple.nix \ - --arg busybox $busybox \ - --out-link $TEST_ROOT/result-from-remote \ + --arg busybox "$busybox" \ + --out-link "$TEST_ROOT/result-from-remote" \ --store mounted-ssh-ng://localhost nix build -Lvf simple.nix \ - --arg busybox $busybox \ - --out-link $TEST_ROOT/result-from-remote-new-cli \ + --arg busybox "$busybox" \ + --out-link "$TEST_ROOT/result-from-remote-new-cli" \ --store 'mounted-ssh-ng://localhost?remote-program=nix daemon' # This verifies that the out link was actually created and valid. The ability # to create out links (permanent gc roots) is the distinguishing feature of # the mounted-ssh-ng store. -cat $TEST_ROOT/result-from-remote/hello | grepQuiet 'Hello World!' -cat $TEST_ROOT/result-from-remote-new-cli/hello | grepQuiet 'Hello World!' +grepQuiet 'Hello World!' < "$TEST_ROOT/result-from-remote/hello" +grepQuiet 'Hello World!' < "$TEST_ROOT/result-from-remote-new-cli/hello" diff --git a/tests/functional/build-remote.sh b/tests/functional/build-remote.sh index d2a2132c1..1a5334577 100644 --- a/tests/functional/build-remote.sh +++ b/tests/functional/build-remote.sh @@ -1,5 +1,9 @@ +# shellcheck shell=bash + +: "${file?must be defined by caller (remote building test case using this)}" + requireSandboxSupport -[[ $busybox =~ busybox ]] || skipTest "no busybox" +[[ "${busybox-}" =~ busybox ]] || skipTest "no busybox" # Avoid store dir being inside sandbox build-dir unset NIX_STORE_DIR @@ -15,50 +19,50 @@ fi builders=( # system-features will automatically be added to the outer URL, but not inner # remote-store URL. - "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=$(join_by "%20" foo ${EXTRA_SYSTEM_FEATURES[@]}) - - 1 1 $(join_by "," foo ${EXTRA_SYSTEM_FEATURES[@]})" - "$TEST_ROOT/machine2 - - 1 1 $(join_by "," bar ${EXTRA_SYSTEM_FEATURES[@]})" - "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=$(join_by "%20" baz ${EXTRA_SYSTEM_FEATURES[@]}) - - 1 1 $(join_by "," baz ${EXTRA_SYSTEM_FEATURES[@]})" + "ssh://localhost?remote-store=$TEST_ROOT/machine1?system-features=$(join_by "%20" foo "${EXTRA_SYSTEM_FEATURES[@]}") - - 1 1 $(join_by "," foo "${EXTRA_SYSTEM_FEATURES[@]}")" + "$TEST_ROOT/machine2 - - 1 1 $(join_by "," bar "${EXTRA_SYSTEM_FEATURES[@]}")" + "ssh-ng://localhost?remote-store=$TEST_ROOT/machine3?system-features=$(join_by "%20" baz "${EXTRA_SYSTEM_FEATURES[@]}") - - 1 1 $(join_by "," baz "${EXTRA_SYSTEM_FEATURES[@]}")" ) -chmod -R +w $TEST_ROOT/machine* || true -rm -rf $TEST_ROOT/machine* || true +chmod -R +w "$TEST_ROOT/machine"* || true +rm -rf "$TEST_ROOT/machine"* || true # Note: ssh://localhost bypasses ssh, directly invoking nix-store as a # child process. This allows us to test LegacySSHStore::buildDerivation(). # ssh-ng://... likewise allows us to test RemoteStore::buildDerivation(). -nix build -L -v -f $file -o $TEST_ROOT/result --max-jobs 0 \ - --arg busybox $busybox \ - --store $TEST_ROOT/machine0 \ +nix build -L -v -f "$file" -o "$TEST_ROOT/result" --max-jobs 0 \ + --arg busybox "$busybox" \ + --store "$TEST_ROOT/machine0" \ --builders "$(join_by '; ' "${builders[@]}")" -outPath=$(readlink -f $TEST_ROOT/result) +outPath=$(readlink -f "$TEST_ROOT/result") -grep 'FOO BAR BAZ' $TEST_ROOT/machine0/$outPath +grep 'FOO BAR BAZ' "$TEST_ROOT/machine0/$outPath" -testPrintOutPath=$(nix build -L -v -f $file --no-link --print-out-paths --max-jobs 0 \ - --arg busybox $busybox \ - --store $TEST_ROOT/machine0 \ +testPrintOutPath=$(nix build -L -v -f "$file" --no-link --print-out-paths --max-jobs 0 \ + --arg busybox "$busybox" \ + --store "$TEST_ROOT/machine0" \ --builders "$(join_by '; ' "${builders[@]}")" ) [[ $testPrintOutPath =~ store.*build-remote ]] # Ensure that input1 was built on store1 due to the required feature. -output=$(nix path-info --store $TEST_ROOT/machine1 --all) +output=$(nix path-info --store "$TEST_ROOT/machine1" --all) echo "$output" | grepQuiet builder-build-remote-input-1.sh echo "$output" | grepQuietInverse builder-build-remote-input-2.sh echo "$output" | grepQuietInverse builder-build-remote-input-3.sh unset output # Ensure that input2 was built on store2 due to the required feature. -output=$(nix path-info --store $TEST_ROOT/machine2 --all) +output=$(nix path-info --store "$TEST_ROOT/machine2" --all) echo "$output" | grepQuietInverse builder-build-remote-input-1.sh echo "$output" | grepQuiet builder-build-remote-input-2.sh echo "$output" | grepQuietInverse builder-build-remote-input-3.sh unset output # Ensure that input3 was built on store3 due to the required feature. -output=$(nix path-info --store $TEST_ROOT/machine3 --all) +output=$(nix path-info --store "$TEST_ROOT/machine3" --all) echo "$output" | grepQuietInverse builder-build-remote-input-1.sh echo "$output" | grepQuietInverse builder-build-remote-input-2.sh echo "$output" | grepQuiet builder-build-remote-input-3.sh @@ -66,7 +70,7 @@ unset output for i in input1 input3; do -nix log --store $TEST_ROOT/machine0 --file "$file" --arg busybox $busybox passthru."$i" | grep hi-$i +nix log --store "$TEST_ROOT/machine0" --file "$file" --arg busybox "$busybox" "passthru.$i" | grep hi-$i done # Behavior of keep-failed @@ -74,9 +78,9 @@ out="$(nix-build 2>&1 failing.nix \ --no-out-link \ --builders "$(join_by '; ' "${builders[@]}")" \ --keep-failed \ - --store $TEST_ROOT/machine0 \ + --store "$TEST_ROOT/machine0" \ -j0 \ - --arg busybox $busybox)" || true + --arg busybox "$busybox")" || true [[ "$out" =~ .*"note: keeping build directory".* ]] diff --git a/tests/functional/build.sh b/tests/functional/build.sh old mode 100644 new mode 100755 index 7fbdb0f07..5396a465f --- a/tests/functional/build.sh +++ b/tests/functional/build.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible # Make sure that 'nix build' returns all outputs by default. nix build -f multiple-outputs.nix --json a b --no-link | jq --exit-status ' @@ -43,6 +45,14 @@ nix build -f multiple-outputs.nix --json e --no-link | jq --exit-status ' (.outputs | keys == ["a_a", "b"])) ' +# Tests that we can handle empty 'outputsToInstall' (assuming that default +# output "out" exists). +nix build -f multiple-outputs.nix --json nothing-to-install --no-link | jq --exit-status ' + (.[0] | + (.drvPath | match(".*nothing-to-install.drv")) and + (.outputs | keys == ["out"])) +' + # But not when it's overriden. nix build -f multiple-outputs.nix --json e^a_a --no-link nix build -f multiple-outputs.nix --json e^a_a --no-link | jq --exit-status ' @@ -130,6 +140,50 @@ nix build --impure -f multiple-outputs.nix --json e --no-link | jq --exit-status (.outputs | keys == ["a_a", "b"])) ' +# Make sure that the 3 types of aliases work +# BaseSettings, BaseSettings, and BaseSettings. +nix build --impure -f multiple-outputs.nix --json e --no-link \ + --build-max-jobs 3 \ + --gc-keep-outputs \ + --build-use-sandbox | \ + jq --exit-status ' + (.[0] | + (.drvPath | match(".*multiple-outputs-e.drv")) and + (.outputs | keys == ["a_a", "b"])) +' + # Make sure that `--stdin` works and does not apply any defaults printf "" | nix build --no-link --stdin --json | jq --exit-status '. == []' printf "%s\n" "$drv^*" | nix build --no-link --stdin --json | jq --exit-status '.[0]|has("drvPath")' + +# --keep-going and FOD +out="$(nix build -f fod-failing.nix -L 2>&1)" && status=0 || status=$? +test "$status" = 1 +# one "hash mismatch" error, one "build of ... failed" +test "$(<<<"$out" grep -E '^error:' | wc -l)" = 2 +<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x1\\.drv'" +<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x3\\.drv'" +<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x2\\.drv'" +<<<"$out" grepQuiet -E "error: build of '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out', '.*-x[1-4]\\.drv\\^out' failed" + +out="$(nix build -f fod-failing.nix -L x1 x2 x3 --keep-going 2>&1)" && status=0 || status=$? +test "$status" = 1 +# three "hash mismatch" errors - for each failing fod, one "build of ... failed" +test "$(<<<"$out" grep -E '^error:' | wc -l)" = 4 +<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x1\\.drv'" +<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x3\\.drv'" +<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x2\\.drv'" +<<<"$out" grepQuiet -E "error: build of '.*-x[1-3]\\.drv\\^out', '.*-x[1-3]\\.drv\\^out', '.*-x[1-3]\\.drv\\^out' failed" + +out="$(nix build -f fod-failing.nix -L x4 2>&1)" && status=0 || status=$? +test "$status" = 1 +test "$(<<<"$out" grep -E '^error:' | wc -l)" = 2 +<<<"$out" grepQuiet -E "error: 1 dependencies of derivation '.*-x4\\.drv' failed to build" +<<<"$out" grepQuiet -E "hash mismatch in fixed-output derivation '.*-x2\\.drv'" + +out="$(nix build -f fod-failing.nix -L x4 --keep-going 2>&1)" && status=0 || status=$? +test "$status" = 1 +test "$(<<<"$out" grep -E '^error:' | wc -l)" = 3 +<<<"$out" grepQuiet -E "error: 2 dependencies of derivation '.*-x4\\.drv' failed to build" +<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x3\\.drv'" +<<<"$out" grepQuiet -vE "hash mismatch in fixed-output derivation '.*-x2\\.drv'" diff --git a/tests/functional/ca/build-cache.sh b/tests/functional/ca/build-cache.sh index 6a4080fec..5cc71823e 100644 --- a/tests/functional/ca/build-cache.sh +++ b/tests/functional/ca/build-cache.sh @@ -26,7 +26,8 @@ copyAttr () { # Note: to copy CA derivations, we need to copy the realisations, which # currently requires naming the installables, not just the derivation output # path. - nix copy --to file://$cacheDir "${args[@]}" + + nix copy --to "file://$cacheDir" "${args[@]}" } testRemoteCacheFor () { @@ -35,7 +36,7 @@ testRemoteCacheFor () { copyAttr "$derivationPath" 1 clearStore # Check nothing gets built. - buildAttr "$derivationPath" 1 --option substituters file://$cacheDir --no-require-sigs |& grepQuietInverse " will be built:" + buildAttr "$derivationPath" 1 --option substituters "file://$cacheDir" --no-require-sigs |& grepQuietInverse " will be built:" } testRemoteCache () { @@ -48,4 +49,4 @@ testRemoteCache () { } clearStore -testRemoteCache \ No newline at end of file +testRemoteCache diff --git a/tests/functional/ca/build.sh b/tests/functional/ca/build.sh index e1a8a7625..e5ad9d2a0 100644 --- a/tests/functional/ca/build.sh +++ b/tests/functional/ca/build.sh @@ -20,11 +20,11 @@ testDeterministicCA () { testCutoffFor () { local out1 out2 - out1=$(buildAttr $1 1) + out1=$(buildAttr "$1" 1) # The seed only changes the root derivation, and not it's output, so the # dependent derivations should only need to be built once. buildAttr rootCA 2 - out2=$(buildAttr $1 2 -j0) + out2=$(buildAttr "$1" 2 -j0) test "$out1" == "$out2" } @@ -41,7 +41,7 @@ testGC () { nix-instantiate ./content-addressed.nix -A rootCA --arg seed 5 nix-collect-garbage --option keep-derivations true clearStore - buildAttr rootCA 1 --out-link $TEST_ROOT/rootCA + buildAttr rootCA 1 --out-link "$TEST_ROOT"/rootCA nix-collect-garbage buildAttr rootCA 1 -j0 } @@ -55,7 +55,7 @@ testNixCommand () { testNormalization () { clearStore outPath=$(buildAttr rootCA 1) - test "$(stat -c %Y $outPath)" -eq 1 + test "$(stat -c %Y "$outPath")" -eq 1 } clearStore diff --git a/tests/functional/ca/common.sh b/tests/functional/ca/common.sh index b104b5a78..48f1ac46b 100644 --- a/tests/functional/ca/common.sh +++ b/tests/functional/ca/common.sh @@ -2,4 +2,6 @@ source ../common.sh enableFeatures "ca-derivations" +TODO_NixOS + restartDaemon diff --git a/tests/functional/ca/derivation-json.sh b/tests/functional/ca/derivation-json.sh index c1480fd17..1e2a8fe35 100644 --- a/tests/functional/ca/derivation-json.sh +++ b/tests/functional/ca/derivation-json.sh @@ -1,29 +1,31 @@ +#!/usr/bin/env bash +# source common.sh export NIX_TESTS_CA_BY_DEFAULT=1 drvPath=$(nix-instantiate ../simple.nix) -nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json +nix derivation show "$drvPath" | jq .[] > "$TEST_HOME"/simple.json -drvPath2=$(nix derivation add < $TEST_HOME/simple.json) +drvPath2=$(nix derivation add < "$TEST_HOME"/simple.json) [[ "$drvPath" = "$drvPath2" ]] # Content-addressed derivations can be renamed. -jq '.name = "foo"' < $TEST_HOME/simple.json > $TEST_HOME/foo.json -drvPath3=$(nix derivation add --dry-run < $TEST_HOME/foo.json) +jq '.name = "foo"' < "$TEST_HOME"/simple.json > "$TEST_HOME"/foo.json +drvPath3=$(nix derivation add --dry-run < "$TEST_HOME"/foo.json) # With --dry-run nothing is actually written [[ ! -e "$drvPath3" ]] # But the JSON is rejected without the experimental feature -expectStderr 1 nix derivation add < $TEST_HOME/foo.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'ca-derivations' is disabled" +expectStderr 1 nix derivation add < "$TEST_HOME"/foo.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'ca-derivations' is disabled" # Without --dry-run it is actually written -drvPath4=$(nix derivation add < $TEST_HOME/foo.json) +drvPath4=$(nix derivation add < "$TEST_HOME"/foo.json) [[ "$drvPath4" = "$drvPath3" ]] [[ -e "$drvPath3" ]] # The modified derivation read back as JSON matches -nix derivation show $drvPath3 | jq .[] > $TEST_HOME/foo-read.json -diff $TEST_HOME/foo.json $TEST_HOME/foo-read.json +nix derivation show "$drvPath3" | jq .[] > "$TEST_HOME"/foo-read.json +diff "$TEST_HOME"/foo.json "$TEST_HOME"/foo-read.json diff --git a/tests/functional/ca/duplicate-realisation-in-closure.sh b/tests/functional/ca/duplicate-realisation-in-closure.sh index da9cd8fb4..0baf15cc2 100644 --- a/tests/functional/ca/duplicate-realisation-in-closure.sh +++ b/tests/functional/ca/duplicate-realisation-in-closure.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source ./common.sh requireDaemonNewerThan "2.4pre20210625" @@ -5,7 +7,7 @@ requireDaemonNewerThan "2.4pre20210625" export REMOTE_STORE_DIR="$TEST_ROOT/remote_store" export REMOTE_STORE="file://$REMOTE_STORE_DIR" -rm -rf $REMOTE_STORE_DIR +rm -rf "$REMOTE_STORE_DIR" clearStore # Build dep1 and push that to the binary cache. diff --git a/tests/functional/ca/import-derivation.sh b/tests/functional/ca/import-from-derivation.sh similarity index 55% rename from tests/functional/ca/import-derivation.sh rename to tests/functional/ca/import-from-derivation.sh index e98e0fbd0..0713619a6 100644 --- a/tests/functional/ca/import-derivation.sh +++ b/tests/functional/ca/import-from-derivation.sh @@ -2,5 +2,5 @@ source common.sh export NIX_TESTS_CA_BY_DEFAULT=1 -cd .. && source import-derivation.sh +cd .. && source import-from-derivation.sh diff --git a/tests/functional/ca/local.mk b/tests/functional/ca/local.mk index 4f86b268f..7c2fcc451 100644 --- a/tests/functional/ca/local.mk +++ b/tests/functional/ca/local.mk @@ -7,7 +7,7 @@ ca-tests := \ $(d)/duplicate-realisation-in-closure.sh \ $(d)/eval-store.sh \ $(d)/gc.sh \ - $(d)/import-derivation.sh \ + $(d)/import-from-derivation.sh \ $(d)/new-build-cmd.sh \ $(d)/nix-copy.sh \ $(d)/nix-run.sh \ diff --git a/tests/functional/ca/meson.build b/tests/functional/ca/meson.build new file mode 100644 index 000000000..00cf8b35f --- /dev/null +++ b/tests/functional/ca/meson.build @@ -0,0 +1,33 @@ +configure_file( + input : 'config.nix.in', + output : 'config.nix', + configuration : test_confdata, +) + +suites += { + 'name': 'ca', + 'deps': [], + 'tests': [ + 'build-with-garbage-path.sh', + 'build.sh', + 'build-cache.sh', + 'concurrent-builds.sh', + 'derivation-json.sh', + 'duplicate-realisation-in-closure.sh', + 'eval-store.sh', + 'gc.sh', + 'import-from-derivation.sh', + 'new-build-cmd.sh', + 'nix-copy.sh', + 'nix-run.sh', + 'nix-shell.sh', + 'post-hook.sh', + 'recursive.sh', + 'repl.sh', + 'selfref-gc.sh', + 'signatures.sh', + 'substitute.sh', + 'why-depends.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/ca/nix-copy.sh b/tests/functional/ca/nix-copy.sh index 7a8307a4e..f77b00030 100755 --- a/tests/functional/ca/nix-copy.sh +++ b/tests/functional/ca/nix-copy.sh @@ -15,13 +15,13 @@ testOneCopy () { rm -rf "$REMOTE_STORE_DIR" attrPath="$1" - nix copy --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix + nix copy --to "$REMOTE_STORE" "$attrPath" --file ./content-addressed.nix ensureCorrectlyCopied "$attrPath" # Ensure that we can copy back what we put in the store clearStore - nix copy --from $REMOTE_STORE \ + nix copy --from "$REMOTE_STORE" \ --file ./content-addressed.nix "$attrPath" \ --no-check-sigs } diff --git a/tests/functional/ca/nix-run.sh b/tests/functional/ca/nix-run.sh index 5f46518e8..920950c11 100755 --- a/tests/functional/ca/nix-run.sh +++ b/tests/functional/ca/nix-run.sh @@ -4,4 +4,4 @@ source common.sh FLAKE_PATH=path:$PWD -nix run --no-write-lock-file $FLAKE_PATH#runnable +nix run --no-write-lock-file "$FLAKE_PATH#runnable" diff --git a/tests/functional/ca/signatures.sh b/tests/functional/ca/signatures.sh index eb18a4130..f69a205d2 100644 --- a/tests/functional/ca/signatures.sh +++ b/tests/functional/ca/signatures.sh @@ -1,10 +1,12 @@ +#!/usr/bin/env bash + source common.sh clearStore clearCache -nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 -pk1=$(cat $TEST_ROOT/pk1) +nix-store --generate-binary-cache-key cache1.example.org "$TEST_ROOT/sk1" "$TEST_ROOT/pk1" +pk1=$(cat "$TEST_ROOT/pk1") export REMOTE_STORE_DIR="$TEST_ROOT/remote_store" export REMOTE_STORE="file://$REMOTE_STORE_DIR" @@ -19,16 +21,16 @@ testOneCopy () { rm -rf "$REMOTE_STORE_DIR" attrPath="$1" - nix copy -vvvv --to $REMOTE_STORE "$attrPath" --file ./content-addressed.nix \ + nix copy -vvvv --to "$REMOTE_STORE" "$attrPath" --file ./content-addressed.nix \ --secret-key-files "$TEST_ROOT/sk1" --show-trace ensureCorrectlyCopied "$attrPath" # Ensure that we can copy back what we put in the store clearStore - nix copy --from $REMOTE_STORE \ + nix copy --from "$REMOTE_STORE" \ --file ./content-addressed.nix "$attrPath" \ - --trusted-public-keys $pk1 + --trusted-public-keys "$pk1" } for attrPath in rootCA dependentCA transitivelyDependentCA dependentNonCA dependentFixedOutput; do diff --git a/tests/functional/ca/substitute.sh b/tests/functional/ca/substitute.sh index ea981adc4..9728470f0 100644 --- a/tests/functional/ca/substitute.sh +++ b/tests/functional/ca/substitute.sh @@ -4,9 +4,10 @@ source common.sh +# shellcheck disable=SC1111 needLocalStore "“--no-require-sigs” can’t be used with the daemon" -rm -rf $TEST_ROOT/binary_cache +rm -rf "$TEST_ROOT/binary_cache" export REMOTE_STORE_DIR=$TEST_ROOT/binary_cache export REMOTE_STORE=file://$REMOTE_STORE_DIR @@ -17,11 +18,11 @@ buildDrvs () { # Populate the remote cache clearStore -nix copy --to $REMOTE_STORE --file ./content-addressed.nix +nix copy --to "$REMOTE_STORE" --file ./content-addressed.nix # Restart the build on an empty store, ensuring that we don't build clearStore -buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 transitivelyDependentCA +buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 transitivelyDependentCA # Check that the thing we’ve just substituted has its realisation stored nix realisation info --file ./content-addressed.nix transitivelyDependentCA # Check that its dependencies have it too @@ -63,9 +64,9 @@ clearStore # Add the realisations of rootCA to the cachecache clearCacheCache export _NIX_FORCE_HTTP=1 -buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 +buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 # Try rebuilding, but remove the realisations from the remote cache to force # using the cachecache clearStore -rm $REMOTE_STORE_DIR/realisations/* -buildDrvs --substitute --substituters $REMOTE_STORE --no-require-sigs -j0 +rm "$REMOTE_STORE_DIR"/realisations/* +buildDrvs --substitute --substituters "$REMOTE_STORE" --no-require-sigs -j0 diff --git a/tests/functional/case-collision.nar b/tests/functional/case-collision.nar new file mode 100644 index 000000000..2eff86901 Binary files /dev/null and b/tests/functional/case-collision.nar differ diff --git a/tests/functional/case-hack.sh b/tests/functional/case-hack.sh deleted file mode 100644 index 61bf9b94b..000000000 --- a/tests/functional/case-hack.sh +++ /dev/null @@ -1,19 +0,0 @@ -source common.sh - -clearStore - -rm -rf $TEST_ROOT/case - -opts="--option use-case-hack true" - -# Check whether restoring and dumping a NAR that contains case -# collisions is round-tripping, even on a case-insensitive system. -nix-store $opts --restore $TEST_ROOT/case < case.nar -nix-store $opts --dump $TEST_ROOT/case > $TEST_ROOT/case.nar -cmp case.nar $TEST_ROOT/case.nar -[ "$(nix-hash $opts --type sha256 $TEST_ROOT/case)" = "$(nix-hash --flat --type sha256 case.nar)" ] - -# Check whether we detect true collisions (e.g. those remaining after -# removal of the suffix). -touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3" -(! nix-store $opts --dump $TEST_ROOT/case > /dev/null) diff --git a/tests/functional/lang-test-infra.sh b/tests/functional/characterisation-test-infra.sh old mode 100644 new mode 100755 similarity index 97% rename from tests/functional/lang-test-infra.sh rename to tests/functional/characterisation-test-infra.sh index 30da8977b..279454550 --- a/tests/functional/lang-test-infra.sh +++ b/tests/functional/characterisation-test-infra.sh @@ -1,7 +1,9 @@ +#!/usr/bin/env bash + # Test the function for lang.sh source common.sh -source lang/framework.sh +source characterisation/framework.sh # We are testing this, so don't want outside world to affect us. unset _NIX_TEST_ACCEPT diff --git a/tests/functional/lang/empty.exp b/tests/functional/characterisation/empty similarity index 100% rename from tests/functional/lang/empty.exp rename to tests/functional/characterisation/empty diff --git a/tests/functional/characterisation/framework.sh b/tests/functional/characterisation/framework.sh new file mode 100644 index 000000000..5ca125ab5 --- /dev/null +++ b/tests/functional/characterisation/framework.sh @@ -0,0 +1,77 @@ +# shellcheck shell=bash + +# Golden test support +# +# Test that the output of the given test matches what is expected. If +# `_NIX_TEST_ACCEPT` is non-empty also update the expected output so +# that next time the test succeeds. +function diffAndAcceptInner() { + local -r testName=$1 + local -r got="$2" + local -r expected="$3" + + # Absence of expected file indicates empty output expected. + if test -e "$expected"; then + local -r expectedOrEmpty="$expected" + else + local -r expectedOrEmpty=characterisation/empty + fi + + # Diff so we get a nice message + if ! diff --color=always --unified "$expectedOrEmpty" "$got"; then + echo "FAIL: evaluation result of $testName not as expected" + # shellcheck disable=SC2034 + badDiff=1 + fi + + # Update expected if `_NIX_TEST_ACCEPT` is non-empty. + if test -n "${_NIX_TEST_ACCEPT-}"; then + cp "$got" "$expected" + # Delete empty expected files to avoid bloating the repo with + # empty files. + if ! test -s "$expected"; then + rm "$expected" + fi + fi +} + +function characterisationTestExit() { + # Make sure shellcheck knows all these will be defined by the caller + : "${badDiff?} ${badExitCode?}" + + if test -n "${_NIX_TEST_ACCEPT-}"; then + if (( "$badDiff" )); then + set +x + echo 'Output did mot match, but accepted output as the persisted expected output.' + echo 'That means the next time the tests are run, they should pass.' + set -x + else + set +x + echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,' + echo 'indicating the unexpected output should be accepted as the expected output going forward,' + echo 'but no tests had unexpected output so there was no expected output to update.' + set -x + fi + if (( "$badExitCode" )); then + exit "$badExitCode" + else + skipTest "regenerating golden masters" + fi + else + if (( "$badDiff" )); then + set +x + echo '' + echo 'You can rerun this test with:' + echo '' + echo " _NIX_TEST_ACCEPT=1 make tests/functional/${TEST_NAME}.sh.test" + echo '' + echo 'to regenerate the files containing the expected output,' + echo 'and then view the git diff to decide whether a change is' + echo 'good/intentional or bad/unintentional.' + echo 'If the diff contains arbitrary or impure information,' + echo 'please improve the normalization that the test applies to the output.' + set -x + fi + exit $(( "$badExitCode" + "$badDiff" )) + fi +} diff --git a/tests/functional/check-refs.sh b/tests/functional/check-refs.sh old mode 100644 new mode 100755 index 3b587d1e5..5c3ac915e --- a/tests/functional/check-refs.sh +++ b/tests/functional/check-refs.sh @@ -1,53 +1,62 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore RESULT=$TEST_ROOT/result -dep=$(nix-build -o $RESULT check-refs.nix -A dep) +dep=$(nix-build -o "$RESULT" check-refs.nix -A dep) # test1 references dep, not itself. -test1=$(nix-build -o $RESULT check-refs.nix -A test1) -nix-store -q --references $test1 | grepQuietInverse $test1 -nix-store -q --references $test1 | grepQuiet $dep +test1=$(nix-build -o "$RESULT" check-refs.nix -A test1) +nix-store -q --references "$test1" | grepQuietInverse "$test1" +nix-store -q --references "$test1" | grepQuiet "$dep" # test2 references src, not itself nor dep. -test2=$(nix-build -o $RESULT check-refs.nix -A test2) -nix-store -q --references $test2 | grepQuietInverse $test2 -nix-store -q --references $test2 | grepQuietInverse $dep -nix-store -q --references $test2 | grepQuiet aux-ref +test2=$(nix-build -o "$RESULT" check-refs.nix -A test2) +nix-store -q --references "$test2" | grepQuietInverse "$test2" +nix-store -q --references "$test2" | grepQuietInverse "$dep" +nix-store -q --references "$test2" | grepQuiet aux-ref # test3 should fail (unallowed ref). -(! nix-build -o $RESULT check-refs.nix -A test3) +(! nix-build -o "$RESULT" check-refs.nix -A test3) # test4 should succeed. -nix-build -o $RESULT check-refs.nix -A test4 +nix-build -o "$RESULT" check-refs.nix -A test4 # test5 should succeed. -nix-build -o $RESULT check-refs.nix -A test5 +nix-build -o "$RESULT" check-refs.nix -A test5 # test6 should fail (unallowed self-ref). -(! nix-build -o $RESULT check-refs.nix -A test6) +(! nix-build -o "$RESULT" check-refs.nix -A test6) # test7 should succeed (allowed self-ref). -nix-build -o $RESULT check-refs.nix -A test7 +nix-build -o "$RESULT" check-refs.nix -A test7 # test8 should fail (toFile depending on derivation output). -(! nix-build -o $RESULT check-refs.nix -A test8) +(! nix-build -o "$RESULT" check-refs.nix -A test8) # test9 should fail (disallowed reference). -(! nix-build -o $RESULT check-refs.nix -A test9) +(! nix-build -o "$RESULT" check-refs.nix -A test9) # test10 should succeed (no disallowed references). -nix-build -o $RESULT check-refs.nix -A test10 +nix-build -o "$RESULT" check-refs.nix -A test10 -if isDaemonNewer 2.12pre20230103; then - if ! isDaemonNewer 2.16.0; then - enableFeatures discard-references - restartDaemon +if ! isTestOnNixOS; then + # If we have full control over our store, we can test some more things. + + if isDaemonNewer 2.12pre20230103; then + if ! isDaemonNewer 2.16.0; then + enableFeatures discard-references + restartDaemon + fi + + # test11 should succeed. + test11=$(nix-build -o "$RESULT" check-refs.nix -A test11) + [[ -z $(nix-store -q --references "$test11") ]] fi - # test11 should succeed. - test11=$(nix-build -o $RESULT check-refs.nix -A test11) - [[ -z $(nix-store -q --references "$test11") ]] fi diff --git a/tests/functional/check-reqs.sh b/tests/functional/check-reqs.sh old mode 100644 new mode 100755 index 856c94cec..34eb133db --- a/tests/functional/check-reqs.sh +++ b/tests/functional/check-reqs.sh @@ -1,16 +1,18 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible RESULT=$TEST_ROOT/result -nix-build -o $RESULT check-reqs.nix -A test1 +nix-build -o "$RESULT" check-reqs.nix -A test1 -(! nix-build -o $RESULT check-reqs.nix -A test2) -(! nix-build -o $RESULT check-reqs.nix -A test3) -(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep1' -(! nix-build -o $RESULT check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep2' -(! nix-build -o $RESULT check-reqs.nix -A test5) -(! nix-build -o $RESULT check-reqs.nix -A test6) +(! nix-build -o "$RESULT" check-reqs.nix -A test2) +(! nix-build -o "$RESULT" check-reqs.nix -A test3) +(! nix-build -o "$RESULT" check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep1' +(! nix-build -o "$RESULT" check-reqs.nix -A test4) 2>&1 | grepQuiet 'check-reqs-dep2' +(! nix-build -o "$RESULT" check-reqs.nix -A test5) +(! nix-build -o "$RESULT" check-reqs.nix -A test6) -nix-build -o $RESULT check-reqs.nix -A test7 +nix-build -o "$RESULT" check-reqs.nix -A test7 diff --git a/tests/functional/check.sh b/tests/functional/check.sh old mode 100644 new mode 100755 index 38883c5d7..9b15dccb6 --- a/tests/functional/check.sh +++ b/tests/functional/check.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh # XXX: This shouldn’t be, but #4813 cause this test to fail @@ -13,6 +15,8 @@ checkBuildTempDirRemoved () # written to build temp directories to verify created by this instance checkBuildId=$(date +%s%N) +TODO_NixOS + clearStore nix-build dependencies.nix --no-out-link @@ -44,7 +48,10 @@ test_custom_build_dir() { --no-out-link --keep-failed --option build-dir "$TEST_ROOT/custom-build-dir" 2> $TEST_ROOT/log || status=$? [ "$status" = "100" ] [[ 1 == "$(count "$customBuildDir/nix-build-"*)" ]] - local buildDir="$customBuildDir/nix-build-"* + local buildDir="$customBuildDir/nix-build-"*"" + if [[ -e $buildDir/build ]]; then + buildDir=$buildDir/build + fi grep $checkBuildId $buildDir/checkBuildId } test_custom_build_dir @@ -74,6 +81,8 @@ grep 'may not be deterministic' $TEST_ROOT/log [ "$status" = "104" ] if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi +TODO_NixOS + clearStore path=$(nix-build check.nix -A fetchurl --no-out-link) diff --git a/tests/functional/chroot-store.sh b/tests/functional/chroot-store.sh old mode 100644 new mode 100755 index 9e589d04b..03803a2b9 --- a/tests/functional/chroot-store.sh +++ b/tests/functional/chroot-store.sh @@ -1,33 +1,35 @@ +#!/usr/bin/env bash + source common.sh -echo example > $TEST_ROOT/example.txt -mkdir -p $TEST_ROOT/x +echo example > "$TEST_ROOT"/example.txt +mkdir -p "$TEST_ROOT/x" export NIX_STORE_DIR=/nix2/store -CORRECT_PATH=$(cd $TEST_ROOT && nix-store --store ./x --add example.txt) +CORRECT_PATH=$(cd "$TEST_ROOT" && nix-store --store ./x --add example.txt) [[ $CORRECT_PATH =~ ^/nix2/store/.*-example.txt$ ]] -PATH1=$(cd $TEST_ROOT && nix path-info --store ./x $CORRECT_PATH) -[ $CORRECT_PATH == $PATH1 ] +PATH1=$(cd "$TEST_ROOT" && nix path-info --store ./x "$CORRECT_PATH") +[ "$CORRECT_PATH" == "$PATH1" ] -PATH2=$(nix path-info --store "$TEST_ROOT/x" $CORRECT_PATH) -[ $CORRECT_PATH == $PATH2 ] +PATH2=$(nix path-info --store "$TEST_ROOT/x" "$CORRECT_PATH") +[ "$CORRECT_PATH" == "$PATH2" ] -PATH3=$(nix path-info --store "local?root=$TEST_ROOT/x" $CORRECT_PATH) -[ $CORRECT_PATH == $PATH3 ] +PATH3=$(nix path-info --store "local?root=$TEST_ROOT/x" "$CORRECT_PATH") +[ "$CORRECT_PATH" == "$PATH3" ] # Ensure store info trusted works with local store -nix --store $TEST_ROOT/x store info --json | jq -e '.trusted' +nix --store "$TEST_ROOT/x" store info --json | jq -e '.trusted' # Test building in a chroot store. if canUseSandbox; then flakeDir=$TEST_ROOT/flake - mkdir -p $flakeDir + mkdir -p "$flakeDir" - cat > $flakeDir/flake.nix < "$flakeDir"/flake.nix <&2 + exit 1 +} + +readLink() { + # TODO fix this + # shellcheck disable=SC2012 + ls -l "$1" | sed 's/.*->\ //' +} + +clearProfiles() { + profiles="$HOME/.local/state/nix/profiles" + rm -rf "$profiles" +} + +# Clear the store, but do not fail if we're in an environment where we can't. +# This allows the test to run in a NixOS test environment, where we use the system store. +# See doc/manual/src/contributing/testing.md / Running functional tests on NixOS. +clearStoreIfPossible() { + if isTestOnNixOS; then + echo "clearStoreIfPossible: Not clearing store, because we're on NixOS. Moving on." + else + doClearStore + fi +} + +clearStore() { + if isTestOnNixOS; then + die "clearStore: not supported when testing on NixOS. If not essential, call clearStoreIfPossible. If really needed, add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + doClearStore +} + +doClearStore() { + echo "clearing store..." + chmod -R +w "$NIX_STORE_DIR" + rm -rf "$NIX_STORE_DIR" + mkdir "$NIX_STORE_DIR" + rm -rf "$NIX_STATE_DIR" + mkdir "$NIX_STATE_DIR" + clearProfiles +} + +clearCache() { + rm -rf "${cacheDir?}" +} + +clearCacheCache() { + rm -f "$TEST_HOME/.cache/nix/binary-cache"* +} + +startDaemon() { + if isTestOnNixOS; then + die "startDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + + # Don’t start the daemon twice, as this would just make it loop indefinitely + if [[ "${_NIX_TEST_DAEMON_PID-}" != '' ]]; then + return + fi + # Start the daemon, wait for the socket to appear. + rm -f "$NIX_DAEMON_SOCKET_PATH" + PATH=$DAEMON_PATH nix --extra-experimental-features 'nix-command' daemon & + _NIX_TEST_DAEMON_PID=$! + export _NIX_TEST_DAEMON_PID + for ((i = 0; i < 300; i++)); do + if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then + DAEMON_STARTED=1 + break; + fi + sleep 0.1 + done + if [[ -z ${DAEMON_STARTED+x} ]]; then + fail "Didn’t manage to start the daemon" + fi + trap "killDaemon" EXIT + # Save for if daemon is killed + NIX_REMOTE_OLD=$NIX_REMOTE + export NIX_REMOTE=daemon +} + +killDaemon() { + if isTestOnNixOS; then + die "killDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + + # Don’t fail trying to stop a non-existant daemon twice + if [[ "${_NIX_TEST_DAEMON_PID-}" == '' ]]; then + return + fi + kill "$_NIX_TEST_DAEMON_PID" + for i in {0..100}; do + kill -0 "$_NIX_TEST_DAEMON_PID" 2> /dev/null || break + sleep 0.1 + done + kill -9 "$_NIX_TEST_DAEMON_PID" 2> /dev/null || true + wait "$_NIX_TEST_DAEMON_PID" || true + rm -f "$NIX_DAEMON_SOCKET_PATH" + # Indicate daemon is stopped + unset _NIX_TEST_DAEMON_PID + # Restore old nix remote + NIX_REMOTE=$NIX_REMOTE_OLD + trap "" EXIT +} + +restartDaemon() { + if isTestOnNixOS; then + die "restartDaemon: not supported when testing on NixOS. Is it really needed? If so add conditionals; e.g. if ! isTestOnNixOS; then ..." + fi + + [[ -z "${_NIX_TEST_DAEMON_PID:-}" ]] && return 0 + + killDaemon + startDaemon +} + +isDaemonNewer () { + [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0 + local requiredVersion="$1" + local daemonVersion + daemonVersion=$("$NIX_DAEMON_PACKAGE/bin/nix" daemon --version | cut -d' ' -f3) + [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]] +} + +skipTest () { + echo "$1, skipping this test..." >&2 + exit 77 +} + +TODO_NixOS() { + if isTestOnNixOS; then + skipTest "This test has not been adapted for NixOS yet" + fi +} + +requireDaemonNewerThan () { + isDaemonNewer "$1" || skipTest "Daemon is too old" +} + +canUseSandbox() { + [[ ${_canUseSandbox-} ]] +} + +requireSandboxSupport () { + canUseSandbox || skipTest "Sandboxing not supported" +} + +requireGit() { + [[ $(type -p git) ]] || skipTest "Git not installed" +} + +fail() { + echo "test failed: $*" >&2 + exit 1 +} + +# Run a command failing if it didn't exit with the expected exit code. +# +# Has two advantages over the built-in `!`: +# +# 1. `!` conflates all non-0 codes. `expect` allows testing for an exact +# code. +# +# 2. `!` unexpectedly negates `set -e`, and cannot be used on individual +# pipeline stages with `set -o pipefail`. It only works on the entire +# pipeline, which is useless if we want, say, `nix ...` invocation to +# *fail*, but a grep on the error message it outputs to *succeed*. +expect() { + local expected res + expected="$1" + shift + "$@" && res=0 || res="$?" + # also match "negative" codes, which wrap around to >127 + if [[ $res -ne $expected && $res -ne $((256 + expected)) ]]; then + echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 + return 1 + fi + return 0 +} + +# Better than just doing `expect ... >&2` because the "Expected..." +# message below will *not* be redirected. +expectStderr() { + local expected res + expected="$1" + shift + "$@" 2>&1 && res=0 || res="$?" + # also match "negative" codes, which wrap around to >127 + if [[ $res -ne $expected && $res -ne $((256 + expected)) ]]; then + echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 + return 1 + fi + return 0 +} + +# Run a command and check whether the stderr matches stdin. +# Show a diff when output does not match. +# Usage: +# +# assertStderr nix profile remove nothing << EOF +# error: This error is expected +# EOF +assertStderr() { + diff -u /dev/stdin <("$@" 2>/dev/null 2>&1) +} + +needLocalStore() { + if [[ "$NIX_REMOTE" == "daemon" ]]; then + skipTest "Can’t run through the daemon ($1)" + fi +} + +# Just to make it easy to find which tests should be fixed +buggyNeedLocalStore() { + needLocalStore "$1" +} + +enableFeatures() { + local features="$1" + sed -i 's/experimental-features .*/& '"$features"'/' "${test_nix_conf?}" +} + +onError() { + set +x + echo "$0: test failed at:" >&2 + for ((i = 1; i < ${#BASH_SOURCE[@]}; i++)); do + if [[ -z ${BASH_SOURCE[i]} ]]; then break; fi + echo " ${FUNCNAME[i]} in ${BASH_SOURCE[i]}:${BASH_LINENO[i-1]}" >&2 + done +} + +# Prints an error message prefix referring to the last call into this file. +# Ignores `expect` and `expectStderr` calls. +# Set a special exit code when test suite functions are misused, so that +# functions like expectStderr won't mistake them for expected Nix CLI errors. +# Suggestion: -101 (negative to indicate very abnormal, and beyond the normal +# range of signals) +# Example (showns as string): 'repl.sh:123: in call to grepQuiet: ' +# This function is inefficient, so it should only be used in error messages. +callerPrefix() { + # Find the closest caller that's not from this file + # using the bash `caller` builtin. + local i file line fn savedFn + # Use `caller` + for i in $(seq 0 100); do + caller "$i" > /dev/null || { + if [[ -n "${file:-}" ]]; then + echo "$file:$line: ${savedFn+in call to $savedFn: }" + fi + break + } + line="$(caller "$i" | cut -d' ' -f1)" + fn="$(caller "$i" | cut -d' ' -f2)" + file="$(caller "$i" | cut -d' ' -f3)" + if [[ $file != "${BASH_SOURCE[0]}" ]]; then + echo "$file:$line: ${savedFn+in call to $savedFn: }" + return + fi + case "$fn" in + # Ignore higher order functions that don't report any misuse of themselves + # This way a misuse of a foo in `expectStderr 1 foo` will be reported as + # calling foo, not expectStderr. + expect|expectStderr|callerPrefix) + ;; + *) + savedFn="$fn" + ;; + esac + done +} + +checkGrepArgs() { + local arg + for arg in "$@"; do + if [[ "$arg" != "${arg//$'\n'/_}" ]]; then + echo "$(callerPrefix)newline not allowed in arguments; grep would try each line individually as if connected by an OR operator" >&2 + return 155 # = -101 mod 256 + fi + done +} + +# `grep -v` doesn't work well for exit codes. We want `!(exist line l. l +# matches)`. It gives us `exist line l. !(l matches)`. +# +# `!` normally doesn't work well with `set -e`, but when we wrap in a +# function it *does*. +# +# `command grep` lets us avoid re-checking the args by going directly to the +# executable. +grepInverse() { + checkGrepArgs "$@" && \ + ! command grep "$@" +} + +# A shorthand, `> /dev/null` is a bit noisy. +# +# `grep -q` would seem to do this, no function necessary, but it is a +# bad fit with pipes and `set -o pipefail`: `-q` will exit after the +# first match, and then subsequent writes will result in broken pipes. +# +# Note that reproducing the above is a bit tricky as it depends on +# non-deterministic properties such as the timing between the match and +# the closing of the pipe, the buffering of the pipe, and the speed of +# the producer into the pipe. But rest assured we've seen it happen in +# CI reliably. +# +# `command grep` lets us avoid re-checking the args by going directly to the +# executable. +grepQuiet() { + checkGrepArgs "$@" && \ + command grep "$@" > /dev/null +} + +# The previous two, combined +grepQuietInverse() { + checkGrepArgs "$@" && \ + ! command grep "$@" > /dev/null +} + +# Wrap grep to remove its newline footgun; see checkGrepArgs. +# Note that we keep the checkGrepArgs calls in the other helpers, because some +# of them are negated and that would defeat this check. +grep() { + checkGrepArgs "$@" && \ + command grep "$@" +} + +# Return the number of arguments +count() { + echo $# +} + +trap onError ERR + +fi # COMMON_FUNCTIONS_SH_SOURCED diff --git a/tests/functional/common/init.sh b/tests/functional/common/init.sh index 74da12651..d849c0734 100755 --- a/tests/functional/common/init.sh +++ b/tests/functional/common/init.sh @@ -1,3 +1,31 @@ +# shellcheck shell=bash + +# for shellcheck +: "${test_nix_conf_dir?}" "${test_nix_conf?}" + +if isTestOnNixOS; then + + mkdir -p "$test_nix_conf_dir" "$TEST_HOME" + + export NIX_USER_CONF_FILES="$test_nix_conf" + mkdir -p "$test_nix_conf_dir" "$TEST_HOME" + ! test -e "$test_nix_conf" + cat > "$test_nix_conf" < /dev/null); then - # Maybe the build directory is symlinked. - export NIX_IGNORE_SYMLINK_STORE=1 - NIX_STORE_DIR=$TEST_ROOT/store -fi -export NIX_LOCALSTATE_DIR=$TEST_ROOT/var -export NIX_LOG_DIR=$TEST_ROOT/var/log/nix -export NIX_STATE_DIR=$TEST_ROOT/var/nix -export NIX_CONF_DIR=$TEST_ROOT/etc -export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket -unset NIX_USER_CONF_FILES -export _NIX_TEST_SHARED=$TEST_ROOT/shared -if [[ -n $NIX_STORE ]]; then - export _NIX_TEST_NO_SANDBOX=1 -fi -export _NIX_IN_TEST=$TEST_ROOT/shared -export _NIX_TEST_NO_LSOF=1 -export NIX_REMOTE=${NIX_REMOTE_-} -unset NIX_PATH -export TEST_HOME=$TEST_ROOT/test-home -export HOME=$TEST_HOME -unset XDG_STATE_HOME -unset XDG_DATA_HOME -unset XDG_CONFIG_HOME -unset XDG_CONFIG_DIRS -unset XDG_CACHE_HOME - -export PATH=@bindir@:$PATH -if [[ -n "${NIX_CLIENT_PACKAGE:-}" ]]; then - export PATH="$NIX_CLIENT_PACKAGE/bin":$PATH -fi -DAEMON_PATH="$PATH" -if [[ -n "${NIX_DAEMON_PACKAGE:-}" ]]; then - DAEMON_PATH="${NIX_DAEMON_PACKAGE}/bin:$DAEMON_PATH" -fi -coreutils=@coreutils@ -lsof=@lsof@ - -export dot=@dot@ -export SHELL="@bash@" -export PAGER=cat -export busybox="@sandbox_shell@" - -export version=@PACKAGE_VERSION@ -export system=@system@ - -export BUILD_SHARED_LIBS=@BUILD_SHARED_LIBS@ - -export IMPURE_VAR1=foo -export IMPURE_VAR2=bar - -cacheDir=$TEST_ROOT/binary-cache - -readLink() { - ls -l "$1" | sed 's/.*->\ //' -} - -clearProfiles() { - profiles="$HOME"/.local/state/nix/profiles - rm -rf "$profiles" -} - -clearStore() { - echo "clearing store..." - chmod -R +w "$NIX_STORE_DIR" - rm -rf "$NIX_STORE_DIR" - mkdir "$NIX_STORE_DIR" - rm -rf "$NIX_STATE_DIR" - mkdir "$NIX_STATE_DIR" - clearProfiles -} - -clearCache() { - rm -rf "$cacheDir" -} - -clearCacheCache() { - rm -f $TEST_HOME/.cache/nix/binary-cache* -} - -startDaemon() { - # Don’t start the daemon twice, as this would just make it loop indefinitely - if [[ "${_NIX_TEST_DAEMON_PID-}" != '' ]]; then - return - fi - # Start the daemon, wait for the socket to appear. - rm -f $NIX_DAEMON_SOCKET_PATH - PATH=$DAEMON_PATH nix --extra-experimental-features 'nix-command' daemon & - _NIX_TEST_DAEMON_PID=$! - export _NIX_TEST_DAEMON_PID - for ((i = 0; i < 300; i++)); do - if [[ -S $NIX_DAEMON_SOCKET_PATH ]]; then - DAEMON_STARTED=1 - break; - fi - sleep 0.1 - done - if [[ -z ${DAEMON_STARTED+x} ]]; then - fail "Didn’t manage to start the daemon" - fi - trap "killDaemon" EXIT - # Save for if daemon is killed - NIX_REMOTE_OLD=$NIX_REMOTE - export NIX_REMOTE=daemon -} - -killDaemon() { - # Don’t fail trying to stop a non-existant daemon twice - if [[ "${_NIX_TEST_DAEMON_PID-}" == '' ]]; then - return - fi - kill $_NIX_TEST_DAEMON_PID - for i in {0..100}; do - kill -0 $_NIX_TEST_DAEMON_PID 2> /dev/null || break - sleep 0.1 - done - kill -9 $_NIX_TEST_DAEMON_PID 2> /dev/null || true - wait $_NIX_TEST_DAEMON_PID || true - rm -f $NIX_DAEMON_SOCKET_PATH - # Indicate daemon is stopped - unset _NIX_TEST_DAEMON_PID - # Restore old nix remote - NIX_REMOTE=$NIX_REMOTE_OLD - trap "" EXIT -} - -restartDaemon() { - [[ -z "${_NIX_TEST_DAEMON_PID:-}" ]] && return 0 - - killDaemon - startDaemon -} - -if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then - _canUseSandbox=1 -fi - -isDaemonNewer () { - [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0 - local requiredVersion="$1" - local daemonVersion=$($NIX_DAEMON_PACKAGE/bin/nix daemon --version | cut -d' ' -f3) - [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]] -} - -skipTest () { - echo "$1, skipping this test..." >&2 - exit 99 -} - -requireDaemonNewerThan () { - isDaemonNewer "$1" || skipTest "Daemon is too old" -} - -canUseSandbox() { - [[ ${_canUseSandbox-} ]] -} - -requireSandboxSupport () { - canUseSandbox || skipTest "Sandboxing not supported" -} - -requireGit() { - [[ $(type -p git) ]] || skipTest "Git not installed" -} - -fail() { - echo "$1" >&2 - exit 1 -} - -# Run a command failing if it didn't exit with the expected exit code. -# -# Has two advantages over the built-in `!`: -# -# 1. `!` conflates all non-0 codes. `expect` allows testing for an exact -# code. -# -# 2. `!` unexpectedly negates `set -e`, and cannot be used on individual -# pipeline stages with `set -o pipefail`. It only works on the entire -# pipeline, which is useless if we want, say, `nix ...` invocation to -# *fail*, but a grep on the error message it outputs to *succeed*. -expect() { - local expected res - expected="$1" - shift - "$@" && res=0 || res="$?" - if [[ $res -ne $expected ]]; then - echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 - return 1 - fi - return 0 -} - -# Better than just doing `expect ... >&2` because the "Expected..." -# message below will *not* be redirected. -expectStderr() { - local expected res - expected="$1" - shift - "$@" 2>&1 && res=0 || res="$?" - if [[ $res -ne $expected ]]; then - echo "Expected exit code '$expected' but got '$res' from command ${*@Q}" >&2 - return 1 - fi - return 0 -} - -# Run a command and check whether the stderr matches stdin. -# Show a diff when output does not match. -# Usage: -# -# assertStderr nix profile remove nothing << EOF -# error: This error is expected -# EOF -assertStderr() { - diff -u /dev/stdin <($@ 2>/dev/null 2>&1) -} - -needLocalStore() { - if [[ "$NIX_REMOTE" == "daemon" ]]; then - skipTest "Can’t run through the daemon ($1)" - fi -} - -# Just to make it easy to find which tests should be fixed -buggyNeedLocalStore() { - needLocalStore "$1" -} - -enableFeatures() { - local features="$1" - sed -i 's/experimental-features .*/& '"$features"'/' "$NIX_CONF_DIR"/nix.conf -} - -set -x - -onError() { - set +x - echo "$0: test failed at:" >&2 - for ((i = 1; i < ${#BASH_SOURCE[@]}; i++)); do - if [[ -z ${BASH_SOURCE[i]} ]]; then break; fi - echo " ${FUNCNAME[i]} in ${BASH_SOURCE[i]}:${BASH_LINENO[i-1]}" >&2 - done -} - -# `grep -v` doesn't work well for exit codes. We want `!(exist line l. l -# matches)`. It gives us `exist line l. !(l matches)`. -# -# `!` normally doesn't work well with `set -e`, but when we wrap in a -# function it *does*. -grepInverse() { - ! grep "$@" -} - -# A shorthand, `> /dev/null` is a bit noisy. -# -# `grep -q` would seem to do this, no function necessary, but it is a -# bad fit with pipes and `set -o pipefail`: `-q` will exit after the -# first match, and then subsequent writes will result in broken pipes. -# -# Note that reproducing the above is a bit tricky as it depends on -# non-deterministic properties such as the timing between the match and -# the closing of the pipe, the buffering of the pipe, and the speed of -# the producer into the pipe. But rest assured we've seen it happen in -# CI reliably. -grepQuiet() { - grep "$@" > /dev/null -} - -# The previous two, combined -grepQuietInverse() { - ! grep "$@" > /dev/null -} - -# Return the number of arguments -count() { - echo $# -} - -trap onError ERR - -fi # COMMON_VARS_AND_FUNCTIONS_SH_SOURCED diff --git a/tests/functional/common/vars.sh b/tests/functional/common/vars.sh new file mode 100644 index 000000000..c99a6a5c2 --- /dev/null +++ b/tests/functional/common/vars.sh @@ -0,0 +1,72 @@ +# shellcheck shell=bash + +set -eu -o pipefail + +if [[ -z "${COMMON_VARS_SH_SOURCED-}" ]]; then + +COMMON_VARS_SH_SOURCED=1 + +commonDir="$(readlink -f "$(dirname "${BASH_SOURCE[0]-$0}")")" + +# Since this is a generated file +# shellcheck disable=SC1091 +source "$commonDir/subst-vars.sh" +# Make sure shellcheck knows all these will be defined by the above generated snippet +: "${bindir?} ${coreutils?} ${dot?} ${SHELL?} ${busybox?} ${version?} ${system?}" +export coreutils dot busybox version system + +export PAGER=cat + +source "$commonDir/paths.sh" +source "$commonDir/test-root.sh" + +test_nix_conf_dir=$TEST_ROOT/etc +# Used in other files +# shellcheck disable=SC2034 +test_nix_conf=$test_nix_conf_dir/nix.conf + +export TEST_HOME=$TEST_ROOT/test-home + +if ! isTestOnNixOS; then + export NIX_STORE_DIR + if ! NIX_STORE_DIR=$(readlink -f "$TEST_ROOT/store" 2> /dev/null); then + # Maybe the build directory is symlinked. + export NIX_IGNORE_SYMLINK_STORE=1 + NIX_STORE_DIR=$TEST_ROOT/store + fi + export NIX_LOCALSTATE_DIR=$TEST_ROOT/var + export NIX_LOG_DIR=$TEST_ROOT/var/log/nix + export NIX_STATE_DIR=$TEST_ROOT/var/nix + export NIX_CONF_DIR=$test_nix_conf_dir + export NIX_DAEMON_SOCKET_PATH=$TEST_ROOT/dSocket + unset NIX_USER_CONF_FILES + export _NIX_TEST_SHARED=$TEST_ROOT/shared + if [[ -n $NIX_STORE ]]; then + export _NIX_TEST_NO_SANDBOX=1 + fi + export _NIX_IN_TEST=$TEST_ROOT/shared + export _NIX_TEST_NO_LSOF=1 + export NIX_REMOTE=${NIX_REMOTE_-} + +fi # ! isTestOnNixOS + +unset NIX_PATH +export HOME=$TEST_HOME +unset XDG_STATE_HOME +unset XDG_DATA_HOME +unset XDG_CONFIG_HOME +unset XDG_CONFIG_DIRS +unset XDG_CACHE_HOME + +export IMPURE_VAR1=foo +export IMPURE_VAR2=bar + +# Used in other files +# shellcheck disable=SC2034 +cacheDir=$TEST_ROOT/binary-cache + +if [[ $(uname) == Linux ]] && [[ -L /proc/self/ns/user ]] && unshare --user true; then + _canUseSandbox=1 +fi + +fi # COMMON_VARS_SH_SOURCED diff --git a/tests/functional/completions.sh b/tests/functional/completions.sh old mode 100644 new mode 100755 index d3d5bbd48..9164c5013 --- a/tests/functional/completions.sh +++ b/tests/functional/completions.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh cd "$TEST_ROOT" diff --git a/tests/functional/compression-levels.sh b/tests/functional/compression-levels.sh old mode 100644 new mode 100755 index 85f12974a..399265f9c --- a/tests/functional/compression-levels.sh +++ b/tests/functional/compression-levels.sh @@ -1,22 +1,24 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible clearCache outPath=$(nix-build dependencies.nix --no-out-link) cacheURI="file://$cacheDir?compression=xz&compression-level=0" -nix copy --to $cacheURI $outPath +nix copy --to "$cacheURI" "$outPath" -FILESIZES=$(cat ${cacheDir}/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}') +FILESIZES=$(cat "${cacheDir}"/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}') clearCache cacheURI="file://$cacheDir?compression=xz&compression-level=5" -nix copy --to $cacheURI $outPath +nix copy --to "$cacheURI" "$outPath" -FILESIZES2=$(cat ${cacheDir}/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}') +FILESIZES2=$(cat "${cacheDir}"/*.narinfo | awk '/FileSize: /{sum+=$2}END{print sum}') [[ $FILESIZES -gt $FILESIZES2 ]] diff --git a/tests/functional/compute-levels.sh b/tests/functional/compute-levels.sh old mode 100644 new mode 100755 index de3da2ebd..a8bd27610 --- a/tests/functional/compute-levels.sh +++ b/tests/functional/compute-levels.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh if [[ $(uname -ms) = "Linux x86_64" ]]; then diff --git a/tests/functional/config.sh b/tests/functional/config.sh old mode 100644 new mode 100755 index efdafa8ca..50858eaa4 --- a/tests/functional/config.sh +++ b/tests/functional/config.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh # Isolate the home for this test. @@ -26,6 +28,8 @@ nix registry remove userhome-with-xdg # Assert the .config folder hasn't been created. [ ! -e "$HOME/.config" ] +TODO_NixOS # Very specific test setup not compatible with the NixOS test environment? + # Test that files are loaded from XDG by default export XDG_CONFIG_HOME=$TEST_ROOT/confighome export XDG_CONFIG_DIRS=$TEST_ROOT/dir1:$TEST_ROOT/dir2 diff --git a/tests/functional/db-migration.sh b/tests/functional/db-migration.sh old mode 100644 new mode 100755 index 44cd16bc0..6feabb90d --- a/tests/functional/db-migration.sh +++ b/tests/functional/db-migration.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Test that we can successfully migrate from an older db schema source common.sh @@ -8,6 +10,8 @@ if [[ -z "${NIX_DAEMON_PACKAGE-}" ]]; then skipTest "not using the Nix daemon" fi +TODO_NixOS + killDaemon # Fill the db using the older Nix diff --git a/tests/functional/debugger.sh b/tests/functional/debugger.sh old mode 100644 new mode 100755 index 63d88cbf3..b96b7e5d3 --- a/tests/functional/debugger.sh +++ b/tests/functional/debugger.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible # regression #9932 echo ":env" | expect 1 nix eval --debugger --expr '(_: throw "oh snap") 42' diff --git a/tests/functional/dependencies.sh b/tests/functional/dependencies.sh old mode 100644 new mode 100755 index b93dacac0..972bc5a9b --- a/tests/functional/dependencies.sh +++ b/tests/functional/dependencies.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible drvPath=$(nix-instantiate dependencies.nix) @@ -31,7 +33,7 @@ nix-store -q --tree "$outPath" | grep '───.*dependencies-input-2' echo "output path is $outPath" -text=$(cat "$outPath"/foobar) +text=$(cat "$outPath/foobar") if test "$text" != "FOOBAR"; then exit 1; fi deps=$(nix-store -quR "$drvPath") @@ -63,6 +65,8 @@ drvPath2=$(nix-instantiate dependencies.nix --argstr hashInvalidator yay) # now --valid-derivers returns both test "$(nix-store -q --valid-derivers "$outPath" | sort)" = "$(sort <<< "$drvPath"$'\n'"$drvPath2")" +TODO_NixOS # The following --delete fails, because it seems to be still alive. This might be caused by a different test using the same path. We should try make the derivations unique, e.g. naming after tests, and adding a timestamp that's constant for that test script run. + # check that nix-store --valid-derivers only returns existing drv nix-store --delete "$drvPath" test "$(nix-store -q --valid-derivers "$outPath")" = "$drvPath2" diff --git a/tests/functional/derivation-advanced-attributes.sh b/tests/functional/derivation-advanced-attributes.sh new file mode 100755 index 000000000..271f17dc6 --- /dev/null +++ b/tests/functional/derivation-advanced-attributes.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +source common/test-root.sh +source common/paths.sh + +set -eu -o pipefail + +source characterisation/framework.sh + +badDiff=0 +badExitCode=0 + +store="$TEST_ROOT/store" + +for nixFile in derivation/*.nix; do + drvPath=$(nix-instantiate --store "$store" --pure-eval --expr "$(< "$nixFile")") + testName=$(basename "$nixFile" .nix) + got="${store}${drvPath}" + expected="derivation/$testName.drv" + diffAndAcceptInner "$testName" "$got" "$expected" +done + +characterisationTestExit diff --git a/tests/functional/derivation-json.sh b/tests/functional/derivation-json.sh old mode 100644 new mode 100755 index b6be5d977..06f934cfe --- a/tests/functional/derivation-json.sh +++ b/tests/functional/derivation-json.sh @@ -1,12 +1,14 @@ +#!/usr/bin/env bash + source common.sh drvPath=$(nix-instantiate simple.nix) -nix derivation show $drvPath | jq .[] > $TEST_HOME/simple.json +nix derivation show "$drvPath" | jq .[] > "$TEST_HOME"/simple.json -drvPath2=$(nix derivation add < $TEST_HOME/simple.json) +drvPath2=$(nix derivation add < "$TEST_HOME"/simple.json) [[ "$drvPath" = "$drvPath2" ]] # Input addressed derivations cannot be renamed. -jq '.name = "foo"' < $TEST_HOME/simple.json | expectStderr 1 nix derivation add | grepQuiet "has incorrect output" +jq '.name = "foo"' < "$TEST_HOME"/simple.json | expectStderr 1 nix derivation add | grepQuiet "has incorrect output" diff --git a/tests/functional/derivation/advanced-attributes-defaults.drv b/tests/functional/derivation/advanced-attributes-defaults.drv new file mode 100644 index 000000000..391c6ab80 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-defaults.drv @@ -0,0 +1 @@ +Derive([("out","/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults","","")],[],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("builder","/bin/bash"),("name","advanced-attributes-defaults"),("out","/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-defaults.nix b/tests/functional/derivation/advanced-attributes-defaults.nix new file mode 100644 index 000000000..51a8d0e7e --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-defaults.nix @@ -0,0 +1,6 @@ +derivation { + name = "advanced-attributes-defaults"; + system = "my-system"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; +} diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv new file mode 100644 index 000000000..9dd402057 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.drv @@ -0,0 +1 @@ +Derive([("dev","/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev","",""),("out","/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults","","")],[],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}"),("dev","/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev"),("out","/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix new file mode 100644 index 000000000..0c13a7691 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs-defaults.nix @@ -0,0 +1,8 @@ +derivation { + name = "advanced-attributes-structured-attrs-defaults"; + system = "my-system"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; + outputs = [ "out" "dev" ]; + __structuredAttrs = true; +} diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs.drv b/tests/functional/derivation/advanced-attributes-structured-attrs.drv new file mode 100644 index 000000000..e47a41ad5 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +Derive([("bin","/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin","",""),("dev","/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev","",""),("out","/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__json","{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}"),("bin","/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin"),("dev","/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev"),("out","/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes-structured-attrs.nix b/tests/functional/derivation/advanced-attributes-structured-attrs.nix new file mode 100644 index 000000000..0044b65fd --- /dev/null +++ b/tests/functional/derivation/advanced-attributes-structured-attrs.nix @@ -0,0 +1,45 @@ +let + system = "my-system"; + foo = derivation { + inherit system; + name = "foo"; + builder = "/bin/bash"; + args = ["-c" "echo foo > $out"]; + }; + bar = derivation { + inherit system; + name = "bar"; + builder = "/bin/bash"; + args = ["-c" "echo bar > $out"]; + }; +in +derivation { + inherit system; + name = "advanced-attributes-structured-attrs"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; + __sandboxProfile = "sandcastle"; + __noChroot = true; + __impureHostDeps = ["/usr/bin/ditto"]; + impureEnvVars = ["UNICORN"]; + __darwinAllowLocalNetworking = true; + outputs = [ "out" "bin" "dev" ]; + __structuredAttrs = true; + outputChecks = { + out = { + allowedReferences = [foo]; + allowedRequisites = [foo]; + }; + bin = { + disallowedReferences = [bar]; + disallowedRequisites = [bar]; + }; + dev = { + maxSize = 789; + maxClosureSize = 5909; + }; + }; + requiredSystemFeatures = ["rainbow" "uid-range"]; + preferLocalBuild = true; + allowSubstitutes = false; +} diff --git a/tests/functional/derivation/advanced-attributes.drv b/tests/functional/derivation/advanced-attributes.drv new file mode 100644 index 000000000..ec3112ab2 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes.drv @@ -0,0 +1 @@ +Derive([("out","/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes","","")],[("/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv",["out"]),("/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv",["out"])],[],"my-system","/bin/bash",["-c","echo hello > $out"],[("__darwinAllowLocalNetworking","1"),("__impureHostDeps","/usr/bin/ditto"),("__noChroot","1"),("__sandboxProfile","sandcastle"),("allowSubstitutes",""),("allowedReferences","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("allowedRequisites","/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"),("builder","/bin/bash"),("disallowedReferences","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("disallowedRequisites","/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"),("impureEnvVars","UNICORN"),("name","advanced-attributes"),("out","/nix/store/33a6fdmn8q9ih9d7npbnrxn2q56a4l8q-advanced-attributes"),("preferLocalBuild","1"),("requiredSystemFeatures","rainbow uid-range"),("system","my-system")]) \ No newline at end of file diff --git a/tests/functional/derivation/advanced-attributes.nix b/tests/functional/derivation/advanced-attributes.nix new file mode 100644 index 000000000..ff680c567 --- /dev/null +++ b/tests/functional/derivation/advanced-attributes.nix @@ -0,0 +1,33 @@ +let + system = "my-system"; + foo = derivation { + inherit system; + name = "foo"; + builder = "/bin/bash"; + args = ["-c" "echo foo > $out"]; + }; + bar = derivation { + inherit system; + name = "bar"; + builder = "/bin/bash"; + args = ["-c" "echo bar > $out"]; + }; +in +derivation { + inherit system; + name = "advanced-attributes"; + builder = "/bin/bash"; + args = [ "-c" "echo hello > $out" ]; + __sandboxProfile = "sandcastle"; + __noChroot = true; + __impureHostDeps = ["/usr/bin/ditto"]; + impureEnvVars = ["UNICORN"]; + __darwinAllowLocalNetworking = true; + allowedReferences = [foo]; + allowedRequisites = [foo]; + disallowedReferences = [bar]; + disallowedRequisites = [bar]; + requiredSystemFeatures = ["rainbow" "uid-range"]; + preferLocalBuild = true; + allowSubstitutes = false; +} diff --git a/tests/functional/dot.nar b/tests/functional/dot.nar new file mode 100644 index 000000000..3a9452f67 Binary files /dev/null and b/tests/functional/dot.nar differ diff --git a/tests/functional/dotdot.nar b/tests/functional/dotdot.nar new file mode 100644 index 000000000..f8d019c39 Binary files /dev/null and b/tests/functional/dotdot.nar differ diff --git a/tests/functional/dump-db.sh b/tests/functional/dump-db.sh old mode 100644 new mode 100755 index 48647f403..14181b4b6 --- a/tests/functional/dump-db.sh +++ b/tests/functional/dump-db.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + needLocalStore "--dump-db requires a local store" clearStore diff --git a/tests/functional/duplicate.nar b/tests/functional/duplicate.nar new file mode 100644 index 000000000..1d0993ed4 Binary files /dev/null and b/tests/functional/duplicate.nar differ diff --git a/tests/functional/dyn-drv/common.sh b/tests/functional/dyn-drv/common.sh index c786f6925..0d95881b6 100644 --- a/tests/functional/dyn-drv/common.sh +++ b/tests/functional/dyn-drv/common.sh @@ -5,4 +5,6 @@ requireDaemonNewerThan "2.16.0pre20230419" enableFeatures "ca-derivations dynamic-derivations" +TODO_NixOS + restartDaemon diff --git a/tests/functional/dyn-drv/meson.build b/tests/functional/dyn-drv/meson.build new file mode 100644 index 000000000..3c671d013 --- /dev/null +++ b/tests/functional/dyn-drv/meson.build @@ -0,0 +1,19 @@ +configure_file( + input : 'config.nix.in', + output : 'config.nix', + configuration : test_confdata, +) + +suites += { + 'name': 'dyn-drv', + 'deps': [], + 'tests': [ + 'text-hashed-output.sh', + 'recursive-mod-json.sh', + 'build-built-drv.sh', + 'eval-outputOf.sh', + 'dep-built-drv.sh', + 'old-daemon-error-hack.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/dyn-drv/text-hashed-output.sh b/tests/functional/dyn-drv/text-hashed-output.sh index f3e5aa93b..2cc877219 100644 --- a/tests/functional/dyn-drv/text-hashed-output.sh +++ b/tests/functional/dyn-drv/text-hashed-output.sh @@ -20,7 +20,7 @@ nix show-derivation "$drvProducingDrv" out1=$(nix-build ./text-hashed-output.nix -A producingDrv --no-out-link) -nix path-info $drv --derivation --json | jq -nix path-info $out1 --derivation --json | jq +nix path-info "$drv" --derivation --json | jq +nix path-info "$out1" --derivation --json | jq -test $out1 == $drv +test "$out1" == "$drv" diff --git a/tests/functional/empty.nar b/tests/functional/empty.nar new file mode 100644 index 000000000..43434f2b4 Binary files /dev/null and b/tests/functional/empty.nar differ diff --git a/tests/functional/eval-store.sh b/tests/functional/eval-store.sh old mode 100644 new mode 100755 index 9937ecbce..202e7b004 --- a/tests/functional/eval-store.sh +++ b/tests/functional/eval-store.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + # Using `--eval-store` with the daemon will eventually copy everything # to the build store, invalidating most of the tests here needLocalStore "“--eval-store” doesn't achieve much with the daemon" diff --git a/tests/functional/eval.sh b/tests/functional/eval.sh old mode 100644 new mode 100755 index c6a475cd0..22d2d02a2 --- a/tests/functional/eval.sh +++ b/tests/functional/eval.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible testStdinHeredoc=$(nix eval -f - <$TEST_ROOT/stdout 2>$TEST_ROOT/stderr -[[ $(cat $TEST_ROOT/stdout) = '' ]] -grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr -grepQuiet "error: could not find setting 'accept-flake-config'" $TEST_ROOT/stderr +' expect 1 nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +[[ $(cat "$TEST_ROOT/stdout") = '' ]] +grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" "$TEST_ROOT/stderr" +grepQuiet "error: could not find setting 'accept-flake-config'" "$TEST_ROOT/stderr" # 'flakes' experimental-feature is disabled after, ignore and warn NIX_CONFIG=' accept-flake-config = true experimental-features = nix-command -' expect 1 nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr -[[ $(cat $TEST_ROOT/stdout) = '' ]] -grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" $TEST_ROOT/stderr -grepQuiet "error: could not find setting 'accept-flake-config'" $TEST_ROOT/stderr +' expect 1 nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +[[ $(cat "$TEST_ROOT/stdout") = '' ]] +grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" "$TEST_ROOT/stderr" +grepQuiet "error: could not find setting 'accept-flake-config'" "$TEST_ROOT/stderr" # 'flakes' experimental-feature is enabled before, process NIX_CONFIG=' experimental-features = nix-command flakes accept-flake-config = true -' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr -grepQuiet "true" $TEST_ROOT/stdout -grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr +' nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +grepQuiet "true" "$TEST_ROOT/stdout" +grepQuietInverse "Ignoring setting 'accept-flake-config'" "$TEST_ROOT/stderr" # 'flakes' experimental-feature is enabled after, process NIX_CONFIG=' accept-flake-config = true experimental-features = nix-command flakes -' nix config show accept-flake-config 1>$TEST_ROOT/stdout 2>$TEST_ROOT/stderr -grepQuiet "true" $TEST_ROOT/stdout -grepQuietInverse "Ignoring setting 'accept-flake-config'" $TEST_ROOT/stderr +' nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +grepQuiet "true" "$TEST_ROOT/stdout" +grepQuietInverse "Ignoring setting 'accept-flake-config'" "$TEST_ROOT/stderr" function exit_code_both_ways { expect 1 nix --experimental-features 'nix-command' "$@" 1>/dev/null diff --git a/tests/functional/export-graph.sh b/tests/functional/export-graph.sh old mode 100644 new mode 100755 index 1f6232a40..b507b6d3a --- a/tests/functional/export-graph.sh +++ b/tests/functional/export-graph.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore clearProfiles diff --git a/tests/functional/export.sh b/tests/functional/export.sh old mode 100644 new mode 100755 index 2238539bc..3e895a540 --- a/tests/functional/export.sh +++ b/tests/functional/export.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore outPath=$(nix-build dependencies.nix --no-out-link) diff --git a/tests/functional/extra-sandbox-profile.sh b/tests/functional/extra-sandbox-profile.sh old mode 100644 new mode 100755 index ac3ca036f..672e5779d --- a/tests/functional/extra-sandbox-profile.sh +++ b/tests/functional/extra-sandbox-profile.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh if [[ $(uname) != Darwin ]]; then skipTest "Need Darwin"; fi diff --git a/tests/functional/fetchClosure.sh b/tests/functional/fetchClosure.sh old mode 100644 new mode 100755 index a02d1ce7a..7ef635d36 --- a/tests/functional/fetchClosure.sh +++ b/tests/functional/fetchClosure.sh @@ -1,7 +1,11 @@ +#!/usr/bin/env bash + source common.sh enableFeatures "fetch-closure" +TODO_NixOS + clearStore clearCacheCache diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh old mode 100644 new mode 100755 index 74d6de4e3..78925b5cd --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -1,8 +1,10 @@ +#!/usr/bin/env bash + source common.sh requireGit -clearStore +clearStoreIfPossible # Intentionally not in a canonical form # See https://github.com/NixOS/nix/issues/6195 diff --git a/tests/functional/fetchGitRefs.sh b/tests/functional/fetchGitRefs.sh old mode 100644 new mode 100755 index d643fea04..ee054fabc --- a/tests/functional/fetchGitRefs.sh +++ b/tests/functional/fetchGitRefs.sh @@ -1,8 +1,10 @@ +#!/usr/bin/env bash + source common.sh requireGit -clearStore +clearStoreIfPossible repo="$TEST_ROOT/git" @@ -12,7 +14,7 @@ git init "$repo" git -C "$repo" config user.email "foobar@example.com" git -C "$repo" config user.name "Foobar" -echo utrecht > "$repo"/hello +echo utrecht > "$repo/hello" git -C "$repo" add hello git -C "$repo" commit -m 'Bla1' diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh old mode 100644 new mode 100755 index bd82a0a17..cd3b51674 --- a/tests/functional/fetchGitSubmodules.sh +++ b/tests/functional/fetchGitSubmodules.sh @@ -1,10 +1,12 @@ +#!/usr/bin/env bash + source common.sh set -u requireGit -clearStore +clearStoreIfPossible rootRepo=$TEST_ROOT/gitSubmodulesRoot subRepo=$TEST_ROOT/gitSubmodulesSub @@ -102,6 +104,27 @@ noSubmoduleRepo=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$subR [[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] +# Test .gitmodules with entries that refer to non-existent objects or objects that are not submodules. +cat >> $rootRepo/.gitmodules < $rootRepo/file +git -C $rootRepo add file +git -C $rootRepo commit -a -m "Add bad submodules" + +rev=$(git -C $rootRepo rev-parse HEAD) + +r=$(nix eval --raw --expr "builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }") + +[[ -f $r/file ]] +[[ ! -e $r/missing ]] + # Test relative submodule URLs. rm $TEST_HOME/.cache/nix/fetcher-cache* rm -rf $rootRepo/.git $rootRepo/.gitmodules $rootRepo/sub diff --git a/tests/functional/fetchGitVerification.sh b/tests/functional/fetchGitVerification.sh old mode 100644 new mode 100755 index b80e061b5..4012d8229 --- a/tests/functional/fetchGitVerification.sh +++ b/tests/functional/fetchGitVerification.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh requireGit @@ -5,7 +7,7 @@ requireGit enableFeatures "verified-fetches" -clearStore +clearStoreIfPossible repo="$TEST_ROOT/git" diff --git a/tests/functional/fetchMercurial.sh b/tests/functional/fetchMercurial.sh old mode 100644 new mode 100755 index 9f7cef7b2..6de192865 --- a/tests/functional/fetchMercurial.sh +++ b/tests/functional/fetchMercurial.sh @@ -1,7 +1,11 @@ +#!/usr/bin/env bash + source common.sh [[ $(type -p hg) ]] || skipTest "Mercurial not installed" +TODO_NixOS + clearStore # Intentionally not in a canonical form diff --git a/tests/functional/fetchPath.sh b/tests/functional/fetchPath.sh old mode 100644 new mode 100755 index 29be38ce2..560a270c1 --- a/tests/functional/fetchPath.sh +++ b/tests/functional/fetchPath.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -touch $TEST_ROOT/foo -t 202211111111 +touch "$TEST_ROOT/foo" -t 202211111111 # We only check whether 2022-11-1* **:**:** is the last modified date since # `lastModified` is transformed into UTC in `builtins.fetchTarball`. [[ "$(nix eval --impure --raw --expr "(builtins.fetchTree \"path://$TEST_ROOT/foo\").lastModifiedDate")" =~ 2022111.* ]] diff --git a/tests/functional/fetchTree-file.sh b/tests/functional/fetchTree-file.sh old mode 100644 new mode 100755 index be698ea35..66be928c7 --- a/tests/functional/fetchTree-file.sh +++ b/tests/functional/fetchTree-file.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore cd "$TEST_ROOT" @@ -88,7 +92,7 @@ EOF EOF # Test tarball URLs on the command line. - [[ $(nix flake metadata --json file://$PWD/test_input_no_ext | jq -r .resolved.type) = tarball ]] + [[ $(nix flake metadata --json "file://$PWD/test_input_no_ext" | jq -r .resolved.type) = tarball ]] popd diff --git a/tests/functional/fetchurl.sh b/tests/functional/fetchurl.sh old mode 100644 new mode 100755 index a3620f52b..5af44fcf2 --- a/tests/functional/fetchurl.sh +++ b/tests/functional/fetchurl.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore # Test fetching a flat file. diff --git a/tests/functional/filter-source.sh b/tests/functional/filter-source.sh old mode 100644 new mode 100755 index ba34d2eac..b32f5b59d --- a/tests/functional/filter-source.sh +++ b/tests/functional/filter-source.sh @@ -1,25 +1,27 @@ +#!/usr/bin/env bash + source common.sh -rm -rf $TEST_ROOT/filterin -mkdir $TEST_ROOT/filterin -mkdir $TEST_ROOT/filterin/foo -touch $TEST_ROOT/filterin/foo/bar -touch $TEST_ROOT/filterin/xyzzy -touch $TEST_ROOT/filterin/b -touch $TEST_ROOT/filterin/bak -touch $TEST_ROOT/filterin/bla.c.bak -ln -s xyzzy $TEST_ROOT/filterin/link +rm -rf "$TEST_ROOT/filterin" +mkdir "$TEST_ROOT/filterin" +mkdir "$TEST_ROOT/filterin/foo" +touch "$TEST_ROOT/filterin/foo/bar" +touch "$TEST_ROOT/filterin/xyzzy" +touch "$TEST_ROOT/filterin/b" +touch "$TEST_ROOT/filterin/bak" +touch "$TEST_ROOT"/filterin/bla.c.bak +ln -s xyzzy "$TEST_ROOT/filterin/link" checkFilter() { - test ! -e $1/foo/bar - test -e $1/xyzzy - test -e $1/bak - test ! -e $1/bla.c.bak - test ! -L $1/link + test ! -e "$1/foo/bar" + test -e "$1/xyzzy" + test -e "$1/bak" + test ! -e "$1"/bla.c.bak + test ! -L "$1/link" } -nix-build ./filter-source.nix -o $TEST_ROOT/filterout1 -checkFilter $TEST_ROOT/filterout1 +nix-build ./filter-source.nix -o "$TEST_ROOT/filterout1" +checkFilter "$TEST_ROOT/filterout1" -nix-build ./path.nix -o $TEST_ROOT/filterout2 -checkFilter $TEST_ROOT/filterout2 +nix-build ./path.nix -o "$TEST_ROOT/filterout2" +checkFilter "$TEST_ROOT/filterout2" diff --git a/tests/functional/fixed.sh b/tests/functional/fixed.sh old mode 100644 new mode 100755 index 7bbecda91..d98769e64 --- a/tests/functional/fixed.sh +++ b/tests/functional/fixed.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore path=$(nix-store -q $(nix-instantiate fixed.nix -A good.0)) diff --git a/tests/functional/flakes/absolute-attr-paths.sh b/tests/functional/flakes/absolute-attr-paths.sh old mode 100644 new mode 100755 index 491adceb7..b0e6225d8 --- a/tests/functional/flakes/absolute-attr-paths.sh +++ b/tests/functional/flakes/absolute-attr-paths.sh @@ -1,9 +1,11 @@ +#!/usr/bin/env bash + source ./common.sh flake1Dir=$TEST_ROOT/flake1 -mkdir -p $flake1Dir -cat > $flake1Dir/flake.nix < "$flake1Dir"/flake.nix < $flake1Dir/flake.nix < $flake1Dir/flake.nix < "$flake1Dir"/flake.nix < $flake1Dir/flake.nix < $flake1Dir/foo +echo bar > "$flake1Dir/foo" -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a1 +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a1" [[ -e $TEST_ROOT/result/simple.nix ]] -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a2 -[[ $(cat $TEST_ROOT/result) = bar ]] +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a2" +[[ $(cat "$TEST_ROOT/result") = bar ]] -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a3 +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a3" -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a4 +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a4" -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a6 +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a6" [[ -e $TEST_ROOT/result/simple.nix ]] -nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a8 -diff common.sh $TEST_ROOT/result +nix build --impure --json --out-link "$TEST_ROOT/result" "$flake1Dir#a8" +diff common.sh "$TEST_ROOT/result" -expectStderr 1 nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a9 \ +expectStderr 1 nix build --impure --json --out-link "$TEST_ROOT/result" "$flake1Dir#a9" \ | grepQuiet "has 0 entries in its context. It should only have exactly one entry" -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a10 -[[ $(readlink -e $TEST_ROOT/result) = *simple.drv ]] +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir"#a10 +[[ $(readlink -e "$TEST_ROOT/result") = *simple.drv ]] -expectStderr 1 nix build --json --out-link $TEST_ROOT/result $flake1Dir#a11 \ +expectStderr 1 nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a11" \ | grepQuiet "has a context which refers to a complete source and binary closure" -nix build --json --out-link $TEST_ROOT/result $flake1Dir#a12 +nix build --json --out-link "$TEST_ROOT/result" "$flake1Dir#a12" [[ -e $TEST_ROOT/result/hello ]] -expectStderr 1 nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a13 \ +expectStderr 1 nix build --impure --json --out-link "$TEST_ROOT/result" "$flake1Dir#a13" \ | grepQuiet "has 2 entries in its context. It should only have exactly one entry" # Test accessing output in installables with `.` (foobarbaz.) -nix build --json --no-link $flake1Dir#a14.foo | jq --exit-status ' +nix build --json --no-link "$flake1Dir"#a14.foo | jq --exit-status ' (.[0] | (.drvPath | match(".*dot-installable.drv")) and (.outputs | keys == ["foo"])) diff --git a/tests/functional/flakes/bundle.sh b/tests/functional/flakes/bundle.sh old mode 100644 new mode 100755 index 67bbb05ac..5e185cbf6 --- a/tests/functional/flakes/bundle.sh +++ b/tests/functional/flakes/bundle.sh @@ -1,8 +1,10 @@ +#!/usr/bin/env bash + source common.sh -cp ../simple.nix ../simple.builder.sh ../config.nix $TEST_HOME +cp ../simple.nix ../simple.builder.sh ../config.nix "$TEST_HOME" -cd $TEST_HOME +cd "$TEST_HOME" cat < flake.nix { @@ -25,8 +27,8 @@ EOF nix build .# nix bundle --bundler .# .# -nix bundle --bundler .#bundlers.$system.default .#packages.$system.default -nix bundle --bundler .#bundlers.$system.simple .#packages.$system.default +nix bundle --bundler .#bundlers."$system".default .#packages."$system".default +nix bundle --bundler .#bundlers."$system".simple .#packages."$system".default -nix bundle --bundler .#bundlers.$system.default .#apps.$system.default -nix bundle --bundler .#bundlers.$system.simple .#apps.$system.default +nix bundle --bundler .#bundlers."$system".default .#apps."$system".default +nix bundle --bundler .#bundlers."$system".simple .#apps."$system".default diff --git a/tests/functional/flakes/check.sh b/tests/functional/flakes/check.sh old mode 100644 new mode 100755 index 0433e5335..27e73444a --- a/tests/functional/flakes/check.sh +++ b/tests/functional/flakes/check.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh flakeDir=$TEST_ROOT/flake3 @@ -89,3 +91,47 @@ nix flake check $flakeDir checkRes=$(nix flake check --all-systems --keep-going $flakeDir 2>&1 && fail "nix flake check --all-systems should have failed" || true) echo "$checkRes" | grepQuiet "packages.system-1.default" echo "$checkRes" | grepQuiet "packages.system-2.default" + +cat > $flakeDir/flake.nix < $flakeDir/flake.nix <&1 && fail "nix flake check --all-systems should have failed" || true) +echo "$checkRes" | grepQuiet "unknown-attr" + +cat > $flakeDir/flake.nix <&1 && fail "nix flake check --all-systems should have failed" || true) +echo "$checkRes" | grepQuiet "formatter.system-1" diff --git a/tests/functional/flakes/circular.sh b/tests/functional/flakes/circular.sh old mode 100644 new mode 100755 index d3bb8e8a3..5304496ba --- a/tests/functional/flakes/circular.sh +++ b/tests/functional/flakes/circular.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Test circular flake dependencies. source ./common.sh @@ -6,10 +8,10 @@ requireGit flakeA=$TEST_ROOT/flakeA flakeB=$TEST_ROOT/flakeB -createGitRepo $flakeA -createGitRepo $flakeB +createGitRepo "$flakeA" +createGitRepo "$flakeB" -cat > $flakeA/flake.nix < "$flakeA"/flake.nix < $flakeA/flake.nix < $flakeB/flake.nix < "$flakeB"/flake.nix < $flakeB/flake.nix <"$flake1Dir/flake.nix" < \$out + ''; + }; + ifd = assert (import self.drv); self.drv; + }; +} +EOF + +git -C "$flake1Dir" add flake.nix +git -C "$flake1Dir" commit -m "Init" + +expect 1 nix build "$flake1Dir#foo.bar" 2>&1 | grepQuiet 'error: breaks' +expect 1 nix build "$flake1Dir#foo.bar" 2>&1 | grepQuiet 'error: breaks' + +# Conditional error should not be cached +expect 1 nix build "$flake1Dir#ifd" --option allow-import-from-derivation false 2>&1 \ + | grepQuiet 'error: cannot build .* during evaluation because the option '\''allow-import-from-derivation'\'' is disabled' +nix build --no-link "$flake1Dir#ifd" diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh old mode 100644 new mode 100755 index 85a4d3389..817f77783 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh # Tests that: @@ -13,6 +15,8 @@ source common.sh requireGit +TODO_NixOS + clearStore # Submodules can't be fetched locally by default. @@ -25,8 +29,8 @@ rootRepo=$TEST_ROOT/rootRepo subRepo=$TEST_ROOT/submodule -createGitRepo $subRepo -cat > $subRepo/flake.nix < "$subRepo"/flake.nix < $subRepo/flake.nix < $subRepo/sub.nix -git -C $subRepo add flake.nix sub.nix -git -C $subRepo commit -m Initial +echo '"expression in submodule"' > "$subRepo"/sub.nix +git -C "$subRepo" add flake.nix sub.nix +git -C "$subRepo" commit -m Initial -createGitRepo $rootRepo +createGitRepo "$rootRepo" -git -C $rootRepo submodule init -git -C $rootRepo submodule add $subRepo submodule -echo '"expression in root repo"' > $rootRepo/root.nix -git -C $rootRepo add root.nix -git -C $rootRepo commit -m "Add root.nix" +git -C "$rootRepo" submodule init +git -C "$rootRepo" submodule add "$subRepo" submodule +echo '"expression in root repo"' > "$rootRepo"/root.nix +git -C "$rootRepo" add root.nix +git -C "$rootRepo" commit -m "Add root.nix" flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule # Flake can live inside a submodule and can be accessed via ?dir=submodule -[[ $(nix eval --json $flakeref#sub ) = '"expression in submodule"' ]] +[[ $(nix eval --json "$flakeref#sub" ) = '"expression in submodule"' ]] # The flake can access content outside of the submodule -[[ $(nix eval --json $flakeref#root ) = '"expression in root repo"' ]] +[[ $(nix eval --json "$flakeref#root" ) = '"expression in root repo"' ]] # Check that dirtying a submodule makes the entire thing dirty. -[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) != null ]] -echo '"foo"' > $rootRepo/submodule/sub.nix -[[ $(nix eval --json $flakeref#sub ) = '"foo"' ]] -[[ $(nix flake metadata --json $flakeref | jq -r .locked.rev) = null ]] +[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) != null ]] +echo '"foo"' > "$rootRepo"/submodule/sub.nix +[[ $(nix eval --json "$flakeref#sub" ) = '"foo"' ]] +[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]] diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh old mode 100644 new mode 100755 index 35b0c5d84..aa4cb1e18 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source ./common.sh +TODO_NixOS + requireGit clearStore @@ -15,13 +19,17 @@ flake7Dir=$TEST_ROOT/flake7 nonFlakeDir=$TEST_ROOT/nonFlake badFlakeDir=$TEST_ROOT/badFlake flakeGitBare=$TEST_ROOT/flakeGitBare +lockfileSummaryFlake=$TEST_ROOT/lockfileSummaryFlake -for repo in "$flake1Dir" "$flake2Dir" "$flake3Dir" "$flake7Dir" "$nonFlakeDir"; do +for repo in "$flake1Dir" "$flake2Dir" "$flake3Dir" "$flake7Dir" "$nonFlakeDir" "$lockfileSummaryFlake"; do # Give one repo a non-main initial branch. extraArgs= if [[ "$repo" == "$flake2Dir" ]]; then extraArgs="--initial-branch=main" fi + if [[ "$repo" == "$lockfileSummaryFlake" ]]; then + extraArgs="--initial-branch=main" + fi createGitRepo "$repo" "$extraArgs" done @@ -176,6 +184,9 @@ nix registry list | grepInverse '^user' # nothing in user registry nix flake metadata flake1 nix flake metadata flake1 | grepQuiet 'Locked URL:.*flake1.*' +# Test 'nix flake metadata' on a chroot store. +nix flake metadata --store $TEST_ROOT/chroot-store flake1 + # Test 'nix flake metadata' on a local flake. (cd "$flake1Dir" && nix flake metadata) | grepQuiet 'URL:.*flake1.*' (cd "$flake1Dir" && nix flake metadata .) | grepQuiet 'URL:.*flake1.*' @@ -187,6 +198,7 @@ json=$(nix flake metadata flake1 --json | jq .) [[ -d $(echo "$json" | jq -r .path) ]] [[ $(echo "$json" | jq -r .lastModified) = $(git -C "$flake1Dir" log -n1 --format=%ct) ]] hash1=$(echo "$json" | jq -r .revision) +[[ -n $(echo "$json" | jq -r .fingerprint) ]] echo foo > "$flake1Dir/foo" git -C "$flake1Dir" add $flake1Dir/foo @@ -640,3 +652,37 @@ expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock [[ $($nonFlakeDir/shebang-inline-expr.sh baz) = "foo"$'\n'"baz" ]] [[ $($nonFlakeDir/shebang-file.sh baz) = "foo"$'\n'"baz" ]] expect 1 $nonFlakeDir/shebang-reject.sh 2>&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' + +# Test that the --commit-lock-file-summary flag and its alias work +cat > "$lockfileSummaryFlake/flake.nix" < $templatesDir/flake.nix < "$templatesDir"/flake.nix < $templatesDir/flake.nix < $templatesDir/trivial/flake.nix < "$templatesDir"/trivial/flake.nix < $templatesDir/trivial/flake.nix < $templatesDir/trivial/a -echo b > $templatesDir/trivial/b +echo a > "$templatesDir/trivial/a" +echo b > "$templatesDir/trivial/b" -git -C $templatesDir add flake.nix trivial/ -git -C $templatesDir commit -m 'Initial' +git -C "$templatesDir" add flake.nix trivial/ +git -C "$templatesDir" commit -m 'Initial' nix flake check templates nix flake show templates nix flake show templates --json | jq -createGitRepo $flakeDir -(cd $flakeDir && nix flake init) -(cd $flakeDir && nix flake init) # check idempotence -git -C $flakeDir add flake.nix -nix flake check $flakeDir -nix flake show $flakeDir -nix flake show $flakeDir --json | jq -git -C $flakeDir commit -a -m 'Initial' +createGitRepo "$flakeDir" +(cd "$flakeDir" && nix flake init) +(cd "$flakeDir" && nix flake init) # check idempotence +git -C "$flakeDir" add flake.nix +nix flake check "$flakeDir" +nix flake show "$flakeDir" +nix flake show "$flakeDir" --json | jq +git -C "$flakeDir" commit -a -m 'Initial' # Test 'nix flake init' with benign conflicts createGitRepo "$flakeDir" -echo a > $flakeDir/a -(cd $flakeDir && nix flake init) # check idempotence +echo a > "$flakeDir/a" +(cd "$flakeDir" && nix flake init) # check idempotence # Test 'nix flake init' with conflicts createGitRepo "$flakeDir" -echo b > $flakeDir/a -pushd $flakeDir +echo b > "$flakeDir/a" +pushd "$flakeDir" (! nix flake init) |& grep "refusing to overwrite existing file '$flakeDir/a'" popd -git -C $flakeDir commit -a -m 'Changed' +git -C "$flakeDir" commit -a -m 'Changed' # Test 'nix flake new'. -rm -rf $flakeDir -nix flake new -t templates#trivial $flakeDir -nix flake new -t templates#trivial $flakeDir # check idempotence -nix flake check $flakeDir +rm -rf "$flakeDir" +nix flake new -t templates#trivial "$flakeDir" +nix flake new -t templates#trivial "$flakeDir" # check idempotence +nix flake check "$flakeDir" diff --git a/tests/functional/flakes/inputs.sh b/tests/functional/flakes/inputs.sh old mode 100644 new mode 100755 index 80620488a..bc0603f1b --- a/tests/functional/flakes/inputs.sh +++ b/tests/functional/flakes/inputs.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source ./common.sh requireGit @@ -6,12 +8,12 @@ requireGit test_subdir_self_path() { baseDir=$TEST_ROOT/$RANDOM flakeDir=$baseDir/b-low - mkdir -p $flakeDir - writeSimpleFlake $baseDir - writeSimpleFlake $flakeDir + mkdir -p "$flakeDir" + writeSimpleFlake "$baseDir" + writeSimpleFlake "$flakeDir" - echo all good > $flakeDir/message - cat > $flakeDir/flake.nix < "$flakeDir/message" + cat > "$flakeDir"/flake.nix < $flakeDir/message - cat > $flakeDir/flake.nix < "$flakeDir/message" + cat > "$flakeDir"/flake.nix < $clientDir/flake.nix < "$clientDir"/flake.nix < $TEST_ROOT/expected-env +nix run -f shell-hello.nix env > $TEST_ROOT/actual-env +# Remove/reset variables we expect to be different. +# - PATH is modified by nix shell +# - we unset TMPDIR on macOS if it contains /var/folders. bad. https://github.com/NixOS/nix/issues/7731 +# - _ is set by bash and is expected to differ because it contains the original command +# - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control +sed -i \ + -e 's/PATH=.*/PATH=.../' \ + -e 's/_=.*/_=.../' \ + -e '/^TMPDIR=\/var\/folders\/.*/d' \ + -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \ + $TEST_ROOT/expected-env $TEST_ROOT/actual-env +sort $TEST_ROOT/expected-env | uniq > $TEST_ROOT/expected-env.sorted +# nix run appears to clear _. I don't understand why. Is this ok? +echo "_=..." >> $TEST_ROOT/actual-env +sort $TEST_ROOT/actual-env | uniq > $TEST_ROOT/actual-env.sorted +diff $TEST_ROOT/expected-env.sorted $TEST_ROOT/actual-env.sorted + clearStore diff --git a/tests/functional/flakes/search-root.sh b/tests/functional/flakes/search-root.sh old mode 100644 new mode 100755 index 6b137aa86..1ee29fac4 --- a/tests/functional/flakes/search-root.sh +++ b/tests/functional/flakes/search-root.sh @@ -1,9 +1,11 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible -writeSimpleFlake $TEST_HOME -cd $TEST_HOME +writeSimpleFlake "$TEST_HOME" +cd "$TEST_HOME" mkdir -p foo/subdir echo '{ outputs = _: {}; }' > foo/flake.nix @@ -25,11 +27,11 @@ success=("" . .# .#test ../subdir ../subdir#test "$PWD") failure=("path:$PWD" "../simple.nix") for i in "${success[@]}"; do - nix build $i || fail "flake should be found by searching up directories" + nix build "$i" || fail "flake should be found by searching up directories" done for i in "${failure[@]}"; do - ! nix build $i || fail "flake should not search up directories when using 'path:'" + ! nix build "$i" || fail "flake should not search up directories when using 'path:'" done popd @@ -43,7 +45,7 @@ if [[ -n $(type -p git) ]]; then pushd subdir git init for i in "${success[@]}" "${failure[@]}"; do - ! nix build $i || fail "flake should not search past a git repository" + ! nix build "$i" || fail "flake should not search past a git repository" done rm -rf .git popd diff --git a/tests/functional/flakes/show.sh b/tests/functional/flakes/show.sh old mode 100644 new mode 100755 index a3d300552..0edc450c3 --- a/tests/functional/flakes/show.sh +++ b/tests/functional/flakes/show.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source ./common.sh flakeDir=$TEST_ROOT/flake @@ -85,3 +87,28 @@ assert show_output.legacyPackages.${builtins.currentSystem}.AAAAAASomeThingsFail assert show_output.legacyPackages.${builtins.currentSystem}.simple.name == "simple"; true ' + +cat >flake.nix< ./show-output.txt +test "$(awk -F '[:] ' '/aNoDescription/{print $NF}' ./show-output.txt)" = "package 'simple'" +test "$(awk -F '[:] ' '/bOneLineDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - 'one line'" +test "$(awk -F '[:] ' '/cMultiLineDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - 'line one'" +test "$(awk -F '[:] ' '/dLongDescription/{print $NF}' ./show-output.txt)" = "package 'simple' - '012345678901234567890123456..." +test "$(awk -F '[:] ' '/eEmptyDescription/{print $NF}' ./show-output.txt)" = "package 'simple'" \ No newline at end of file diff --git a/tests/functional/flakes/unlocked-override.sh b/tests/functional/flakes/unlocked-override.sh old mode 100644 new mode 100755 index 8abc8b7d3..a17a0c2af --- a/tests/functional/flakes/unlocked-override.sh +++ b/tests/functional/flakes/unlocked-override.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source ./common.sh requireGit @@ -5,26 +7,26 @@ requireGit flake1Dir=$TEST_ROOT/flake1 flake2Dir=$TEST_ROOT/flake2 -createGitRepo $flake1Dir -cat > $flake1Dir/flake.nix < "$flake1Dir"/flake.nix < $flake1Dir/x.nix -git -C $flake1Dir add flake.nix x.nix -git -C $flake1Dir commit -m Initial +echo 123 > "$flake1Dir"/x.nix +git -C "$flake1Dir" add flake.nix x.nix +git -C "$flake1Dir" commit -m Initial -createGitRepo $flake2Dir -cat > $flake2Dir/flake.nix < "$flake2Dir"/flake.nix < $flake1Dir/x.nix +echo 456 > "$flake1Dir"/x.nix -[[ $(nix eval --json $flake2Dir#x --override-input flake1 $TEST_ROOT/flake1) = 456 ]] +[[ $(nix eval --json "$flake2Dir#x" --override-input flake1 "$TEST_ROOT/flake1") = 456 ]] diff --git a/tests/functional/fmt.sh b/tests/functional/fmt.sh old mode 100644 new mode 100755 index 3c1bd9989..4e0fd57a5 --- a/tests/functional/fmt.sh +++ b/tests/functional/fmt.sh @@ -1,13 +1,17 @@ +#!/usr/bin/env bash + source common.sh -clearStore -rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local +TODO_NixOS # Provide a `shell` variable. Try not to `export` it, perhaps. -cp ./simple.nix ./simple.builder.sh ./fmt.simple.sh ./config.nix $TEST_HOME +clearStoreIfPossible +rm -rf "$TEST_HOME"/.cache "$TEST_HOME"/.config "$TEST_HOME"/.local -cd $TEST_HOME +cp ./simple.nix ./simple.builder.sh ./fmt.simple.sh ./config.nix "$TEST_HOME" -nix fmt --help | grep "Format" +cd "$TEST_HOME" + +nix fmt --help | grep "forward" cat << EOF > flake.nix { @@ -26,8 +30,9 @@ cat << EOF > flake.nix }; } EOF -nix fmt ./file ./folder | grep 'Formatting: ./file ./folder' +# No arguments check +[[ "$(nix fmt)" = "Formatting(0):" ]] +# Argument forwarding check +nix fmt ./file ./folder | grep 'Formatting(2): ./file ./folder' nix flake check nix flake show | grep -P "package 'formatter'" - -clearStore diff --git a/tests/functional/fmt.simple.sh b/tests/functional/fmt.simple.sh index 4c8c67ebb..e53f6c9be 100755 --- a/tests/functional/fmt.simple.sh +++ b/tests/functional/fmt.simple.sh @@ -1 +1,2 @@ -echo Formatting: "${@}" +#!/usr/bin/env bash +echo "Formatting(${#}):" "${@}" diff --git a/tests/functional/fod-failing.nix b/tests/functional/fod-failing.nix new file mode 100644 index 000000000..37c04fe12 --- /dev/null +++ b/tests/functional/fod-failing.nix @@ -0,0 +1,39 @@ +with import ./config.nix; +rec { + x1 = mkDerivation { + name = "x1"; + builder = builtins.toFile "builder.sh" + '' + echo $name > $out + ''; + outputHashMode = "recursive"; + outputHash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + }; + x2 = mkDerivation { + name = "x2"; + builder = builtins.toFile "builder.sh" + '' + echo $name > $out + ''; + outputHashMode = "recursive"; + outputHash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + }; + x3 = mkDerivation { + name = "x3"; + builder = builtins.toFile "builder.sh" + '' + echo $name > $out + ''; + outputHashMode = "recursive"; + outputHash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="; + }; + x4 = mkDerivation { + name = "x4"; + inherit x2 x3; + builder = builtins.toFile "builder.sh" + '' + echo $x2 $x3 + exit 1 + ''; + }; +} diff --git a/tests/functional/function-trace.sh b/tests/functional/function-trace.sh index bd804bf18..7524afdf2 100755 --- a/tests/functional/function-trace.sh +++ b/tests/functional/function-trace.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh set +x @@ -19,12 +21,12 @@ expect_trace() { <(echo "$expect") \ <(echo "$actual") ) && result=0 || result=$? - if [ $result -eq 0 ]; then + if [ "$result" -eq 0 ]; then echo " ok." else echo " failed. difference:" echo "$msg" - return $result + return "$result" fi } diff --git a/tests/functional/gc-auto.sh b/tests/functional/gc-auto.sh old mode 100644 new mode 100755 index 281eef20d..8f25be3e9 --- a/tests/functional/gc-auto.sh +++ b/tests/functional/gc-auto.sh @@ -1,7 +1,11 @@ +#!/usr/bin/env bash + source common.sh needLocalStore "“min-free” and “max-free” are daemon options" +TODO_NixOS + clearStore garbage1=$(nix store add-path --name garbage1 ./nar-access.sh) diff --git a/tests/functional/gc-concurrent.sh b/tests/functional/gc-concurrent.sh old mode 100644 new mode 100755 index 2c6622c62..df180b14f --- a/tests/functional/gc-concurrent.sh +++ b/tests/functional/gc-concurrent.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore lockFifo1=$TEST_ROOT/test1.fifo @@ -18,8 +22,8 @@ outPath3=$(nix-store -r $drvPath3) touch $outPath3.lock rm -f "$NIX_STATE_DIR"/gcroots/foo* -ln -s $drvPath2 "$NIX_STATE_DIR"/gcroots/foo -ln -s $outPath3 "$NIX_STATE_DIR"/gcroots/foo2 +ln -s $drvPath2 "$NIX_STATE_DIR/gcroots/foo" +ln -s $outPath3 "$NIX_STATE_DIR/gcroots/foo2" # Start build #1 in the background. It starts immediately. nix-store -rvv "$drvPath1" & diff --git a/tests/functional/gc-non-blocking.sh b/tests/functional/gc-non-blocking.sh old mode 100644 new mode 100755 index ec280badb..de10837eb --- a/tests/functional/gc-non-blocking.sh +++ b/tests/functional/gc-non-blocking.sh @@ -1,7 +1,11 @@ +#!/usr/bin/env bash + # Test whether the collector is non-blocking, i.e. a build can run in # parallel with it. source common.sh +TODO_NixOS + needLocalStore "the GC test needs a synchronisation point" clearStore diff --git a/tests/functional/gc-runtime.sh b/tests/functional/gc-runtime.sh old mode 100644 new mode 100755 index dc1826a55..0cccaaf16 --- a/tests/functional/gc-runtime.sh +++ b/tests/functional/gc-runtime.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh case $system in @@ -9,28 +11,30 @@ esac set -m # enable job control, needed for kill +TODO_NixOS + profiles="$NIX_STATE_DIR"/profiles -rm -rf $profiles +rm -rf "$profiles" -nix-env -p $profiles/test -f ./gc-runtime.nix -i gc-runtime +nix-env -p "$profiles/test" -f ./gc-runtime.nix -i gc-runtime -outPath=$(nix-env -p $profiles/test -q --no-name --out-path gc-runtime) -echo $outPath +outPath=$(nix-env -p "$profiles/test" -q --no-name --out-path gc-runtime) +echo "$outPath" echo "backgrounding program..." -$profiles/test/program & +"$profiles"/test/program & sleep 2 # hack - wait for the program to get started child=$! echo PID=$child -nix-env -p $profiles/test -e gc-runtime -nix-env -p $profiles/test --delete-generations old +nix-env -p "$profiles/test" -e gc-runtime +nix-env -p "$profiles/test" --delete-generations old nix-store --gc kill -- -$child -if ! test -e $outPath; then +if ! test -e "$outPath"; then echo "running program was garbage collected!" exit 1 fi diff --git a/tests/functional/gc.sh b/tests/functional/gc.sh old mode 100644 new mode 100755 index ad09a8b39..7707a7e27 --- a/tests/functional/gc.sh +++ b/tests/functional/gc.sh @@ -1,13 +1,17 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore drvPath=$(nix-instantiate dependencies.nix) outPath=$(nix-store -rvv "$drvPath") # Set a GC root. -rm -f "$NIX_STATE_DIR"/gcroots/foo -ln -sf $outPath "$NIX_STATE_DIR"/gcroots/foo +rm -f "$NIX_STATE_DIR/gcroots/foo" +ln -sf $outPath "$NIX_STATE_DIR/gcroots/foo" [ "$(nix-store -q --roots $outPath)" = "$NIX_STATE_DIR/gcroots/foo -> $outPath" ] @@ -40,7 +44,7 @@ cat $outPath/reference-to-input-2/bar # Check that the derivation has been GC'd. if test -e $drvPath; then false; fi -rm "$NIX_STATE_DIR"/gcroots/foo +rm "$NIX_STATE_DIR/gcroots/foo" nix-collect-garbage diff --git a/tests/functional/git-hashing/common.sh b/tests/functional/git-hashing/common.sh index 572cea438..29c518fea 100644 --- a/tests/functional/git-hashing/common.sh +++ b/tests/functional/git-hashing/common.sh @@ -1,5 +1,7 @@ source ../common.sh +TODO_NixOS # Need to enable git hashing feature and make sure test is ok for store we don't clear + clearStore clearCache diff --git a/tests/functional/git-hashing/meson.build b/tests/functional/git-hashing/meson.build new file mode 100644 index 000000000..7486bfb8f --- /dev/null +++ b/tests/functional/git-hashing/meson.build @@ -0,0 +1,8 @@ +suites += { + 'name': 'git-hashing', + 'deps': [], + 'tests': [ + 'simple.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/hash-convert.sh b/tests/functional/hash-convert.sh old mode 100644 new mode 100755 index 9b3afc10b..3a099950f --- a/tests/functional/hash-convert.sh +++ b/tests/functional/hash-convert.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh # Conversion with `nix hash` `nix-hash` and `nix hash convert` diff --git a/tests/functional/hash-path.sh b/tests/functional/hash-path.sh old mode 100644 new mode 100755 index 4ad9f8ff2..86d782a95 --- a/tests/functional/hash-path.sh +++ b/tests/functional/hash-path.sh @@ -1,7 +1,9 @@ +#!/usr/bin/env bash + source common.sh try () { - printf "%s" "$2" > $TEST_ROOT/vector + printf "%s" "$2" > "$TEST_ROOT/vector" hash="$(nix-hash --flat ${FORMAT+--$FORMAT} --type "$1" "$TEST_ROOT/vector")" if ! (( "${NO_TEST_CLASSIC-}" )) && test "$hash" != "$3"; then echo "try nix-hash: hash $1, expected $3, got $hash" @@ -59,7 +61,7 @@ NO_TEST_NIX_COMMAND=1 try sha512 "abc" "ddaf35a193617abacc417349ae20413112e6fa4e NO_TEST_CLASSIC=1 try sha512 "abc" "sha512-3a81oZNherrMQXNJriBBMRLm+k6JqX6iCp7u5ktV05ohkpkqJ0/BqDa6PCOj/uu9RU1EI2Q86A4qmslPpUyknw==" try2 () { - hash=$(nix-hash --type "$1" $TEST_ROOT/hash-path) + hash=$(nix-hash --type "$1" "$TEST_ROOT/hash-path") if test "$hash" != "$2"; then echo "try nix-hash; hash $1, expected $2, got $hash" exit 1 @@ -71,22 +73,22 @@ try2 () { fi } -rm -rf $TEST_ROOT/hash-path -mkdir $TEST_ROOT/hash-path -echo "Hello World" > $TEST_ROOT/hash-path/hello +rm -rf "$TEST_ROOT/hash-path" +mkdir "$TEST_ROOT/hash-path" +echo "Hello World" > "$TEST_ROOT/hash-path/hello" try2 md5 "ea9b55537dd4c7e104515b2ccfaf4100" # Execute bit matters. -chmod +x $TEST_ROOT/hash-path/hello +chmod +x "$TEST_ROOT/hash-path/hello" try2 md5 "20f3ffe011d4cfa7d72bfabef7882836" # Mtime and other bits don't. -touch -r . $TEST_ROOT/hash-path/hello -chmod 744 $TEST_ROOT/hash-path/hello +touch -r . "$TEST_ROOT/hash-path/hello" +chmod 744 "$TEST_ROOT/hash-path/hello" try2 md5 "20f3ffe011d4cfa7d72bfabef7882836" # File type (e.g., symlink) does. -rm $TEST_ROOT/hash-path/hello -ln -s x $TEST_ROOT/hash-path/hello +rm "$TEST_ROOT/hash-path/hello" +ln -s x "$TEST_ROOT/hash-path/hello" try2 md5 "f78b733a68f5edbdf9413899339eaa4a" diff --git a/tests/functional/help.sh b/tests/functional/help.sh old mode 100644 new mode 100755 index 868f5d2e9..61efc1cb2 --- a/tests/functional/help.sh +++ b/tests/functional/help.sh @@ -1,6 +1,6 @@ -source common.sh +#!/usr/bin/env bash -clearStore +source common.sh # test help output diff --git a/tests/functional/import-derivation.nix b/tests/functional/import-derivation.nix deleted file mode 100644 index 44fa9a45d..000000000 --- a/tests/functional/import-derivation.nix +++ /dev/null @@ -1,26 +0,0 @@ -with import ./config.nix; - -let - - bar = mkDerivation { - name = "bar"; - builder = builtins.toFile "builder.sh" - '' - echo 'builtins.add 123 456' > $out - ''; - }; - - value = - # Test that pathExists can check the existence of /nix/store paths - assert builtins.pathExists bar; - import bar; - -in - -mkDerivation { - name = "foo"; - builder = builtins.toFile "builder.sh" - '' - echo -n FOO${toString value} > $out - ''; -} diff --git a/tests/functional/import-derivation.sh b/tests/functional/import-derivation.sh deleted file mode 100644 index 98d61ef49..000000000 --- a/tests/functional/import-derivation.sh +++ /dev/null @@ -1,12 +0,0 @@ -source common.sh - -clearStore - -if nix-instantiate --readonly-mode ./import-derivation.nix; then - echo "read-only evaluation of an imported derivation unexpectedly failed" - exit 1 -fi - -outPath=$(nix-build ./import-derivation.nix --no-out-link) - -[ "$(cat $outPath)" = FOO579 ] diff --git a/tests/functional/import-from-derivation.nix b/tests/functional/import-from-derivation.nix new file mode 100644 index 000000000..cc53451cf --- /dev/null +++ b/tests/functional/import-from-derivation.nix @@ -0,0 +1,33 @@ +with import ./config.nix; + +rec { + bar = mkDerivation { + name = "bar"; + builder = builtins.toFile "builder.sh" + '' + echo 'builtins.add 123 456' > $out + ''; + }; + + value = + # Test that pathExists can check the existence of /nix/store paths + assert builtins.pathExists bar; + import bar; + + result = mkDerivation { + name = "foo"; + builder = builtins.toFile "builder.sh" + '' + echo -n FOO${toString value} > $out + ''; + }; + + addPath = mkDerivation { + name = "add-path"; + src = builtins.filterSource (path: type: true) result; + builder = builtins.toFile "builder.sh" + '' + echo -n BLA$(cat $src) > $out + ''; + }; +} diff --git a/tests/functional/import-from-derivation.sh b/tests/functional/import-from-derivation.sh new file mode 100755 index 000000000..83ef92a6f --- /dev/null +++ b/tests/functional/import-from-derivation.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +clearStoreIfPossible + +if nix-instantiate --readonly-mode ./import-from-derivation.nix -A result; then + echo "read-only evaluation of an imported derivation unexpectedly failed" + exit 1 +fi + +outPath=$(nix-build ./import-from-derivation.nix -A result --no-out-link) + +[ "$(cat "$outPath")" = FOO579 ] + +# FIXME: the next tests are broken on CA. +if [[ -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + exit 0 +fi + +# Test filterSource on the result of a derivation. +outPath2=$(nix-build ./import-from-derivation.nix -A addPath --no-out-link) +[[ "$(cat "$outPath2")" = BLAFOO579 ]] + +# Test that IFD works with a chroot store. +if canUseSandbox; then + + store2="$TEST_ROOT/store2" + store2_url="$store2?store=$NIX_STORE_DIR" + + # Copy the derivation outputs to the chroot store to avoid having + # to actually build anything, as that would fail due to the lack + # of a shell in the sandbox. We only care about testing the IFD + # semantics. + for i in bar result addPath; do + nix copy --to "$store2_url" --no-check-sigs "$(nix-build ./import-from-derivation.nix -A "$i" --no-out-link)" + done + + clearStore + + outPath_check=$(nix-build ./import-from-derivation.nix -A result --no-out-link --store "$store2_url") + [[ "$outPath" = "$outPath_check" ]] + [[ ! -e "$outPath" ]] + [[ -e "$store2/nix/store/$(basename "$outPath")" ]] + + outPath2_check=$(nix-build ./import-from-derivation.nix -A addPath --no-out-link --store "$store2_url") + [[ "$outPath2" = "$outPath2_check" ]] +fi diff --git a/tests/functional/impure-derivations.sh b/tests/functional/impure-derivations.sh old mode 100644 new mode 100755 index 54ed6f5dd..5dea220fe --- a/tests/functional/impure-derivations.sh +++ b/tests/functional/impure-derivations.sh @@ -1,11 +1,15 @@ +#!/usr/bin/env bash + source common.sh requireDaemonNewerThan "2.8pre20220311" +TODO_NixOS + enableFeatures "ca-derivations impure-derivations" restartDaemon -clearStore +clearStoreIfPossible # Basic test of impure derivations: building one a second time should not use the previous result. printf 0 > $TEST_ROOT/counter diff --git a/tests/functional/impure-env.sh b/tests/functional/impure-env.sh old mode 100644 new mode 100755 index cfea4cae9..ca32c1030 --- a/tests/functional/impure-env.sh +++ b/tests/functional/impure-env.sh @@ -1,8 +1,12 @@ +#!/usr/bin/env bash + source common.sh # Needs the config option 'impure-env' to work requireDaemonNewerThan "2.19.0" +TODO_NixOS + enableFeatures "configurable-impure-env" restartDaemon @@ -18,13 +22,13 @@ startDaemon varTest env_name value --impure-env env_name=value -echo 'impure-env = set_in_config=config_value' >> "$NIX_CONF_DIR/nix.conf" +echo 'impure-env = set_in_config=config_value' >> "$test_nix_conf" set_in_config=daemon_value restartDaemon varTest set_in_config config_value varTest set_in_config client_value --impure-env set_in_config=client_value -sed -i -e '/^trusted-users =/d' "$NIX_CONF_DIR/nix.conf" +sed -i -e '/^trusted-users =/d' "$test_nix_conf" env_name=daemon_value restartDaemon diff --git a/tests/functional/impure-eval.sh b/tests/functional/impure-eval.sh old mode 100644 new mode 100755 index 6c72f01d7..33a5ea409 --- a/tests/functional/impure-eval.sh +++ b/tests/functional/impure-eval.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh export REMOTE_STORE="dummy://" diff --git a/tests/functional/lang-gc.sh b/tests/functional/lang-gc.sh new file mode 100644 index 000000000..1746fa4c1 --- /dev/null +++ b/tests/functional/lang-gc.sh @@ -0,0 +1,36 @@ +# shellcheck shell=bash + +# Regression tests for the evaluator +# These are not in lang.sh because they generally only need to run in CI, +# whereas lang.sh is often run locally during development + + +source common.sh + +set -o pipefail + +skipTest "Too memory instensive for CI. Attempt to reduce memory usage was unsuccessful, because it made detection of the bug unreliable." + +# Regression test for #11141. The stack pointer corrector assigned the base +# instead of the top (which resides at the low end of the stack). Sounds confusing? +# Stacks grow downwards, so that's why this mistake happened. +# My manual testing did not uncover this, because it didn't rely on the stack enough. +# https://github.com/NixOS/nix/issues/11141 +test_issue_11141() { + mkdir -p "$TEST_ROOT/issue-11141/src" + cp lang-gc/issue-11141-gc-coroutine-test.nix "$TEST_ROOT/issue-11141/" + ( + set +x; + n=10 + echo "populating $TEST_ROOT/issue-11141/src with $((n*100)) files..." + for i in $(seq 0 $n); do + touch "$TEST_ROOT/issue-11141/src/file-$i"{0,1,2,3,4,5,6,7,8,9}{0,1,2,3,4,5,6,7,8,9} + done + ) + + GC_INITIAL_HEAP_SIZE=$((1024 * 1024)) \ + NIX_SHOW_STATS=1 \ + nix eval -vvv\ + -f "$TEST_ROOT/issue-11141/issue-11141-gc-coroutine-test.nix" +} +test_issue_11141 diff --git a/tests/functional/lang-gc/issue-11141-gc-coroutine-test.nix b/tests/functional/lang-gc/issue-11141-gc-coroutine-test.nix new file mode 100644 index 000000000..4f311af75 --- /dev/null +++ b/tests/functional/lang-gc/issue-11141-gc-coroutine-test.nix @@ -0,0 +1,65 @@ + +# Run: +# GC_INITIAL_HEAP_SIZE=$[1024 * 1024] NIX_SHOW_STATS=1 nix eval -f gc-coroutine-test.nix -vvvv + +let + inherit (builtins) + foldl' + isList + ; + + # Generate a tree of numbers, n deep, such that the numbers add up to (1 + salt) * 10^n. + # The salting makes the numbers all different, increasing the likelihood of catching + # any memory corruptions that might be caused by the GC or otherwise. + garbage = salt: n: + if n == 0 + then [(1 + salt)] + else [ + (garbage (10 * salt + 1) (n - 1)) + (garbage (10 * salt - 1) (n - 1)) + (garbage (10 * salt + 2) (n - 1)) + (garbage (10 * salt - 2) (n - 1)) + (garbage (10 * salt + 3) (n - 1)) + (garbage (10 * salt - 3) (n - 1)) + (garbage (10 * salt + 4) (n - 1)) + (garbage (10 * salt - 4) (n - 1)) + (garbage (10 * salt + 5) (n - 1)) + (garbage (10 * salt - 5) (n - 1)) + ]; + + pow = base: n: + if n == 0 + then 1 + else base * (pow base (n - 1)); + + sumNestedLists = l: + if isList l + then foldl' (a: b: a + sumNestedLists b) 0 l + else l; + +in + assert sumNestedLists (garbage 0 3) == pow 10 3; + assert sumNestedLists (garbage 0 6) == pow 10 6; + builtins.foldl' + (a: b: + assert + "${ + builtins.path { + path = ./src; + filter = path: type: + # We're not doing common subexpression elimination, so this reallocates + # the fairly big tree over and over, producing a lot of garbage during + # source filtering, whose filter runs in a coroutine. + assert sumNestedLists (garbage 0 3) == pow 10 3; + true; + } + }" + == "${./src}"; + + # These asserts don't seem necessary, as the lambda value get corrupted first + assert a.okay; + assert b.okay; + { okay = true; } + ) + { okay = true; } + [ { okay = true; } { okay = true; } { okay = true; } ] diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index c45326473..46cf3f1fe 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -1,8 +1,10 @@ +#!/usr/bin/env bash + source common.sh set -o pipefail -source lang/framework.sh +source characterisation/framework.sh # specialize function a bit function diffAndAccept() { @@ -24,6 +26,9 @@ nix-instantiate --eval -E 'builtins.traceVerbose "Hello" 123' 2>&1 | grepQuietIn nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" 123' 2>&1 | grepQuietInverse Hello expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello" (throw "Foo")' | grepQuiet Hello expectStderr 1 nix-instantiate --show-trace --eval -E 'builtins.addErrorContext "Hello %" (throw "Foo")' | grepQuiet 'Hello %' +# Relies on parsing the expression derivation as a derivation, can't use --eval +expectStderr 1 nix-instantiate --show-trace lang/non-eval-fail-bad-drvPath.nix | grepQuiet "store path '8qlfcic10lw5304gqm8q45nr7g7jl62b-cachix-1.7.3-bin' is not a valid derivation path" + nix-instantiate --eval -E 'let x = builtins.trace { x = x; } true; in x' \ 2>&1 | grepQuiet -E 'trace: { x = «potential infinite recursion»; }' @@ -31,16 +36,39 @@ nix-instantiate --eval -E 'let x = builtins.trace { x = x; } true; in x' \ nix-instantiate --eval -E 'let x = { repeating = x; tracing = builtins.trace x true; }; in x.tracing'\ 2>&1 | grepQuiet -F 'trace: { repeating = «repeated»; tracing = «potential infinite recursion»; }' +nix-instantiate --eval -E 'builtins.warn "Hello" 123' 2>&1 | grepQuiet 'warning: Hello' +nix-instantiate --eval -E 'builtins.addErrorContext "while doing ${"something"} interesting" (builtins.warn "Hello" 123)' 2>/dev/null | grepQuiet 123 + +# warn does not accept non-strings for now +expectStderr 1 nix-instantiate --eval -E 'let x = builtins.warn { x = x; } true; in x' \ + | grepQuiet "expected a string but found a set" +expectStderr 1 nix-instantiate --eval --abort-on-warn -E 'builtins.warn "Hello" 123' | grepQuiet Hello +NIX_ABORT_ON_WARN=1 expectStderr 1 nix-instantiate --eval -E 'builtins.addErrorContext "while doing ${"something"} interesting" (builtins.warn "Hello" 123)' | grepQuiet "while doing something interesting" + set +x badDiff=0 badExitCode=0 +# Extra post-processing that's specific to each test case +postprocess() { + if [[ -e "lang/$1.postprocess" ]]; then + ( + # We could allow arbitrary interpreters in .postprocess, but that + # just exposes us to the complexity of not having /usr/bin/env in + # the sandbox. So let's just hardcode bash for now. + set -x; + bash "lang/$1.postprocess" "lang/$1" + ) + fi +} + for i in lang/parse-fail-*.nix; do echo "parsing $i (should fail)"; i=$(basename "$i" .nix) if expectStderr 1 nix-instantiate --parse - < "lang/$i.nix" > "lang/$i.err" then + postprocess "$i" diffAndAccept "$i" err err.exp else echo "FAIL: $i shouldn't parse" @@ -57,6 +85,7 @@ for i in lang/parse-okay-*.nix; do 2> "lang/$i.err" then sed "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + postprocess "$i" diffAndAccept "$i" out exp diffAndAccept "$i" err err.exp else @@ -80,6 +109,7 @@ for i in lang/eval-fail-*.nix; do expectStderr 1 nix-instantiate $flags "lang/$i.nix" \ | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" then + postprocess "$i" diffAndAccept "$i" err err.exp else echo "FAIL: $i shouldn't evaluate" @@ -95,6 +125,7 @@ for i in lang/eval-okay-*.nix; do if expect 0 nix-instantiate --eval --xml --no-location --strict \ "lang/$i.nix" > "lang/$i.out.xml" then + postprocess "$i" diffAndAccept "$i" out.xml exp.xml else echo "FAIL: $i should evaluate" @@ -115,6 +146,7 @@ for i in lang/eval-okay-*.nix; do 2> "lang/$i.err" then sed -i "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err" + postprocess "$i" diffAndAccept "$i" out exp diffAndAccept "$i" err err.exp else @@ -124,32 +156,4 @@ for i in lang/eval-okay-*.nix; do fi done -if test -n "${_NIX_TEST_ACCEPT-}"; then - if (( "$badDiff" )); then - echo 'Output did mot match, but accepted output as the persisted expected output.' - echo 'That means the next time the tests are run, they should pass.' - else - echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,' - echo 'indicating the unexpected output should be accepted as the expected output going forward,' - echo 'but no tests had unexpected output so there was no expected output to update.' - fi - if (( "$badExitCode" )); then - exit "$badExitCode" - else - skipTest "regenerating golden masters" - fi -else - if (( "$badDiff" )); then - echo '' - echo 'You can rerun this test with:' - echo '' - echo ' _NIX_TEST_ACCEPT=1 make tests/functional/lang.sh.test' - echo '' - echo 'to regenerate the files containing the expected output,' - echo 'and then view the git diff to decide whether a change is' - echo 'good/intentional or bad/unintentional.' - echo 'If the diff contains arbitrary or impure information,' - echo 'please improve the normalization that the test applies to the output.' - fi - exit $(( "$badExitCode" + "$badDiff" )) -fi +characterisationTestExit diff --git a/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.err.exp b/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.err.exp new file mode 100644 index 000000000..4b68d97c2 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '({ a = true; } == { a = true; b = true; })' + at /pwd/lang/eval-fail-assert-equal-attrs-names-2.nix:1:1: + 1| assert { a = true; } == { a = true; b = true; }; + | ^ + 2| throw "unreachable" + + error: attribute names of attribute set '{ a = true; }' differs from attribute set '{ a = true; b = true; }' diff --git a/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.nix b/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.nix new file mode 100644 index 000000000..8e7ac9cf2 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-attrs-names-2.nix @@ -0,0 +1,2 @@ +assert { a = true; } == { a = true; b = true; }; +throw "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-attrs-names.err.exp b/tests/functional/lang/eval-fail-assert-equal-attrs-names.err.exp new file mode 100644 index 000000000..bc61ca63a --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-attrs-names.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '({ a = true; b = true; } == { a = true; })' + at /pwd/lang/eval-fail-assert-equal-attrs-names.nix:1:1: + 1| assert { a = true; b = true; } == { a = true; }; + | ^ + 2| throw "unreachable" + + error: attribute names of attribute set '{ a = true; b = true; }' differs from attribute set '{ a = true; }' diff --git a/tests/functional/lang/eval-fail-assert-equal-attrs-names.nix b/tests/functional/lang/eval-fail-assert-equal-attrs-names.nix new file mode 100644 index 000000000..e2f53a85a --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-attrs-names.nix @@ -0,0 +1,2 @@ +assert { a = true; b = true; } == { a = true; }; +throw "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-derivations-extra.err.exp b/tests/functional/lang/eval-fail-assert-equal-derivations-extra.err.exp new file mode 100644 index 000000000..7f4924074 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-derivations-extra.err.exp @@ -0,0 +1,26 @@ +error: + … while evaluating the condition of the assertion '({ foo = { outPath = "/nix/store/0"; type = "derivation"; }; } == { foo = { devious = true; outPath = "/nix/store/1"; type = "derivation"; }; })' + at /pwd/lang/eval-fail-assert-equal-derivations-extra.nix:1:1: + 1| assert + | ^ + 2| { foo = { type = "derivation"; outPath = "/nix/store/0"; }; } + + … while comparing attribute 'foo' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-derivations-extra.nix:2:5: + 1| assert + 2| { foo = { type = "derivation"; outPath = "/nix/store/0"; }; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-derivations-extra.nix:4:5: + 3| == + 4| { foo = { type = "derivation"; outPath = "/nix/store/1"; devious = true; }; }; + | ^ + 5| throw "unreachable" + + … while comparing a derivation by its 'outPath' attribute + + error: string '"/nix/store/0"' is not equal to string '"/nix/store/1"' diff --git a/tests/functional/lang/eval-fail-assert-equal-derivations-extra.nix b/tests/functional/lang/eval-fail-assert-equal-derivations-extra.nix new file mode 100644 index 000000000..fd8bc3f26 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-derivations-extra.nix @@ -0,0 +1,5 @@ +assert + { foo = { type = "derivation"; outPath = "/nix/store/0"; }; } + == + { foo = { type = "derivation"; outPath = "/nix/store/1"; devious = true; }; }; +throw "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-derivations.err.exp b/tests/functional/lang/eval-fail-assert-equal-derivations.err.exp new file mode 100644 index 000000000..d7f0face0 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-derivations.err.exp @@ -0,0 +1,26 @@ +error: + … while evaluating the condition of the assertion '({ foo = { ignored = (abort "not ignored"); outPath = "/nix/store/0"; type = "derivation"; }; } == { foo = { ignored = (abort "not ignored"); outPath = "/nix/store/1"; type = "derivation"; }; })' + at /pwd/lang/eval-fail-assert-equal-derivations.nix:1:1: + 1| assert + | ^ + 2| { foo = { type = "derivation"; outPath = "/nix/store/0"; ignored = abort "not ignored"; }; } + + … while comparing attribute 'foo' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-derivations.nix:2:5: + 1| assert + 2| { foo = { type = "derivation"; outPath = "/nix/store/0"; ignored = abort "not ignored"; }; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-derivations.nix:4:5: + 3| == + 4| { foo = { type = "derivation"; outPath = "/nix/store/1"; ignored = abort "not ignored"; }; }; + | ^ + 5| throw "unreachable" + + … while comparing a derivation by its 'outPath' attribute + + error: string '"/nix/store/0"' is not equal to string '"/nix/store/1"' diff --git a/tests/functional/lang/eval-fail-assert-equal-derivations.nix b/tests/functional/lang/eval-fail-assert-equal-derivations.nix new file mode 100644 index 000000000..c648eae37 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-derivations.nix @@ -0,0 +1,5 @@ +assert + { foo = { type = "derivation"; outPath = "/nix/store/0"; ignored = abort "not ignored"; }; } + == + { foo = { type = "derivation"; outPath = "/nix/store/1"; ignored = abort "not ignored"; }; }; +throw "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-floats.err.exp b/tests/functional/lang/eval-fail-assert-equal-floats.err.exp new file mode 100644 index 000000000..d8545e2db --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-floats.err.exp @@ -0,0 +1,22 @@ +error: + … while evaluating the condition of the assertion '({ b = 1; } == { b = 1.01; })' + at /pwd/lang/eval-fail-assert-equal-floats.nix:1:1: + 1| assert { b = 1.0; } == { b = 1.01; }; + | ^ + 2| abort "unreachable" + + … while comparing attribute 'b' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-floats.nix:1:10: + 1| assert { b = 1.0; } == { b = 1.01; }; + | ^ + 2| abort "unreachable" + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-floats.nix:1:26: + 1| assert { b = 1.0; } == { b = 1.01; }; + | ^ + 2| abort "unreachable" + + error: a float with value '1' is not equal to a float with value '1.01' diff --git a/tests/functional/lang/eval-fail-assert-equal-floats.nix b/tests/functional/lang/eval-fail-assert-equal-floats.nix new file mode 100644 index 000000000..438e85abf --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-floats.nix @@ -0,0 +1,2 @@ +assert { b = 1.0; } == { b = 1.01; }; +abort "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-function-direct.err.exp b/tests/functional/lang/eval-fail-assert-equal-function-direct.err.exp new file mode 100644 index 000000000..f06d79698 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-function-direct.err.exp @@ -0,0 +1,9 @@ +error: + … while evaluating the condition of the assertion '((x: x) == (x: x))' + at /pwd/lang/eval-fail-assert-equal-function-direct.nix:3:1: + 2| # This only compares a direct comparison and makes no claims about functions in nested structures. + 3| assert + | ^ + 4| (x: x) + + error: distinct functions and immediate comparisons of identical functions compare as unequal diff --git a/tests/functional/lang/eval-fail-assert-equal-function-direct.nix b/tests/functional/lang/eval-fail-assert-equal-function-direct.nix new file mode 100644 index 000000000..68e5e3908 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-function-direct.nix @@ -0,0 +1,7 @@ +# Note: functions in nested structures, e.g. attributes, may be optimized away by pointer identity optimization. +# This only compares a direct comparison and makes no claims about functions in nested structures. +assert + (x: x) + == + (x: x); +abort "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-int-float.err.exp b/tests/functional/lang/eval-fail-assert-equal-int-float.err.exp new file mode 100644 index 000000000..c927e38d6 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-int-float.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '(1 == 1.1)' + at /pwd/lang/eval-fail-assert-equal-int-float.nix:1:1: + 1| assert 1 == 1.1; + | ^ + 2| throw "unreachable" + + error: an integer with value '1' is not equal to a float with value '1.1' diff --git a/tests/functional/lang/eval-fail-assert-equal-int-float.nix b/tests/functional/lang/eval-fail-assert-equal-int-float.nix new file mode 100644 index 000000000..1dfdf2bda --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-int-float.nix @@ -0,0 +1,2 @@ +assert 1 == 1.1; +throw "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-ints.err.exp b/tests/functional/lang/eval-fail-assert-equal-ints.err.exp new file mode 100644 index 000000000..d6219e200 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-ints.err.exp @@ -0,0 +1,22 @@ +error: + … while evaluating the condition of the assertion '({ b = 1; } == { b = 2; })' + at /pwd/lang/eval-fail-assert-equal-ints.nix:1:1: + 1| assert { b = 1; } == { b = 2; }; + | ^ + 2| abort "unreachable" + + … while comparing attribute 'b' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-ints.nix:1:10: + 1| assert { b = 1; } == { b = 2; }; + | ^ + 2| abort "unreachable" + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-ints.nix:1:24: + 1| assert { b = 1; } == { b = 2; }; + | ^ + 2| abort "unreachable" + + error: an integer with value '1' is not equal to an integer with value '2' diff --git a/tests/functional/lang/eval-fail-assert-equal-ints.nix b/tests/functional/lang/eval-fail-assert-equal-ints.nix new file mode 100644 index 000000000..645258ea6 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-ints.nix @@ -0,0 +1,2 @@ +assert { b = 1; } == { b = 2; }; +abort "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-list-length.err.exp b/tests/functional/lang/eval-fail-assert-equal-list-length.err.exp new file mode 100644 index 000000000..90108552c --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-list-length.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '([ (1) (0) ] == [ (10) ])' + at /pwd/lang/eval-fail-assert-equal-list-length.nix:1:1: + 1| assert [ 1 0 ] == [ 10 ]; + | ^ + 2| throw "unreachable" + + error: list of size '2' is not equal to list of size '1', left hand side is '[ 1 0 ]', right hand side is '[ 10 ]' diff --git a/tests/functional/lang/eval-fail-assert-equal-list-length.nix b/tests/functional/lang/eval-fail-assert-equal-list-length.nix new file mode 100644 index 000000000..6d40f4d8e --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-list-length.nix @@ -0,0 +1,2 @@ +assert [ 1 0 ] == [ 10 ]; +throw "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-paths.err.exp b/tests/functional/lang/eval-fail-assert-equal-paths.err.exp new file mode 100644 index 000000000..66c34e971 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-paths.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '(/pwd/lang/foo == /pwd/lang/bar)' + at /pwd/lang/eval-fail-assert-equal-paths.nix:1:1: + 1| assert ./foo == ./bar; + | ^ + 2| throw "unreachable" + + error: path '/pwd/lang/foo' is not equal to path '/pwd/lang/bar' diff --git a/tests/functional/lang/eval-fail-assert-equal-paths.nix b/tests/functional/lang/eval-fail-assert-equal-paths.nix new file mode 100644 index 000000000..ef0b67024 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-paths.nix @@ -0,0 +1,2 @@ +assert ./foo == ./bar; +throw "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert-equal-type-nested.err.exp b/tests/functional/lang/eval-fail-assert-equal-type-nested.err.exp new file mode 100644 index 000000000..f78badd25 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-type-nested.err.exp @@ -0,0 +1,22 @@ +error: + … while evaluating the condition of the assertion '({ ding = false; } == { ding = null; })' + at /pwd/lang/eval-fail-assert-equal-type-nested.nix:1:1: + 1| assert { ding = false; } == { ding = null; }; + | ^ + 2| abort "unreachable" + + … while comparing attribute 'ding' + + … where left hand side is + at /pwd/lang/eval-fail-assert-equal-type-nested.nix:1:10: + 1| assert { ding = false; } == { ding = null; }; + | ^ + 2| abort "unreachable" + + … where right hand side is + at /pwd/lang/eval-fail-assert-equal-type-nested.nix:1:31: + 1| assert { ding = false; } == { ding = null; }; + | ^ + 2| abort "unreachable" + + error: a Boolean of value 'false' is not equal to null of value 'null' diff --git a/tests/functional/lang/eval-fail-assert-equal-type-nested.nix b/tests/functional/lang/eval-fail-assert-equal-type-nested.nix new file mode 100644 index 000000000..3fbd14ce6 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-type-nested.nix @@ -0,0 +1,2 @@ +assert { ding = false; } == { ding = null; }; +abort "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-equal-type.err.exp b/tests/functional/lang/eval-fail-assert-equal-type.err.exp new file mode 100644 index 000000000..4dc3f2ece --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-type.err.exp @@ -0,0 +1,8 @@ +error: + … while evaluating the condition of the assertion '(false == null)' + at /pwd/lang/eval-fail-assert-equal-type.nix:1:1: + 1| assert false == null; + | ^ + 2| abort "unreachable" + + error: a Boolean of value 'false' is not equal to null of value 'null' diff --git a/tests/functional/lang/eval-fail-assert-equal-type.nix b/tests/functional/lang/eval-fail-assert-equal-type.nix new file mode 100644 index 000000000..7023ea007 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-equal-type.nix @@ -0,0 +1,2 @@ +assert false == null; +abort "unreachable" diff --git a/tests/functional/lang/eval-fail-assert-nested-bool.err.exp b/tests/functional/lang/eval-fail-assert-nested-bool.err.exp new file mode 100644 index 000000000..1debb668c --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-nested-bool.err.exp @@ -0,0 +1,74 @@ +error: + … while evaluating the condition of the assertion '({ a = { b = [ ({ c = { d = true; }; }) ]; }; } == { a = { b = [ ({ c = { d = false; }; }) ]; }; })' + at /pwd/lang/eval-fail-assert-nested-bool.nix:1:1: + 1| assert + | ^ + 2| { a.b = [ { c.d = true; } ]; } + + … while comparing attribute 'a' + + … where left hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:2:5: + 1| assert + 2| { a.b = [ { c.d = true; } ]; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:4:5: + 3| == + 4| { a.b = [ { c.d = false; } ]; }; + | ^ + 5| + + … while comparing attribute 'b' + + … where left hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:2:5: + 1| assert + 2| { a.b = [ { c.d = true; } ]; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:4:5: + 3| == + 4| { a.b = [ { c.d = false; } ]; }; + | ^ + 5| + + … while comparing list element 0 + + … while comparing attribute 'c' + + … where left hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:2:15: + 1| assert + 2| { a.b = [ { c.d = true; } ]; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:4:15: + 3| == + 4| { a.b = [ { c.d = false; } ]; }; + | ^ + 5| + + … while comparing attribute 'd' + + … where left hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:2:15: + 1| assert + 2| { a.b = [ { c.d = true; } ]; } + | ^ + 3| == + + … where right hand side is + at /pwd/lang/eval-fail-assert-nested-bool.nix:4:15: + 3| == + 4| { a.b = [ { c.d = false; } ]; }; + | ^ + 5| + + error: boolean 'true' is not equal to boolean 'false' diff --git a/tests/functional/lang/eval-fail-assert-nested-bool.nix b/tests/functional/lang/eval-fail-assert-nested-bool.nix new file mode 100644 index 000000000..228576983 --- /dev/null +++ b/tests/functional/lang/eval-fail-assert-nested-bool.nix @@ -0,0 +1,6 @@ +assert + { a.b = [ { c.d = true; } ]; } + == + { a.b = [ { c.d = false; } ]; }; + +abort "unreachable" \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-assert.err.exp b/tests/functional/lang/eval-fail-assert.err.exp index 0656ec81c..7be9e2387 100644 --- a/tests/functional/lang/eval-fail-assert.err.exp +++ b/tests/functional/lang/eval-fail-assert.err.exp @@ -20,9 +20,11 @@ error: | ^ 3| - error: assertion '(arg == "y")' failed - at /pwd/lang/eval-fail-assert.nix:2:12: + … while evaluating the condition of the assertion '(arg == "y")' + at /pwd/lang/eval-fail-assert.nix:2:12: 1| let { 2| x = arg: assert arg == "y"; 123; | ^ 3| + + error: string '"x"' is not equal to string '"y"' diff --git a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp index 6f907106b..b262e814d 100644 --- a/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp +++ b/tests/functional/lang/eval-fail-bad-string-interpolation-4.err.exp @@ -6,4 +6,4 @@ error: | ^ 10| - error: cannot coerce a set to a string: { a = { a = { a = { a = "ha"; b = "ha"; c = "ha"; d = "ha"; e = "ha"; f = "ha"; g = "ha"; h = "ha"; j = "ha"; }; «4294967295 attributes elided» }; «4294967294 attributes elided» }; «4294967293 attributes elided» } + error: cannot coerce a set to a string: { a = { a = { a = { a = "ha"; b = "ha"; c = "ha"; d = "ha"; e = "ha"; f = "ha"; g = "ha"; h = "ha"; j = "ha"; }; «8 attributes elided» }; «8 attributes elided» }; «8 attributes elided» } diff --git a/tests/functional/lang/eval-fail-derivation-name.err.exp b/tests/functional/lang/eval-fail-derivation-name.err.exp new file mode 100644 index 000000000..0ef98674d --- /dev/null +++ b/tests/functional/lang/eval-fail-derivation-name.err.exp @@ -0,0 +1,26 @@ +error: + … while evaluating the attribute 'outPath' + at ::: + | value = commonAttrs // { + | outPath = builtins.getAttr outputName strict; + | ^ + | drvPath = strict.drvPath; + + … while calling the 'getAttr' builtin + at ::: + | value = commonAttrs // { + | outPath = builtins.getAttr outputName strict; + | ^ + | drvPath = strict.drvPath; + + … while calling the 'derivationStrict' builtin + at ::: + | + | strict = derivationStrict drvAttrs; + | ^ + | + + … while evaluating derivation '~jiggle~' + whose name attribute is located at /pwd/lang/eval-fail-derivation-name.nix:: + + error: invalid derivation name: name '~jiggle~' contains illegal character '~'. Please pass a different 'name'. diff --git a/tests/functional/lang/eval-fail-derivation-name.nix b/tests/functional/lang/eval-fail-derivation-name.nix new file mode 100644 index 000000000..e779ad6ff --- /dev/null +++ b/tests/functional/lang/eval-fail-derivation-name.nix @@ -0,0 +1,5 @@ +derivation { + name = "~jiggle~"; + system = "some-system"; + builder = "/dontcare"; +} diff --git a/tests/functional/lang/eval-fail-derivation-name.postprocess b/tests/functional/lang/eval-fail-derivation-name.postprocess new file mode 100644 index 000000000..ffbc2b5d4 --- /dev/null +++ b/tests/functional/lang/eval-fail-derivation-name.postprocess @@ -0,0 +1,9 @@ +# shellcheck shell=bash +set -euo pipefail +testcaseBasename=$1 + +# Line numbers change when derivation.nix docs are updated. +sed -i "$testcaseBasename.err" \ + -e 's/[0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9 ][0-9]\([^0-9]\)/\1/g' \ + -e 's/[0-9][0-9]*//g' \ + ; diff --git a/tests/functional/lang/eval-fail-fetchTree-negative.err.exp b/tests/functional/lang/eval-fail-fetchTree-negative.err.exp new file mode 100644 index 000000000..d9ba1f0b2 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchTree-negative.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchTree' builtin + at /pwd/lang/eval-fail-fetchTree-negative.nix:1:1: + 1| builtins.fetchTree { + | ^ + 2| type = "file"; + + error: negative value given for fetchTree attr owner: -1 diff --git a/tests/functional/lang/eval-fail-fetchTree-negative.nix b/tests/functional/lang/eval-fail-fetchTree-negative.nix new file mode 100644 index 000000000..90bcab5d8 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchTree-negative.nix @@ -0,0 +1,5 @@ +builtins.fetchTree { + type = "file"; + url = "file://eval-fail-fetchTree-negative.nix"; + owner = -1; +} diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.err.exp b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.err.exp new file mode 100644 index 000000000..30f8b6a35 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchurl' builtin + at /pwd/lang/eval-fail-fetchurl-baseName-attrs-name.nix:1:1: + 1| builtins.fetchurl { url = "https://example.com/foo.tar.gz"; name = "~wobble~"; } + | ^ + 2| + + error: invalid store path name when fetching URL 'https://example.com/foo.tar.gz': name '~wobble~' contains illegal character '~'. Please change the value for the 'name' attribute passed to 'fetchurl', so that it can create a valid store path. diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.nix b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.nix new file mode 100644 index 000000000..583805539 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs-name.nix @@ -0,0 +1 @@ +builtins.fetchurl { url = "https://example.com/foo.tar.gz"; name = "~wobble~"; } diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.err.exp b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.err.exp new file mode 100644 index 000000000..cef532e94 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchurl' builtin + at /pwd/lang/eval-fail-fetchurl-baseName-attrs.nix:1:1: + 1| builtins.fetchurl { url = "https://example.com/~wiggle~"; } + | ^ + 2| + + error: invalid store path name when fetching URL 'https://example.com/~wiggle~': name '~wiggle~' contains illegal character '~'. Please add a valid 'name' attribute to the argument for 'fetchurl', so that it can create a valid store path. diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.nix b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.nix new file mode 100644 index 000000000..068120edb --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName-attrs.nix @@ -0,0 +1 @@ +builtins.fetchurl { url = "https://example.com/~wiggle~"; } diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName.err.exp b/tests/functional/lang/eval-fail-fetchurl-baseName.err.exp new file mode 100644 index 000000000..0950e8e70 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fetchurl' builtin + at /pwd/lang/eval-fail-fetchurl-baseName.nix:1:1: + 1| builtins.fetchurl "https://example.com/~wiggle~" + | ^ + 2| + + error: invalid store path name when fetching URL 'https://example.com/~wiggle~': name '~wiggle~' contains illegal character '~'. Please pass an attribute set with 'url' and 'name' attributes to 'fetchurl', so that it can create a valid store path. diff --git a/tests/functional/lang/eval-fail-fetchurl-baseName.nix b/tests/functional/lang/eval-fail-fetchurl-baseName.nix new file mode 100644 index 000000000..965093843 --- /dev/null +++ b/tests/functional/lang/eval-fail-fetchurl-baseName.nix @@ -0,0 +1 @@ +builtins.fetchurl "https://example.com/~wiggle~" diff --git a/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.err.exp b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.err.exp new file mode 100644 index 000000000..25c8d7eaa --- /dev/null +++ b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.err.exp @@ -0,0 +1,14 @@ +error: + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-flake-ref-to-string-negative-integer.nix:1:16: + 1| let n = -1; in builtins.seq n (builtins.flakeRefToString { + | ^ + 2| type = "github"; + + … while calling the 'flakeRefToString' builtin + at /pwd/lang/eval-fail-flake-ref-to-string-negative-integer.nix:1:32: + 1| let n = -1; in builtins.seq n (builtins.flakeRefToString { + | ^ + 2| type = "github"; + + error: negative value given for flake ref attr repo: -1 diff --git a/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.nix b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.nix new file mode 100644 index 000000000..e0208eb25 --- /dev/null +++ b/tests/functional/lang/eval-fail-flake-ref-to-string-negative-integer.nix @@ -0,0 +1,7 @@ +let n = -1; in builtins.seq n (builtins.flakeRefToString { + type = "github"; + owner = "NixOS"; + repo = n; + ref = "23.05"; + dir = "lib"; +}) diff --git a/tests/functional/lang/eval-fail-fromJSON-overflowing.err.exp b/tests/functional/lang/eval-fail-fromJSON-overflowing.err.exp new file mode 100644 index 000000000..a39082b45 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-overflowing.err.exp @@ -0,0 +1,8 @@ +error: + … while calling the 'fromJSON' builtin + at /pwd/lang/eval-fail-fromJSON-overflowing.nix:1:1: + 1| builtins.fromJSON ''{"attr": 18446744073709551615}'' + | ^ + 2| + + error: unsigned json number 18446744073709551615 outside of Nix integer range diff --git a/tests/functional/lang/eval-fail-fromJSON-overflowing.nix b/tests/functional/lang/eval-fail-fromJSON-overflowing.nix new file mode 100644 index 000000000..6dfbce3f6 --- /dev/null +++ b/tests/functional/lang/eval-fail-fromJSON-overflowing.nix @@ -0,0 +1 @@ +builtins.fromJSON ''{"attr": 18446744073709551615}'' diff --git a/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp b/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp index 5d843d827..712dd75a8 100644 --- a/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp +++ b/tests/functional/lang/eval-fail-infinite-recursion-lambda.err.exp @@ -29,7 +29,7 @@ error: | ^ 2| - (19997 duplicate frames omitted) + (197 duplicate frames omitted) error: stack overflow; max-call-depth exceeded at /pwd/lang/eval-fail-infinite-recursion-lambda.nix:1:14: diff --git a/tests/functional/lang/eval-fail-infinite-recursion-lambda.flags b/tests/functional/lang/eval-fail-infinite-recursion-lambda.flags new file mode 100644 index 000000000..59e20ec9c --- /dev/null +++ b/tests/functional/lang/eval-fail-infinite-recursion-lambda.flags @@ -0,0 +1 @@ +--max-call-depth 100 \ No newline at end of file diff --git a/tests/functional/lang/eval-fail-nested-list-items.err.exp b/tests/functional/lang/eval-fail-nested-list-items.err.exp new file mode 100644 index 000000000..90d439061 --- /dev/null +++ b/tests/functional/lang/eval-fail-nested-list-items.err.exp @@ -0,0 +1,9 @@ +error: + … while evaluating a path segment + at /pwd/lang/eval-fail-nested-list-items.nix:11:6: + 10| + 11| "" + (let v = [ [ 1 2 3 4 5 6 7 8 ] [1 2 3 4]]; in builtins.deepSeq v v) + | ^ + 12| + + error: cannot coerce a list to a string: [ [ 1 2 3 4 5 6 7 8 ] [ 1 «3 items elided» ] ] diff --git a/tests/functional/lang/eval-fail-nested-list-items.nix b/tests/functional/lang/eval-fail-nested-list-items.nix new file mode 100644 index 000000000..af45b1dd4 --- /dev/null +++ b/tests/functional/lang/eval-fail-nested-list-items.nix @@ -0,0 +1,11 @@ +# This reproduces https://github.com/NixOS/nix/issues/10993, for lists +# $ nix run nix/2.23.1 -- eval --expr '"" + (let v = [ [ 1 2 3 4 5 6 7 8 ] [1 2 3 4]]; in builtins.deepSeq v v)' +# error: +# … while evaluating a path segment +# at «string»:1:6: +# 1| "" + (let v = [ [ 1 2 3 4 5 6 7 8 ] [1 2 3 4]]; in builtins.deepSeq v v) +# | ^ +# +# error: cannot coerce a list to a string: [ [ 1 2 3 4 5 6 7 8 ] [ 1 «4294967290 items elided» ] ] + +"" + (let v = [ [ 1 2 3 4 5 6 7 8 ] [1 2 3 4]]; in builtins.deepSeq v v) diff --git a/tests/functional/lang/eval-fail-overflowing-add.err.exp b/tests/functional/lang/eval-fail-overflowing-add.err.exp new file mode 100644 index 000000000..6458cf1c9 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-add.err.exp @@ -0,0 +1,6 @@ +error: integer overflow in adding 9223372036854775807 + 1 + at /pwd/lang/eval-fail-overflowing-add.nix:4:8: + 3| b = 1; + 4| in a + b + | ^ + 5| diff --git a/tests/functional/lang/eval-fail-overflowing-add.nix b/tests/functional/lang/eval-fail-overflowing-add.nix new file mode 100644 index 000000000..24258fc20 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-add.nix @@ -0,0 +1,4 @@ +let + a = 9223372036854775807; + b = 1; +in a + b diff --git a/tests/functional/lang/eval-fail-overflowing-div.err.exp b/tests/functional/lang/eval-fail-overflowing-div.err.exp new file mode 100644 index 000000000..8ce07d4d6 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-div.err.exp @@ -0,0 +1,23 @@ +error: + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-overflowing-div.nix:7:4: + 6| b = -1; + 7| in builtins.seq intMin (builtins.seq b (intMin / b)) + | ^ + 8| + + … while calling the 'seq' builtin + at /pwd/lang/eval-fail-overflowing-div.nix:7:25: + 6| b = -1; + 7| in builtins.seq intMin (builtins.seq b (intMin / b)) + | ^ + 8| + + … while calling the 'div' builtin + at /pwd/lang/eval-fail-overflowing-div.nix:7:48: + 6| b = -1; + 7| in builtins.seq intMin (builtins.seq b (intMin / b)) + | ^ + 8| + + error: integer overflow in dividing -9223372036854775808 / -1 diff --git a/tests/functional/lang/eval-fail-overflowing-div.nix b/tests/functional/lang/eval-fail-overflowing-div.nix new file mode 100644 index 000000000..44fbe9d7e --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-div.nix @@ -0,0 +1,7 @@ +let + # lol, this has to be written as an expression like this because negative + # numbers use unary negation rather than parsing directly, and 2**63 is out + # of range + intMin = -9223372036854775807 - 1; + b = -1; +in builtins.seq intMin (builtins.seq b (intMin / b)) diff --git a/tests/functional/lang/eval-fail-overflowing-mul.err.exp b/tests/functional/lang/eval-fail-overflowing-mul.err.exp new file mode 100644 index 000000000..f42b39d4d --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-mul.err.exp @@ -0,0 +1,16 @@ +error: + … while calling the 'mul' builtin + at /pwd/lang/eval-fail-overflowing-mul.nix:3:10: + 2| a = 4294967297; + 3| in a * a * a + | ^ + 4| + + … while calling the 'mul' builtin + at /pwd/lang/eval-fail-overflowing-mul.nix:3:6: + 2| a = 4294967297; + 3| in a * a * a + | ^ + 4| + + error: integer overflow in multiplying 4294967297 * 4294967297 diff --git a/tests/functional/lang/eval-fail-overflowing-mul.nix b/tests/functional/lang/eval-fail-overflowing-mul.nix new file mode 100644 index 000000000..6081d9c7b --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-mul.nix @@ -0,0 +1,3 @@ +let + a = 4294967297; +in a * a * a diff --git a/tests/functional/lang/eval-fail-overflowing-sub.err.exp b/tests/functional/lang/eval-fail-overflowing-sub.err.exp new file mode 100644 index 000000000..66a3a03f8 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-sub.err.exp @@ -0,0 +1,9 @@ +error: + … while calling the 'sub' builtin + at /pwd/lang/eval-fail-overflowing-sub.nix:4:6: + 3| b = 2; + 4| in a - b + | ^ + 5| + + error: integer overflow in subtracting -9223372036854775807 - 2 diff --git a/tests/functional/lang/eval-fail-overflowing-sub.nix b/tests/functional/lang/eval-fail-overflowing-sub.nix new file mode 100644 index 000000000..229b8c6d2 --- /dev/null +++ b/tests/functional/lang/eval-fail-overflowing-sub.nix @@ -0,0 +1,4 @@ +let + a = -9223372036854775807; + b = 2; +in a - b diff --git a/tests/functional/lang/eval-fail-pipe-operators.err.exp b/tests/functional/lang/eval-fail-pipe-operators.err.exp new file mode 100644 index 000000000..49f3fa8ad --- /dev/null +++ b/tests/functional/lang/eval-fail-pipe-operators.err.exp @@ -0,0 +1,5 @@ +error: experimental Nix feature 'pipe-operators' is disabled; add '--extra-experimental-features pipe-operators' to enable it + at /pwd/lang/eval-fail-pipe-operators.nix:1:3: + 1| 1 |> 2 + | ^ + 2| diff --git a/tests/functional/lang/eval-fail-pipe-operators.nix b/tests/functional/lang/eval-fail-pipe-operators.nix new file mode 100644 index 000000000..433e0fd7f --- /dev/null +++ b/tests/functional/lang/eval-fail-pipe-operators.nix @@ -0,0 +1 @@ +1 |> 2 diff --git a/tests/functional/lang/eval-okay-deprecate-cursed-or.err.exp b/tests/functional/lang/eval-okay-deprecate-cursed-or.err.exp new file mode 100644 index 000000000..4a656827a --- /dev/null +++ b/tests/functional/lang/eval-okay-deprecate-cursed-or.err.exp @@ -0,0 +1,12 @@ +warning: at /pwd/lang/eval-okay-deprecate-cursed-or.nix:3:47: This expression uses `or` as an identifier in a way that will change in a future Nix release. +Wrap this entire expression in parentheses to preserve its current meaning: + ((x: x) or) +Give feedback at https://github.com/NixOS/nix/pull/11121 +warning: at /pwd/lang/eval-okay-deprecate-cursed-or.nix:4:39: This expression uses `or` as an identifier in a way that will change in a future Nix release. +Wrap this entire expression in parentheses to preserve its current meaning: + ((x: x + 1) or) +Give feedback at https://github.com/NixOS/nix/pull/11121 +warning: at /pwd/lang/eval-okay-deprecate-cursed-or.nix:5:44: This expression uses `or` as an identifier in a way that will change in a future Nix release. +Wrap this entire expression in parentheses to preserve its current meaning: + ((x: x) or) +Give feedback at https://github.com/NixOS/nix/pull/11121 diff --git a/tests/functional/lang/eval-okay-deprecate-cursed-or.exp b/tests/functional/lang/eval-okay-deprecate-cursed-or.exp new file mode 100644 index 000000000..573541ac9 --- /dev/null +++ b/tests/functional/lang/eval-okay-deprecate-cursed-or.exp @@ -0,0 +1 @@ +0 diff --git a/tests/functional/lang/eval-okay-deprecate-cursed-or.nix b/tests/functional/lang/eval-okay-deprecate-cursed-or.nix new file mode 100644 index 000000000..a4f9e747f --- /dev/null +++ b/tests/functional/lang/eval-okay-deprecate-cursed-or.nix @@ -0,0 +1,11 @@ +let + # These are cursed and should warn + cursed0 = builtins.length (let or = 1; in [ (x: x) or ]); + cursed1 = let or = 1; in (x: x * 2) (x: x + 1) or; + cursed2 = let or = 1; in { a = 2; }.a or (x: x) or; + + # These are uses of `or` as an identifier that are not cursed + allowed0 = let or = (x: x); in map or []; + allowed1 = let f = (x: x); or = f; in f (f or); +in +0 diff --git a/tests/functional/lang/eval-okay-derivation-legacy.err.exp b/tests/functional/lang/eval-okay-derivation-legacy.err.exp new file mode 100644 index 000000000..94f0854dd --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.err.exp @@ -0,0 +1,6 @@ +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks..allowedReferences' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks..allowedRequisites' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks..disallowedReferences' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks..disallowedRequisites' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks..maxClosureSize' instead +warning: In a derivation named 'eval-okay-derivation-legacy', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks..maxSize' instead diff --git a/tests/functional/lang/eval-okay-derivation-legacy.exp b/tests/functional/lang/eval-okay-derivation-legacy.exp new file mode 100644 index 000000000..4f374a1aa --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.exp @@ -0,0 +1 @@ +"/nix/store/mzgwvrjjir216ra58mwwizi8wj6y9ddr-eval-okay-derivation-legacy" diff --git a/tests/functional/lang/eval-okay-derivation-legacy.nix b/tests/functional/lang/eval-okay-derivation-legacy.nix new file mode 100644 index 000000000..b529cdf90 --- /dev/null +++ b/tests/functional/lang/eval-okay-derivation-legacy.nix @@ -0,0 +1,12 @@ +(builtins.derivationStrict { + name = "eval-okay-derivation-legacy"; + system = "x86_64-linux"; + builder = "/dontcare"; + __structuredAttrs = true; + allowedReferences = [ ]; + disallowedReferences = [ ]; + allowedRequisites = [ ]; + disallowedRequisites = [ ]; + maxSize = 1234; + maxClosureSize = 12345; +}).out diff --git a/tests/functional/lang/framework.sh b/tests/functional/lang/framework.sh deleted file mode 100644 index 9b886e983..000000000 --- a/tests/functional/lang/framework.sh +++ /dev/null @@ -1,33 +0,0 @@ -# Golden test support -# -# Test that the output of the given test matches what is expected. If -# `_NIX_TEST_ACCEPT` is non-empty also update the expected output so -# that next time the test succeeds. -function diffAndAcceptInner() { - local -r testName=$1 - local -r got="$2" - local -r expected="$3" - - # Absence of expected file indicates empty output expected. - if test -e "$expected"; then - local -r expectedOrEmpty="$expected" - else - local -r expectedOrEmpty=lang/empty.exp - fi - - # Diff so we get a nice message - if ! diff --color=always --unified "$expectedOrEmpty" "$got"; then - echo "FAIL: evaluation result of $testName not as expected" - badDiff=1 - fi - - # Update expected if `_NIX_TEST_ACCEPT` is non-empty. - if test -n "${_NIX_TEST_ACCEPT-}"; then - cp "$got" "$expected" - # Delete empty expected files to avoid bloating the repo with - # empty files. - if ! test -s "$expected"; then - rm "$expected" - fi - fi -} diff --git a/tests/functional/lang/non-eval-fail-bad-drvPath.nix b/tests/functional/lang/non-eval-fail-bad-drvPath.nix new file mode 100644 index 000000000..23639bc54 --- /dev/null +++ b/tests/functional/lang/non-eval-fail-bad-drvPath.nix @@ -0,0 +1,14 @@ +let + package = { + type = "derivation"; + name = "cachix-1.7.3"; + system = builtins.currentSystem; + outputs = [ "out" ]; + # Illegal, because does not end in `.drv` + drvPath = "${builtins.storeDir}/8qlfcic10lw5304gqm8q45nr7g7jl62b-cachix-1.7.3-bin"; + outputName = "out"; + outPath = "${builtins.storeDir}/8qlfcic10lw5304gqm8q45nr7g7jl62b-cachix-1.7.3-bin"; + out = package; + }; +in +package diff --git a/tests/functional/lang/parse-fail-undef-var-2.err.exp b/tests/functional/lang/parse-fail-undef-var-2.err.exp index 393c454dd..96e87b2aa 100644 --- a/tests/functional/lang/parse-fail-undef-var-2.err.exp +++ b/tests/functional/lang/parse-fail-undef-var-2.err.exp @@ -1,4 +1,4 @@ -error: syntax error, unexpected ':', expecting '}' +error: syntax error, unexpected ':', expecting '}' or ',' at «stdin»:3:13: 2| 3| f = {x, y : ["baz" "bar" z "bat"]}: x + y; diff --git a/tests/functional/lang/parse-okay-ind-string.exp b/tests/functional/lang/parse-okay-ind-string.exp new file mode 100644 index 000000000..82e9940a2 --- /dev/null +++ b/tests/functional/lang/parse-okay-ind-string.exp @@ -0,0 +1 @@ +(let string = "str"; in [ (/some/path) ((/some/path)) ((/some/path)) ((/some/path + "\n end")) (string) ((string)) ((string)) ((string + "\n end")) ("") ("") ("end") ]) diff --git a/tests/functional/lang/parse-okay-ind-string.nix b/tests/functional/lang/parse-okay-ind-string.nix new file mode 100644 index 000000000..97c9de3cd --- /dev/null +++ b/tests/functional/lang/parse-okay-ind-string.nix @@ -0,0 +1,31 @@ +let + string = "str"; +in [ + /some/path + + ''${/some/path}'' + + '' + ${/some/path}'' + + ''${/some/path} + end'' + + string + + ''${string}'' + + '' + ${string}'' + + ''${string} + end'' + + '''' + + '' + '' + + '' + end'' +] diff --git a/tests/functional/legacy-ssh-store.sh b/tests/functional/legacy-ssh-store.sh old mode 100644 new mode 100755 index 56b4c2d20..3a1a7b022 --- a/tests/functional/legacy-ssh-store.sh +++ b/tests/functional/legacy-ssh-store.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh store_uri="ssh://localhost?remote-store=$TEST_ROOT/other-store" diff --git a/tests/functional/linux-sandbox.sh b/tests/functional/linux-sandbox.sh old mode 100644 new mode 100755 index e553791d9..653a3873f --- a/tests/functional/linux-sandbox.sh +++ b/tests/functional/linux-sandbox.sh @@ -1,7 +1,11 @@ +#!/usr/bin/env bash + source common.sh needLocalStore "the sandbox only runs on the builder side, so it makes no sense to test it with the daemon" +TODO_NixOS + clearStore requireSandboxSupport diff --git a/tests/functional/local-overlay-store/bad-uris.sh b/tests/functional/local-overlay-store/bad-uris.sh index 42a6d47f7..b7930e32e 100644 --- a/tests/functional/local-overlay-store/bad-uris.sh +++ b/tests/functional/local-overlay-store/bad-uris.sh @@ -15,6 +15,8 @@ declare -a storesBad=( "$storeBadRoot" "$storeBadLower" "$storeBadUpper" ) +TODO_NixOS + for i in "${storesBad[@]}"; do echo $i unshare --mount --map-root-user bash <> "$NIX_CONF_DIR/nix.conf" + echo "$1" >> "$test_nix_conf" } setupConfig () { @@ -66,7 +69,7 @@ mountOverlayfs () { || skipTest "overlayfs is not supported" cleanupOverlay () { - umount "$storeBRoot/nix/store" + umount -n "$storeBRoot/nix/store" rm -r $storeVolume/workdir } trap cleanupOverlay EXIT diff --git a/tests/functional/local-overlay-store/gc-inner.sh b/tests/functional/local-overlay-store/gc-inner.sh index ea92154d2..687fed897 100644 --- a/tests/functional/local-overlay-store/gc-inner.sh +++ b/tests/functional/local-overlay-store/gc-inner.sh @@ -20,8 +20,8 @@ outPath=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg # Set a GC root. mkdir -p "$stateB" -rm -f "$stateB"/gcroots/foo -ln -sf $outPath "$stateB"/gcroots/foo +rm -f "$stateB/gcroots/foo" +ln -sf $outPath "$stateB/gcroots/foo" [ "$(nix-store -q --roots $outPath)" = "$stateB/gcroots/foo -> $outPath" ] @@ -46,7 +46,7 @@ nix-collect-garbage # Check that the root and its dependencies haven't been deleted. cat "$storeBRoot/$outPath" -rm "$stateB"/gcroots/foo +rm "$stateB/gcroots/foo" nix-collect-garbage diff --git a/tests/functional/local-overlay-store/meson.build b/tests/functional/local-overlay-store/meson.build new file mode 100644 index 000000000..6ff5d3169 --- /dev/null +++ b/tests/functional/local-overlay-store/meson.build @@ -0,0 +1,18 @@ +suites += { + 'name': 'local-overlay-store', + 'deps': [], + 'tests': [ + 'check-post-init.sh', + 'redundant-add.sh', + 'build.sh', + 'bad-uris.sh', + 'add-lower.sh', + 'delete-refs.sh', + 'delete-duplicate.sh', + 'gc.sh', + 'verify.sh', + 'optimise.sh', + 'stale-file-handle.sh', + ], + 'workdir': meson.current_build_dir(), +} diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 18b920f7d..3f796291a 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -1,27 +1,12 @@ nix_tests = \ test-infra.sh \ - flakes/flakes.sh \ - flakes/develop.sh \ - flakes/run.sh \ - flakes/mercurial.sh \ - flakes/circular.sh \ - flakes/init.sh \ - flakes/inputs.sh \ - flakes/follow-paths.sh \ - flakes/bundle.sh \ - flakes/check.sh \ - flakes/unlocked-override.sh \ - flakes/absolute-paths.sh \ - flakes/absolute-attr-paths.sh \ - flakes/build-paths.sh \ - flakes/flake-in-submodule.sh \ - flakes/prefetch.sh \ gc.sh \ nix-collect-garbage-d.sh \ remote-store.sh \ legacy-ssh-store.sh \ lang.sh \ - lang-test-infra.sh \ + lang-gc.sh \ + characterisation-test-infra.sh \ experimental-features.sh \ fetchMercurial.sh \ gc-auto.sh \ @@ -58,7 +43,6 @@ nix_tests = \ restricted.sh \ fetchGitSubmodules.sh \ fetchGitVerification.sh \ - flakes/search-root.sh \ readfile-context.sh \ nix-channel.sh \ recursive.sh \ @@ -99,14 +83,14 @@ nix_tests = \ nix-copy-ssh-ng.sh \ post-hook.sh \ function-trace.sh \ - flakes/config.sh \ fmt.sh \ eval-store.sh \ why-depends.sh \ derivation-json.sh \ - import-derivation.sh \ + derivation-advanced-attributes.sh \ + import-from-derivation.sh \ nix_path.sh \ - case-hack.sh \ + nars.sh \ placeholders.sh \ ssh-relay.sh \ build.sh \ @@ -121,7 +105,6 @@ nix_tests = \ store-info.sh \ fetchClosure.sh \ completions.sh \ - flakes/show.sh \ impure-derivations.sh \ path-from-hash-part.sh \ path-info.sh \ @@ -153,7 +136,7 @@ $(d)/plugins.sh.test $(d)/plugins.sh.test-debug: \ install-tests += $(foreach x, $(nix_tests), $(d)/$(x)) test-clean-files := \ - $(d)/common/vars-and-functions.sh \ + $(d)/common/subst-vars.sh \ $(d)/config.nix clean-files += $(test-clean-files) diff --git a/tests/functional/logging.sh b/tests/functional/logging.sh old mode 100644 new mode 100755 index 1ccc21d0b..bd80a9163 --- a/tests/functional/logging.sh +++ b/tests/functional/logging.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore path=$(nix-build dependencies.nix --no-out-link) diff --git a/tests/functional/meson.build b/tests/functional/meson.build new file mode 100644 index 000000000..54f3e7a01 --- /dev/null +++ b/tests/functional/meson.build @@ -0,0 +1,267 @@ +project('nix-functional-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.3', + license : 'LGPL-2.1-or-later', +) + +fs = import('fs') + +# Need to combine source and build trees +run_command( + 'rsync', + '-a', + '--copy-unsafe-links', + meson.current_source_dir() / '', + meson.current_build_dir() / '', +) +# This current-source-escaping relative is no good because we don't know +# where the build directory will be, therefore we fix it up. Once the +# Make build system is gone, we should think about doing this better. +scripts_dir = fs.relative_to( + meson.current_source_dir() / '..' / '..' / 'scripts', + meson.current_build_dir(), +) +run_command( + 'sed', + '-i', meson.current_build_dir() / 'bash-profile.sh', + '-e', 's^../../scripts^@0@^'.format(scripts_dir), +) + +nix = find_program('nix') +bash = find_program('bash', native : true) +busybox = find_program('busybox', native : true, required : false) +coreutils = find_program('coreutils', native : true) +dot = find_program('dot', native : true, required : false) + +nix_bin_dir = fs.parent(nix.full_path()) + +test_confdata = { + 'bindir': nix_bin_dir, + 'coreutils': fs.parent(coreutils.full_path()), + 'dot': dot.found() ? dot.full_path() : '', + 'bash': bash.full_path(), + 'sandbox_shell': busybox.found() ? busybox.full_path() : '', + 'PACKAGE_VERSION': meson.project_version(), + 'system': host_machine.cpu_family() + '-' + host_machine.system(), +} + +# Just configures `common/vars-and-functions.sh.in`. +# Done as a subdir() so Meson places it under `common` in the build directory as well. +subdir('common') + +config_nix_in = configure_file( + input : 'config.nix.in', + output : 'config.nix', + configuration : test_confdata, +) + +suites = [ + { + 'name' : 'main', + 'deps': [], + 'tests': [ + 'test-infra.sh', + 'gc.sh', + 'nix-collect-garbage-d.sh', + 'remote-store.sh', + 'legacy-ssh-store.sh', + 'lang.sh', + 'lang-gc.sh', + 'characterisation-test-infra.sh', + 'experimental-features.sh', + 'fetchMercurial.sh', + 'gc-auto.sh', + 'user-envs.sh', + 'user-envs-migration.sh', + 'binary-cache.sh', + 'multiple-outputs.sh', + 'nix-build.sh', + 'gc-concurrent.sh', + 'repair.sh', + 'fixed.sh', + 'export-graph.sh', + 'timeout.sh', + 'fetchGitRefs.sh', + 'gc-runtime.sh', + 'tarball.sh', + 'fetchGit.sh', + 'fetchurl.sh', + 'fetchPath.sh', + 'fetchTree-file.sh', + 'simple.sh', + 'referrers.sh', + 'optimise-store.sh', + 'substitute-with-invalid-ca.sh', + 'signing.sh', + 'hash-convert.sh', + 'hash-path.sh', + 'gc-non-blocking.sh', + 'check.sh', + 'nix-shell.sh', + 'check-refs.sh', + 'build-remote-input-addressed.sh', + 'secure-drv-outputs.sh', + 'restricted.sh', + 'fetchGitSubmodules.sh', + 'fetchGitVerification.sh', + 'readfile-context.sh', + 'nix-channel.sh', + 'recursive.sh', + 'dependencies.sh', + 'check-reqs.sh', + 'build-remote-content-addressed-fixed.sh', + 'build-remote-content-addressed-floating.sh', + 'build-remote-trustless-should-pass-0.sh', + 'build-remote-trustless-should-pass-1.sh', + 'build-remote-trustless-should-pass-2.sh', + 'build-remote-trustless-should-pass-3.sh', + 'build-remote-trustless-should-fail-0.sh', + 'build-remote-with-mounted-ssh-ng.sh', + 'nar-access.sh', + 'impure-eval.sh', + 'pure-eval.sh', + 'eval.sh', + 'repl.sh', + 'binary-cache-build-remote.sh', + 'search.sh', + 'logging.sh', + 'export.sh', + 'config.sh', + 'add.sh', + 'chroot-store.sh', + 'filter-source.sh', + 'misc.sh', + 'dump-db.sh', + 'linux-sandbox.sh', + 'supplementary-groups.sh', + 'build-dry.sh', + 'structured-attrs.sh', + 'shell.sh', + 'brotli.sh', + 'zstd.sh', + 'compression-levels.sh', + 'nix-copy-ssh.sh', + 'nix-copy-ssh-ng.sh', + 'post-hook.sh', + 'function-trace.sh', + 'fmt.sh', + 'eval-store.sh', + 'why-depends.sh', + 'derivation-json.sh', + 'derivation-advanced-attributes.sh', + 'import-from-derivation.sh', + 'nix_path.sh', + 'nars.sh', + 'placeholders.sh', + 'ssh-relay.sh', + 'build.sh', + 'build-delete.sh', + 'output-normalization.sh', + 'selfref-gc.sh', + 'db-migration.sh', + 'bash-profile.sh', + 'pass-as-file.sh', + 'nix-profile.sh', + 'suggestions.sh', + 'store-info.sh', + 'fetchClosure.sh', + 'completions.sh', + 'impure-derivations.sh', + 'path-from-hash-part.sh', + 'path-info.sh', + 'toString-path.sh', + 'read-only-store.sh', + 'nested-sandboxing.sh', + 'impure-env.sh', + 'debugger.sh', + 'extra-sandbox-profile.sh', + 'help.sh', + ], + 'workdir': meson.current_build_dir(), + }, +] + +nix_store = dependency('nix-store', required : false) +if nix_store.found() + subdir('test-libstoreconsumer') + suites += { + 'name': 'libstoreconsumer', + 'deps': [ + libstoreconsumer_tester, + ], + 'tests': [ + 'test-libstoreconsumer.sh', + ], + 'workdir': meson.current_build_dir(), + } + +endif + +# Plugin tests require shared libraries support. +nix_expr = dependency('nix-expr', required : false) +if nix_expr.found() and get_option('default_library') != 'static' + subdir('plugins') + suites += { + 'name': 'plugins', + 'deps': [ + libplugintest, + ], + 'tests': [ + 'plugins.sh', + ], + 'workdir': meson.current_build_dir(), + } +endif + +subdir('ca') +subdir('dyn-drv') +subdir('flakes') +subdir('git-hashing') +subdir('local-overlay-store') + +foreach suite : suites + foreach script : suite['tests'] + workdir = suite['workdir'] + prefix = fs.relative_to(workdir, meson.project_build_root()) + + script = script + # Turns, e.g., `tests/functional/flakes/show.sh` into a Meson test target called + # `functional-flakes-show`. + name = fs.replace_suffix(prefix / script, '') + + test( + name, + bash, + args: [ + '-x', + '-e', + '-u', + '-o', 'pipefail', + script, + ], + suite : suite['name'], + env : { + 'TEST_NAME': name, + 'NIX_REMOTE': '', + 'PS4': '+(${BASH_SOURCE[0]-$0}:$LINENO) ', + }, + # Some tests take 15+ seconds even on an otherwise idle machine; + # on a loaded machine this can easily drive them to failure. Give + # them more time than the default of 30 seconds. + timeout : 300, + # Used for target dependency/ordering tracking, not adding compiler flags or anything. + depends : suite['deps'], + workdir : workdir, + # Won't pass until man pages are generated + should_fail : suite['name'] == 'main' and script == 'help.sh' + ) + endforeach +endforeach diff --git a/tests/functional/misc.sh b/tests/functional/misc.sh old mode 100644 new mode 100755 index af96d20bd..7d63756b7 --- a/tests/functional/misc.sh +++ b/tests/functional/misc.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh # Tests miscellaneous commands. @@ -11,6 +13,9 @@ source common.sh # Can we ask for the version number? nix-env --version | grep "$version" +nix_env=$(type -P nix-env) +(PATH=""; ! $nix_env --help 2>&1 ) | grepQuiet -F "The 'man' command was not found, but it is needed for 'nix-env' and some other 'nix-*' commands' help text. Perhaps you could install the 'man' command?" + # Usage errors. expect 1 nix-env --foo 2>&1 | grep "no operation" expect 1 nix-env -q --foo 2>&1 | grep "unknown flag" @@ -30,3 +35,12 @@ expectStderr 1 nix-instantiate --eval -E '[]' -A 'x' | grepQuiet "should be a se expectStderr 1 nix-instantiate --eval -E '{}' -A '1' | grepQuiet "should be a list" expectStderr 1 nix-instantiate --eval -E '{}' -A '.' | grepQuiet "empty attribute name" expectStderr 1 nix-instantiate --eval -E '[]' -A '1' | grepQuiet "out of range" + +# Unknown setting warning +# NOTE(cole-h): behavior is different depending on the order, which is why we test an unknown option +# before and after the `'{}'`! +out="$(expectStderr 0 nix-instantiate --option foobar baz --expr '{}')" +[[ "$(echo "$out" | grep foobar | wc -l)" = 1 ]] + +out="$(expectStderr 0 nix-instantiate '{}' --option foobar baz --expr )" +[[ "$(echo "$out" | grep foobar | wc -l)" = 1 ]] diff --git a/tests/functional/multiple-outputs.nix b/tests/functional/multiple-outputs.nix index 413d392e4..6ba7c523d 100644 --- a/tests/functional/multiple-outputs.nix +++ b/tests/functional/multiple-outputs.nix @@ -96,6 +96,12 @@ rec { buildCommand = "mkdir $a_a $b $c"; }; + nothing-to-install = mkDerivation { + name = "nothing-to-install"; + meta.outputsToInstall = [ ]; + buildCommand = "mkdir $out"; + }; + independent = mkDerivation { name = "multiple-outputs-independent"; outputs = [ "first" "second" ]; diff --git a/tests/functional/multiple-outputs.sh b/tests/functional/multiple-outputs.sh old mode 100644 new mode 100755 index 330600d08..35a78d152 --- a/tests/functional/multiple-outputs.sh +++ b/tests/functional/multiple-outputs.sh @@ -1,6 +1,10 @@ +#!/usr/bin/env bash + source common.sh -clearStore +TODO_NixOS + +clearStoreIfPossible rm -f $TEST_ROOT/result* @@ -33,7 +37,7 @@ outPath=$(nix-store -q $drvPath) echo "building b..." outPath=$(nix-build multiple-outputs.nix -A b --no-out-link) echo "output path is $outPath" -[ "$(cat "$outPath"/file)" = "success" ] +[ "$(cat "$outPath/file")" = "success" ] # Test nix-build on a derivation with multiple outputs. outPath1=$(nix-build multiple-outputs.nix -A a -o $TEST_ROOT/result) diff --git a/tests/functional/name-after-node.nar b/tests/functional/name-after-node.nar new file mode 100644 index 000000000..3f5cd9d0b Binary files /dev/null and b/tests/functional/name-after-node.nar differ diff --git a/tests/functional/nar-access.sh b/tests/functional/nar-access.sh old mode 100644 new mode 100755 index 87981e7d9..b254081cf --- a/tests/functional/nar-access.sh +++ b/tests/functional/nar-access.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh echo "building test path" @@ -25,6 +27,8 @@ diff -u baz.cat-nar $storePath/foo/baz nix store cat $storePath/foo/baz > baz.cat-nar diff -u baz.cat-nar $storePath/foo/baz +TODO_NixOS + # Check that 'nix store cat' fails on invalid store paths. invalidPath="$(dirname $storePath)/99999999999999999999999999999999-foo" cp -r $storePath $invalidPath diff --git a/tests/functional/nars.sh b/tests/functional/nars.sh new file mode 100755 index 000000000..39d9389db --- /dev/null +++ b/tests/functional/nars.sh @@ -0,0 +1,161 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +clearStore + +# Check that NARs with duplicate directory entries are rejected. +rm -rf "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "NAR directory is not sorted" + +# Check that nix-store --restore fails if the output already exists. +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "path '.*/out' already exists" + +rm -rf "$TEST_ROOT/out" +echo foo > "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "File exists" + +rm -rf "$TEST_ROOT/out" +ln -s "$TEST_ROOT/out2" "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "File exists" + +mkdir -p "$TEST_ROOT/out2" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < duplicate.nar | grepQuiet "path '.*/out' already exists" + +# The same, but for a regular file. +nix-store --dump ./nars.sh > "$TEST_ROOT/tmp.nar" + +rm -rf "$TEST_ROOT/out" +nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" + +rm -rf "$TEST_ROOT/out" +mkdir -p "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" + +rm -rf "$TEST_ROOT/out" +ln -s "$TEST_ROOT/out2" "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" + +mkdir -p "$TEST_ROOT/out2" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" + +# The same, but for a symlink. +ln -sfn foo "$TEST_ROOT/symlink" +nix-store --dump "$TEST_ROOT/symlink" > "$TEST_ROOT/tmp.nar" + +rm -rf "$TEST_ROOT/out" +nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" +[[ -L "$TEST_ROOT/out" ]] +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" + +rm -rf "$TEST_ROOT/out" +mkdir -p "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" + +rm -rf "$TEST_ROOT/out" +ln -s "$TEST_ROOT/out2" "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" + +mkdir -p "$TEST_ROOT/out2" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < "$TEST_ROOT/tmp.nar" | grepQuiet "File exists" + +# Check whether restoring and dumping a NAR that contains case +# collisions is round-tripping, even on a case-insensitive system. +rm -rf "$TEST_ROOT/case" +opts=("--option" "use-case-hack" "true") +nix-store "${opts[@]}" --restore "$TEST_ROOT/case" < case.nar +[[ -e "$TEST_ROOT/case/xt_CONNMARK.h" ]] +[[ -e "$TEST_ROOT/case/xt_CONNmark.h~nix~case~hack~1" ]] +[[ -e "$TEST_ROOT/case/xt_connmark.h~nix~case~hack~2" ]] +[[ -e "$TEST_ROOT/case/x/FOO" ]] +[[ -d "$TEST_ROOT/case/x/Foo~nix~case~hack~1" ]] +[[ -e "$TEST_ROOT/case/x/foo~nix~case~hack~2/a~nix~case~hack~1/foo" ]] +nix-store "${opts[@]}" --dump "$TEST_ROOT/case" > "$TEST_ROOT/case.nar" +cmp case.nar "$TEST_ROOT/case.nar" +[ "$(nix-hash "${opts[@]}" --type sha256 "$TEST_ROOT/case")" = "$(nix-hash --flat --type sha256 case.nar)" ] + +# Check whether we detect true collisions (e.g. those remaining after +# removal of the suffix). +touch "$TEST_ROOT/case/xt_CONNMARK.h~nix~case~hack~3" +(! nix-store "${opts[@]}" --dump "$TEST_ROOT/case" > /dev/null) + +# Detect NARs that have a directory entry that after case-hacking +# collides with another entry (e.g. a directory containing 'Test', +# 'Test~nix~case~hack~1' and 'test'). +rm -rf "$TEST_ROOT/case" +expectStderr 1 nix-store "${opts[@]}" --restore "$TEST_ROOT/case" < case-collision.nar | grepQuiet "NAR contains file name 'test' that collides with case-hacked file name 'Test~nix~case~hack~1'" + +# Deserializing a NAR that contains file names that Unicode-normalize to the +# same name should fail on macOS and specific Linux setups (typically ZFS with +# `utf8only` enabled and `normalization` set to anything else than `none`). The +# deserialization should succeed on most Linux, where file names aren't +# unicode-normalized. +# +# We test that: +# +# 1. It either succeeds or fails with "already exists" error. +# 2. Nix has the same behavior with respect to unicode normalization than +# $TEST_ROOT's filesystem (when using basic Unix commands) +rm -rf "$TEST_ROOT/out" +set +e +unicodeTestOut=$(nix-store --restore "$TEST_ROOT/out" < unnormalized.nar 2>&1) +unicodeTestCode=$? +set -e + +touch "$TEST_ROOT/unicode-â" # non-canonical version +touch "$TEST_ROOT/unicode-â" + +touchFilesCount=$(find "$TEST_ROOT" -maxdepth 1 -name "unicode-*" -type f | wc -l) + +if (( unicodeTestCode == 1 )); then + # If the command failed (MacOS or ZFS + normalization), checks that it failed + # with the expected "already exists" error, and that this is the same + # behavior as `touch` + echo "$unicodeTestOut" | grepQuiet "path '.*/out/â' already exists" + + (( touchFilesCount == 1 )) +elif (( unicodeTestCode == 0 )); then + # If the command succeeded, check that both files are present, and that this + # is the same behavior as `touch` + [[ -e $TEST_ROOT/out/â ]] + [[ -e $TEST_ROOT/out/â ]] + + (( touchFilesCount == 2 )) +else + # if the return code is neither 0 or 1, fail the test. + echo "NAR deserialization of files with the same Unicode normalization failed with unexpected return code $unicodeTestCode" >&2 + exit 1 +fi + +rm -f "$TEST_ROOT/unicode-*" + +# Unpacking a NAR with a NUL character in a file name should fail. +rm -rf "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < nul.nar | grepQuiet "NAR contains invalid file name 'f" + +# Likewise for a '.' filename. +rm -rf "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < dot.nar | grepQuiet "NAR contains invalid file name '.'" + +# Likewise for a '..' filename. +rm -rf "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < dotdot.nar | grepQuiet "NAR contains invalid file name '..'" + +# Likewise for a filename containing a slash. +rm -rf "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < slash.nar | grepQuiet "NAR contains invalid file name 'x/y'" + +# Likewise for an empty filename. +rm -rf "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < empty.nar | grepQuiet "NAR contains invalid file name ''" + +# Test that the 'executable' field cannot come before the 'contents' field. +rm -rf "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < executable-after-contents.nar | grepQuiet "expected tag ')', got 'executable'" + +# Test that the 'name' field cannot come before the 'node' field in a directory entry. +rm -rf "$TEST_ROOT/out" +expectStderr 1 nix-store --restore "$TEST_ROOT/out" < name-after-node.nar | grepQuiet "expected tag 'name'" diff --git a/tests/functional/nested-sandboxing.sh b/tests/functional/nested-sandboxing.sh old mode 100644 new mode 100755 index 61fe043c6..ae0256de2 --- a/tests/functional/nested-sandboxing.sh +++ b/tests/functional/nested-sandboxing.sh @@ -1,7 +1,11 @@ +#!/usr/bin/env bash + source common.sh # This test is run by `tests/functional/nested-sandboxing/runner.nix` in an extra layer of sandboxing. [[ -d /nix/store ]] || skipTest "running this test without Nix's deps being drawn from /nix/store is not yet supported" +TODO_NixOS + requireSandboxSupport source ./nested-sandboxing/command.sh diff --git a/tests/functional/nested-sandboxing/command.sh b/tests/functional/nested-sandboxing/command.sh index 69366486c..e9c40a5d9 100644 --- a/tests/functional/nested-sandboxing/command.sh +++ b/tests/functional/nested-sandboxing/command.sh @@ -1,3 +1,5 @@ +set -eu -o pipefail + export NIX_BIN_DIR=$(dirname $(type -p nix)) # TODO Get Nix and its closure more flexibly export EXTRA_SANDBOX="/nix/store $(dirname $NIX_BIN_DIR)" diff --git a/tests/functional/nested-sandboxing/runner.nix b/tests/functional/nested-sandboxing/runner.nix index 9a5822c88..1e79d5065 100644 --- a/tests/functional/nested-sandboxing/runner.nix +++ b/tests/functional/nested-sandboxing/runner.nix @@ -6,7 +6,10 @@ mkDerivation { name = "nested-sandboxing"; busybox = builtins.getEnv "busybox"; EXTRA_SANDBOX = builtins.getEnv "EXTRA_SANDBOX"; - buildCommand = if altitude == 0 then '' + buildCommand = '' + set -x + set -eu -o pipefail + '' + (if altitude == 0 then '' echo Deep enough! > $out '' else '' cp -r ${../common} ./common @@ -20,5 +23,5 @@ mkDerivation { source ./nested-sandboxing/command.sh runNixBuild ${storeFun} ${toString altitude} >> $out - ''; + ''); } diff --git a/tests/functional/nix-build.sh b/tests/functional/nix-build.sh old mode 100644 new mode 100755 index 44a5a14cd..091e429e0 --- a/tests/functional/nix-build.sh +++ b/tests/functional/nix-build.sh @@ -1,6 +1,10 @@ +#!/usr/bin/env bash + source common.sh -clearStore +TODO_NixOS + +clearStoreIfPossible outPath=$(nix-build dependencies.nix -o $TEST_ROOT/result) test "$(cat $TEST_ROOT/result/foobar)" = FOOBAR diff --git a/tests/functional/nix-channel.sh b/tests/functional/nix-channel.sh old mode 100644 new mode 100755 index ca5df3bdd..a4870e7a8 --- a/tests/functional/nix-channel.sh +++ b/tests/functional/nix-channel.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh clearProfiles diff --git a/tests/functional/nix-collect-garbage-d.sh b/tests/functional/nix-collect-garbage-d.sh old mode 100644 new mode 100755 index bf30f8938..119efe629 --- a/tests/functional/nix-collect-garbage-d.sh +++ b/tests/functional/nix-collect-garbage-d.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore ## Test `nix-collect-garbage -d` diff --git a/tests/functional/nix-copy-ssh-common.sh b/tests/functional/nix-copy-ssh-common.sh index cc8314ff7..5eea9612d 100644 --- a/tests/functional/nix-copy-ssh-common.sh +++ b/tests/functional/nix-copy-ssh-common.sh @@ -2,6 +2,8 @@ proto=$1 shift (( $# == 0 )) +TODO_NixOS + clearStore clearCache diff --git a/tests/functional/nix-copy-ssh-ng.sh b/tests/functional/nix-copy-ssh-ng.sh old mode 100644 new mode 100755 index 62e99cd24..41958c2c3 --- a/tests/functional/nix-copy-ssh-ng.sh +++ b/tests/functional/nix-copy-ssh-ng.sh @@ -1,7 +1,11 @@ +#!/usr/bin/env bash + source common.sh source nix-copy-ssh-common.sh "ssh-ng" +TODO_NixOS + clearStore clearRemoteStore diff --git a/tests/functional/nix-copy-ssh.sh b/tests/functional/nix-copy-ssh.sh old mode 100644 new mode 100755 index 12e8346bc..1dc256e49 --- a/tests/functional/nix-copy-ssh.sh +++ b/tests/functional/nix-copy-ssh.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh source nix-copy-ssh-common.sh "ssh" diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh old mode 100644 new mode 100755 index 7c4da6283..e2f19b99e --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore clearProfiles diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh old mode 100644 new mode 100755 index 04c83138e..b9625eb66 --- a/tests/functional/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible if [[ -n ${CONTENT_ADDRESSED:-} ]]; then shellDotNix="$PWD/ca-shell.nix" @@ -19,6 +21,10 @@ output=$(nix-shell --pure "$shellDotNix" -A shellDrv --run \ [ "$output" = " - foo - bar - true" ] +output=$(nix-shell --pure "$shellDotNix" -A shellDrv --option nix-shell-always-looks-for-shell-nix false --run \ + 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') +[ "$output" = " - foo - bar - true" ] + # Test --keep output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR "$shellDotNix" -A shellDrv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $SELECTED_IMPURE_VAR"') @@ -59,6 +65,25 @@ chmod a+rx $TEST_ROOT/shell.shebang.sh output=$($TEST_ROOT/shell.shebang.sh abc def) [ "$output" = "foo bar abc def" ] +# Test nix-shell shebang mode with an alternate working directory +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.expr > $TEST_ROOT/shell.shebang.expr +chmod a+rx $TEST_ROOT/shell.shebang.expr +# Should fail due to expressions using relative path +! $TEST_ROOT/shell.shebang.expr bar +cp shell.nix config.nix $TEST_ROOT +# Should succeed +echo "cwd: $PWD" +output=$($TEST_ROOT/shell.shebang.expr bar) +[ "$output" = foo ] + +# Test nix-shell shebang mode with an alternate working directory +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.legacy.expr > $TEST_ROOT/shell.shebang.legacy.expr +chmod a+rx $TEST_ROOT/shell.shebang.legacy.expr +# Should fail due to expressions using relative path +mkdir -p "$TEST_ROOT/somewhere-unrelated" +output="$(cd "$TEST_ROOT/somewhere-unrelated"; $TEST_ROOT/shell.shebang.legacy.expr bar;)" +[[ $(realpath "$output") = $(realpath "$TEST_ROOT/somewhere-unrelated") ]] + # Test nix-shell shebang mode again with metacharacters in the filename. # First word of filename is chosen to not match any file in the test root. sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh @@ -89,6 +114,55 @@ sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.nix > $TEST_ROOT/shell.sheba chmod a+rx $TEST_ROOT/shell.shebang.nix $TEST_ROOT/shell.shebang.nix +mkdir $TEST_ROOT/lookup-test $TEST_ROOT/empty + +echo "import $shellDotNix" > $TEST_ROOT/lookup-test/shell.nix +cp config.nix $TEST_ROOT/lookup-test/ +echo 'abort "do not load default.nix!"' > $TEST_ROOT/lookup-test/default.nix + +nix-shell $TEST_ROOT/lookup-test -A shellDrv --run 'echo "it works"' | grepQuiet "it works" +# https://github.com/NixOS/nix/issues/4529 +nix-shell -I "testRoot=$TEST_ROOT" '' -A shellDrv --run 'echo "it works"' | grepQuiet "it works" + +expectStderr 1 nix-shell $TEST_ROOT/lookup-test -A shellDrv --run 'echo "it works"' --option nix-shell-always-looks-for-shell-nix false \ + | grepQuiet -F "do not load default.nix!" # we did, because we chose to enable legacy behavior +expectStderr 1 nix-shell $TEST_ROOT/lookup-test -A shellDrv --run 'echo "it works"' --option nix-shell-always-looks-for-shell-nix false \ + | grepQuiet "Skipping .*lookup-test/shell\.nix.*, because the setting .*nix-shell-always-looks-for-shell-nix.* is disabled. This is a deprecated behavior\. Consider enabling .*nix-shell-always-looks-for-shell-nix.*" + +( + cd $TEST_ROOT/empty; + expectStderr 1 nix-shell | \ + grepQuiet "error.*no argument specified and no .*shell\.nix.* or .*default\.nix.* file found in the working directory" +) + +expectStderr 1 nix-shell -I "testRoot=$TEST_ROOT" '' | + grepQuiet "error.*neither .*shell\.nix.* nor .*default\.nix.* found in .*/empty" + +cat >$TEST_ROOT/lookup-test/shebangscript < $TEST_ROOT/marco/shell.nix +cat >$TEST_ROOT/marco/polo/default.nix <' --restrict-eval [[ $(nix-instantiate --find-file by-absolute-path/simple.nix) = $PWD/simple.nix ]] [[ $(nix-instantiate --find-file by-relative-path/simple.nix) = $PWD/simple.nix ]] + +# this is the human-readable specification for the following test cases of interactions between various ways of specifying NIX_PATH. +# TODO: the actual tests are incomplete and too manual. +# there should be 43 of them, since the table has 9 rows and columns, and 2 interactions are meaningless +# ideally they would work off the table programmatically. +# +# | precedence | hard-coded | nix-path in file | extra-nix-path in file | nix-path in env | extra-nix-path in env | NIX_PATH | nix-path | extra-nix-path | -I | +# |------------------------|------------|------------------|------------------------|-----------------|-----------------------|-----------|-----------|-----------------|-----------------| +# | hard-coded | x | ^override | ^append | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | nix-path in file | | last wins | ^append | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | extra-nix-path in file | | | append in order | ^override | ^append | ^override | ^override | ^append | ^prepend | +# | nix-path in env | | | | last wins | ^append | ^override | ^override | ^append | ^prepend | +# | extra-nix-path in env | | | | | append in order | ^override | ^override | ^append | ^prepend | +# | NIX_PATH | | | | | | x | ^override | ^append | ^prepend | +# | nix-path | | | | | | | last wins | ^append | ^prepend | +# | extra-nix-path | | | | | | | | append in order | append in order | +# | -I | | | | | | | | | append in order | + +unset NIX_PATH + +mkdir -p $TEST_ROOT/{from-nix-path-file,from-NIX_PATH,from-nix-path,from-extra-nix-path,from-I} +for i in from-nix-path-file from-NIX_PATH from-nix-path from-extra-nix-path from-I; do + touch $TEST_ROOT/$i/only-$i.nix +done + +# finding something that's not in any of the default paths fails +( ! $(nix-instantiate --find-file test) ) + +echo "nix-path = test=$TEST_ROOT/from-nix-path-file" >> "$test_nix_conf" + +# Use nix.conf in absence of NIX_PATH +[[ $(nix-instantiate --find-file test) = $TEST_ROOT/from-nix-path-file ]] + +# NIX_PATH overrides nix.conf +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --find-file test) = $TEST_ROOT/from-NIX_PATH ]] +# if NIX_PATH does not have the desired entry, it fails +(! NIX_PATH=test=$TEST_ROOT nix-instantiate --find-file test/only-from-nix-path-file.nix) + +# -I extends nix.conf +[[ $(nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +# if -I does not have the desired entry, the value from nix.conf is used +[[ $(nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-nix-path-file.nix) = $TEST_ROOT/from-nix-path-file/only-from-nix-path-file.nix ]] + +# -I extends NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +# -I takes precedence over NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test) = $TEST_ROOT/from-I ]] +# if -I does not have the desired entry, the value from NIX_PATH is used +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] + +# --extra-nix-path extends NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-extra-nix-path.nix) = $TEST_ROOT/from-extra-nix-path/only-from-extra-nix-path.nix ]] +# if --extra-nix-path does not have the desired entry, the value from NIX_PATH is used +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] + +# --nix-path overrides NIX_PATH +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test) = $TEST_ROOT/from-nix-path ]] +# if --nix-path does not have the desired entry, it fails +(! NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test/only-from-NIX_PATH.nix) + +# --nix-path overrides nix.conf +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test) = $TEST_ROOT/from-nix-path ]] +(! nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test/only-from-nix-path-file.nix) + +# --extra-nix-path extends nix.conf +[[ $(nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-extra-nix-path.nix) = $TEST_ROOT/from-extra-nix-path/only-from-extra-nix-path.nix ]] +# if --extra-nix-path does not have the desired entry, it is taken from nix.conf +[[ $(nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test) = $TEST_ROOT/from-nix-path-file ]] + +# -I extends --nix-path +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path -I test=$TEST_ROOT/from-I --find-file test/only-from-nix-path.nix) = $TEST_ROOT/from-nix-path/only-from-nix-path.nix ]] diff --git a/tests/functional/nul.nar b/tests/functional/nul.nar new file mode 100644 index 000000000..9ae48baf6 Binary files /dev/null and b/tests/functional/nul.nar differ diff --git a/tests/functional/optimise-store.sh b/tests/functional/optimise-store.sh old mode 100644 new mode 100755 index 8c2d05cd5..0bedafc43 --- a/tests/functional/optimise-store.sh +++ b/tests/functional/optimise-store.sh @@ -1,10 +1,15 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible outPath1=$(echo 'with import ./config.nix; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) outPath2=$(echo 'with import ./config.nix; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) +TODO_NixOS # ignoring the client-specified setting 'auto-optimise-store', because it is a restricted setting and you are not a trusted user + # TODO: only continue when trusted user or root + inode1="$(stat --format=%i $outPath1/foo)" inode2="$(stat --format=%i $outPath2/foo)" if [ "$inode1" != "$inode2" ]; then diff --git a/tests/functional/output-normalization.sh b/tests/functional/output-normalization.sh old mode 100644 new mode 100755 index 0f6df5e31..c55f1b1d1 --- a/tests/functional/output-normalization.sh +++ b/tests/functional/output-normalization.sh @@ -1,6 +1,9 @@ +#!/usr/bin/env bash + source common.sh testNormalization () { + TODO_NixOS clearStore outPath=$(nix-build ./simple.nix --no-out-link) test "$(stat -c %Y $outPath)" -eq 1 diff --git a/tests/functional/package.nix b/tests/functional/package.nix new file mode 100644 index 000000000..a0c1f249f --- /dev/null +++ b/tests/functional/package.nix @@ -0,0 +1,110 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config +, rsync + +, jq +, git +, mercurial +, util-linux + +, nix-store +, nix-expr +, nix-cli + +, rapidcheck +, gtest +, runCommand + +, busybox-sandbox-shell ? null + +# Configuration Options + +, version + +# For running the functional tests against a different pre-built Nix. +, test-daemon ? null +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-functional-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../scripts/nix-profile.sh.in + ../../.version + ../../tests/functional + ./. + ]; + + # Hack for sake of the dev shell + passthru.externalNativeBuildInputs = [ + meson + ninja + pkg-config + rsync + + jq + git + mercurial + ] ++ lib.optionals stdenv.hostPlatform.isLinux [ + # For various sandboxing tests that needs a statically-linked shell, + # etc. + busybox-sandbox-shell + # For Overlay FS tests need `mount`, `umount`, and `unshare`. + # TODO use `unixtools` to be precise over which executables instead? + util-linux + ]; + + nativeBuildInputs = finalAttrs.passthru.externalNativeBuildInputs ++ [ + nix-cli + ]; + + buildInputs = [ + nix-store + nix-expr + ]; + + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + '' + # TEMP hack for Meson before make is gone, where + # `src/nix-functional-tests` is during the transition a symlink and + # not the actual directory directory. + + '' + cd $(readlink -e $PWD) + echo $PWD | grep tests/functional + ''; + + mesonCheckFlags = [ + "--print-errorlogs" + ]; + + doCheck = true; + + installPhase = '' + mkdir $out + ''; + + meta = { + platforms = lib.platforms.unix; + }; + +} // lib.optionalAttrs (test-daemon != null) { + NIX_DAEMON_PACKAGE = test-daemon; +}) diff --git a/tests/functional/parallel.sh b/tests/functional/parallel.sh index 3b7bbe5a2..7e420688d 100644 --- a/tests/functional/parallel.sh +++ b/tests/functional/parallel.sh @@ -4,6 +4,8 @@ source common.sh # First, test that -jN performs builds in parallel. echo "testing nix-build -j..." +TODO_NixOS + clearStore rm -f $_NIX_TEST_SHARED.cur $_NIX_TEST_SHARED.max diff --git a/tests/functional/pass-as-file.sh b/tests/functional/pass-as-file.sh old mode 100644 new mode 100755 index 2c0bc5031..6487bfffd --- a/tests/functional/pass-as-file.sh +++ b/tests/functional/pass-as-file.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible outPath=$(nix-build --no-out-link -E " with import ./config.nix; diff --git a/tests/functional/path-from-hash-part.sh b/tests/functional/path-from-hash-part.sh old mode 100644 new mode 100755 index bdd104434..41d1b7410 --- a/tests/functional/path-from-hash-part.sh +++ b/tests/functional/path-from-hash-part.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh path=$(nix build --no-link --print-out-paths -f simple.nix) diff --git a/tests/functional/path-info.sh b/tests/functional/path-info.sh old mode 100644 new mode 100755 index 763935eb7..8597de683 --- a/tests/functional/path-info.sh +++ b/tests/functional/path-info.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh echo foo > $TEST_ROOT/foo diff --git a/tests/functional/placeholders.sh b/tests/functional/placeholders.sh old mode 100644 new mode 100755 index cd1bb7bc2..33ec0c2b7 --- a/tests/functional/placeholders.sh +++ b/tests/functional/placeholders.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible nix-build --no-out-link -E ' with import ./config.nix; diff --git a/tests/functional/plugins.sh b/tests/functional/plugins.sh old mode 100644 new mode 100755 index baf71a362..fc2d1907c --- a/tests/functional/plugins.sh +++ b/tests/functional/plugins.sh @@ -1,9 +1,12 @@ +#!/usr/bin/env bash + source common.sh -if [[ $BUILD_SHARED_LIBS != 1 ]]; then - skipTest "Plugins are not supported" -fi +for ext in so dylib; do + plugin="$PWD/plugins/libplugintest.$ext" + [[ -f "$plugin" ]] && break +done -res=$(nix --option setting-set true --option plugin-files $PWD/plugins/libplugintest* eval --expr builtins.anotherNull) +res=$(nix --option setting-set true --option plugin-files "$plugin" eval --expr builtins.anotherNull) [ "$res"x = "nullx" ] diff --git a/tests/functional/plugins/meson.build b/tests/functional/plugins/meson.build new file mode 100644 index 000000000..3d6b2f0e1 --- /dev/null +++ b/tests/functional/plugins/meson.build @@ -0,0 +1,16 @@ +libplugintest = shared_module( + 'plugintest', + 'plugintest.cc', + cpp_args : [ + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.hh', + '-include', 'config-expr.hh', + ], + dependencies : [ + dependency('nix-expr'), + ], + build_by_default : false, +) diff --git a/tests/functional/plugins/plugintest.cc b/tests/functional/plugins/plugintest.cc index e02fd68d5..7433ad190 100644 --- a/tests/functional/plugins/plugintest.cc +++ b/tests/functional/plugins/plugintest.cc @@ -1,4 +1,4 @@ -#include "config.hh" +#include "config-global.hh" #include "primops.hh" using namespace nix; diff --git a/tests/functional/post-hook.sh b/tests/functional/post-hook.sh old mode 100644 new mode 100755 index 752f8220c..94a6d0d69 --- a/tests/functional/post-hook.sh +++ b/tests/functional/post-hook.sh @@ -1,11 +1,15 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore rm -f $TEST_ROOT/result export REMOTE_STORE=file:$TEST_ROOT/remote_store -echo 'require-sigs = false' >> $NIX_CONF_DIR/nix.conf +echo 'require-sigs = false' >> $test_nix_conf restartDaemon diff --git a/tests/functional/pure-eval.sh b/tests/functional/pure-eval.sh old mode 100644 new mode 100755 index 5334bf28e..250381099 --- a/tests/functional/pure-eval.sh +++ b/tests/functional/pure-eval.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible nix eval --expr 'assert 1 + 2 == 3; true' diff --git a/tests/functional/read-only-store.sh b/tests/functional/read-only-store.sh old mode 100644 new mode 100755 index d63920c19..f6b6eaf32 --- a/tests/functional/read-only-store.sh +++ b/tests/functional/read-only-store.sh @@ -1,15 +1,22 @@ +#!/usr/bin/env bash + source common.sh enableFeatures "read-only-local-store" needLocalStore "cannot open store read-only when daemon has already opened it writeable" +TODO_NixOS + clearStore happy () { # We can do a read-only query just fine with a read-only store nix --store local?read-only=true path-info $dummyPath - + + # `local://` also works. + nix --store local://?read-only=true path-info $dummyPath + # We can "write" an already-present store-path a read-only store, because no IO is actually required nix-store --store local?read-only=true --add dummy } diff --git a/tests/functional/readfile-context.sh b/tests/functional/readfile-context.sh old mode 100644 new mode 100755 index 31e70ddb1..cb9ef6234 --- a/tests/functional/readfile-context.sh +++ b/tests/functional/readfile-context.sh @@ -1,11 +1,15 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS # NixOS doesn't provide $NIX_STATE_DIR (and shouldn't) + clearStore outPath=$(nix-build --no-out-link readfile-context.nix) # Set a GC root. -ln -s $outPath "$NIX_STATE_DIR"/gcroots/foo +ln -s $outPath "$NIX_STATE_DIR/gcroots/foo" # Check that file exists. [ "$(cat $(cat $outPath))" = "Hello World!" ] diff --git a/tests/functional/recursive.sh b/tests/functional/recursive.sh old mode 100644 new mode 100755 index 0bf00f8fa..640fb92d2 --- a/tests/functional/recursive.sh +++ b/tests/functional/recursive.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS # can't enable a sandbox feature easily + enableFeatures 'recursive-nix' restartDaemon diff --git a/tests/functional/referrers.sh b/tests/functional/referrers.sh old mode 100644 new mode 100755 index 81323c280..411cdb7c1 --- a/tests/functional/referrers.sh +++ b/tests/functional/referrers.sh @@ -1,7 +1,11 @@ +#!/usr/bin/env bash + source common.sh needLocalStore "uses some low-level store manipulations that aren’t available through the daemon" +TODO_NixOS + clearStore max=500 @@ -29,7 +33,7 @@ echo "registering..." nix-store --register-validity < $TEST_ROOT/reg_info echo "collecting garbage..." -ln -sfn $reference "$NIX_STATE_DIR"/gcroots/ref +ln -sfn $reference "$NIX_STATE_DIR/gcroots/ref" nix-store --gc if [ -n "$(type -p sqlite3)" -a "$(sqlite3 $NIX_STATE_DIR/db/db.sqlite 'select count(*) from Refs')" -ne 0 ]; then diff --git a/tests/functional/remote-store.sh b/tests/functional/remote-store.sh old mode 100644 new mode 100755 index e2c16f18a..841b6b27a --- a/tests/functional/remote-store.sh +++ b/tests/functional/remote-store.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore # Ensure "fake ssh" remote store works just as legacy fake ssh would. diff --git a/tests/functional/repair.sh b/tests/functional/repair.sh old mode 100644 new mode 100755 index c8f07b1c6..1f6004b2c --- a/tests/functional/repair.sh +++ b/tests/functional/repair.sh @@ -1,7 +1,11 @@ +#!/usr/bin/env bash + source common.sh needLocalStore "--repair needs a local store" +TODO_NixOS + clearStore path=$(nix-build dependencies.nix -o $TEST_ROOT/result) diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh old mode 100644 new mode 100755 index f11fa7140..706e0f5db --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -1,4 +1,7 @@ +#!/usr/bin/env bash + source common.sh +source characterisation/framework.sh testDir="$PWD" cd "$TEST_ROOT" @@ -20,12 +23,17 @@ replUndefinedVariable=" import $testDir/undefined-variable.nix " +TODO_NixOS + testRepl () { - local nixArgs=("$@") + local nixArgs + nixArgs=("$@") rm -rf repl-result-out || true # cleanup from other runs backed by a foreign nix store - local replOutput="$(nix repl "${nixArgs[@]}" <<< "$replCmds")" + local replOutput + replOutput="$(nix repl "${nixArgs[@]}" <<< "$replCmds")" echo "$replOutput" - local outPath=$(echo "$replOutput" |& + local outPath + outPath=$(echo "$replOutput" |& grep -o -E "$NIX_STORE_DIR/\w*-simple") nix path-info "${nixArgs[@]}" "$outPath" [ "$(realpath ./repl-result-out)" == "$outPath" ] || fail "nix repl :bl doesn't make a symlink" @@ -34,11 +42,11 @@ testRepl () { # simple.nix prints a PATH during build echo "$replOutput" | grepQuiet -s 'PATH=' || fail "nix repl :log doesn't output logs" - local replOutput="$(nix repl "${nixArgs[@]}" <<< "$replFailingCmds" 2>&1)" + replOutput="$(nix repl "${nixArgs[@]}" <<< "$replFailingCmds" 2>&1)" echo "$replOutput" echo "$replOutput" | grepQuiet -s 'This should fail' \ || fail "nix repl :log doesn't output logs for a failed derivation" - local replOutput="$(nix repl --show-trace "${nixArgs[@]}" <<< "$replUndefinedVariable" 2>&1)" + replOutput="$(nix repl --show-trace "${nixArgs[@]}" <<< "$replUndefinedVariable" 2>&1)" echo "$replOutput" echo "$replOutput" | grepQuiet -s "while evaluating the file" \ || fail "nix repl --show-trace doesn't show the trace" @@ -48,7 +56,7 @@ testRepl () { nix repl "${nixArgs[@]}" 2>&1 <<< "builtins.currentSystem" \ | grep "$(nix-instantiate --eval -E 'builtins.currentSystem')" - expectStderr 1 nix repl ${testDir}/simple.nix \ + expectStderr 1 nix repl "${testDir}/simple.nix" \ | grepQuiet -s "error: path '$testDir/simple.nix' is not a flake" } @@ -63,23 +71,34 @@ stripColors () { } testReplResponseGeneral () { - local grepMode="$1"; shift - local commands="$1"; shift - local expectedResponse="$1"; shift - local response="$(nix repl "$@" <<< "$commands" | stripColors)" - echo "$response" | grepQuiet "$grepMode" -s "$expectedResponse" \ - || fail "repl command set: + local grepMode commands expectedResponse response + grepMode="$1"; shift + commands="$1"; shift + # Expected response can contain newlines. + # grep can't handle multiline patterns, so replace newlines with TEST_NEWLINE + # in both expectedResponse and response. + # awk ORS always adds a trailing record separator, so we strip it with sed. + expectedResponse="$(printf '%s' "$1" | awk 1 ORS=TEST_NEWLINE | sed 's/TEST_NEWLINE$//')"; shift + # We don't need to strip trailing record separator here, since extra data is ok. + response="$(nix repl "$@" <<< "$commands" 2>&1 | stripColors | awk 1 ORS=TEST_NEWLINE)" + printf '%s' "$response" | grepQuiet "$grepMode" -s "$expectedResponse" \ + || fail "$(echo "repl command set: $commands does not respond with: +--- $expectedResponse +--- but with: +--- $response -" +--- + +" | sed 's/TEST_NEWLINE/\n/g')" } testReplResponse () { @@ -91,6 +110,8 @@ testReplResponseNoRegex () { } # :a uses the newest version of a symbol +# +# shellcheck disable=SC2016 testReplResponse ' :a { a = "1"; } :a { a = "2"; } @@ -101,6 +122,8 @@ testReplResponse ' # note the escaped \, # \\ # because the second argument is a regex +# +# shellcheck disable=SC2016 testReplResponseNoRegex ' "$" + "{hi}" ' '"\${hi}"' @@ -108,12 +131,12 @@ testReplResponseNoRegex ' testReplResponse ' drvPath ' '".*-simple.drv"' \ ---file $testDir/simple.nix +--file "$testDir/simple.nix" testReplResponse ' drvPath ' '".*-simple.drv"' \ ---file $testDir/simple.nix --experimental-features 'ca-derivations' +--file "$testDir/simple.nix" --experimental-features 'ca-derivations' mkdir -p flake && cat < flake/flake.nix { @@ -177,7 +200,7 @@ testReplResponseNoRegex ' let x = { y = { a = 1; }; inherit x; }; in x ' \ '{ - x = { ... }; + x = «repeated»; y = { ... }; } ' @@ -229,6 +252,70 @@ testReplResponseNoRegex ' ' \ '{ x = «repeated»; - y = { a = 1 }; + y = { a = 1; }; } ' + +# TODO: move init to characterisation/framework.sh +badDiff=0 +badExitCode=0 + +nixVersion="$(nix eval --impure --raw --expr 'builtins.nixVersion' --extra-experimental-features nix-command)" + +# TODO: write a repl interacter for testing. Papering over the differences between readline / editline and between platforms is a pain. + +# I couldn't get readline and editline to agree on the newline before the prompt, +# so let's just force it to be one empty line. +stripEmptyLinesBeforePrompt() { + # --null-data: treat input as NUL-terminated instead of newline-terminated + sed --null-data 's/\n\n*nix-repl>/\n\nnix-repl>/g' +} + +# We don't get a final prompt on darwin, so we strip this as well. +stripFinalPrompt() { + # Strip the final prompt and/or any trailing spaces + sed --null-data \ + -e 's/\(.*[^\n]\)\n\n*nix-repl>[ \n]*$/\1/' \ + -e 's/[ \n]*$/\n/' +} + +runRepl () { + + # That is right, we are also filtering out the testdir _without underscores_. + # This is crazy, but without it, GHA will fail to run the tests, showing paths + # _with_ underscores in the set -x log, but _without_ underscores in the + # supposed nix repl output. I have looked in a number of places, but I cannot + # find a mechanism that could cause this to happen. + local testDirNoUnderscores + testDirNoUnderscores="${testDir//_/}" + + # TODO: pass arguments to nix repl; see lang.sh + _NIX_TEST_RAW_MARKDOWN=1 \ + _NIX_TEST_REPL_ECHO=1 \ + nix repl 2>&1 \ + | stripColors \ + | tr -d '\0' \ + | stripEmptyLinesBeforePrompt \ + | stripFinalPrompt \ + | sed \ + -e "s@$testDir@/path/to/tests/functional@g" \ + -e "s@$testDirNoUnderscores@/path/to/tests/functional@g" \ + -e "s@$nixVersion@@g" \ + -e "s@Added [0-9]* variables@Added variables@g" \ + | grep -vF $'warning: you don\'t have Internet access; disabling some network-dependent features' \ + ; +} + +for test in $(cd "$testDir/repl"; echo *.in); do + test="$(basename "$test" .in)" + in="$testDir/repl/$test.in" + actual="$testDir/repl/$test.actual" + expected="$testDir/repl/$test.expected" + (cd "$testDir/repl"; set +x; runRepl 2>&1) < "$in" > "$actual" || { + echo "FAIL: $test (exit code $?)" >&2 + badExitCode=1 + } + diffAndAcceptInner "$test" "$actual" "$expected" +done + +characterisationTestExit diff --git a/tests/functional/repl/characterisation/empty b/tests/functional/repl/characterisation/empty new file mode 100644 index 000000000..e69de29bb diff --git a/tests/functional/repl/doc-comment-curried-args.expected b/tests/functional/repl/doc-comment-curried-args.expected new file mode 100644 index 000000000..56607e911 --- /dev/null +++ b/tests/functional/repl/doc-comment-curried-args.expected @@ -0,0 +1,28 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> :doc curriedArgs +Function `curriedArgs`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:48:5 + +A documented function. + +nix-repl> x = curriedArgs 1 + +nix-repl> "Note that users may not expect this to behave as it currently does" +"Note that users may not expect this to behave as it currently does" + +nix-repl> :doc x +Function `curriedArgs`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:50:5 + +The function returned by applying once + +nix-repl> "This won't produce docs; no support for arbitrary values" +"This won't produce docs; no support for arbitrary values" + +nix-repl> :doc x 2 +error: value does not have documentation diff --git a/tests/functional/repl/doc-comment-curried-args.in b/tests/functional/repl/doc-comment-curried-args.in new file mode 100644 index 000000000..06ba21dcc --- /dev/null +++ b/tests/functional/repl/doc-comment-curried-args.in @@ -0,0 +1,7 @@ +:l doc-comments.nix +:doc curriedArgs +x = curriedArgs 1 +"Note that users may not expect this to behave as it currently does" +:doc x +"This won't produce docs; no support for arbitrary values" +:doc x 2 diff --git a/tests/functional/repl/doc-comment-formals.expected b/tests/functional/repl/doc-comment-formals.expected new file mode 100644 index 000000000..1024919f4 --- /dev/null +++ b/tests/functional/repl/doc-comment-formals.expected @@ -0,0 +1,14 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> "Note that this is not yet complete" +"Note that this is not yet complete" + +nix-repl> :doc documentedFormals +Function `documentedFormals`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:57:5 + +Finds x diff --git a/tests/functional/repl/doc-comment-formals.in b/tests/functional/repl/doc-comment-formals.in new file mode 100644 index 000000000..e32fb8ab1 --- /dev/null +++ b/tests/functional/repl/doc-comment-formals.in @@ -0,0 +1,3 @@ +:l doc-comments.nix +"Note that this is not yet complete" +:doc documentedFormals diff --git a/tests/functional/repl/doc-comment-function.expected b/tests/functional/repl/doc-comment-function.expected new file mode 100644 index 000000000..3889c4f78 --- /dev/null +++ b/tests/functional/repl/doc-comment-function.expected @@ -0,0 +1,7 @@ +Nix +Type :? for help. + +nix-repl> :doc import ./doc-comment-function.nix +Function defined at /path/to/tests/functional/repl/doc-comment-function.nix:2:1 + +A doc comment for a file that only contains a function diff --git a/tests/functional/repl/doc-comment-function.in b/tests/functional/repl/doc-comment-function.in new file mode 100644 index 000000000..8f3c1388a --- /dev/null +++ b/tests/functional/repl/doc-comment-function.in @@ -0,0 +1 @@ +:doc import ./doc-comment-function.nix diff --git a/tests/functional/repl/doc-comment-function.nix b/tests/functional/repl/doc-comment-function.nix new file mode 100644 index 000000000..cdd241347 --- /dev/null +++ b/tests/functional/repl/doc-comment-function.nix @@ -0,0 +1,3 @@ +/** A doc comment for a file that only contains a function */ +{ ... }: +{ } diff --git a/tests/functional/repl/doc-comments.nix b/tests/functional/repl/doc-comments.nix new file mode 100644 index 000000000..e91ee0b51 --- /dev/null +++ b/tests/functional/repl/doc-comments.nix @@ -0,0 +1,60 @@ +{ + /** + Perform *arithmetic* multiplication. It's kind of like repeated **addition**, very neat. + + ```nix + multiply 2 3 + => 6 + ``` + */ + multiply = x: y: x * y; + + /**👈 precisely this wide 👉*/ + measurement = x: x; + + floatedIn = /** This also works. */ + x: y: x; + + compact=/**boom*/x: x; + + # https://github.com/NixOS/rfcs/blob/master/rfcs/0145-doc-strings.md#ambiguous-placement + /** Ignore!!! */ + unambiguous = + /** Very close */ + x: x; + + /** Firmly rigid. */ + constant = true; + + /** Immovably fixed. */ + lib.version = "9000"; + + /** Unchangeably constant. */ + lib.attr.empty = { }; + + lib.attr.undocumented = { }; + + nonStrict = /** My syntax is not strict, but I'm strict anyway. */ x: x; + strict = /** I don't have to be strict, but I am anyway. */ { ... }: null; + # Note that pre and post are the same here. I just had to name them somehow. + strictPre = /** Here's one way to do this */ a@{ ... }: a; + strictPost = /** Here's another way to do this */ { ... }@a: a; + + # TODO + + /** You won't see this. */ + curriedArgs = + /** A documented function. */ + x: + /** The function returned by applying once */ + y: + /** A function body performing summation of two items */ + x + y; + + /** Documented formals (but you won't see this comment) */ + documentedFormals = + /** Finds x */ + { /** The x attribute */ + x + }: x; +} diff --git a/tests/functional/repl/doc-compact.expected b/tests/functional/repl/doc-compact.expected new file mode 100644 index 000000000..79f1fd44f --- /dev/null +++ b/tests/functional/repl/doc-compact.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> :doc compact +Function `compact`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:18:20 + +boom diff --git a/tests/functional/repl/doc-compact.in b/tests/functional/repl/doc-compact.in new file mode 100644 index 000000000..c87c4e7ab --- /dev/null +++ b/tests/functional/repl/doc-compact.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc compact diff --git a/tests/functional/repl/doc-constant.expected b/tests/functional/repl/doc-constant.expected new file mode 100644 index 000000000..5787e04dc --- /dev/null +++ b/tests/functional/repl/doc-constant.expected @@ -0,0 +1,110 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> :doc constant +error: value does not have documentation + +nix-repl> :doc lib.version +Attribute `version` + + … defined at /path/to/tests/functional/repl/doc-comments.nix:30:3 + +Immovably fixed. + +nix-repl> :doc lib.attr.empty +Attribute `empty` + + … defined at /path/to/tests/functional/repl/doc-comments.nix:33:3 + +Unchangeably constant. + +nix-repl> :doc lib.attr.undocument +error: + … while evaluating the attribute 'attr.undocument' + at /path/to/tests/functional/repl/doc-comments.nix:33:3: + 32| /** Unchangeably constant. */ + 33| lib.attr.empty = { }; + | ^ + 34| + + error: attribute 'undocument' missing + at «string»:1:1: + 1| lib.attr.undocument + | ^ + Did you mean undocumented? + +nix-repl> :doc (import ./doc-comments.nix).constant +Attribute `constant` + + … defined at /path/to/tests/functional/repl/doc-comments.nix:27:3 + +Firmly rigid. + +nix-repl> :doc (import ./doc-comments.nix).lib.version +Attribute `version` + + … defined at /path/to/tests/functional/repl/doc-comments.nix:30:3 + +Immovably fixed. + +nix-repl> :doc (import ./doc-comments.nix).lib.attr.empty +Attribute `empty` + + … defined at /path/to/tests/functional/repl/doc-comments.nix:33:3 + +Unchangeably constant. + +nix-repl> :doc (import ./doc-comments.nix).lib.attr.undocumented +Attribute `undocumented` + + … defined at /path/to/tests/functional/repl/doc-comments.nix:35:3 + +No documentation found. + +nix-repl> :doc missing +error: undefined variable 'missing' + at «string»:1:1: + 1| missing + | ^ + +nix-repl> :doc constanz +error: undefined variable 'constanz' + at «string»:1:1: + 1| constanz + | ^ + +nix-repl> :doc missing.attr +error: undefined variable 'missing' + at «string»:1:1: + 1| missing.attr + | ^ + +nix-repl> :doc lib.missing +error: attribute 'missing' missing + at «string»:1:1: + 1| lib.missing + | ^ + +nix-repl> :doc lib.missing.attr +error: attribute 'missing' missing + at «string»:1:1: + 1| lib.missing.attr + | ^ + +nix-repl> :doc lib.attr.undocumental +error: + … while evaluating the attribute 'attr.undocumental' + at /path/to/tests/functional/repl/doc-comments.nix:33:3: + 32| /** Unchangeably constant. */ + 33| lib.attr.empty = { }; + | ^ + 34| + + error: attribute 'undocumental' missing + at «string»:1:1: + 1| lib.attr.undocumental + | ^ + Did you mean undocumented? diff --git a/tests/functional/repl/doc-constant.in b/tests/functional/repl/doc-constant.in new file mode 100644 index 000000000..9c0dde5e1 --- /dev/null +++ b/tests/functional/repl/doc-constant.in @@ -0,0 +1,15 @@ +:l doc-comments.nix +:doc constant +:doc lib.version +:doc lib.attr.empty +:doc lib.attr.undocument +:doc (import ./doc-comments.nix).constant +:doc (import ./doc-comments.nix).lib.version +:doc (import ./doc-comments.nix).lib.attr.empty +:doc (import ./doc-comments.nix).lib.attr.undocumented +:doc missing +:doc constanz +:doc missing.attr +:doc lib.missing +:doc lib.missing.attr +:doc lib.attr.undocumental diff --git a/tests/functional/repl/doc-floatedIn.expected b/tests/functional/repl/doc-floatedIn.expected new file mode 100644 index 000000000..82bb80b95 --- /dev/null +++ b/tests/functional/repl/doc-floatedIn.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> :doc floatedIn +Function `floatedIn`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:16:5 + +This also works. diff --git a/tests/functional/repl/doc-floatedIn.in b/tests/functional/repl/doc-floatedIn.in new file mode 100644 index 000000000..97c12408e --- /dev/null +++ b/tests/functional/repl/doc-floatedIn.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc floatedIn diff --git a/tests/functional/repl/doc-functor.expected b/tests/functional/repl/doc-functor.expected new file mode 100644 index 000000000..8cb2706ef --- /dev/null +++ b/tests/functional/repl/doc-functor.expected @@ -0,0 +1,101 @@ +Nix +Type :? for help. + +nix-repl> :l doc-functor.nix +Added variables. + +nix-repl> :doc multiplier +Function `__functor`\ + … defined at /path/to/tests/functional/repl/doc-functor.nix:12:23 + + +Multiply the argument by the factor stored in the factor attribute. + +nix-repl> :doc doubler +Function `multiply`\ + … defined at /path/to/tests/functional/repl/doc-functor.nix:5:17 + + +Look, it's just like a function! + +nix-repl> :doc recursive +Function `__functor`\ + … defined at /path/to/tests/functional/repl/doc-functor.nix:77:23 + + +This looks bad, but the docs are ok because of the eta expansion. + +nix-repl> :doc recursive2 +error: + … while partially calling '__functor' to retrieve documentation + + … while calling '__functor' + at /path/to/tests/functional/repl/doc-functor.nix:85:17: + 84| */ + 85| __functor = self: self.__functor self; + | ^ + 86| }; + + … from call site + at /path/to/tests/functional/repl/doc-functor.nix:85:23: + 84| */ + 85| __functor = self: self.__functor self; + | ^ + 86| }; + + (19999 duplicate frames omitted) + + error: stack overflow; max-call-depth exceeded + at /path/to/tests/functional/repl/doc-functor.nix:85:23: + 84| */ + 85| __functor = self: self.__functor self; + | ^ + 86| }; + +nix-repl> :doc diverging +error: + … while partially calling '__functor' to retrieve documentation + + (10000 duplicate frames omitted) + + … while calling '__functor' + at /path/to/tests/functional/repl/doc-functor.nix:97:19: + 96| f = x: { + 97| __functor = self: (f (x + 1)); + | ^ + 98| }; + + error: stack overflow; max-call-depth exceeded + at /path/to/tests/functional/repl/doc-functor.nix:97:26: + 96| f = x: { + 97| __functor = self: (f (x + 1)); + | ^ + 98| }; + +nix-repl> :doc helper +Function `square`\ + … defined at /path/to/tests/functional/repl/doc-functor.nix:36:12 + + +Compute x^2 + +nix-repl> :doc helper2 +Function `__functor`\ + … defined at /path/to/tests/functional/repl/doc-functor.nix:45:23 + + +This is a function that can be overridden. + +nix-repl> :doc lib.helper3 +Function `__functor`\ + … defined at /path/to/tests/functional/repl/doc-functor.nix:45:23 + + +This is a function that can be overridden. + +nix-repl> :doc helper3 +Function `__functor`\ + … defined at /path/to/tests/functional/repl/doc-functor.nix:45:23 + + +This is a function that can be overridden. diff --git a/tests/functional/repl/doc-functor.in b/tests/functional/repl/doc-functor.in new file mode 100644 index 000000000..d2bb57a02 --- /dev/null +++ b/tests/functional/repl/doc-functor.in @@ -0,0 +1,10 @@ +:l doc-functor.nix +:doc multiplier +:doc doubler +:doc recursive +:doc recursive2 +:doc diverging +:doc helper +:doc helper2 +:doc lib.helper3 +:doc helper3 diff --git a/tests/functional/repl/doc-functor.nix b/tests/functional/repl/doc-functor.nix new file mode 100644 index 000000000..f526f453f --- /dev/null +++ b/tests/functional/repl/doc-functor.nix @@ -0,0 +1,101 @@ +rec { + /** + Look, it's just like a function! + */ + multiply = p: q: p * q; + + multiplier = { + factor = 2; + /** + Multiply the argument by the factor stored in the factor attribute. + */ + __functor = self: x: x * self.factor; + }; + + doubler = { + description = "bla"; + /** + Multiply by two. This doc probably won't be rendered because the + returned partial application won't have any reference to this location; + only pointing to the second lambda in the multiply function. + */ + __functor = self: multiply 2; + }; + + makeOverridable = f: { + /** + This is a function that can be overridden. + */ + __functor = self: f; + override = throw "not implemented"; + }; + + /** + Compute x^2 + */ + square = x: x * x; + + helper = makeOverridable square; + + # Somewhat analogous to the Nixpkgs makeOverridable function. + makeVeryOverridable = f: { + /** + This is a function that can be overridden. + */ + __functor = self: arg: f arg // { override = throw "not implemented"; overrideAttrs = throw "not implemented"; }; + override = throw "not implemented"; + }; + + helper2 = makeVeryOverridable square; + + # The RFC might be ambiguous here. The doc comment from makeVeryOverridable + # is "inner" in terms of values, but not inner in terms of expressions. + # Returning the following attribute comment might be allowed. + # TODO: I suppose we could look whether the attribute value expression + # contains a doc, and if not, return the attribute comment anyway? + + /** + Compute x^3 + */ + lib.helper3 = makeVeryOverridable (x: x * x * x); + + /** + Compute x^3... + */ + helper3 = makeVeryOverridable (x: x * x * x); + + + # ------ + + # getDoc traverses a potentially infinite structure in case of __functor, so + # we need to test with recursive inputs and diverging inputs. + + recursive = { + /** + This looks bad, but the docs are ok because of the eta expansion. + */ + __functor = self: x: self x; + }; + + recursive2 = { + /** + Docs probably won't work in this case, because the "partial" application + of self results in an infinite recursion. + */ + __functor = self: self.__functor self; + }; + + diverging = let + /** + Docs probably won't work in this case, because the "partial" application + of self results in an diverging computation that causes a stack overflow. + It's not an infinite recursion because each call is different. + This must be handled by the documentation retrieval logic, as it + reimplements the __functor invocation to be partial. + */ + f = x: { + __functor = self: (f (x + 1)); + }; + in f null; + +} diff --git a/tests/functional/repl/doc-lambda-flavors.expected b/tests/functional/repl/doc-lambda-flavors.expected new file mode 100644 index 000000000..ab5c95639 --- /dev/null +++ b/tests/functional/repl/doc-lambda-flavors.expected @@ -0,0 +1,29 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> :doc nonStrict +Function `nonStrict`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:37:70 + +My syntax is not strict, but I'm strict anyway. + +nix-repl> :doc strict +Function `strict`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:38:63 + +I don't have to be strict, but I am anyway. + +nix-repl> :doc strictPre +Function `strictPre`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:40:48 + +Here's one way to do this + +nix-repl> :doc strictPost +Function `strictPost`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:41:53 + +Here's another way to do this diff --git a/tests/functional/repl/doc-lambda-flavors.in b/tests/functional/repl/doc-lambda-flavors.in new file mode 100644 index 000000000..760c99636 --- /dev/null +++ b/tests/functional/repl/doc-lambda-flavors.in @@ -0,0 +1,5 @@ +:l doc-comments.nix +:doc nonStrict +:doc strict +:doc strictPre +:doc strictPost diff --git a/tests/functional/repl/doc-measurement.expected b/tests/functional/repl/doc-measurement.expected new file mode 100644 index 000000000..555cac9a2 --- /dev/null +++ b/tests/functional/repl/doc-measurement.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> :doc measurement +Function `measurement`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:13:17 + +👈 precisely this wide 👉 diff --git a/tests/functional/repl/doc-measurement.in b/tests/functional/repl/doc-measurement.in new file mode 100644 index 000000000..fecd5f9d2 --- /dev/null +++ b/tests/functional/repl/doc-measurement.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc measurement diff --git a/tests/functional/repl/doc-multiply.expected b/tests/functional/repl/doc-multiply.expected new file mode 100644 index 000000000..21523e24c --- /dev/null +++ b/tests/functional/repl/doc-multiply.expected @@ -0,0 +1,17 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> :doc multiply +Function `multiply`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:10:14 + + +Perform *arithmetic* multiplication. It's kind of like repeated **addition**, very neat. + +```nix +multiply 2 3 +=> 6 +``` diff --git a/tests/functional/repl/doc-multiply.in b/tests/functional/repl/doc-multiply.in new file mode 100644 index 000000000..bffc6696f --- /dev/null +++ b/tests/functional/repl/doc-multiply.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc multiply diff --git a/tests/functional/repl/doc-unambiguous.expected b/tests/functional/repl/doc-unambiguous.expected new file mode 100644 index 000000000..0db5505d7 --- /dev/null +++ b/tests/functional/repl/doc-unambiguous.expected @@ -0,0 +1,11 @@ +Nix +Type :? for help. + +nix-repl> :l doc-comments.nix +Added variables. + +nix-repl> :doc unambiguous +Function `unambiguous`\ + … defined at /path/to/tests/functional/repl/doc-comments.nix:24:5 + +Very close diff --git a/tests/functional/repl/doc-unambiguous.in b/tests/functional/repl/doc-unambiguous.in new file mode 100644 index 000000000..8282a5cb9 --- /dev/null +++ b/tests/functional/repl/doc-unambiguous.in @@ -0,0 +1,2 @@ +:l doc-comments.nix +:doc unambiguous diff --git a/tests/functional/repl/pretty-print-idempotent.expected b/tests/functional/repl/pretty-print-idempotent.expected new file mode 100644 index 000000000..311855dae --- /dev/null +++ b/tests/functional/repl/pretty-print-idempotent.expected @@ -0,0 +1,37 @@ +Nix +Type :? for help. + +nix-repl> :l pretty-print-idempotent.nix +Added variables. + +nix-repl> oneDeep +{ homepage = "https://example.com"; } + +nix-repl> oneDeep +{ homepage = "https://example.com"; } + +nix-repl> twoDeep +{ + layerOne = { ... }; +} + +nix-repl> twoDeep +{ + layerOne = { ... }; +} + +nix-repl> oneDeepList +[ "https://example.com" ] + +nix-repl> oneDeepList +[ "https://example.com" ] + +nix-repl> twoDeepList +[ + [ ... ] +] + +nix-repl> twoDeepList +[ + [ ... ] +] diff --git a/tests/functional/repl/pretty-print-idempotent.in b/tests/functional/repl/pretty-print-idempotent.in new file mode 100644 index 000000000..5f865316f --- /dev/null +++ b/tests/functional/repl/pretty-print-idempotent.in @@ -0,0 +1,9 @@ +:l pretty-print-idempotent.nix +oneDeep +oneDeep +twoDeep +twoDeep +oneDeepList +oneDeepList +twoDeepList +twoDeepList diff --git a/tests/functional/repl/pretty-print-idempotent.nix b/tests/functional/repl/pretty-print-idempotent.nix new file mode 100644 index 000000000..68929f387 --- /dev/null +++ b/tests/functional/repl/pretty-print-idempotent.nix @@ -0,0 +1,19 @@ +{ + oneDeep = { + homepage = "https://" + "example.com"; + }; + twoDeep = { + layerOne = { + homepage = "https://" + "example.com"; + }; + }; + + oneDeepList = [ + ("https://" + "example.com") + ]; + twoDeepList = [ + [ + ("https://" + "example.com") + ] + ]; +} diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh old mode 100644 new mode 100755 index 3de26eb36..e5fe9c136 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible nix-instantiate --restrict-eval --eval -E '1 + 2' (! nix-instantiate --eval --restrict-eval ./restricted.nix) @@ -8,12 +10,12 @@ nix-instantiate --restrict-eval --eval -E '1 + 2' nix-instantiate --restrict-eval ./simple.nix -I src=. nix-instantiate --restrict-eval ./simple.nix -I src1=simple.nix -I src2=config.nix -I src3=./simple.builder.sh +# no default NIX_PATH +(unset NIX_PATH; ! nix-instantiate --restrict-eval --find-file .) + (! nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix') nix-instantiate --restrict-eval --eval -E 'builtins.readFile ./simple.nix' -I src=../.. -(! nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel') -nix-instantiate --restrict-eval --eval -E 'builtins.readDir ../../src/nix-channel' -I src=../../src - expectStderr 1 nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' | grepQuiet "forbidden in restricted mode" nix-instantiate --restrict-eval --eval -E 'let __nixPath = [ { prefix = "foo"; path = ./.; } ]; in builtins.readFile ' -I src=. diff --git a/tests/functional/search.sh b/tests/functional/search.sh old mode 100644 new mode 100755 index d9c7a75da..3fadecd02 --- a/tests/functional/search.sh +++ b/tests/functional/search.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible clearCache (( $(nix search -f search.nix '' hello | wc -l) > 0 )) diff --git a/tests/functional/secure-drv-outputs.sh b/tests/functional/secure-drv-outputs.sh old mode 100644 new mode 100755 index 50a9c4428..5cc4af435 --- a/tests/functional/secure-drv-outputs.sh +++ b/tests/functional/secure-drv-outputs.sh @@ -1,9 +1,13 @@ +#!/usr/bin/env bash + # Test that users cannot register specially-crafted derivations that # produce output paths belonging to other derivations. This could be # used to inject malware into the store. source common.sh +TODO_NixOS + clearStore startDaemon diff --git a/tests/functional/selfref-gc.sh b/tests/functional/selfref-gc.sh old mode 100644 new mode 100755 index 3f1f50eea..518aea66b --- a/tests/functional/selfref-gc.sh +++ b/tests/functional/selfref-gc.sh @@ -1,8 +1,10 @@ +#!/usr/bin/env bash + source common.sh requireDaemonNewerThan "2.6.0pre20211215" -clearStore +clearStoreIfPossible nix-build --no-out-link -E ' with import ./config.nix; diff --git a/tests/functional/shell-hello.nix b/tests/functional/shell-hello.nix index c46fdec8a..c920d7cb4 100644 --- a/tests/functional/shell-hello.nix +++ b/tests/functional/shell-hello.nix @@ -55,4 +55,26 @@ rec { chmod +x $out/bin/hello ''; }; + + # execs env from PATH, so that we can probe the environment + # does not allow arguments, because we don't need them + env = mkDerivation { + name = "env"; + outputs = [ "out" ]; + buildCommand = + '' + mkdir -p $out/bin + + cat > $out/bin/env <&2 + exit 1 + fi + exec env + EOF + chmod +x $out/bin/env + ''; + }; + } diff --git a/tests/functional/shell.nix b/tests/functional/shell.nix index 6a7dd7ad1..9cae14b78 100644 --- a/tests/functional/shell.nix +++ b/tests/functional/shell.nix @@ -26,6 +26,9 @@ let pkgs = rec { fun() { echo blabla } + runHook() { + eval "''${!1}" + } ''; stdenv = mkDerivation { @@ -43,8 +46,21 @@ let pkgs = rec { ASCII_PERCENT = "%"; ASCII_AT = "@"; TEST_inNixShell = if inNixShell then "true" else "false"; + FOO = fooContents; inherit stdenv; outputs = ["dev" "out"]; + } // { + shellHook = abort "Ignore non-drv shellHook attr"; + }; + + # https://github.com/NixOS/nix/issues/5431 + # See nix-shell.sh + polo = mkDerivation { + name = "polo"; + inherit stdenv; + shellHook = '' + echo Polo + ''; }; # Used by nix-shell -p diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh old mode 100644 new mode 100755 index 8a3fef3e7..04a03eef5 --- a/tests/functional/shell.sh +++ b/tests/functional/shell.sh @@ -1,8 +1,15 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore clearCache +# nix shell is an alias for nix env shell. We'll use the shorter form in the rest of the test. +nix env shell -f shell-hello.nix hello -c hello | grep 'Hello World' + nix shell -f shell-hello.nix hello -c hello | grep 'Hello World' nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS' @@ -16,6 +23,27 @@ nix shell -f shell-hello.nix hello-symlink -c hello | grep 'Hello World' # Test that symlinks outside of the store don't work. expect 1 nix shell -f shell-hello.nix forbidden-symlink -c hello 2>&1 | grepQuiet "is not in the Nix store" +# Test that we're not setting any more environment variables than necessary. +# For instance, we might set an environment variable temporarily to affect some +# initialization or whatnot, but this must not leak into the environment of the +# command being run. +env > $TEST_ROOT/expected-env +nix shell -f shell-hello.nix hello -c env > $TEST_ROOT/actual-env +# Remove/reset variables we expect to be different. +# - PATH is modified by nix shell +# - we unset TMPDIR on macOS if it contains /var/folders +# - _ is set by bash and is expectedf to differ because it contains the original command +# - __CF_USER_TEXT_ENCODING is set by macOS and is beyond our control +sed -i \ + -e 's/PATH=.*/PATH=.../' \ + -e 's/_=.*/_=.../' \ + -e '/^TMPDIR=\/var\/folders\/.*/d' \ + -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \ + $TEST_ROOT/expected-env $TEST_ROOT/actual-env +sort $TEST_ROOT/expected-env > $TEST_ROOT/expected-env.sorted +sort $TEST_ROOT/actual-env > $TEST_ROOT/actual-env.sorted +diff $TEST_ROOT/expected-env.sorted $TEST_ROOT/actual-env.sorted + if isDaemonNewer "2.20.0pre20231220"; then # Test that command line attribute ordering is reflected in the PATH # https://github.com/NixOS/nix/issues/7905 diff --git a/tests/functional/shell.shebang.expr b/tests/functional/shell.shebang.expr new file mode 100755 index 000000000..c602dedbf --- /dev/null +++ b/tests/functional/shell.shebang.expr @@ -0,0 +1,9 @@ +#! @ENV_PROG@ nix-shell +#! nix-shell "{ script, path, ... }: assert path == ./shell.nix; script { }" +#! nix-shell --no-substitute +#! nix-shell --expr +#! nix-shell --arg script "import ./shell.nix" +#! nix-shell --arg path "./shell.nix" +#! nix-shell -A shellDrv +#! nix-shell -i bash +echo "$FOO" diff --git a/tests/functional/shell.shebang.legacy.expr b/tests/functional/shell.shebang.legacy.expr new file mode 100755 index 000000000..490542f43 --- /dev/null +++ b/tests/functional/shell.shebang.legacy.expr @@ -0,0 +1,10 @@ +#! @ENV_PROG@ nix-shell +#! nix-shell "{ script, path, ... }: assert path == ./shell.nix; script { fooContents = toString ./.; }" +#! nix-shell --no-substitute +#! nix-shell --expr +#! nix-shell --arg script "import ((builtins.getEnv ''TEST_ROOT'')+''/shell.nix'')" +#! nix-shell --arg path "./shell.nix" +#! nix-shell -A shellDrv +#! nix-shell -i bash +#! nix-shell --option nix-shell-shebang-arguments-relative-to-script false +echo "$FOO" diff --git a/tests/functional/signing.sh b/tests/functional/signing.sh old mode 100644 new mode 100755 index 942b51630..8ec093a48 --- a/tests/functional/signing.sh +++ b/tests/functional/signing.sh @@ -1,109 +1,112 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible clearCache -nix-store --generate-binary-cache-key cache1.example.org $TEST_ROOT/sk1 $TEST_ROOT/pk1 -pk1=$(cat $TEST_ROOT/pk1) -nix-store --generate-binary-cache-key cache2.example.org $TEST_ROOT/sk2 $TEST_ROOT/pk2 -pk2=$(cat $TEST_ROOT/pk2) +nix-store --generate-binary-cache-key cache1.example.org "$TEST_ROOT"/sk1 "$TEST_ROOT"/pk1 +pk1=$(cat "$TEST_ROOT"/pk1) +nix-store --generate-binary-cache-key cache2.example.org "$TEST_ROOT"/sk2 "$TEST_ROOT"/pk2 +pk2=$(cat "$TEST_ROOT"/pk2) # Build a path. outPath=$(nix-build dependencies.nix --no-out-link --secret-key-files "$TEST_ROOT/sk1 $TEST_ROOT/sk2") # Verify that the path got signed. -info=$(nix path-info --json $outPath) -[[ $info =~ '"ultimate":true' ]] -[[ $info =~ 'cache1.example.org' ]] -[[ $info =~ 'cache2.example.org' ]] +info=$(nix path-info --json "$outPath") +echo "$info" | jq -e '.[] | .ultimate == true' +TODO_NixOS # looks like an actual bug? Following line fails on NixOS: +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' # Test "nix store verify". -nix store verify -r $outPath +nix store verify -r "$outPath" -expect 2 nix store verify -r $outPath --sigs-needed 1 +expect 2 nix store verify -r "$outPath" --sigs-needed 1 -nix store verify -r $outPath --sigs-needed 1 --trusted-public-keys $pk1 +nix store verify -r "$outPath" --sigs-needed 1 --trusted-public-keys "$pk1" -expect 2 nix store verify -r $outPath --sigs-needed 2 --trusted-public-keys $pk1 +expect 2 nix store verify -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1" -nix store verify -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" +nix store verify -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" nix store verify --all --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" # Build something unsigned. outPath2=$(nix-build simple.nix --no-out-link) -nix store verify -r $outPath +nix store verify -r "$outPath" # Verify that the path did not get signed but does have the ultimate bit. -info=$(nix path-info --json $outPath2) -[[ $info =~ '"ultimate":true' ]] -(! [[ $info =~ 'signatures' ]]) +info=$(nix path-info --json "$outPath2") +echo "$info" | jq -e '.[] | .ultimate == true' +echo "$info" | jq -e '.[] | .signatures == []' # Test "nix store verify". -nix store verify -r $outPath2 +nix store verify -r "$outPath2" -expect 2 nix store verify -r $outPath2 --sigs-needed 1 +expect 2 nix store verify -r "$outPath2" --sigs-needed 1 -expect 2 nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 +expect 2 nix store verify -r "$outPath2" --sigs-needed 1 --trusted-public-keys "$pk1" # Test "nix store sign". -nix store sign --key-file $TEST_ROOT/sk1 $outPath2 +nix store sign --key-file "$TEST_ROOT"/sk1 "$outPath2" -nix store verify -r $outPath2 --sigs-needed 1 --trusted-public-keys $pk1 +nix store verify -r "$outPath2" --sigs-needed 1 --trusted-public-keys "$pk1" # Build something content-addressed. outPathCA=$(IMPURE_VAR1=foo IMPURE_VAR2=bar nix-build ./fixed.nix -A good.0 --no-out-link) -[[ $(nix path-info --json $outPathCA) =~ '"ca":"fixed:md5:' ]] +nix path-info --json "$outPathCA" | jq -e '.[] | .ca | startswith("fixed:md5:")' # Content-addressed paths don't need signatures, so they verify # regardless of --sigs-needed. -nix store verify $outPathCA -nix store verify $outPathCA --sigs-needed 1000 +nix store verify "$outPathCA" +nix store verify "$outPathCA" --sigs-needed 1000 # Check that signing a content-addressed path doesn't overflow validSigs -nix store sign --key-file $TEST_ROOT/sk1 $outPathCA -nix store verify -r $outPathCA --sigs-needed 1000 --trusted-public-keys $pk1 +nix store sign --key-file "$TEST_ROOT"/sk1 "$outPathCA" +nix store verify -r "$outPathCA" --sigs-needed 1000 --trusted-public-keys "$pk1" # Copy to a binary cache. -nix copy --to file://$cacheDir $outPath2 +nix copy --to file://"$cacheDir" "$outPath2" # Verify that signatures got copied. -info=$(nix path-info --store file://$cacheDir --json $outPath2) -(! [[ $info =~ '"ultimate":true' ]]) -[[ $info =~ 'cache1.example.org' ]] -(! [[ $info =~ 'cache2.example.org' ]]) +info=$(nix path-info --store file://"$cacheDir" --json "$outPath2") +echo "$info" | jq -e '.[] | .ultimate == false' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' +echo "$info" | expect 4 jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' # Verify that adding a signature to a path in a binary cache works. -nix store sign --store file://$cacheDir --key-file $TEST_ROOT/sk2 $outPath2 -info=$(nix path-info --store file://$cacheDir --json $outPath2) -[[ $info =~ 'cache1.example.org' ]] -[[ $info =~ 'cache2.example.org' ]] +nix store sign --store file://"$cacheDir" --key-file "$TEST_ROOT"/sk2 "$outPath2" +info=$(nix path-info --store file://"$cacheDir" --json "$outPath2") +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache1.example.org"))' +echo "$info" | jq -e '.[] | .signatures.[] | select(startswith("cache2.example.org"))' # Copying to a diverted store should fail due to a lack of signatures by trusted keys. -chmod -R u+w $TEST_ROOT/store0 || true -rm -rf $TEST_ROOT/store0 +chmod -R u+w "$TEST_ROOT"/store0 || true +rm -rf "$TEST_ROOT"/store0 # Fails or very flaky only on GHA + macOS: # expectStderr 1 nix copy --to $TEST_ROOT/store0 $outPath | grepQuiet -E 'cannot add path .* because it lacks a signature by a trusted key' # but this works: -(! nix copy --to $TEST_ROOT/store0 $outPath) +(! nix copy --to "$TEST_ROOT"/store0 "$outPath") # But succeed if we supply the public keys. -nix copy --to $TEST_ROOT/store0 $outPath --trusted-public-keys $pk1 +nix copy --to "$TEST_ROOT"/store0 "$outPath" --trusted-public-keys "$pk1" -expect 2 nix store verify --store $TEST_ROOT/store0 -r $outPath +expect 2 nix store verify --store "$TEST_ROOT"/store0 -r "$outPath" -nix store verify --store $TEST_ROOT/store0 -r $outPath --trusted-public-keys $pk1 -nix store verify --store $TEST_ROOT/store0 -r $outPath --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" +nix store verify --store "$TEST_ROOT"/store0 -r "$outPath" --trusted-public-keys "$pk1" +nix store verify --store "$TEST_ROOT"/store0 -r "$outPath" --sigs-needed 2 --trusted-public-keys "$pk1 $pk2" # It should also succeed if we disable signature checking. -(! nix copy --to $TEST_ROOT/store0 $outPath2) -nix copy --to $TEST_ROOT/store0?require-sigs=false $outPath2 +(! nix copy --to "$TEST_ROOT"/store0 "$outPath2") +nix copy --to "$TEST_ROOT"/store0?require-sigs=false "$outPath2" # But signatures should still get copied. -nix store verify --store $TEST_ROOT/store0 -r $outPath2 --trusted-public-keys $pk1 +nix store verify --store "$TEST_ROOT"/store0 -r "$outPath2" --trusted-public-keys "$pk1" # Content-addressed stuff can be copied without signatures. -nix copy --to $TEST_ROOT/store0 $outPathCA +nix copy --to "$TEST_ROOT"/store0 "$outPathCA" diff --git a/tests/functional/simple.builder.sh b/tests/functional/simple.builder.sh index 569e8ca88..97abf0676 100644 --- a/tests/functional/simple.builder.sh +++ b/tests/functional/simple.builder.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + echo "PATH=$PATH" # Verify that the PATH is empty. @@ -5,7 +7,6 @@ if mkdir foo 2> /dev/null; then exit 1; fi # Set a PATH (!!! impure). export PATH=$goodPath +mkdir "$out" -mkdir $out - -echo "Hello World!" > $out/hello \ No newline at end of file +echo "Hello World!" > "$out"/hello diff --git a/tests/functional/simple.nix b/tests/functional/simple.nix index 4223c0f23..2035ca294 100644 --- a/tests/functional/simple.nix +++ b/tests/functional/simple.nix @@ -5,4 +5,5 @@ mkDerivation { builder = ./simple.builder.sh; PATH = ""; goodPath = path; + meta.position = "${__curPos.file}:${toString __curPos.line}"; } diff --git a/tests/functional/simple.sh b/tests/functional/simple.sh old mode 100644 new mode 100755 index 50d44f93f..8afa369c2 --- a/tests/functional/simple.sh +++ b/tests/functional/simple.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh drvPath=$(nix-instantiate simple.nix) @@ -10,23 +12,25 @@ outPath=$(nix-store -rvv "$drvPath") echo "output path is $outPath" -(! [ -w $outPath ]) +[[ ! -w $outPath ]] -text=$(cat "$outPath"/hello) +text=$(cat "$outPath/hello") if test "$text" != "Hello World!"; then exit 1; fi +TODO_NixOS + # Directed delete: $outPath is not reachable from a root, so it should # be deleteable. -nix-store --delete $outPath -(! [ -e $outPath/hello ]) +nix-store --delete "$outPath" +[[ ! -e $outPath/hello ]] -outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode hash-check.nix)" +outPath="$(NIX_REMOTE='local?store=/foo&real='"$TEST_ROOT"'/real-store' nix-instantiate --readonly-mode hash-check.nix)" if test "$outPath" != "/foo/lfy1s6ca46rm5r6w4gg9hc0axiakjcnm-dependencies.drv"; then echo "hashDerivationModulo appears broken, got $outPath" exit 1 fi -outPath="$(NIX_REMOTE=local?store=/foo\&real=$TEST_ROOT/real-store nix-instantiate --readonly-mode big-derivation-attr.nix)" +outPath="$(NIX_REMOTE='local?store=/foo&real='"$TEST_ROOT"'/real-store' nix-instantiate --readonly-mode big-derivation-attr.nix)" if test "$outPath" != "/foo/xxiwa5zlaajv6xdjynf9yym9g319d6mn-big-derivation-attr.drv"; then echo "big-derivation-attr.nix hash appears broken, got $outPath. Memory corruption in large drv attr?" exit 1 diff --git a/tests/functional/slash.nar b/tests/functional/slash.nar new file mode 100644 index 000000000..118a60216 Binary files /dev/null and b/tests/functional/slash.nar differ diff --git a/tests/functional/ssh-relay.sh b/tests/functional/ssh-relay.sh old mode 100644 new mode 100755 index 053b2f00d..71b8ae9ab --- a/tests/functional/ssh-relay.sh +++ b/tests/functional/ssh-relay.sh @@ -1,9 +1,11 @@ +#!/usr/bin/env bash + source common.sh -echo foo > $TEST_ROOT/hello.sh +echo foo > "$TEST_ROOT"/hello.sh ssh_localhost=ssh://localhost -remote_store=?remote-store=$ssh_localhost +remote_store="?remote-store=$ssh_localhost" store=$ssh_localhost @@ -11,6 +13,6 @@ store+=$remote_store store+=$remote_store store+=$remote_store -out=$(nix store add-path --store "$store" $TEST_ROOT/hello.sh) +out=$(nix store add-path --store "$store" "$TEST_ROOT"/hello.sh) -[ foo = $(< $out) ] +[ foo = "$(< "$out")" ] diff --git a/tests/functional/store-info.sh b/tests/functional/store-info.sh old mode 100644 new mode 100755 index 18a8131a9..beecc2dd9 --- a/tests/functional/store-info.sh +++ b/tests/functional/store-info.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh STORE_INFO=$(nix store info 2>&1) @@ -6,12 +8,14 @@ STORE_INFO_JSON=$(nix store info --json) echo "$STORE_INFO" | grep "Store URL: ${NIX_REMOTE}" if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then - DAEMON_VERSION=$($NIX_DAEMON_PACKAGE/bin/nix daemon --version | cut -d' ' -f3) + DAEMON_VERSION=$("$NIX_DAEMON_PACKAGE"/bin/nix daemon --version | cut -d' ' -f3) echo "$STORE_INFO" | grep "Version: $DAEMON_VERSION" [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]] fi -expect 127 NIX_REMOTE=unix:$PWD/store nix store info || \ +expect 127 NIX_REMOTE=unix:"$PWD"/store nix store info || \ fail "nix store info on a non-existent store should fail" +TODO_NixOS + [[ "$(echo "$STORE_INFO_JSON" | jq -r ".url")" == "${NIX_REMOTE:-local}" ]] diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh old mode 100644 new mode 100755 index 6711efbb4..64d136e99 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -1,27 +1,34 @@ +#!/usr/bin/env bash + source common.sh # 27ce722638 required some incompatible changes to the nix file, so skip this # tests for the older versions requireDaemonNewerThan "2.4pre20210712" -clearStore +clearStoreIfPossible -rm -f $TEST_ROOT/result +rm -f "$TEST_ROOT"/result -nix-build structured-attrs.nix -A all -o $TEST_ROOT/result +nix-build structured-attrs.nix -A all -o "$TEST_ROOT"/result -[[ $(cat $TEST_ROOT/result/foo) = bar ]] -[[ $(cat $TEST_ROOT/result-dev/foo) = foo ]] +[[ $(cat "$TEST_ROOT"/result/foo) = bar ]] +[[ $(cat "$TEST_ROOT"/result-dev/foo) = foo ]] export NIX_BUILD_SHELL=$SHELL +# shellcheck disable=SC2016 env NIX_PATH=nixpkgs=shell.nix nix-shell structured-attrs-shell.nix \ --run 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' +# shellcheck disable=SC2016 nix develop -f structured-attrs-shell.nix -c bash -c 'test "3" = "$(jq ".my.list|length" < $NIX_ATTRS_JSON_FILE)"' +TODO_NixOS # following line fails. + # `nix develop` is a slightly special way of dealing with environment vars, it parses # these from a shell-file exported from a derivation. This is to test especially `outputs` # (which is an associative array in thsi case) being fine. +# shellcheck disable=SC2016 nix develop -f structured-attrs-shell.nix -c bash -c 'test -n "$out"' nix print-dev-env -f structured-attrs-shell.nix | grepQuiet 'NIX_ATTRS_JSON_FILE=' diff --git a/tests/functional/substitute-with-invalid-ca.sh b/tests/functional/substitute-with-invalid-ca.sh old mode 100644 new mode 100755 index 4d0b01e0f..33432e95d --- a/tests/functional/substitute-with-invalid-ca.sh +++ b/tests/functional/substitute-with-invalid-ca.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh BINARY_CACHE=file://$cacheDir @@ -9,16 +11,16 @@ getRemoteNarInfo () { echo "$cacheDir/$(getHash "$1").narinfo" } -cat < $TEST_HOME/good.txt +cat < "$TEST_HOME"/good.txt I’m a good path EOF -cat < $TEST_HOME/bad.txt +cat < "$TEST_HOME"/bad.txt I’m a bad path EOF -good=$(nix-store --add $TEST_HOME/good.txt) -bad=$(nix-store --add $TEST_HOME/bad.txt) +good=$(nix-store --add "$TEST_HOME"/good.txt) +bad=$(nix-store --add "$TEST_HOME"/bad.txt) nix copy --to "$BINARY_CACHE" "$good" nix copy --to "$BINARY_CACHE" "$bad" nix-collect-garbage >/dev/null 2>&1 diff --git a/tests/functional/suggestions.sh b/tests/functional/suggestions.sh old mode 100644 new mode 100755 index f18fefef9..fbca93da8 --- a/tests/functional/suggestions.sh +++ b/tests/functional/suggestions.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible cd "$TEST_HOME" @@ -35,7 +37,7 @@ NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION=$(! nix build .\#bar 2>&1 1>/dev/null) [[ ! "$NIX_BUILD_STDERR_WITH_NO_CLOSE_SUGGESTION" =~ "Did you mean" ]] || \ fail "The nix build stderr shouldn’t suggest anything if there’s nothing relevant to suggest" -NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '(builtins.getFlake (builtins.toPath ./.)).packages.'$system'.fob' 2>&1 1>/dev/null) +NIX_EVAL_STDERR_WITH_SUGGESTIONS=$(! nix build --impure --expr '(builtins.getFlake (builtins.toPath ./.)).packages.'"$system"'.fob' 2>&1 1>/dev/null) [[ "$NIX_EVAL_STDERR_WITH_SUGGESTIONS" =~ "Did you mean one of fo1, fo2, foo or fooo?" ]] || \ fail "The evaluator should suggest the three closest possiblities" diff --git a/tests/functional/supplementary-groups.sh b/tests/functional/supplementary-groups.sh old mode 100644 new mode 100755 index d18fb2414..50259a3e1 --- a/tests/functional/supplementary-groups.sh +++ b/tests/functional/supplementary-groups.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh requireSandboxSupport @@ -5,7 +7,9 @@ requireSandboxSupport if ! command -p -v unshare; then skipTest "Need unshare"; fi needLocalStore "The test uses --store always so we would just be bypassing the daemon" -unshare --mount --map-root-user bash < $tarball + (cd "$TEST_ROOT" && GNUTAR_REPRODUCIBLE=1 tar --mtime="$tarroot"/default.nix --owner=0 --group=0 --numeric-owner --sort=name -c -f - tarball) | $compressor > "$tarball" - nix-env -f file://$tarball -qa --out-path | grepQuiet dependencies + nix-env -f file://"$tarball" -qa --out-path | grepQuiet dependencies - nix-build -o $TEST_ROOT/result file://$tarball + nix-build -o "$TEST_ROOT"/result file://"$tarball" - nix-build -o $TEST_ROOT/result '' -I foo=file://$tarball + nix-build -o "$TEST_ROOT"/result '' -I foo=file://"$tarball" - nix-build -o $TEST_ROOT/result -E "import (fetchTarball file://$tarball)" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTarball file://$tarball)" # Do not re-fetch paths already present - nix-build -o $TEST_ROOT/result -E "import (fetchTarball { url = file:///does-not-exist/must-remain-unused/$tarball; sha256 = \"$hash\"; })" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTarball { url = file:///does-not-exist/must-remain-unused/$tarball; sha256 = \"$hash\"; })" - nix-build -o $TEST_ROOT/result -E "import (fetchTree file://$tarball)" - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" - # Do not re-fetch paths already present - nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file:///does-not-exist/must-remain-unused/$tarball; narHash = \"$hash\"; })" - expectStderr 102 nix-build -o $TEST_ROOT/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"sha256-xdKv2pq/IiwLSnBBJXW8hNowI4MrdZfW+SYqDQs7Tzc=\"; })" | grep 'NAR hash mismatch in input' + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree file://$tarball)" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; })" + nix-build -o "$TEST_ROOT"/result -E "import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })" [[ $(nix eval --impure --expr "(fetchTree file://$tarball).lastModified") = 1000000000 ]] nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" >&2 nix-instantiate --strict --eval -E "!((import (fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; })) ? submodules)" 2>&1 | grep 'true' - nix-instantiate --eval -E '1 + 2' -I fnord=file:///no-such-tarball.tar$ext - nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file:///no-such-tarball$ext - (! nix-instantiate --eval -E ' 1' -I fnord=file:///no-such-tarball$ext) + nix-instantiate --eval -E '1 + 2' -I fnord=file:///no-such-tarball.tar"$ext" + nix-instantiate --eval -E 'with ; 1 + 2' -I fnord=file:///no-such-tarball"$ext" + (! nix-instantiate --eval -E ' 1' -I fnord=file:///no-such-tarball"$ext") - nix-instantiate --eval -E '' -I fnord=file:///no-such-tarball$ext -I fnord=. + nix-instantiate --eval -E '' -I fnord=file:///no-such-tarball"$ext" -I fnord=. # Ensure that the `name` attribute isn’t accepted as that would mess # with the content-addressing (! nix-instantiate --eval -E "fetchTree { type = \"tarball\"; url = file://$tarball; narHash = \"$hash\"; name = \"foo\"; }") + store_path=$(nix store prefetch-file --json "file://$tarball" | jq -r .storePath) + if ! cmp -s "$store_path" "$tarball"; then + echo "prefetched tarball differs from original: $store_path vs $tarball" >&2 + exit 1 + fi + store_path2=$(nix store prefetch-file --json --unpack "file://$tarball" | jq -r .storePath) + diff_output=$(diff -r "$store_path2" "$tarroot") + if [ -n "$diff_output" ]; then + echo "prefetched tarball differs from original: $store_path2 vs $tarroot" >&2 + echo "$diff_output" + exit 1 + fi } test_tarball '' cat test_tarball .xz xz test_tarball .gz gzip + +# Test hard links. +# All entries in tree.tar.gz refer to the same file, and all have the same inode when unpacked by GNU tar. +# We don't preserve the hard links, because that's an optimization we think is not worth the complexity, +# so we only make sure that the contents are copied correctly. +path="$(nix flake prefetch --json "tarball+file://$(pwd)/tree.tar.gz" | jq -r .storePath)" +[[ $(cat "$path/a/b/foo") = bar ]] +[[ $(cat "$path/a/b/xyzzy") = bar ]] +[[ $(cat "$path/a/yyy") = bar ]] +[[ $(cat "$path/a/zzz") = bar ]] +[[ $(cat "$path/c/aap") = bar ]] +[[ $(cat "$path/fnord") = bar ]] + +# Test a tarball that has multiple top-level directories. +rm -rf "$TEST_ROOT/tar_root" +mkdir -p "$TEST_ROOT/tar_root" "$TEST_ROOT/tar_root/foo" "$TEST_ROOT/tar_root/bar" +tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . +path="$(nix flake prefetch --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" +[[ -d "$path/foo" ]] +[[ -d "$path/bar" ]] + +# Test a tarball that has a single regular file. +rm -rf "$TEST_ROOT/tar_root" +mkdir -p "$TEST_ROOT/tar_root" +echo bar > "$TEST_ROOT/tar_root/foo" +chmod +x "$TEST_ROOT/tar_root/foo" +tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . +path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" +[[ $(cat "$path/foo") = bar ]] + +# Test a tarball with non-contiguous directory entries. +rm -rf "$TEST_ROOT/tar_root" +mkdir -p "$TEST_ROOT/tar_root/a/b" +echo foo > "$TEST_ROOT/tar_root/a/b/foo" +echo bla > "$TEST_ROOT/tar_root/bla" +tar cvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" . +echo abc > "$TEST_ROOT/tar_root/bla" +echo xyzzy > "$TEST_ROOT/tar_root/a/b/xyzzy" +tar rvf "$TEST_ROOT/tar.tar" -C "$TEST_ROOT/tar_root" ./a/b/xyzzy ./bla +path="$(nix flake prefetch --refresh --json "tarball+file://$TEST_ROOT/tar.tar" | jq -r .storePath)" +[[ $(cat "$path/a/b/xyzzy") = xyzzy ]] +[[ $(cat "$path/a/b/foo") = foo ]] +[[ $(cat "$path/bla") = abc ]] diff --git a/tests/functional/test-infra.sh b/tests/functional/test-infra.sh old mode 100644 new mode 100755 index 54ae120e7..2da26b08c --- a/tests/functional/test-infra.sh +++ b/tests/functional/test-infra.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Test the functions for testing themselves! # Also test some assumptions on how bash works that they rely on. source common.sh @@ -11,6 +13,25 @@ expect 1 false # `expect` will fail when we get it wrong expect 1 expect 0 false +function ret() { + return "$1" +} + +# `expect` can call functions, not just executables +expect 0 ret 0 +expect 1 ret 1 + +# `expect` supports negative exit codes +expect -1 ret -1 + +# or high positive ones, equivalent to negative ones +expect 255 ret 255 +expect 255 ret -1 +expect -1 ret 255 + +# but it doesn't confuse negative exit codes with positive ones +expect 1 expect -10 ret 10 + noisyTrue () { echo YAY! >&2 true @@ -27,6 +48,7 @@ expectStderr 1 noisyFalse | grepQuiet NAY # `set -o pipefile` is enabled +# shellcheck disable=SC2317# shellcheck disable=SC2317 pipefailure () { # shellcheck disable=SC2216 true | false | true @@ -34,6 +56,7 @@ pipefailure () { expect 1 pipefailure unset pipefailure +# shellcheck disable=SC2317 pipefailure () { # shellcheck disable=SC2216 false | true | true @@ -61,12 +84,17 @@ expect 1 useUnbound # ! alone unfortunately negates `set -e`, but it works in functions: # shellcheck disable=SC2251 ! true +# shellcheck disable=SC2317 funBang () { ! true } expect 1 funBang unset funBang +# callerPrefix can be used by the test framework to improve error messages +# it reports about our call site here +echo "<[$(callerPrefix)]>" | grepQuiet -F "<[test-infra.sh:$LINENO: ]>" + # `grep -v -q` is not what we want for exit codes, but `grepInverse` is # Avoid `grep -v -q`. The following line proves the point, and if it fails, # we'll know that `grep` had a breaking change or `-v -q` may not be portable. @@ -83,3 +111,12 @@ unset res res=$(set -eu -o pipefail; echo foo | expect 1 grepQuietInverse foo | wc -c) (( res == 0 )) unset res + +# `grepQuiet` does not allow newlines in its arguments, because grep quietly +# treats them as multiple queries. +{ echo foo; echo bar; } | expectStderr -101 grepQuiet $'foo\nbar' \ + | grepQuiet -E 'test-infra\.sh:[0-9]+: in call to grepQuiet: newline not allowed in arguments; grep would try each line individually as if connected by an OR operator' + +# We took the blue pill and woke up in a world where `grep` is moderately safe. +expectStderr -101 grep $'foo\nbar' \ + | grepQuiet -E 'test-infra\.sh:[0-9]+: in call to grep: newline not allowed in arguments; grep would try each line individually as if connected by an OR operator' diff --git a/tests/functional/test-libstoreconsumer.sh b/tests/functional/test-libstoreconsumer.sh old mode 100644 new mode 100755 index 8a77cf5a1..2adead1c0 --- a/tests/functional/test-libstoreconsumer.sh +++ b/tests/functional/test-libstoreconsumer.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh drv="$(nix-instantiate simple.nix)" cat "$drv" out="$(./test-libstoreconsumer/test-libstoreconsumer "$drv")" -cat "$out/hello" | grep -F "Hello World!" +grep -F "Hello World!" < "$out/hello" diff --git a/tests/functional/test-libstoreconsumer/meson.build b/tests/functional/test-libstoreconsumer/meson.build new file mode 100644 index 000000000..7076127f7 --- /dev/null +++ b/tests/functional/test-libstoreconsumer/meson.build @@ -0,0 +1,14 @@ +libstoreconsumer_tester = executable( + 'test-libstoreconsumer', + 'main.cc', + cpp_args : [ + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + ], + dependencies : [ + dependency('nix-store'), + ], + build_by_default : false, +) diff --git a/tests/functional/timeout.sh b/tests/functional/timeout.sh old mode 100644 new mode 100755 index b179b79a2..ae47fdc96 --- a/tests/functional/timeout.sh +++ b/tests/functional/timeout.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Test the `--timeout' option. source common.sh @@ -7,8 +9,12 @@ needLocalStore "see #4813" messages=$(nix-build -Q timeout.nix -A infiniteLoop --timeout 2 2>&1) && status=0 || status=$? -if [ $status -ne 101 ]; then +if [ "$status" -ne 101 ]; then echo "error: 'nix-store' exited with '$status'; should have exited 101" + + # FIXME: https://github.com/NixOS/nix/issues/4813 + skipTest "Do not block CI until fixed" + exit 1 fi diff --git a/tests/functional/toString-path.sh b/tests/functional/toString-path.sh old mode 100644 new mode 100755 index 07eb87465..d790109f4 --- a/tests/functional/toString-path.sh +++ b/tests/functional/toString-path.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source common.sh mkdir -p $TEST_ROOT/foo diff --git a/tests/functional/tree.tar.gz b/tests/functional/tree.tar.gz new file mode 100644 index 000000000..f1f1d996d Binary files /dev/null and b/tests/functional/tree.tar.gz differ diff --git a/tests/functional/unnormalized.nar b/tests/functional/unnormalized.nar new file mode 100644 index 000000000..4b7edb17e Binary files /dev/null and b/tests/functional/unnormalized.nar differ diff --git a/tests/functional/user-envs-migration.sh b/tests/functional/user-envs-migration.sh old mode 100644 new mode 100755 index 187372b16..0f33074e1 --- a/tests/functional/user-envs-migration.sh +++ b/tests/functional/user-envs-migration.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + # Test that the migration of user environments # (https://github.com/NixOS/nix/pull/5226) does preserve everything @@ -11,6 +13,8 @@ fi killDaemon unset NIX_REMOTE +TODO_NixOS + clearStore clearProfiles rm -rf ~/.nix-profile diff --git a/tests/functional/user-envs-test-case.sh b/tests/functional/user-envs-test-case.sh index f4a90a675..117c6c7a4 100644 --- a/tests/functional/user-envs-test-case.sh +++ b/tests/functional/user-envs-test-case.sh @@ -11,6 +11,8 @@ outPath10=$(nix-env -f ./user-envs.nix -qa --out-path --no-name '*' | grep foo-1 drvPath10=$(nix-env -f ./user-envs.nix -qa --drv-path --no-name '*' | grep foo-1.0) [ -n "$outPath10" -a -n "$drvPath10" ] +TODO_NixOS + # Query with json nix-env -f ./user-envs.nix -qa --json | jq -e '.[] | select(.name == "bar-0.1") | [ .outputName == "out", diff --git a/tests/functional/user-envs.sh b/tests/functional/user-envs.sh old mode 100644 new mode 100755 index a849d5439..ec9d036f8 --- a/tests/functional/user-envs.sh +++ b/tests/functional/user-envs.sh @@ -1,3 +1,5 @@ +#!/usr/bin/env bash + source ./common.sh source ./user-envs-test-case.sh diff --git a/tests/functional/why-depends.sh b/tests/functional/why-depends.sh old mode 100644 new mode 100755 index 9680bf80e..ce53546d8 --- a/tests/functional/why-depends.sh +++ b/tests/functional/why-depends.sh @@ -1,6 +1,8 @@ +#!/usr/bin/env bash + source common.sh -clearStore +clearStoreIfPossible cp ./dependencies.nix ./dependencies.builder0.sh ./config.nix $TEST_HOME diff --git a/tests/functional/zstd.sh b/tests/functional/zstd.sh old mode 100644 new mode 100755 index ba7c20501..90fe58539 --- a/tests/functional/zstd.sh +++ b/tests/functional/zstd.sh @@ -1,5 +1,9 @@ +#!/usr/bin/env bash + source common.sh +TODO_NixOS + clearStore clearCache diff --git a/tests/installer/default.nix b/tests/installer/default.nix index 238c6ac8e..4aed6eae4 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -13,6 +13,17 @@ let ''; }; + install-both-profile-links = { + script = '' + tar -xf ./nix.tar.xz + mv ./nix-* nix + ln -s $HOME/.local/state/nix/profiles/a-profile $HOME/.nix-profile + mkdir -p $HOME/.local/state/nix + ln -s $HOME/.local/state/nix/profiles/b-profile $HOME/.local/state/nix/profile + ./nix/install --no-channel-add + ''; + }; + install-force-no-daemon = { script = '' tar -xf ./nix.tar.xz diff --git a/tests/nixos/cgroups/default.nix b/tests/nixos/cgroups/default.nix new file mode 100644 index 000000000..b8febbf4b --- /dev/null +++ b/tests/nixos/cgroups/default.nix @@ -0,0 +1,40 @@ +{ nixpkgs, ... }: + +{ + name = "cgroups"; + + nodes = + { + host = + { config, pkgs, ... }: + { virtualisation.additionalPaths = [ pkgs.stdenvNoCC ]; + nix.extraOptions = + '' + extra-experimental-features = nix-command auto-allocate-uids cgroups + extra-system-features = uid-range + ''; + nix.settings.use-cgroups = true; + nix.nixPath = [ "nixpkgs=${nixpkgs}" ]; + }; + }; + + testScript = { nodes }: '' + start_all() + + host.wait_for_unit("multi-user.target") + + # Start build in background + host.execute("NIX_REMOTE=daemon nix build --auto-allocate-uids --file ${./hang.nix} >&2 &") + service = "/sys/fs/cgroup/system.slice/nix-daemon.service" + + # Wait for cgroups to be created + host.succeed(f"until [ -e {service}/nix-daemon ]; do sleep 1; done", timeout=30) + host.succeed(f"until [ -e {service}/nix-build-uid-* ]; do sleep 1; done", timeout=30) + + # Check that there aren't processes where there shouldn't be, and that there are where there should be + host.succeed(f'[ -z "$(cat {service}/cgroup.procs)" ]') + host.succeed(f'[ -n "$(cat {service}/nix-daemon/cgroup.procs)" ]') + host.succeed(f'[ -n "$(cat {service}/nix-build-uid-*/cgroup.procs)" ]') + ''; + +} diff --git a/tests/nixos/cgroups/hang.nix b/tests/nixos/cgroups/hang.nix new file mode 100644 index 000000000..cefe2d031 --- /dev/null +++ b/tests/nixos/cgroups/hang.nix @@ -0,0 +1,10 @@ +{ }: + +with import {}; + +runCommand "hang" + { requiredSystemFeatures = "uid-range"; + } + '' + sleep infinity + '' diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 4edf40c16..c61a2888f 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -1,17 +1,27 @@ -{ lib, nixpkgs, nixpkgsFor }: +{ lib, nixpkgs, nixpkgsFor, self }: let nixos-lib = import (nixpkgs + "/nixos/lib") { }; + noTests = pkg: pkg.overrideAttrs ( + finalAttrs: prevAttrs: { + doCheck = false; + doInstallCheck = false; + }); + # https://nixos.org/manual/nixos/unstable/index.html#sec-calling-nixos-tests runNixOSTestFor = system: test: (nixos-lib.runTest { - imports = [ test ]; + imports = [ + test + ]; + hostPkgs = nixpkgsFor.${system}.native; defaults = { nixpkgs.pkgs = nixpkgsFor.${system}.native; nix.checkAllErrors = false; + nix.package = noTests nixpkgsFor.${system}.native.nix; }; _module.args.nixpkgs = nixpkgs; _module.args.system = system; @@ -23,6 +33,9 @@ let forNix = nixVersion: runNixOSTestFor system { imports = [test]; defaults.nixpkgs.overlays = [(curr: prev: { + # NOTE: noTests pkg might not have been built yet for some older versions of the package + # and in versions before 2.25, the untested build wasn't shared with the tested build yet + # Add noTests here when those versions become irrelevant. nix = (builtins.getFlake "nix/${nixVersion}").packages.${system}.nix; })]; }; @@ -33,8 +46,30 @@ let checkOverrideNixVersion = { pkgs, lib, ... }: { # pkgs.nix: The new Nix in this repo # We disallow it, to make sure we don't accidentally use it. - system.forbiddenDependenciesRegex = lib.strings.escapeRegex "nix-${pkgs.nix.version}"; + system.forbiddenDependenciesRegexes = [ + (lib.strings.escapeRegex "nix-${pkgs.nix.version}") + ]; }; + + otherNixes.nix_2_3.setNixPackage = { lib, pkgs, ... }: { + imports = [ checkOverrideNixVersion ]; + nix.package = lib.mkForce pkgs.nixVersions.nix_2_3; + }; + + otherNixes.nix_2_13.setNixPackage = { lib, pkgs, ... }: { + imports = [ checkOverrideNixVersion ]; + nix.package = lib.mkForce ( + self.inputs.nixpkgs-23-11.legacyPackages.${pkgs.stdenv.hostPlatform.system}.nixVersions.nix_2_13.overrideAttrs (o: { + meta = o.meta // { knownVulnerabilities = []; }; + }) + ); + }; + + otherNixes.nix_2_18.setNixPackage = { lib, pkgs, ... }: { + imports = [ checkOverrideNixVersion ]; + nix.package = lib.mkForce pkgs.nixVersions.nix_2_18; + }; + in { @@ -42,100 +77,48 @@ in remoteBuilds = runNixOSTestFor "x86_64-linux" ./remote-builds.nix; - # Test our Nix as a client against remotes that are older - - remoteBuilds_remote_2_3 = runNixOSTestFor "x86_64-linux" { - name = "remoteBuilds_remote_2_3"; - imports = [ ./remote-builds.nix ]; - builders.config = { lib, pkgs, ... }: { - imports = [ checkOverrideNixVersion ]; - nix.package = lib.mkForce pkgs.nixVersions.nix_2_3; - }; - }; - - remoteBuilds_remote_2_13 = runNixOSTestFor "x86_64-linux" ({ lib, pkgs, ... }: { - name = "remoteBuilds_remote_2_13"; - imports = [ ./remote-builds.nix ]; - builders.config = { lib, pkgs, ... }: { - imports = [ checkOverrideNixVersion ]; - nix.package = lib.mkForce pkgs.nixVersions.nix_2_3; - }; - }); - - # TODO: (nixpkgs update) remoteBuilds_remote_2_18 = ... - - # Test our Nix as a builder for clients that are older - - remoteBuilds_local_2_3 = runNixOSTestFor "x86_64-linux" ({ lib, pkgs, ... }: { - name = "remoteBuilds_local_2_3"; - imports = [ ./remote-builds.nix ]; - nodes.client = { lib, pkgs, ... }: { - imports = [ checkOverrideNixVersion ]; - nix.package = lib.mkForce pkgs.nixVersions.nix_2_3; - }; - }); - - remoteBuilds_local_2_13 = runNixOSTestFor "x86_64-linux" ({ lib, pkgs, ... }: { - name = "remoteBuilds_local_2_13"; - imports = [ ./remote-builds.nix ]; - nodes.client = { lib, pkgs, ... }: { - imports = [ checkOverrideNixVersion ]; - nix.package = lib.mkForce pkgs.nixVersions.nix_2_13; - }; - }); - - # TODO: (nixpkgs update) remoteBuilds_local_2_18 = ... - - # End remoteBuilds tests - remoteBuildsSshNg = runNixOSTestFor "x86_64-linux" ./remote-builds-ssh-ng.nix; - # Test our Nix as a client against remotes that are older - - remoteBuildsSshNg_remote_2_3 = runNixOSTestFor "x86_64-linux" { - name = "remoteBuildsSshNg_remote_2_3"; - imports = [ ./remote-builds-ssh-ng.nix ]; - builders.config = { lib, pkgs, ... }: { - imports = [ checkOverrideNixVersion ]; - nix.package = lib.mkForce pkgs.nixVersions.nix_2_3; +} +// lib.concatMapAttrs ( + nixVersion: { setNixPackage, ... }: + { + "remoteBuilds_remote_${nixVersion}" = runNixOSTestFor "x86_64-linux" { + name = "remoteBuilds_remote_${nixVersion}"; + imports = [ ./remote-builds.nix ]; + builders.config = { lib, pkgs, ... }: { + imports = [ setNixPackage ]; + }; }; - }; - remoteBuildsSshNg_remote_2_13 = runNixOSTestFor "x86_64-linux" { - name = "remoteBuildsSshNg_remote_2_13"; - imports = [ ./remote-builds-ssh-ng.nix ]; - builders.config = { lib, pkgs, ... }: { - imports = [ checkOverrideNixVersion ]; - nix.package = lib.mkForce pkgs.nixVersions.nix_2_13; + "remoteBuilds_local_${nixVersion}" = runNixOSTestFor "x86_64-linux" { + name = "remoteBuilds_local_${nixVersion}"; + imports = [ ./remote-builds.nix ]; + nodes.client = { lib, pkgs, ... }: { + imports = [ setNixPackage ]; + }; }; - }; - # TODO: (nixpkgs update) remoteBuildsSshNg_remote_2_18 = ... - - # Test our Nix as a builder for clients that are older - - # FIXME: these tests don't work yet - /* - remoteBuildsSshNg_local_2_3 = runNixOSTestFor "x86_64-linux" ({ lib, pkgs, ... }: { - name = "remoteBuildsSshNg_local_2_3"; - imports = [ ./remote-builds-ssh-ng.nix ]; - nodes.client = { lib, pkgs, ... }: { - imports = [ checkOverrideNixVersion ]; - nix.package = lib.mkForce pkgs.nixVersions.nix_2_3; + "remoteBuildsSshNg_remote_${nixVersion}" = runNixOSTestFor "x86_64-linux" { + name = "remoteBuildsSshNg_remote_${nixVersion}"; + imports = [ ./remote-builds-ssh-ng.nix ]; + builders.config = { lib, pkgs, ... }: { + imports = [ setNixPackage ]; + }; }; - }); - remoteBuildsSshNg_local_2_13 = runNixOSTestFor "x86_64-linux" ({ lib, pkgs, ... }: { - name = "remoteBuildsSshNg_local_2_13"; - imports = [ ./remote-builds-ssh-ng.nix ]; - nodes.client = { lib, pkgs, ... }: { - imports = [ checkOverrideNixVersion ]; - nix.package = lib.mkForce pkgs.nixVersions.nix_2_13; - }; - }); + # FIXME: these tests don't work yet - # TODO: (nixpkgs update) remoteBuildsSshNg_local_2_18 = ... - */ + # "remoteBuildsSshNg_local_${nixVersion}" = runNixOSTestFor "x86_64-linux" { + # name = "remoteBuildsSshNg_local_${nixVersion}"; + # imports = [ ./remote-builds-ssh-ng.nix ]; + # nodes.client = { lib, pkgs, ... }: { + # imports = [ overridingModule ]; + # }; + # }; + } +) otherNixes +// { nix-copy-closure = runNixOSTestFor "x86_64-linux" ./nix-copy-closure.nix; @@ -154,7 +137,7 @@ in containers = runNixOSTestFor "x86_64-linux" ./containers/containers.nix; setuid = lib.genAttrs - ["i686-linux" "x86_64-linux"] + ["x86_64-linux"] (system: runNixOSTestFor system ./setuid.nix); fetch-git = runNixOSTestFor "x86_64-linux" ./fetch-git; @@ -162,4 +145,20 @@ in ca-fd-leak = runNixOSTestFor "x86_64-linux" ./ca-fd-leak; gzip-content-encoding = runNixOSTestFor "x86_64-linux" ./gzip-content-encoding.nix; + + functional_user = runNixOSTestFor "x86_64-linux" ./functional/as-user.nix; + + functional_trusted = runNixOSTestFor "x86_64-linux" ./functional/as-trusted-user.nix; + + functional_root = runNixOSTestFor "x86_64-linux" ./functional/as-root.nix; + + user-sandboxing = runNixOSTestFor "x86_64-linux" ./user-sandboxing; + + s3-binary-cache-store = runNixOSTestFor "x86_64-linux" ./s3-binary-cache-store.nix; + + fsync = runNixOSTestFor "x86_64-linux" ./fsync.nix; + + cgroups = runNixOSTestFor "x86_64-linux" ./cgroups; + + fetchurl = runNixOSTestFor "x86_64-linux" ./fetchurl.nix; } diff --git a/tests/nixos/fetch-git/test-cases/fetchTree-shallow/default.nix b/tests/nixos/fetch-git/test-cases/fetchTree-shallow/default.nix new file mode 100644 index 000000000..f635df1f8 --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/fetchTree-shallow/default.nix @@ -0,0 +1,45 @@ +{ + description = "fetchTree fetches git repos shallowly by default"; + script = '' + # purge nix git cache to make sure we start with a clean slate + client.succeed("rm -rf ~/.cache/nix") + + # add two commits to the repo: + # - one with a large file (2M) + # - another one making the file small again + client.succeed(f""" + dd if=/dev/urandom of={repo.path}/thailand bs=1M count=2 \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit1' \ + && echo 'ThaigerSprint' > {repo.path}/thailand \ + && {repo.git} add thailand \ + && {repo.git} commit -m 'commit2' \ + && {repo.git} push origin main + """) + + # memoize the revision + commit2_rev = client.succeed(f""" + {repo.git} rev-parse HEAD + """).strip() + + # construct the fetcher call + fetchGit_expr = f""" + builtins.fetchTree {{ + type = "git"; + url = "{repo.remote}"; + rev = "{commit2_rev}"; + }} + """ + + # fetch the repo via nix + fetched1 = client.succeed(f""" + nix eval --impure --raw --expr '({fetchGit_expr}).outPath' + """) + + # check that the size of ~/.cache/nix is less than 1M + cache_size = client.succeed(""" + du -s ~/.cache/nix + """).strip().split()[0] + assert int(cache_size) < 1024, f"cache size is {cache_size}K which is larger than 1M" + ''; +} diff --git a/tests/nixos/fetchurl.nix b/tests/nixos/fetchurl.nix new file mode 100644 index 000000000..243c0cacc --- /dev/null +++ b/tests/nixos/fetchurl.nix @@ -0,0 +1,84 @@ +# Test whether builtin:fetchurl properly performs TLS certificate +# checks on HTTPS servers. + +{ pkgs, ... }: + +let + + makeTlsCert = name: pkgs.runCommand name { + nativeBuildInputs = with pkgs; [ openssl ]; + } '' + mkdir -p $out + openssl req -x509 \ + -subj '/CN=${name}/' -days 49710 \ + -addext 'subjectAltName = DNS:${name}' \ + -keyout "$out/key.pem" -newkey ed25519 \ + -out "$out/cert.pem" -noenc + ''; + + goodCert = makeTlsCert "good"; + badCert = makeTlsCert "bad"; + +in + +{ + name = "nss-preload"; + + nodes = { + machine = { pkgs, ... }: { + services.nginx = { + enable = true; + + virtualHosts."good" = { + addSSL = true; + sslCertificate = "${goodCert}/cert.pem"; + sslCertificateKey = "${goodCert}/key.pem"; + root = pkgs.runCommand "nginx-root" {} '' + mkdir "$out" + echo 'hello world' > "$out/index.html" + ''; + }; + + virtualHosts."bad" = { + addSSL = true; + sslCertificate = "${badCert}/cert.pem"; + sslCertificateKey = "${badCert}/key.pem"; + root = pkgs.runCommand "nginx-root" {} '' + mkdir "$out" + echo 'foobar' > "$out/index.html" + ''; + }; + }; + + security.pki.certificateFiles = [ "${goodCert}/cert.pem" ]; + + networking.hosts."127.0.0.1" = [ "good" "bad" ]; + + virtualisation.writableStore = true; + + nix.settings.experimental-features = "nix-command"; + }; + }; + + testScript = '' + machine.wait_for_unit("nginx") + machine.wait_for_open_port(443) + + out = machine.succeed("curl https://good/index.html") + assert out == "hello world\n" + + out = machine.succeed("cat ${badCert}/cert.pem > /tmp/cafile.pem; curl --cacert /tmp/cafile.pem https://bad/index.html") + assert out == "foobar\n" + + # Fetching from a server with a trusted cert should work. + machine.succeed("nix build --no-substitute --expr 'import { url = \"https://good/index.html\"; hash = \"sha256-qUiQTy8PR5uPgZdpSzAYSw0u0cHNKh7A+4XSmaGSpEc=\"; }'") + + # Fetching from a server with an untrusted cert should fail. + err = machine.fail("nix build --no-substitute --expr 'import { url = \"https://bad/index.html\"; hash = \"sha256-rsBwZF/lPuOzdjBZN2E08FjMM3JHyXit0Xi2zN+wAZ8=\"; }' 2>&1") + print(err) + assert "SSL certificate problem: self-signed certificate" in err + + # Fetching from a server with a trusted cert should work via environment variable override. + machine.succeed("NIX_SSL_CERT_FILE=/tmp/cafile.pem nix build --no-substitute --expr 'import { url = \"https://bad/index.html\"; hash = \"sha256-rsBwZF/lPuOzdjBZN2E08FjMM3JHyXit0Xi2zN+wAZ8=\"; }'") + ''; +} diff --git a/tests/nixos/fsync.nix b/tests/nixos/fsync.nix new file mode 100644 index 000000000..99ac2b25d --- /dev/null +++ b/tests/nixos/fsync.nix @@ -0,0 +1,39 @@ +{ lib, config, nixpkgs, pkgs, ... }: + +let + pkg1 = pkgs.go; +in + +{ + name = "fsync"; + + nodes.machine = + { config, lib, pkgs, ... }: + { virtualisation.emptyDiskImages = [ 1024 ]; + environment.systemPackages = [ pkg1 ]; + nix.settings.experimental-features = [ "nix-command" ]; + nix.settings.fsync-store-paths = true; + nix.settings.require-sigs = false; + boot.supportedFilesystems = [ "ext4" "btrfs" "xfs" ]; + }; + + testScript = { nodes }: '' + # fmt: off + for fs in ("ext4", "btrfs", "xfs"): + machine.succeed("mkfs.{} {} /dev/vdb".format(fs, "-F" if fs == "ext4" else "-f")) + machine.succeed("mkdir -p /mnt") + machine.succeed("mount /dev/vdb /mnt") + machine.succeed("sync") + machine.succeed("nix copy --offline ${pkg1} --to /mnt") + machine.crash() + + machine.start() + machine.wait_for_unit("multi-user.target") + machine.succeed("mkdir -p /mnt") + machine.succeed("mount /dev/vdb /mnt") + machine.succeed("nix path-info --offline --store /mnt ${pkg1}") + machine.succeed("nix store verify --all --store /mnt --no-trust") + + machine.succeed("umount /dev/vdb") + ''; +} diff --git a/tests/nixos/functional/as-root.nix b/tests/nixos/functional/as-root.nix new file mode 100644 index 000000000..96be3d593 --- /dev/null +++ b/tests/nixos/functional/as-root.nix @@ -0,0 +1,12 @@ +{ + name = "functional-tests-on-nixos_root"; + + imports = [ ./common.nix ]; + + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + run-test-suite >&2 + """) + ''; +} diff --git a/tests/nixos/functional/as-trusted-user.nix b/tests/nixos/functional/as-trusted-user.nix new file mode 100644 index 000000000..d6f825697 --- /dev/null +++ b/tests/nixos/functional/as-trusted-user.nix @@ -0,0 +1,18 @@ +{ + name = "functional-tests-on-nixos_trusted-user"; + + imports = [ ./common.nix ]; + + nodes.machine = { + users.users.alice = { isNormalUser = true; }; + nix.settings.trusted-users = [ "alice" ]; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + export TEST_TRUSTED_USER=1 + su --login --command "run-test-suite" alice >&2 + """) + ''; +} \ No newline at end of file diff --git a/tests/nixos/functional/as-user.nix b/tests/nixos/functional/as-user.nix new file mode 100644 index 000000000..1443f6e6c --- /dev/null +++ b/tests/nixos/functional/as-user.nix @@ -0,0 +1,16 @@ +{ + name = "functional-tests-on-nixos_user"; + + imports = [ ./common.nix ]; + + nodes.machine = { + users.users.alice = { isNormalUser = true; }; + }; + + testScript = '' + machine.wait_for_unit("multi-user.target") + machine.succeed(""" + su --login --command "run-test-suite" alice >&2 + """) + ''; +} diff --git a/tests/nixos/functional/common.nix b/tests/nixos/functional/common.nix new file mode 100644 index 000000000..51fd76884 --- /dev/null +++ b/tests/nixos/functional/common.nix @@ -0,0 +1,71 @@ +{ lib, ... }: + +let + # FIXME (roberth) reference issue + inputDerivation = pkg: (pkg.overrideAttrs (o: { + disallowedReferences = [ ]; + })).inputDerivation; + +in +{ + # We rarely change the script in a way that benefits from type checking, so + # we skip it to save time. + skipTypeCheck = true; + + nodes.machine = { config, pkgs, ... }: { + + virtualisation.writableStore = true; + system.extraDependencies = [ + (inputDerivation config.nix.package) + ]; + + nix.settings.substituters = lib.mkForce []; + + environment.systemPackages = let + run-test-suite = pkgs.writeShellApplication { + name = "run-test-suite"; + runtimeInputs = [ pkgs.gnumake pkgs.jq pkgs.git ]; + text = '' + set -x + cat /proc/sys/fs/file-max + ulimit -Hn + ulimit -Sn + cd ~ + cp -r ${pkgs.nix.overrideAttrs (o: { + name = "nix-configured-source"; + outputs = [ "out" ]; + separateDebugInfo = false; + disallowedReferences = [ ]; + buildPhase = ":"; + checkPhase = ":"; + installPhase = '' + cp -r . $out + ''; + installCheckPhase = ":"; + fixupPhase = ":"; + doInstallCheck = true; + })} nix + chmod -R +w nix + cd nix + + # Tests we don't need + echo >tests/functional/plugins/local.mk + sed -i tests/functional/local.mk \ + -e 's!nix_tests += plugins\.sh!!' \ + -e 's!nix_tests += test-libstoreconsumer\.sh!!' \ + ; + + export isTestOnNixOS=1 + export version=${config.nix.package.version} + export NIX_REMOTE_=daemon + export NIX_REMOTE=daemon + export NIX_STORE=${builtins.storeDir} + make -j1 installcheck --keep-going + ''; + }; + in [ + run-test-suite + pkgs.git + ]; + }; +} diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index 221045009..8e646f6dd 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -181,8 +181,14 @@ in print(out) info = json.loads(out) assert info["revision"] == "${private-flake-rev}", f"revision mismatch: {info['revision']} != ${private-flake-rev}" + assert info["fingerprint"] cat_log() + # Fetching with the resolved URL should produce the same result. + info2 = json.loads(client.succeed(f"nix flake metadata {info['url']} --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0")) + print(info["fingerprint"], info2["fingerprint"]) + assert info["fingerprint"] == info2["fingerprint"], "fingerprint mismatch" + client.succeed("nix registry pin nixpkgs") client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2") diff --git a/tests/nixos/nix-copy-closure.nix b/tests/nixos/nix-copy-closure.nix index 66cbfb033..b9daa0a1f 100644 --- a/tests/nixos/nix-copy-closure.nix +++ b/tests/nixos/nix-copy-closure.nix @@ -1,6 +1,6 @@ # Test ‘nix-copy-closure’. -{ lib, config, nixpkgs, hostPkgs, ... }: +{ lib, config, nixpkgs, ... }: let pkgs = config.nodes.client.nixpkgs.pkgs; diff --git a/tests/nixos/remote-builds.nix b/tests/nixos/remote-builds.nix index 1661203ec..ab159eaad 100644 --- a/tests/nixos/remote-builds.nix +++ b/tests/nixos/remote-builds.nix @@ -81,6 +81,17 @@ in virtualisation.additionalPaths = [ config.system.build.extraUtils ]; nix.settings.substituters = lib.mkForce [ ]; programs.ssh.extraConfig = "ConnectTimeout 30"; + environment.systemPackages = [ + # `bad-shell` is used to make sure Nix works an environment with a misbehaving shell. + # + # More realistically, a bad shell would still run the command ("echo started") + # but considering that our solution is to avoid this shell (set via $SHELL), we + # don't need to bother with a more functional mock shell. + (pkgs.writeScriptBin "bad-shell" '' + #!${pkgs.runtimeShell} + echo "Hello, I am a broken shell" + '') + ]; }; }; @@ -104,11 +115,23 @@ in builder.succeed("mkdir -p -m 700 /root/.ssh") builder.copy_from_host("key.pub", "/root/.ssh/authorized_keys") builder.wait_for_unit("sshd") - client.succeed(f"ssh -o StrictHostKeyChecking=no {builder.name} 'echo hello world'") + # Make sure the builder can handle our login correctly + builder.wait_for_unit("multi-user.target") + # Make sure there's no funny business on the client either + # (should not be necessary, but we have reason to be careful) + client.wait_for_unit("multi-user.target") + client.succeed(f""" + ssh -o StrictHostKeyChecking=no {builder.name} \ + 'echo hello world on $(hostname)' >&2 + """) + + # Check that SSH uses SHELL for LocalCommand, as expected, and check that + # our test setup here is working. The next test will use this bad SHELL. + client.succeed(f"SHELL=$(which bad-shell) ssh -oLocalCommand='true' -oPermitLocalCommand=yes {builder1.name} 'echo hello world' | grep -F 'Hello, I am a broken shell'") # Perform a build and check that it was performed on the builder. out = client.succeed( - "nix-build ${expr nodes.client 1} 2> build-output", + "SHELL=$(which bad-shell) nix-build ${expr nodes.client 1} 2> build-output", "grep -q Hello build-output" ) builder1.succeed(f"test -e {out}") diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix new file mode 100644 index 000000000..6ae2e3572 --- /dev/null +++ b/tests/nixos/s3-binary-cache-store.nix @@ -0,0 +1,66 @@ +{ lib, config, nixpkgs, ... }: + +let + pkgs = config.nodes.client.nixpkgs.pkgs; + + pkgA = pkgs.cowsay; + + accessKey = "BKIKJAA5BMMU2RHO6IBB"; + secretKey = "V7f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12"; + env = "AWS_ACCESS_KEY_ID=${accessKey} AWS_SECRET_ACCESS_KEY=${secretKey}"; + + storeUrl = "s3://my-cache?endpoint=http://server:9000®ion=eu-west-1"; + +in { + name = "nix-copy-closure"; + + nodes = + { server = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + virtualisation.additionalPaths = [ pkgA ]; + environment.systemPackages = [ pkgs.minio-client ]; + nix.extraOptions = "experimental-features = nix-command"; + services.minio = { + enable = true; + region = "eu-west-1"; + rootCredentialsFile = pkgs.writeText "minio-credentials-full" '' + MINIO_ROOT_USER=${accessKey} + MINIO_ROOT_PASSWORD=${secretKey} + ''; + }; + networking.firewall.allowedTCPPorts = [ 9000 ]; + }; + + client = + { config, pkgs, ... }: + { virtualisation.writableStore = true; + nix.extraOptions = "experimental-features = nix-command"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + start_all() + + # Create a binary cache. + server.wait_for_unit("minio") + + server.succeed("mc config host add minio http://localhost:9000 ${accessKey} ${secretKey} --api s3v4") + server.succeed("mc mb minio/my-cache") + + server.succeed("${env} nix copy --to '${storeUrl}' ${pkgA}") + + # Test fetchurl on s3:// URLs while we're at it. + client.succeed("${env} nix eval --impure --expr 'builtins.fetchurl { name = \"foo\"; url = \"s3://my-cache/nix-cache-info?endpoint=http://server:9000®ion=eu-west-1\"; }'") + + # Copy a package from the binary cache. + client.fail("nix path-info ${pkgA}") + + client.succeed("${env} nix store info --store '${storeUrl}' >&2") + + client.succeed("${env} nix copy --no-check-sigs --from '${storeUrl}' ${pkgA}") + + client.succeed("nix path-info ${pkgA}") + ''; +} diff --git a/tests/nixos/tarball-flakes.nix b/tests/nixos/tarball-flakes.nix index e30d15739..84cf377ec 100644 --- a/tests/nixos/tarball-flakes.nix +++ b/tests/nixos/tarball-flakes.nix @@ -5,7 +5,7 @@ let root = pkgs.runCommand "nixpkgs-flake" {} '' - mkdir -p $out/stable + mkdir -p $out/{stable,tags} set -x dir=nixpkgs-${nixpkgs.shortRev} @@ -14,9 +14,13 @@ let find $dir -print0 | xargs -0 touch -h -t ${builtins.substring 0 12 nixpkgs.lastModifiedDate}.${builtins.substring 12 2 nixpkgs.lastModifiedDate} -- tar cfz $out/stable/${nixpkgs.rev}.tar.gz $dir --hard-dereference - echo 'Redirect "/latest.tar.gz" "/stable/${nixpkgs.rev}.tar.gz"' > $out/.htaccess - - echo 'Header set Link "; rel=\"immutable\""' > $out/stable/.htaccess + # Set the "Link" header on the redirect but not the final response to + # simulate an S3-like serving environment where the final host cannot set + # arbitrary headers. + cat >$out/tags/.htaccess <; rel=\"immutable\"" + EOF ''; in @@ -59,26 +63,32 @@ in machine.wait_for_unit("httpd.service") - out = machine.succeed("nix flake metadata --json http://localhost/latest.tar.gz") + out = machine.succeed("nix flake metadata --json http://localhost/tags/latest.tar.gz") print(out) info = json.loads(out) # Check that we got redirected to the immutable URL. assert info["locked"]["url"] == "http://localhost/stable/${nixpkgs.rev}.tar.gz" + # Check that we got a fingerprint for caching. + assert info["fingerprint"] + # Check that we got the rev and revCount attributes. assert info["revision"] == "${nixpkgs.rev}" assert info["revCount"] == 1234 + # Check that a 0-byte HTTP 304 "Not modified" result works. + machine.succeed("nix flake metadata --refresh --json http://localhost/tags/latest.tar.gz") + # Check that fetching with rev/revCount/narHash succeeds. - machine.succeed("nix flake metadata --json http://localhost/latest.tar.gz?rev=" + info["revision"]) - machine.succeed("nix flake metadata --json http://localhost/latest.tar.gz?revCount=" + str(info["revCount"])) - machine.succeed("nix flake metadata --json http://localhost/latest.tar.gz?narHash=" + info["locked"]["narHash"]) + machine.succeed("nix flake metadata --json http://localhost/tags/latest.tar.gz?rev=" + info["revision"]) + machine.succeed("nix flake metadata --json http://localhost/tags/latest.tar.gz?revCount=" + str(info["revCount"])) + machine.succeed("nix flake metadata --json http://localhost/tags/latest.tar.gz?narHash=" + info["locked"]["narHash"]) # Check that fetching fails if we provide incorrect attributes. - machine.fail("nix flake metadata --json http://localhost/latest.tar.gz?rev=493300eb13ae6fb387fbd47bf54a85915acc31c0") - machine.fail("nix flake metadata --json http://localhost/latest.tar.gz?revCount=789") - machine.fail("nix flake metadata --json http://localhost/latest.tar.gz?narHash=sha256-tbudgBSg+bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw=") + machine.fail("nix flake metadata --json http://localhost/tags/latest.tar.gz?rev=493300eb13ae6fb387fbd47bf54a85915acc31c0") + machine.fail("nix flake metadata --json http://localhost/tags/latest.tar.gz?revCount=789") + machine.fail("nix flake metadata --json http://localhost/tags/latest.tar.gz?narHash=sha256-tbudgBSg+bHWHiHnlteNzN8TUvI80ygS9IULh4rklEw=") ''; } diff --git a/tests/nixos/user-sandboxing/attacker.c b/tests/nixos/user-sandboxing/attacker.c new file mode 100644 index 000000000..3bd729c04 --- /dev/null +++ b/tests/nixos/user-sandboxing/attacker.c @@ -0,0 +1,82 @@ +#define _GNU_SOURCE + +#include +#include +#include +#include +#include +#include + +#define SYS_fchmodat2 452 + +int fchmodat2(int dirfd, const char *pathname, mode_t mode, int flags) { + return syscall(SYS_fchmodat2, dirfd, pathname, mode, flags); +} + +int main(int argc, char **argv) { + if (argc <= 1) { + // stage 1: place the setuid-builder executable + + // make the build directory world-accessible first + chmod(".", 0755); + + if (fchmodat2(AT_FDCWD, "attacker", 06755, AT_SYMLINK_NOFOLLOW) < 0) { + perror("Setting the suid bit on attacker"); + exit(-1); + } + + } else { + // stage 2: corrupt the victim derivation while it's building + + // prevent the kill + if (setresuid(-1, -1, getuid())) { + perror("setresuid"); + exit(-1); + } + + if (fork() == 0) { + + // wait for the victim to build + int fd = inotify_init(); + inotify_add_watch(fd, argv[1], IN_CREATE); + int dirfd = open(argv[1], O_DIRECTORY); + if (dirfd < 0) { + perror("opening the global build directory"); + exit(-1); + } + char buf[4096]; + fprintf(stderr, "Entering the inotify loop\n"); + for (;;) { + ssize_t len = read(fd, buf, sizeof(buf)); + struct inotify_event *ev; + for (char *pe = buf; pe < buf + len; + pe += sizeof(struct inotify_event) + ev->len) { + ev = (struct inotify_event *)pe; + fprintf(stderr, "folder %s created\n", ev->name); + // wait a bit to prevent racing against the creation + sleep(1); + int builddir = openat(dirfd, ev->name, O_DIRECTORY); + if (builddir < 0) { + perror("opening the build directory"); + continue; + } + int resultfile = openat(builddir, "build/result", O_WRONLY | O_TRUNC); + if (resultfile < 0) { + perror("opening the hijacked file"); + continue; + } + int writeres = write(resultfile, "bad\n", 4); + if (writeres < 0) { + perror("writing to the hijacked file"); + continue; + } + fprintf(stderr, "Hijacked the build for %s\n", ev->name); + return 0; + } + } + } + + exit(0); + } +} + diff --git a/tests/nixos/user-sandboxing/default.nix b/tests/nixos/user-sandboxing/default.nix new file mode 100644 index 000000000..8a16f44e8 --- /dev/null +++ b/tests/nixos/user-sandboxing/default.nix @@ -0,0 +1,129 @@ +{ config, ... }: + +let + pkgs = config.nodes.machine.nixpkgs.pkgs; + + attacker = pkgs.runCommandWith { + name = "attacker"; + stdenv = pkgs.pkgsStatic.stdenv; + } '' + $CC -static -o $out ${./attacker.c} + ''; + + try-open-build-dir = pkgs.writeScript "try-open-build-dir" '' + export PATH=${pkgs.coreutils}/bin:$PATH + + set -x + + chmod 700 . + # Shouldn't be able to open the root build directory + (! chmod 700 ..) + + touch foo + + # Synchronisation point: create a world-writable fifo and wait for someone + # to write into it + mkfifo syncPoint + chmod 777 syncPoint + cat syncPoint + + touch $out + + set +x + ''; + + create-hello-world = pkgs.writeScript "create-hello-world" '' + export PATH=${pkgs.coreutils}/bin:$PATH + + set -x + + echo "hello, world" > result + + # Synchronisation point: create a world-writable fifo and wait for someone + # to write into it + mkfifo syncPoint + chmod 777 syncPoint + cat syncPoint + + cp result $out + + set +x + ''; + +in +{ + name = "sandbox-setuid-leak"; + + nodes.machine = + { config, lib, pkgs, ... }: + { virtualisation.writableStore = true; + nix.settings.substituters = lib.mkForce [ ]; + nix.nrBuildUsers = 1; + virtualisation.additionalPaths = [ pkgs.busybox-sandbox-shell attacker try-open-build-dir create-hello-world pkgs.socat ]; + boot.kernelPackages = pkgs.linuxPackages_latest; + users.users.alice = { + isNormalUser = true; + }; + }; + + testScript = { nodes }: '' + start_all() + + with subtest("A builder can't give access to its build directory"): + # Make sure that a builder can't change the permissions on its build + # directory to the point of opening it up to external users + + # A derivation whose builder tries to make its build directory as open + # as possible and wait for someone to hijack it + machine.succeed(r""" + nix-build -v -E ' + builtins.derivation { + name = "open-build-dir"; + system = builtins.currentSystem; + builder = "${pkgs.busybox-sandbox-shell}/bin/sh"; + args = [ (builtins.storePath "${try-open-build-dir}") ]; + }' >&2 & + """.strip()) + + # Wait for the build to be ready + # This is OK because it runs as root, so we can access everything + machine.wait_for_file("/tmp/nix-build-open-build-dir.drv-0/build/syncPoint") + + # But Alice shouldn't be able to access the build directory + machine.fail("su alice -c 'ls /tmp/nix-build-open-build-dir.drv-0/build'") + machine.fail("su alice -c 'touch /tmp/nix-build-open-build-dir.drv-0/build/bar'") + machine.fail("su alice -c 'cat /tmp/nix-build-open-build-dir.drv-0/build/foo'") + + # Tell the user to finish the build + machine.succeed("echo foo > /tmp/nix-build-open-build-dir.drv-0/build/syncPoint") + + with subtest("Being able to execute stuff as the build user doesn't give access to the build dir"): + machine.succeed(r""" + nix-build -E ' + builtins.derivation { + name = "innocent"; + system = builtins.currentSystem; + builder = "${pkgs.busybox-sandbox-shell}/bin/sh"; + args = [ (builtins.storePath "${create-hello-world}") ]; + }' >&2 & + """.strip()) + machine.wait_for_file("/tmp/nix-build-innocent.drv-0/build/syncPoint") + + # The build ran as `nixbld1` (which is the only build user on the + # machine), but a process running as `nixbld1` outside the sandbox + # shouldn't be able to touch the build directory regardless + machine.fail("su nixbld1 --shell ${pkgs.busybox-sandbox-shell}/bin/sh -c 'ls /tmp/nix-build-innocent.drv-0/build'") + machine.fail("su nixbld1 --shell ${pkgs.busybox-sandbox-shell}/bin/sh -c 'echo pwned > /tmp/nix-build-innocent.drv-0/build/result'") + + # Finish the build + machine.succeed("echo foo > /tmp/nix-build-innocent.drv-0/build/syncPoint") + + # Check that the build was not affected + machine.succeed(r""" + cat ./result + test "$(cat ./result)" = "hello, world" + """.strip()) + ''; + +} + diff --git a/tests/repl-completion.nix b/tests/repl-completion.nix new file mode 100644 index 000000000..3ba198a98 --- /dev/null +++ b/tests/repl-completion.nix @@ -0,0 +1,40 @@ +{ runCommand, nix, expect }: + +# We only use expect when necessary, e.g. for testing tab completion in nix repl. +# See also tests/functional/repl.sh + +runCommand "repl-completion" { + nativeBuildInputs = [ + expect + nix + ]; + expectScript = '' + # Regression https://github.com/NixOS/nix/pull/10778 + spawn nix repl --offline --extra-experimental-features nix-command + expect "nix-repl>" + send "foo = import ./does-not-exist.nix\n" + expect "nix-repl>" + send "foo.\t" + expect { + "nix-repl>" { + puts "Got another prompt. Good." + } + eof { + puts "Got EOF. Bad." + exit 1 + } + } + exit 0 + ''; + passAsFile = [ "expectScript" ]; +} +'' + export NIX_STORE=$TMPDIR/store + export NIX_STATE_DIR=$TMPDIR/state + export HOME=$TMPDIR/home + mkdir $HOME + + nix-store --init + expect $expectScriptPath + touch $out +'' \ No newline at end of file diff --git a/tests/unit/libexpr-support/.version b/tests/unit/libexpr-support/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libexpr-support/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libexpr-support/build-utils-meson b/tests/unit/libexpr-support/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libexpr-support/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libexpr-support/meson.build b/tests/unit/libexpr-support/meson.build new file mode 100644 index 000000000..4f50478aa --- /dev/null +++ b/tests/unit/libexpr-support/meson.build @@ -0,0 +1,76 @@ +project('nix-expr-test-support', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-util-test-support'), + dependency('nix-store'), + dependency('nix-store-test-support'), + dependency('nix-expr'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +rapidcheck = dependency('rapidcheck') +deps_public += rapidcheck + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'tests/value/context.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'tests/libexpr.hh', + 'tests/nix_api_expr.hh', + 'tests/value/context.hh', +) + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nix-expr-test-support', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326 + # is available. See also ../libutil/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/tests/unit/libexpr-support/package.nix b/tests/unit/libexpr-support/package.nix new file mode 100644 index 000000000..f53aa842f --- /dev/null +++ b/tests/unit/libexpr-support/package.nix @@ -0,0 +1,77 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-store-test-support +, nix-expr + +, rapidcheck + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-util-test-support"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-store-test-support + nix-expr + rapidcheck + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/tests/unit/libexpr-support/tests/libexpr.hh b/tests/unit/libexpr-support/tests/libexpr.hh index 1a4313990..045607e87 100644 --- a/tests/unit/libexpr-support/tests/libexpr.hh +++ b/tests/unit/libexpr-support/tests/libexpr.hh @@ -4,12 +4,14 @@ #include #include +#include "fetch-settings.hh" #include "value.hh" #include "nixexpr.hh" +#include "nixexpr.hh" #include "eval.hh" +#include "eval-gc.hh" #include "eval-inline.hh" #include "eval-settings.hh" -#include "store-api.hh" #include "tests/libstore.hh" @@ -19,14 +21,14 @@ namespace nix { static void SetUpTestSuite() { LibStoreTest::SetUpTestSuite(); initGC(); - evalSettings.nixPath = {}; } protected: LibExprTest() : LibStoreTest() - , state({}, store) + , state({}, store, fetchSettings, evalSettings, nullptr) { + evalSettings.nixPath = {}; } Value eval(std::string input, bool forceValue = true) { Value v; @@ -42,6 +44,9 @@ namespace nix { return state.symbols.create(value); } + bool readOnlyMode = true; + fetchers::Settings fetchSettings{}; + EvalSettings evalSettings{readOnlyMode}; EvalState state; }; @@ -80,7 +85,7 @@ namespace nix { if (arg.type() != nInt) { return false; } - return arg.integer() == v; + return arg.integer().value == v; } MATCHER_P(IsFloatEq, v, fmt("The float is equal to \"%1%\"", v)) { diff --git a/tests/unit/libexpr-support/tests/nix_api_expr.hh b/tests/unit/libexpr-support/tests/nix_api_expr.hh index d1840d034..6ddca0d14 100644 --- a/tests/unit/libexpr-support/tests/nix_api_expr.hh +++ b/tests/unit/libexpr-support/tests/nix_api_expr.hh @@ -25,7 +25,7 @@ protected: } EvalState * state; - Value * value; + nix_value * value; }; } diff --git a/tests/unit/libexpr/.version b/tests/unit/libexpr/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libexpr/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libexpr/build-utils-meson b/tests/unit/libexpr/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libexpr/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libexpr/data/.gitkeep b/tests/unit/libexpr/data/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk index c59191db4..1617e2823 100644 --- a/tests/unit/libexpr/local.mk +++ b/tests/unit/libexpr/local.mk @@ -4,7 +4,7 @@ programs += libexpr-tests libexpr-tests_NAME := libnixexpr-tests -libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data +libexpr-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libexpr-tests.xml libexpr-tests_DIR := $(d) @@ -38,3 +38,8 @@ libexpr-tests_LIBS = \ libexpr libexprc libfetchers libstore libstorec libutil libutilc libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock + +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + libexpr-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif diff --git a/tests/unit/libexpr/main.cc b/tests/unit/libexpr/main.cc index cf7fcf5a3..e3412d9ef 100644 --- a/tests/unit/libexpr/main.cc +++ b/tests/unit/libexpr/main.cc @@ -34,6 +34,9 @@ int main (int argc, char **argv) { setEnv("_NIX_TEST_NO_SANDBOX", "1"); #endif + // For pipe operator tests in trivial.cc + experimentalFeatureSettings.set("experimental-features", "pipe-operators"); + ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } diff --git a/tests/unit/libexpr/meson.build b/tests/unit/libexpr/meson.build new file mode 100644 index 000000000..21c321334 --- /dev/null +++ b/tests/unit/libexpr/meson.build @@ -0,0 +1,92 @@ +project('nix-expr-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-expr'), + dependency('nix-expr-c'), + dependency('nix-expr-test-support'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +subdir('build-utils-meson/export-all-symbols') + +rapidcheck = dependency('rapidcheck') +deps_private += rapidcheck + +gtest = dependency('gtest') +deps_private += gtest + +gtest = dependency('gmock') +deps_private += gtest + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + '-include', 'config-util.h', + '-include', 'config-store.h', + '-include', 'config-expr.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'derived-path.cc', + 'error_traces.cc', + 'eval.cc', + 'json.cc', + 'main.cc', + 'nix_api_expr.cc', + 'nix_api_external.cc', + 'nix_api_value.cc', + 'primops.cc', + 'search-path.cc', + 'trivial.cc', + 'value/context.cc', + 'value/print.cc', + 'value/value.cc', +) + +include_dirs = [include_directories('.')] + + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + # TODO: -lrapidcheck, see ../libutil-support/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + install : true, +) + +test( + meson.project_name(), + this_exe, + env : { + '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + }, + protocol : 'gtest', +) diff --git a/tests/unit/libexpr/nix_api_expr.cc b/tests/unit/libexpr/nix_api_expr.cc index 0818f1cab..b37ac44b3 100644 --- a/tests/unit/libexpr/nix_api_expr.cc +++ b/tests/unit/libexpr/nix_api_expr.cc @@ -8,7 +8,7 @@ #include "tests/nix_api_expr.hh" #include "tests/string_callback.hh" -#include "gmock/gmock.h" +#include #include namespace nixC { @@ -39,12 +39,12 @@ TEST_F(nix_api_expr_test, nix_expr_eval_drv) ASSERT_EQ(NIX_TYPE_ATTRS, nix_get_type(nullptr, value)); EvalState * stateFn = nix_state_create(nullptr, nullptr, store); - Value * valueFn = nix_alloc_value(nullptr, state); + nix_value * valueFn = nix_alloc_value(nullptr, state); nix_expr_eval_from_string(nullptr, stateFn, "builtins.toString", ".", valueFn); ASSERT_EQ(NIX_TYPE_FUNCTION, nix_get_type(nullptr, valueFn)); EvalState * stateResult = nix_state_create(nullptr, nullptr, store); - Value * valueResult = nix_alloc_value(nullptr, stateResult); + nix_value * valueResult = nix_alloc_value(nullptr, stateResult); nix_value_call(ctx, stateResult, valueFn, value, valueResult); ASSERT_EQ(NIX_TYPE_STRING, nix_get_type(nullptr, valueResult)); @@ -70,7 +70,7 @@ TEST_F(nix_api_expr_test, nix_build_drv) })"; nix_expr_eval_from_string(nullptr, state, expr, ".", value); - Value * drvPathValue = nix_get_attr_byname(nullptr, value, state, "drvPath"); + nix_value * drvPathValue = nix_get_attr_byname(nullptr, value, state, "drvPath"); std::string drvPath; nix_get_string(nullptr, drvPathValue, OBSERVE_STRING(drvPath)); @@ -84,7 +84,7 @@ TEST_F(nix_api_expr_test, nix_build_drv) StorePath * drvStorePath = nix_store_parse_path(ctx, store, drvPath.c_str()); ASSERT_EQ(true, nix_store_is_valid_path(ctx, store, drvStorePath)); - Value * outPathValue = nix_get_attr_byname(ctx, value, state, "outPath"); + nix_value * outPathValue = nix_get_attr_byname(ctx, value, state, "outPath"); std::string outPath; nix_get_string(ctx, outPathValue, OBSERVE_STRING(outPath)); @@ -191,4 +191,214 @@ TEST_F(nix_api_expr_test, nix_expr_realise_context) nix_realised_string_free(r); } +const char * SAMPLE_USER_DATA = "whatever"; + +static void +primop_square(void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) +{ + assert(context); + assert(state); + assert(user_data == SAMPLE_USER_DATA); + auto i = nix_get_int(context, args[0]); + nix_init_int(context, ret, i * i); +} + +TEST_F(nix_api_expr_test, nix_expr_primop) +{ + PrimOp * primop = + nix_alloc_primop(ctx, primop_square, 1, "square", nullptr, "square an integer", (void *) SAMPLE_USER_DATA); + assert_ctx_ok(); + nix_value * primopValue = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_primop(ctx, primopValue, primop); + assert_ctx_ok(); + + nix_value * three = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_int(ctx, three, 3); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_value_call(ctx, state, primopValue, three, result); + assert_ctx_ok(); + + auto r = nix_get_int(ctx, result); + ASSERT_EQ(9, r); +} + +static void +primop_repeat(void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) +{ + assert(context); + assert(state); + assert(user_data == SAMPLE_USER_DATA); + + // Get the string to repeat + std::string s; + if (nix_get_string(context, args[0], OBSERVE_STRING(s)) != NIX_OK) + return; + + // Get the number of times to repeat + auto n = nix_get_int(context, args[1]); + if (nix_err_code(context) != NIX_OK) + return; + + // Repeat the string + std::string result; + for (int i = 0; i < n; ++i) + result += s; + + nix_init_string(context, ret, result.c_str()); +} + +TEST_F(nix_api_expr_test, nix_expr_primop_arity_2_multiple_calls) +{ + PrimOp * primop = + nix_alloc_primop(ctx, primop_repeat, 2, "repeat", nullptr, "repeat a string", (void *) SAMPLE_USER_DATA); + assert_ctx_ok(); + nix_value * primopValue = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_primop(ctx, primopValue, primop); + assert_ctx_ok(); + + nix_value * hello = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_string(ctx, hello, "hello"); + assert_ctx_ok(); + + nix_value * three = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_int(ctx, three, 3); + assert_ctx_ok(); + + nix_value * partial = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_value_call(ctx, state, primopValue, hello, partial); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_value_call(ctx, state, partial, three, result); + assert_ctx_ok(); + + std::string r; + nix_get_string(ctx, result, OBSERVE_STRING(r)); + ASSERT_STREQ("hellohellohello", r.c_str()); +} + +TEST_F(nix_api_expr_test, nix_expr_primop_arity_2_single_call) +{ + PrimOp * primop = + nix_alloc_primop(ctx, primop_repeat, 2, "repeat", nullptr, "repeat a string", (void *) SAMPLE_USER_DATA); + assert_ctx_ok(); + nix_value * primopValue = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_primop(ctx, primopValue, primop); + assert_ctx_ok(); + + nix_value * hello = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_string(ctx, hello, "hello"); + assert_ctx_ok(); + + nix_value * three = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_int(ctx, three, 3); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + assert_ctx_ok(); + NIX_VALUE_CALL(ctx, state, result, primopValue, hello, three); + assert_ctx_ok(); + + std::string r; + nix_get_string(ctx, result, OBSERVE_STRING(r)); + assert_ctx_ok(); + + ASSERT_STREQ("hellohellohello", r.c_str()); +} + +static void +primop_bad_no_return(void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) +{ +} + +TEST_F(nix_api_expr_test, nix_expr_primop_bad_no_return) +{ + PrimOp * primop = + nix_alloc_primop(ctx, primop_bad_no_return, 1, "badNoReturn", nullptr, "a broken primop", nullptr); + assert_ctx_ok(); + nix_value * primopValue = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_primop(ctx, primopValue, primop); + assert_ctx_ok(); + + nix_value * three = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_int(ctx, three, 3); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_value_call(ctx, state, primopValue, three, result); + ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); + ASSERT_THAT( + ctx->last_err, + testing::Optional( + testing::HasSubstr("Implementation error in custom function: return value was not initialized"))); + ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("badNoReturn"))); +} + +static void primop_bad_return_thunk( + void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) +{ + nix_init_apply(context, ret, args[0], args[1]); +} +TEST_F(nix_api_expr_test, nix_expr_primop_bad_return_thunk) +{ + PrimOp * primop = + nix_alloc_primop(ctx, primop_bad_return_thunk, 2, "badReturnThunk", nullptr, "a broken primop", nullptr); + assert_ctx_ok(); + nix_value * primopValue = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_primop(ctx, primopValue, primop); + assert_ctx_ok(); + + nix_value * toString = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_expr_eval_from_string(ctx, state, "builtins.toString", ".", toString); + assert_ctx_ok(); + + nix_value * four = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_int(ctx, four, 4); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + assert_ctx_ok(); + NIX_VALUE_CALL(ctx, state, result, primopValue, toString, four); + + ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); + ASSERT_THAT( + ctx->last_err, + testing::Optional( + testing::HasSubstr("Implementation error in custom function: return value must not be a thunk"))); + ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("badReturnThunk"))); +} + +TEST_F(nix_api_expr_test, nix_value_call_multi_no_args) +{ + nix_value * n = nix_alloc_value(ctx, state); + nix_init_int(ctx, n, 3); + assert_ctx_ok(); + + nix_value * r = nix_alloc_value(ctx, state); + nix_value_call_multi(ctx, state, n, 0, nullptr, r); + assert_ctx_ok(); + + auto rInt = nix_get_int(ctx, r); + assert_ctx_ok(); + ASSERT_EQ(3, rInt); +} } // namespace nixC diff --git a/tests/unit/libexpr/nix_api_external.cc b/tests/unit/libexpr/nix_api_external.cc index 2391f8317..81ff285a4 100644 --- a/tests/unit/libexpr/nix_api_external.cc +++ b/tests/unit/libexpr/nix_api_external.cc @@ -49,10 +49,10 @@ TEST_F(nix_api_expr_test, nix_expr_eval_external) nix_init_external(ctx, value, val); EvalState * stateResult = nix_state_create(nullptr, nullptr, store); - Value * valueResult = nix_alloc_value(nullptr, stateResult); + nix_value * valueResult = nix_alloc_value(nullptr, stateResult); EvalState * stateFn = nix_state_create(nullptr, nullptr, store); - Value * valueFn = nix_alloc_value(nullptr, stateFn); + nix_value * valueFn = nix_alloc_value(nullptr, stateFn); nix_expr_eval_from_string(nullptr, state, "builtins.typeOf", ".", valueFn); diff --git a/tests/unit/libexpr/nix_api_value.cc b/tests/unit/libexpr/nix_api_value.cc index 6e1131e10..7fc8b4f64 100644 --- a/tests/unit/libexpr/nix_api_value.cc +++ b/tests/unit/libexpr/nix_api_value.cc @@ -4,16 +4,26 @@ #include "nix_api_util_internal.h" #include "nix_api_expr.h" #include "nix_api_value.h" +#include "nix_api_expr_internal.h" #include "tests/nix_api_expr.hh" #include "tests/string_callback.hh" #include "gmock/gmock.h" +#include #include #include namespace nixC { +TEST_F(nix_api_expr_test, as_nix_value_ptr) +{ + // nix_alloc_value casts nix::Value to nix_value + // It should be obvious from the decl that that works, but if it doesn't, + // the whole implementation would be utterly broken. + ASSERT_EQ(sizeof(nix::Value), sizeof(nix_value)); +} + TEST_F(nix_api_expr_test, nix_value_get_int_invalid) { ASSERT_EQ(0, nix_get_int(ctx, nullptr)); @@ -138,8 +148,8 @@ TEST_F(nix_api_expr_test, nix_build_and_init_list) int size = 10; ListBuilder * builder = nix_make_list_builder(ctx, state, size); - Value * intValue = nix_alloc_value(ctx, state); - Value * intValue2 = nix_alloc_value(ctx, state); + nix_value * intValue = nix_alloc_value(ctx, state); + nix_value * intValue2 = nix_alloc_value(ctx, state); // `init` and `insert` can be called in any order nix_init_int(ctx, intValue, 42); @@ -194,10 +204,10 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr) BindingsBuilder * builder = nix_make_bindings_builder(ctx, state, size); - Value * intValue = nix_alloc_value(ctx, state); + nix_value * intValue = nix_alloc_value(ctx, state); nix_init_int(ctx, intValue, 42); - Value * stringValue = nix_alloc_value(ctx, state); + nix_value * stringValue = nix_alloc_value(ctx, state); nix_init_string(ctx, stringValue, "foo"); nix_bindings_builder_insert(ctx, builder, "a", intValue); @@ -207,7 +217,7 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr) ASSERT_EQ(2, nix_get_attrs_size(ctx, value)); - Value * out_value = nix_get_attr_byname(ctx, value, state, "a"); + nix_value * out_value = nix_get_attr_byname(ctx, value, state, "a"); ASSERT_EQ(42, nix_get_int(ctx, out_value)); nix_gc_decref(ctx, out_value); @@ -251,21 +261,24 @@ TEST_F(nix_api_expr_test, nix_value_init) // two = 2; // f = a: a * a; - Value * two = nix_alloc_value(ctx, state); + nix_value * two = nix_alloc_value(ctx, state); nix_init_int(ctx, two, 2); - Value * f = nix_alloc_value(ctx, state); + nix_value * f = nix_alloc_value(ctx, state); nix_expr_eval_from_string( - ctx, state, R"( + ctx, + state, + R"( a: a * a )", - "", f); + "", + f); // Test // r = f two; - Value * r = nix_alloc_value(ctx, state); + nix_value * r = nix_alloc_value(ctx, state); nix_init_apply(ctx, r, f, two); assert_ctx_ok(); @@ -294,11 +307,11 @@ TEST_F(nix_api_expr_test, nix_value_init) TEST_F(nix_api_expr_test, nix_value_init_apply_error) { - Value * some_string = nix_alloc_value(ctx, state); + nix_value * some_string = nix_alloc_value(ctx, state); nix_init_string(ctx, some_string, "some string"); assert_ctx_ok(); - Value * v = nix_alloc_value(ctx, state); + nix_value * v = nix_alloc_value(ctx, state); nix_init_apply(ctx, v, some_string, some_string); assert_ctx_ok(); @@ -323,22 +336,28 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) // r = f e // r should not throw an exception, because e is not evaluated - Value * f = nix_alloc_value(ctx, state); + nix_value * f = nix_alloc_value(ctx, state); nix_expr_eval_from_string( - ctx, state, R"( + ctx, + state, + R"( a: { foo = a; } )", - "", f); + "", + f); assert_ctx_ok(); - Value * e = nix_alloc_value(ctx, state); + nix_value * e = nix_alloc_value(ctx, state); { - Value * g = nix_alloc_value(ctx, state); + nix_value * g = nix_alloc_value(ctx, state); nix_expr_eval_from_string( - ctx, state, R"( + ctx, + state, + R"( _ignore: throw "error message for test case nix_value_init_apply_lazy_arg" )", - "", g); + "", + g); assert_ctx_ok(); nix_init_apply(ctx, e, g, g); @@ -346,7 +365,7 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) nix_gc_decref(ctx, g); } - Value * r = nix_alloc_value(ctx, state); + nix_value * r = nix_alloc_value(ctx, state); nix_init_apply(ctx, r, f, e); assert_ctx_ok(); @@ -358,7 +377,7 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) ASSERT_EQ(1, n); // nix_get_attr_byname isn't lazy (it could have been) so it will throw the exception - Value * foo = nix_get_attr_byname(ctx, r, state, "foo"); + nix_value * foo = nix_get_attr_byname(ctx, r, state, "foo"); ASSERT_EQ(nullptr, foo); ASSERT_THAT(ctx->last_err.value(), testing::HasSubstr("error message for test case nix_value_init_apply_lazy_arg")); @@ -369,7 +388,7 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) TEST_F(nix_api_expr_test, nix_copy_value) { - Value * source = nix_alloc_value(ctx, state); + nix_value * source = nix_alloc_value(ctx, state); nix_init_int(ctx, source, 42); nix_copy_value(ctx, value, source); diff --git a/tests/unit/libexpr/package.nix b/tests/unit/libexpr/package.nix new file mode 100644 index 000000000..e70ed7836 --- /dev/null +++ b/tests/unit/libexpr/package.nix @@ -0,0 +1,98 @@ +{ lib +, buildPackages +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-expr +, nix-expr-c +, nix-expr-test-support + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +, resolvePath +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-expr-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-expr + nix-expr-c + nix-expr-test-support + rapidcheck + gtest + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru = { + tests = { + run = runCommand "${finalAttrs.pname}-run" { + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' + export _NIX_TEST_UNIT_DATA=${resolvePath ./data} + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} + touch $out + ''); + }; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; + }; + +}) diff --git a/tests/unit/libexpr/trivial.cc b/tests/unit/libexpr/trivial.cc index 61ea71a0f..e455a571b 100644 --- a/tests/unit/libexpr/trivial.cc +++ b/tests/unit/libexpr/trivial.cc @@ -182,6 +182,60 @@ namespace nix { ASSERT_THAT(v, IsIntEq(15)); } + TEST_F(TrivialExpressionTest, forwardPipe) { + auto v = eval("1 |> builtins.add 2 |> builtins.mul 3"); + ASSERT_THAT(v, IsIntEq(9)); + } + + TEST_F(TrivialExpressionTest, backwardPipe) { + auto v = eval("builtins.add 1 <| builtins.mul 2 <| 3"); + ASSERT_THAT(v, IsIntEq(7)); + } + + TEST_F(TrivialExpressionTest, forwardPipeEvaluationOrder) { + auto v = eval("1 |> null |> (x: 2)"); + ASSERT_THAT(v, IsIntEq(2)); + } + + TEST_F(TrivialExpressionTest, backwardPipeEvaluationOrder) { + auto v = eval("(x: 1) <| null <| 2"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsDoNotAssociate) { + ASSERT_THROW(eval("(x: 1) <| 2 |> (x: 3)"), ParseError); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsParensLeft) { + auto v = eval("((x: 1) <| 2) |> (x: 3)"); + ASSERT_THAT(v, IsIntEq(3)); + } + + TEST_F(TrivialExpressionTest, differentPipeOperatorsParensRight) { + auto v = eval("(x: 1) <| (2 |> (x: 3))"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, forwardPipeLowestPrecedence) { + auto v = eval("false -> true |> (x: !x)"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(TrivialExpressionTest, backwardPipeLowestPrecedence) { + auto v = eval("(x: !x) <| false -> true"); + ASSERT_THAT(v, IsFalse()); + } + + TEST_F(TrivialExpressionTest, forwardPipeStrongerThanElse) { + auto v = eval("if true then 1 else 2 |> 3"); + ASSERT_THAT(v, IsIntEq(1)); + } + + TEST_F(TrivialExpressionTest, backwardPipeStrongerThanElse) { + auto v = eval("if true then 1 else 2 <| 3"); + ASSERT_THAT(v, IsIntEq(1)); + } + TEST_F(TrivialExpressionTest, bindOr) { auto v = eval("{ or = 1; }"); ASSERT_THAT(v, IsAttrsOfSize(1)); diff --git a/tests/unit/libfetchers/.version b/tests/unit/libfetchers/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libfetchers/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libfetchers/build-utils-meson b/tests/unit/libfetchers/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libfetchers/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libfetchers/git-utils.cc b/tests/unit/libfetchers/git-utils.cc new file mode 100644 index 000000000..0bf3076dc --- /dev/null +++ b/tests/unit/libfetchers/git-utils.cc @@ -0,0 +1,112 @@ +#include "git-utils.hh" +#include "file-system.hh" +#include "gmock/gmock.h" +#include +#include +#include +#include +#include "fs-sink.hh" +#include "serialise.hh" + +namespace nix { + +class GitUtilsTest : public ::testing::Test +{ + // We use a single repository for all tests. + Path tmpDir; + std::unique_ptr delTmpDir; + +public: + void SetUp() override + { + tmpDir = createTempDir(); + delTmpDir = std::make_unique(tmpDir, true); + + // Create the repo with libgit2 + git_libgit2_init(); + git_repository * repo = nullptr; + auto r = git_repository_init(&repo, tmpDir.c_str(), 0); + ASSERT_EQ(r, 0); + git_repository_free(repo); + } + + void TearDown() override + { + // Destroy the AutoDelete, triggering removal + // not AutoDelete::reset(), which would cancel the deletion. + delTmpDir.reset(); + } + + ref openRepo() + { + return GitRepo::openRepo(tmpDir, true, false); + } +}; + +void writeString(CreateRegularFileSink & fileSink, std::string contents, bool executable) +{ + if (executable) + fileSink.isExecutable(); + fileSink.preallocateContents(contents.size()); + fileSink(contents); +} + +TEST_F(GitUtilsTest, sink_basic) +{ + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + // TODO/Question: It seems a little odd that we use the tarball-like convention of requiring a top-level directory + // here + // The sync method does not document this behavior, should probably renamed because it's not very + // general, and I can't imagine that "non-conventional" archives or any other source to be handled by + // this sink. + + sink->createDirectory(CanonPath("foo-1.1")); + + sink->createRegularFile(CanonPath("foo-1.1/hello"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "hello world", false); + }); + sink->createRegularFile(CanonPath("foo-1.1/bye"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "thanks for all the fish", false); + }); + sink->createSymlink(CanonPath("foo-1.1/bye-link"), "bye"); + sink->createDirectory(CanonPath("foo-1.1/empty")); + sink->createDirectory(CanonPath("foo-1.1/links")); + sink->createHardlink(CanonPath("foo-1.1/links/foo"), CanonPath("foo-1.1/hello")); + + // sink->createHardlink("foo-1.1/links/foo-2", CanonPath("foo-1.1/hello")); + + auto result = repo->dereferenceSingletonDirectory(sink->flush()); + auto accessor = repo->getAccessor(result, false); + auto entries = accessor->readDirectory(CanonPath::root); + ASSERT_EQ(entries.size(), 5); + ASSERT_EQ(accessor->readFile(CanonPath("hello")), "hello world"); + ASSERT_EQ(accessor->readFile(CanonPath("bye")), "thanks for all the fish"); + ASSERT_EQ(accessor->readLink(CanonPath("bye-link")), "bye"); + ASSERT_EQ(accessor->readDirectory(CanonPath("empty")).size(), 0); + ASSERT_EQ(accessor->readFile(CanonPath("links/foo")), "hello world"); +}; + +TEST_F(GitUtilsTest, sink_hardlink) +{ + auto repo = openRepo(); + auto sink = repo->getFileSystemObjectSink(); + + sink->createDirectory(CanonPath("foo-1.1")); + + sink->createRegularFile(CanonPath("foo-1.1/hello"), [](CreateRegularFileSink & fileSink) { + writeString(fileSink, "hello world", false); + }); + + try { + sink->createHardlink(CanonPath("foo-1.1/link"), CanonPath("hello")); + FAIL() << "Expected an exception"; + } catch (const nix::Error & e) { + ASSERT_THAT(e.msg(), testing::HasSubstr("cannot find hard link target")); + ASSERT_THAT(e.msg(), testing::HasSubstr("/hello")); + ASSERT_THAT(e.msg(), testing::HasSubstr("foo-1.1/link")); + } +}; + +} // namespace nix diff --git a/tests/unit/libfetchers/local.mk b/tests/unit/libfetchers/local.mk index e9f659fd7..30aa142a5 100644 --- a/tests/unit/libfetchers/local.mk +++ b/tests/unit/libfetchers/local.mk @@ -4,7 +4,7 @@ programs += libfetchers-tests libfetchers-tests_NAME = libnixfetchers-tests -libfetchers-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data +libfetchers-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libfetchers-tests.xml libfetchers-tests_DIR := $(d) @@ -29,4 +29,9 @@ libfetchers-tests_LIBS = \ libstore-test-support libutil-test-support \ libfetchers libstore libutil -libfetchers-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) +libfetchers-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) $(LIBGIT2_LIBS) + +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + libfetchers-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif diff --git a/tests/unit/libfetchers/meson.build b/tests/unit/libfetchers/meson.build new file mode 100644 index 000000000..dc9818e27 --- /dev/null +++ b/tests/unit/libfetchers/meson.build @@ -0,0 +1,73 @@ +project('nix-fetchers-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-store-test-support'), + dependency('nix-fetchers'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +subdir('build-utils-meson/export-all-symbols') + +rapidcheck = dependency('rapidcheck') +deps_private += rapidcheck + +gtest = dependency('gtest', main : true) +deps_private += gtest + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + # '-include', 'config-fetchers.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'public-key.cc', +) + +include_dirs = [include_directories('.')] + + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + # TODO: -lrapidcheck, see ../libutil-support/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + # get main from gtest + install : true, +) + +test( + meson.project_name(), + this_exe, + env : { + '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + }, + protocol : 'gtest', +) diff --git a/tests/unit/libfetchers/package.nix b/tests/unit/libfetchers/package.nix new file mode 100644 index 000000000..ad512f562 --- /dev/null +++ b/tests/unit/libfetchers/package.nix @@ -0,0 +1,96 @@ +{ lib +, buildPackages +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-fetchers +, nix-store-test-support + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +, resolvePath +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-fetchers-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-fetchers + nix-store-test-support + rapidcheck + gtest + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru = { + tests = { + run = runCommand "${finalAttrs.pname}-run" { + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' + export _NIX_TEST_UNIT_DATA=${resolvePath ./data} + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} + touch $out + ''); + }; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; + }; + +}) diff --git a/tests/unit/libfetchers/public-key.cc b/tests/unit/libfetchers/public-key.cc index 8a639da9f..80796bd0f 100644 --- a/tests/unit/libfetchers/public-key.cc +++ b/tests/unit/libfetchers/public-key.cc @@ -10,11 +10,11 @@ using nlohmann::json; class PublicKeyTest : public CharacterizationTest { - Path unitTestData = getUnitTestData() + "/public-key"; + std::filesystem::path unitTestData = getUnitTestData() / "public-key"; public: - Path goldenMaster(std::string_view testStem) const override { - return unitTestData + "/" + testStem; + std::filesystem::path goldenMaster(std::string_view testStem) const override { + return unitTestData / testStem; } }; diff --git a/tests/unit/libflake/.version b/tests/unit/libflake/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libflake/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libflake/build-utils-meson b/tests/unit/libflake/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libflake/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libflake/data/.gitkeep b/tests/unit/libflake/data/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/libexpr/flake/flakeref.cc b/tests/unit/libflake/flakeref.cc similarity index 82% rename from tests/unit/libexpr/flake/flakeref.cc rename to tests/unit/libflake/flakeref.cc index 2b7809b93..d704a26d3 100644 --- a/tests/unit/libexpr/flake/flakeref.cc +++ b/tests/unit/libflake/flakeref.cc @@ -1,5 +1,6 @@ #include +#include "fetch-settings.hh" #include "flake/flakeref.hh" namespace nix { @@ -11,8 +12,9 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(to_string, doesntReencodeUrl) { + fetchers::Settings fetchSettings; auto s = "http://localhost:8181/test/+3d.tar.gz"; - auto flakeref = parseFlakeRef(s); + auto flakeref = parseFlakeRef(fetchSettings, s); auto parsed = flakeref.to_string(); auto expected = "http://localhost:8181/test/%2B3d.tar.gz"; diff --git a/tests/unit/libflake/local.mk b/tests/unit/libflake/local.mk new file mode 100644 index 000000000..590bcf7c0 --- /dev/null +++ b/tests/unit/libflake/local.mk @@ -0,0 +1,43 @@ +check: libflake-tests_RUN + +programs += libflake-tests + +libflake-tests_NAME := libnixflake-tests + +libflake-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libflake-tests.xml + +libflake-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libflake-tests_INSTALL_DIR := $(checkbindir) +else + libflake-tests_INSTALL_DIR := +endif + +libflake-tests_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/value/*.cc) \ + $(wildcard $(d)/flake/*.cc) + +libflake-tests_EXTRA_INCLUDES = \ + -I tests/unit/libflake-support \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + $(INCLUDE_libflake) \ + $(INCLUDE_libexpr) \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) \ + $(INCLUDE_libutil) \ + +libflake-tests_CXXFLAGS += $(libflake-tests_EXTRA_INCLUDES) + +libflake-tests_LIBS = \ + libexpr-test-support libstore-test-support libutil-test-support \ + libflake libexpr libfetchers libstore libutil + +libflake-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock + +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + libflake-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif diff --git a/tests/unit/libflake/meson.build b/tests/unit/libflake/meson.build new file mode 100644 index 000000000..c022d7f41 --- /dev/null +++ b/tests/unit/libflake/meson.build @@ -0,0 +1,74 @@ +project('nix-flake-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-expr-test-support'), + dependency('nix-flake'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +subdir('build-utils-meson/export-all-symbols') + +rapidcheck = dependency('rapidcheck') +deps_private += rapidcheck + +gtest = dependency('gtest', main : true) +deps_private += gtest + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-expr.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'flakeref.cc', + 'url-name.cc', +) + +include_dirs = [include_directories('.')] + + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + # TODO: -lrapidcheck, see ../libutil-support/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + # get main from gtest + install : true, +) + +test( + meson.project_name(), + this_exe, + env : { + '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + }, + protocol : 'gtest', +) diff --git a/tests/unit/libflake/package.nix b/tests/unit/libflake/package.nix new file mode 100644 index 000000000..0d63d2ff7 --- /dev/null +++ b/tests/unit/libflake/package.nix @@ -0,0 +1,96 @@ +{ lib +, buildPackages +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-flake +, nix-expr-test-support + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +, resolvePath +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-flake-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-flake + nix-expr-test-support + rapidcheck + gtest + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru = { + tests = { + run = runCommand "${finalAttrs.pname}-run" { + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' + export _NIX_TEST_UNIT_DATA=${resolvePath ./data} + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} + touch $out + ''); + }; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; + }; + +}) diff --git a/tests/unit/libexpr/flake/url-name.cc b/tests/unit/libflake/url-name.cc similarity index 100% rename from tests/unit/libexpr/flake/url-name.cc rename to tests/unit/libflake/url-name.cc diff --git a/tests/unit/libstore-support/.version b/tests/unit/libstore-support/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libstore-support/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libstore-support/build-utils-meson b/tests/unit/libstore-support/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libstore-support/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libstore-support/meson.build b/tests/unit/libstore-support/meson.build new file mode 100644 index 000000000..f09d26a31 --- /dev/null +++ b/tests/unit/libstore-support/meson.build @@ -0,0 +1,78 @@ +project('nix-store-test-support', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-util-test-support'), + dependency('nix-store'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +rapidcheck = dependency('rapidcheck') +deps_public += rapidcheck + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'tests/derived-path.cc', + 'tests/outputs-spec.cc', + 'tests/path.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'tests/derived-path.hh', + 'tests/libstore.hh', + 'tests/nix_api_store.hh', + 'tests/outputs-spec.hh', + 'tests/path.hh', + 'tests/protocol.hh', +) + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nix-store-test-support', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326 + # is available. See also ../libutil/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/tests/unit/libstore-support/package.nix b/tests/unit/libstore-support/package.nix new file mode 100644 index 000000000..f512db3ee --- /dev/null +++ b/tests/unit/libstore-support/package.nix @@ -0,0 +1,77 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util-test-support +, nix-store + +, rapidcheck + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-store-test-support"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util-test-support + nix-store + rapidcheck + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/tests/unit/libstore-support/tests/libstore.hh b/tests/unit/libstore-support/tests/libstore.hh index 267188224..84be52c23 100644 --- a/tests/unit/libstore-support/tests/libstore.hh +++ b/tests/unit/libstore-support/tests/libstore.hh @@ -8,19 +8,27 @@ namespace nix { -class LibStoreTest : public virtual ::testing::Test { - public: - static void SetUpTestSuite() { - initLibStore(false); - } +class LibStoreTest : public virtual ::testing::Test +{ +public: + static void SetUpTestSuite() + { + initLibStore(false); + } - protected: - LibStoreTest() - : store(openStore("dummy://")) - { } +protected: + LibStoreTest() + : store(openStore({ + .variant = + StoreReference::Specified{ + .scheme = "dummy", + }, + .params = {}, + })) + { + } - ref store; + ref store; }; - } /* namespace nix */ diff --git a/tests/unit/libstore-support/tests/nix_api_store.hh b/tests/unit/libstore-support/tests/nix_api_store.hh index a2d35d083..b7d5c2c33 100644 --- a/tests/unit/libstore-support/tests/nix_api_store.hh +++ b/tests/unit/libstore-support/tests/nix_api_store.hh @@ -3,6 +3,7 @@ #include "tests/nix_api_util.hh" #include "file-system.hh" +#include #include "nix_api_store.h" #include "nix_api_store_internal.h" @@ -10,7 +11,7 @@ #include #include -namespace fs = std::filesystem; +namespace fs { using namespace std::filesystem; } namespace nixC { class nix_api_store_test : public nix_api_util_context @@ -47,7 +48,9 @@ protected: if (fs::create_directory(nixDir)) break; } #else - auto tmpl = nix::defaultTempDir() + "/tests_nix-store.XXXXXX"; + // resolve any symlinks in i.e. on macOS /tmp -> /private/tmp + // because this is not allowed for a nix store. + auto tmpl = nix::absPath(std::filesystem::path(nix::defaultTempDir()) / "tests_nix-store.XXXXXX", true); nixDir = mkdtemp((char *) tmpl.c_str()); #endif @@ -61,6 +64,10 @@ protected: const char ** params[] = {p1, p2, p3, nullptr}; store = nix_store_open(ctx, "local", params); + if (!store) { + std::string errMsg = nix_err_msg(nullptr, ctx, nullptr); + ASSERT_NE(store, nullptr) << "Could not open store: " << errMsg; + }; } }; } diff --git a/tests/unit/libstore-support/tests/protocol.hh b/tests/unit/libstore-support/tests/protocol.hh index 3c9e52c11..3f6799d1c 100644 --- a/tests/unit/libstore-support/tests/protocol.hh +++ b/tests/unit/libstore-support/tests/protocol.hh @@ -12,10 +12,10 @@ namespace nix { template class ProtoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/" + protocolDir; + std::filesystem::path unitTestData = getUnitTestData() / protocolDir; - Path goldenMaster(std::string_view testStem) const override { - return unitTestData + "/" + testStem + ".bin"; + std::filesystem::path goldenMaster(std::string_view testStem) const override { + return unitTestData / (std::string { testStem + ".bin" }); } }; diff --git a/tests/unit/libstore/.version b/tests/unit/libstore/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libstore/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libstore/build-utils-meson b/tests/unit/libstore/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libstore/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libstore/common-protocol.cc b/tests/unit/libstore/common-protocol.cc index d23805fc3..c8f6dd002 100644 --- a/tests/unit/libstore/common-protocol.cc +++ b/tests/unit/libstore/common-protocol.cc @@ -83,15 +83,15 @@ CHARACTERIZATION_TEST( "content-address", (std::tuple { ContentAddress { - .method = TextIngestionMethod {}, + .method = ContentAddressMethod::Raw::Text, .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -178,7 +178,7 @@ CHARACTERIZATION_TEST( std::nullopt, std::optional { ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, diff --git a/tests/unit/libstore/content-address.cc b/tests/unit/libstore/content-address.cc index cc1c7fcc6..72eb84fec 100644 --- a/tests/unit/libstore/content-address.cc +++ b/tests/unit/libstore/content-address.cc @@ -9,11 +9,11 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(ContentAddressMethod, testRoundTripPrintParse_1) { - for (const ContentAddressMethod & cam : { - ContentAddressMethod { TextIngestionMethod {} }, - ContentAddressMethod { FileIngestionMethod::Flat }, - ContentAddressMethod { FileIngestionMethod::Recursive }, - ContentAddressMethod { FileIngestionMethod::Git }, + for (ContentAddressMethod cam : { + ContentAddressMethod::Raw::Text, + ContentAddressMethod::Raw::Flat, + ContentAddressMethod::Raw::NixArchive, + ContentAddressMethod::Raw::Git, }) { EXPECT_EQ(ContentAddressMethod::parse(cam.render()), cam); } diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv new file mode 120000 index 000000000..353090ad8 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes-defaults.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-defaults.json b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.json new file mode 100644 index 000000000..d58e7d5b5 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-defaults.json @@ -0,0 +1,22 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "builder": "/bin/bash", + "name": "advanced-attributes-defaults", + "out": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults", + "system": "my-system" + }, + "inputDrvs": {}, + "inputSrcs": [], + "name": "advanced-attributes-defaults", + "outputs": { + "out": { + "path": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults" + } + }, + "system": "my-system" +} diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv new file mode 120000 index 000000000..11713da12 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes-structured-attrs-defaults.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json new file mode 100644 index 000000000..473d006ac --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs-defaults.json @@ -0,0 +1,24 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__json": "{\"builder\":\"/bin/bash\",\"name\":\"advanced-attributes-structured-attrs-defaults\",\"outputs\":[\"out\",\"dev\"],\"system\":\"my-system\"}", + "dev": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev", + "out": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" + }, + "inputDrvs": {}, + "inputSrcs": [], + "name": "advanced-attributes-structured-attrs-defaults", + "outputs": { + "dev": { + "path": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev" + }, + "out": { + "path": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" + } + }, + "system": "my-system" +} diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv new file mode 120000 index 000000000..962f8ea3f --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes-structured-attrs.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json new file mode 100644 index 000000000..324428124 --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes-structured-attrs.json @@ -0,0 +1,41 @@ +{ + "args": [ + "-c", + "echo hello > $out" + ], + "builder": "/bin/bash", + "env": { + "__json": "{\"__darwinAllowLocalNetworking\":true,\"__impureHostDeps\":[\"/usr/bin/ditto\"],\"__noChroot\":true,\"__sandboxProfile\":\"sandcastle\",\"allowSubstitutes\":false,\"builder\":\"/bin/bash\",\"impureEnvVars\":[\"UNICORN\"],\"name\":\"advanced-attributes-structured-attrs\",\"outputChecks\":{\"bin\":{\"disallowedReferences\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"],\"disallowedRequisites\":[\"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar\"]},\"dev\":{\"maxClosureSize\":5909,\"maxSize\":789},\"out\":{\"allowedReferences\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"],\"allowedRequisites\":[\"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo\"]}},\"outputs\":[\"out\",\"bin\",\"dev\"],\"preferLocalBuild\":true,\"requiredSystemFeatures\":[\"rainbow\",\"uid-range\"],\"system\":\"my-system\"}", + "bin": "/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin", + "dev": "/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev", + "out": "/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs" + }, + "inputDrvs": { + "/nix/store/4xm4wccqsvagz9gjksn24s7rip2fdy7v-foo.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + }, + "/nix/store/plsq5jbr5nhgqwcgb2qxw7jchc09dnl8-bar.drv": { + "dynamicOutputs": {}, + "outputs": [ + "out" + ] + } + }, + "inputSrcs": [], + "name": "advanced-attributes-structured-attrs", + "outputs": { + "bin": { + "path": "/nix/store/pbzb48v0ycf80jgligcp4n8z0rblna4n-advanced-attributes-structured-attrs-bin" + }, + "dev": { + "path": "/nix/store/7xapi8jv7flcz1qq8jhw55ar8ag8hldh-advanced-attributes-structured-attrs-dev" + }, + "out": { + "path": "/nix/store/mpq3l1l1qc2yr50q520g08kprprwv79f-advanced-attributes-structured-attrs" + } + }, + "system": "my-system" +} diff --git a/tests/unit/libstore/data/derivation/advanced-attributes.drv b/tests/unit/libstore/data/derivation/advanced-attributes.drv new file mode 120000 index 000000000..2a53a05ca --- /dev/null +++ b/tests/unit/libstore/data/derivation/advanced-attributes.drv @@ -0,0 +1 @@ +../../../../functional/derivation/advanced-attributes.drv \ No newline at end of file diff --git a/tests/unit/libstore/data/derivation/output-caFixedFlat.json b/tests/unit/libstore/data/derivation/output-caFixedFlat.json index fe000ea36..7001ea0a9 100644 --- a/tests/unit/libstore/data/derivation/output-caFixedFlat.json +++ b/tests/unit/libstore/data/derivation/output-caFixedFlat.json @@ -1,5 +1,6 @@ { "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", "hashAlgo": "sha256", + "method": "flat", "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" } diff --git a/tests/unit/libstore/data/derivation/output-caFixedNAR.json b/tests/unit/libstore/data/derivation/output-caFixedNAR.json index 1afd60223..54eb306e6 100644 --- a/tests/unit/libstore/data/derivation/output-caFixedNAR.json +++ b/tests/unit/libstore/data/derivation/output-caFixedNAR.json @@ -1,5 +1,6 @@ { "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "hashAlgo": "r:sha256", + "hashAlgo": "sha256", + "method": "nar", "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" } diff --git a/tests/unit/libstore/data/derivation/output-caFixedText.json b/tests/unit/libstore/data/derivation/output-caFixedText.json index 0b2cc8bbc..e8a651860 100644 --- a/tests/unit/libstore/data/derivation/output-caFixedText.json +++ b/tests/unit/libstore/data/derivation/output-caFixedText.json @@ -1,5 +1,6 @@ { "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", - "hashAlgo": "text:sha256", + "hashAlgo": "sha256", + "method": "text", "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" } diff --git a/tests/unit/libstore/data/derivation/output-caFloating.json b/tests/unit/libstore/data/derivation/output-caFloating.json index 9115de851..8b9b5f681 100644 --- a/tests/unit/libstore/data/derivation/output-caFloating.json +++ b/tests/unit/libstore/data/derivation/output-caFloating.json @@ -1,3 +1,4 @@ { - "hashAlgo": "r:sha256" + "hashAlgo": "sha256", + "method": "nar" } diff --git a/tests/unit/libstore/data/derivation/output-impure.json b/tests/unit/libstore/data/derivation/output-impure.json index 62b61cdca..bec03702b 100644 --- a/tests/unit/libstore/data/derivation/output-impure.json +++ b/tests/unit/libstore/data/derivation/output-impure.json @@ -1,4 +1,5 @@ { - "hashAlgo": "r:sha256", - "impure": true + "hashAlgo": "sha256", + "impure": true, + "method": "nar" } diff --git a/tests/unit/libstore/test-data/machines.bad_format b/tests/unit/libstore/data/machines/bad_format similarity index 100% rename from tests/unit/libstore/test-data/machines.bad_format rename to tests/unit/libstore/data/machines/bad_format diff --git a/tests/unit/libstore/test-data/machines.valid b/tests/unit/libstore/data/machines/valid similarity index 100% rename from tests/unit/libstore/test-data/machines.valid rename to tests/unit/libstore/data/machines/valid diff --git a/tests/unit/libstore/data/path-info/empty_impure.json b/tests/unit/libstore/data/path-info/empty_impure.json new file mode 100644 index 000000000..be982dcef --- /dev/null +++ b/tests/unit/libstore/data/path-info/empty_impure.json @@ -0,0 +1,10 @@ +{ + "ca": null, + "deriver": null, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 0, + "references": [], + "registrationTime": null, + "signatures": [], + "ultimate": false +} diff --git a/tests/unit/libstore/data/path-info/empty_pure.json b/tests/unit/libstore/data/path-info/empty_pure.json new file mode 100644 index 000000000..10d9f508a --- /dev/null +++ b/tests/unit/libstore/data/path-info/empty_pure.json @@ -0,0 +1,6 @@ +{ + "ca": null, + "narHash": "sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=", + "narSize": 0, + "references": [] +} diff --git a/tests/unit/libstore/data/store-reference/auto.txt b/tests/unit/libstore/data/store-reference/auto.txt new file mode 100644 index 000000000..4d18c3e59 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/auto.txt @@ -0,0 +1 @@ +auto \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/auto_param.txt b/tests/unit/libstore/data/store-reference/auto_param.txt new file mode 100644 index 000000000..54adabb25 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/auto_param.txt @@ -0,0 +1 @@ +auto?root=/foo/bar/baz \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/local_1.txt b/tests/unit/libstore/data/store-reference/local_1.txt new file mode 100644 index 000000000..74b1b9677 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/local_1.txt @@ -0,0 +1 @@ +local://?root=/foo/bar/baz \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/local_2.txt b/tests/unit/libstore/data/store-reference/local_2.txt new file mode 100644 index 000000000..8b5593fb1 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/local_2.txt @@ -0,0 +1 @@ +local:///foo/bar/baz?trusted=true \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/local_shorthand_1.txt b/tests/unit/libstore/data/store-reference/local_shorthand_1.txt new file mode 100644 index 000000000..896189be9 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/local_shorthand_1.txt @@ -0,0 +1 @@ +local?root=/foo/bar/baz \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/local_shorthand_2.txt b/tests/unit/libstore/data/store-reference/local_shorthand_2.txt new file mode 100644 index 000000000..7a9dad3b3 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/local_shorthand_2.txt @@ -0,0 +1 @@ +/foo/bar/baz?trusted=true \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/ssh.txt b/tests/unit/libstore/data/store-reference/ssh.txt new file mode 100644 index 000000000..8c61010ec --- /dev/null +++ b/tests/unit/libstore/data/store-reference/ssh.txt @@ -0,0 +1 @@ +ssh://localhost \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/unix.txt b/tests/unit/libstore/data/store-reference/unix.txt new file mode 100644 index 000000000..195489048 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/unix.txt @@ -0,0 +1 @@ +unix://?max-connections=7&trusted=true \ No newline at end of file diff --git a/tests/unit/libstore/data/store-reference/unix_shorthand.txt b/tests/unit/libstore/data/store-reference/unix_shorthand.txt new file mode 100644 index 000000000..0300337e9 --- /dev/null +++ b/tests/unit/libstore/data/store-reference/unix_shorthand.txt @@ -0,0 +1 @@ +daemon?max-connections=7&trusted=true \ No newline at end of file diff --git a/tests/unit/libstore/data/worker-protocol/build-mode.bin b/tests/unit/libstore/data/worker-protocol/build-mode.bin new file mode 100644 index 000000000..51b239409 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/build-mode.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_30.bin b/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_30.bin new file mode 100644 index 000000000..e69de29bb diff --git a/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_33.bin b/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_33.bin new file mode 100644 index 000000000..96c6efafc Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_33.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_35.bin b/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_35.bin new file mode 100644 index 000000000..e877159aa Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/client-handshake-info_1_35.bin differ diff --git a/tests/unit/libstore/data/worker-protocol/handshake-to-client.bin b/tests/unit/libstore/data/worker-protocol/handshake-to-client.bin new file mode 100644 index 000000000..bee94fbe5 Binary files /dev/null and b/tests/unit/libstore/data/worker-protocol/handshake-to-client.bin differ diff --git a/tests/unit/libstore/derivation-advanced-attrs.cc b/tests/unit/libstore/derivation-advanced-attrs.cc new file mode 100644 index 000000000..9d2c64ef3 --- /dev/null +++ b/tests/unit/libstore/derivation-advanced-attrs.cc @@ -0,0 +1,234 @@ +#include +#include + +#include "experimental-features.hh" +#include "derivations.hh" + +#include "tests/libstore.hh" +#include "tests/characterization.hh" +#include "parsed-derivations.hh" +#include "types.hh" +#include "json-utils.hh" + +namespace nix { + +using nlohmann::json; + +class DerivationAdvancedAttrsTest : public CharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "derivation"; + +public: + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +#define TEST_ATERM_JSON(STEM, NAME) \ + TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_from_json) \ + { \ + readTest(NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(goldenMaster(NAME ".drv")); \ + auto expected = parseDerivation(*store, std::move(aterm), NAME); \ + Derivation got = Derivation::fromJSON(*store, encoded); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_to_json) \ + { \ + writeTest( \ + NAME ".json", \ + [&]() -> json { \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(goldenMaster(NAME ".drv")); \ + return parseDerivation(*store, std::move(aterm), NAME).toJSON(*store); \ + }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } \ + \ + TEST_F(DerivationAdvancedAttrsTest, Derivation_##STEM##_from_aterm) \ + { \ + readTest(NAME ".drv", [&](auto encoded) { \ + /* Use JSON file instead of C++ literal as source of truth. */ \ + auto json = json::parse(readFile(goldenMaster(NAME ".json"))); \ + auto expected = Derivation::fromJSON(*store, json); \ + auto got = parseDerivation(*store, std::move(encoded), NAME); \ + EXPECT_EQ(got.toJSON(*store), expected.toJSON(*store)); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ + /* No corresponding write test, because we need to read the drv to write the json file */ + +TEST_ATERM_JSON(advancedAttributes_defaults, "advanced-attributes-defaults"); +TEST_ATERM_JSON(advancedAttributes, "advanced-attributes-defaults"); +TEST_ATERM_JSON(advancedAttributes_structuredAttrs_defaults, "advanced-attributes-structured-attrs"); +TEST_ATERM_JSON(advancedAttributes_structuredAttrs, "advanced-attributes-structured-attrs-defaults"); + +#undef TEST_ATERM_JSON + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_defaults) +{ + readTest("advanced-attributes-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), ""); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), false); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), false); + EXPECT_EQ(parsedDrv.getStringsAttr("allowedReferences"), std::nullopt); + EXPECT_EQ(parsedDrv.getStringsAttr("allowedRequisites"), std::nullopt); + EXPECT_EQ(parsedDrv.getStringsAttr("disallowedReferences"), std::nullopt); + EXPECT_EQ(parsedDrv.getStringsAttr("disallowedRequisites"), std::nullopt); + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), StringSet()); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), true); + EXPECT_EQ(parsedDrv.useUidRange(), false); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes) +{ + readTest("advanced-attributes.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + StringSet systemFeatures{"rainbow", "uid-range"}; + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), "sandcastle"); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), true); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings{"/usr/bin/ditto"}); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings{"UNICORN"}); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), true); + EXPECT_EQ( + parsedDrv.getStringsAttr("allowedReferences"), Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + EXPECT_EQ( + parsedDrv.getStringsAttr("allowedRequisites"), Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + EXPECT_EQ( + parsedDrv.getStringsAttr("disallowedReferences"), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + EXPECT_EQ( + parsedDrv.getStringsAttr("disallowedRequisites"), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), systemFeatures); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), false); + EXPECT_EQ(parsedDrv.useUidRange(), true); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttrs_defaults) +{ + readTest("advanced-attributes-structured-attrs-defaults.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), ""); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), false); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings()); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), false); + + { + auto structuredAttrs_ = parsedDrv.getStructuredAttrs(); + ASSERT_TRUE(structuredAttrs_); + auto & structuredAttrs = *structuredAttrs_; + + auto outputChecks_ = get(structuredAttrs, "outputChecks"); + ASSERT_FALSE(outputChecks_); + } + + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), StringSet()); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), true); + EXPECT_EQ(parsedDrv.useUidRange(), false); + }); +}; + +TEST_F(DerivationAdvancedAttrsTest, Derivation_advancedAttributes_structuredAttrs) +{ + readTest("advanced-attributes-structured-attrs.drv", [&](auto encoded) { + auto got = parseDerivation(*store, std::move(encoded), "foo"); + + auto drvPath = writeDerivation(*store, got, NoRepair, true); + + ParsedDerivation parsedDrv(drvPath, got); + + StringSet systemFeatures{"rainbow", "uid-range"}; + + EXPECT_EQ(parsedDrv.getStringAttr("__sandboxProfile").value_or(""), "sandcastle"); + EXPECT_EQ(parsedDrv.getBoolAttr("__noChroot"), true); + EXPECT_EQ(parsedDrv.getStringsAttr("__impureHostDeps").value_or(Strings()), Strings{"/usr/bin/ditto"}); + EXPECT_EQ(parsedDrv.getStringsAttr("impureEnvVars").value_or(Strings()), Strings{"UNICORN"}); + EXPECT_EQ(parsedDrv.getBoolAttr("__darwinAllowLocalNetworking"), true); + + { + auto structuredAttrs_ = parsedDrv.getStructuredAttrs(); + ASSERT_TRUE(structuredAttrs_); + auto & structuredAttrs = *structuredAttrs_; + + auto outputChecks_ = get(structuredAttrs, "outputChecks"); + ASSERT_TRUE(outputChecks_); + auto & outputChecks = *outputChecks_; + + { + auto output_ = get(outputChecks, "out"); + ASSERT_TRUE(output_); + auto & output = *output_; + EXPECT_EQ( + get(output, "allowedReferences")->get(), + Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + EXPECT_EQ( + get(output, "allowedRequisites")->get(), + Strings{"/nix/store/3c08bzb71z4wiag719ipjxr277653ynp-foo"}); + } + + { + auto output_ = get(outputChecks, "bin"); + ASSERT_TRUE(output_); + auto & output = *output_; + EXPECT_EQ( + get(output, "disallowedReferences")->get(), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + EXPECT_EQ( + get(output, "disallowedRequisites")->get(), + Strings{"/nix/store/7rhsm8i393hm1wcsmph782awg1hi2f7x-bar"}); + } + + { + auto output_ = get(outputChecks, "dev"); + ASSERT_TRUE(output_); + auto & output = *output_; + EXPECT_EQ(get(output, "maxSize")->get(), 789); + EXPECT_EQ(get(output, "maxClosureSize")->get(), 5909); + } + } + + EXPECT_EQ(parsedDrv.getRequiredSystemFeatures(), systemFeatures); + EXPECT_EQ(parsedDrv.canBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.willBuildLocally(*store), false); + EXPECT_EQ(parsedDrv.substitutesAllowed(), false); + EXPECT_EQ(parsedDrv.useUidRange(), true); + }); +}; + +} diff --git a/tests/unit/libstore/derivation.cc b/tests/unit/libstore/derivation.cc index 7a4b1403a..14652921a 100644 --- a/tests/unit/libstore/derivation.cc +++ b/tests/unit/libstore/derivation.cc @@ -13,11 +13,11 @@ using nlohmann::json; class DerivationTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/derivation"; + std::filesystem::path unitTestData = getUnitTestData() / "derivation"; public: - Path goldenMaster(std::string_view testStem) const override { - return unitTestData + "/" + testStem; + std::filesystem::path goldenMaster(std::string_view testStem) const override { + return unitTestData / testStem; } /** @@ -108,7 +108,7 @@ TEST_JSON(DerivationTest, inputAddressed, TEST_JSON(DerivationTest, caFixedFlat, (DerivationOutput::CAFixed { .ca = { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), @@ -117,7 +117,7 @@ TEST_JSON(DerivationTest, caFixedFlat, TEST_JSON(DerivationTest, caFixedNAR, (DerivationOutput::CAFixed { .ca = { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), @@ -126,6 +126,7 @@ TEST_JSON(DerivationTest, caFixedNAR, TEST_JSON(DynDerivationTest, caFixedText, (DerivationOutput::CAFixed { .ca = { + .method = ContentAddressMethod::Raw::Text, .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), }, }), @@ -133,7 +134,7 @@ TEST_JSON(DynDerivationTest, caFixedText, TEST_JSON(CaDerivationTest, caFloating, (DerivationOutput::CAFloating { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hashAlgo = HashAlgorithm::SHA256, }), "drv-name", "output-name") @@ -144,7 +145,7 @@ TEST_JSON(DerivationTest, deferred, TEST_JSON(ImpureDerivationTest, impure, (DerivationOutput::Impure { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hashAlgo = HashAlgorithm::SHA256, }), "drv-name", "output-name") diff --git a/tests/unit/libstore/http-binary-cache-store.cc b/tests/unit/libstore/http-binary-cache-store.cc new file mode 100644 index 000000000..1e415f625 --- /dev/null +++ b/tests/unit/libstore/http-binary-cache-store.cc @@ -0,0 +1,21 @@ +#include + +#include "http-binary-cache-store.hh" + +namespace nix { + +TEST(HttpBinaryCacheStore, constructConfig) +{ + HttpBinaryCacheStoreConfig config{"http", "foo.bar.baz", {}}; + + EXPECT_EQ(config.cacheUri, "http://foo.bar.baz"); +} + +TEST(HttpBinaryCacheStore, constructConfigNoTrailingSlash) +{ + HttpBinaryCacheStoreConfig config{"https", "foo.bar.baz/a/b/", {}}; + + EXPECT_EQ(config.cacheUri, "https://foo.bar.baz/a/b"); +} + +} // namespace nix diff --git a/tests/unit/libstore/legacy-ssh-store.cc b/tests/unit/libstore/legacy-ssh-store.cc new file mode 100644 index 000000000..eb31a2408 --- /dev/null +++ b/tests/unit/libstore/legacy-ssh-store.cc @@ -0,0 +1,26 @@ +#include + +#include "legacy-ssh-store.hh" + +namespace nix { + +TEST(LegacySSHStore, constructConfig) +{ + LegacySSHStoreConfig config{ + "ssh", + "localhost", + StoreConfig::Params{ + { + "remote-program", + // TODO #11106, no more split on space + "foo bar", + }, + }}; + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); +} +} diff --git a/tests/unit/libstore/local-binary-cache-store.cc b/tests/unit/libstore/local-binary-cache-store.cc new file mode 100644 index 000000000..2e840228d --- /dev/null +++ b/tests/unit/libstore/local-binary-cache-store.cc @@ -0,0 +1,14 @@ +#include + +#include "local-binary-cache-store.hh" + +namespace nix { + +TEST(LocalBinaryCacheStore, constructConfig) +{ + LocalBinaryCacheStoreConfig config{"local", "/foo/bar/baz", {}}; + + EXPECT_EQ(config.binaryCacheDir, "/foo/bar/baz"); +} + +} // namespace nix diff --git a/tests/unit/libstore/local-overlay-store.cc b/tests/unit/libstore/local-overlay-store.cc new file mode 100644 index 000000000..b34ca9237 --- /dev/null +++ b/tests/unit/libstore/local-overlay-store.cc @@ -0,0 +1,34 @@ +// FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 +# include + +# include "local-overlay-store.hh" + +namespace nix { + +TEST(LocalOverlayStore, constructConfig_rootQueryParam) +{ + LocalOverlayStoreConfig config{ + "local-overlay", + "", + { + { + "root", + "/foo/bar", + }, + }, + }; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +TEST(LocalOverlayStore, constructConfig_rootPath) +{ + LocalOverlayStoreConfig config{"local-overlay", "/foo/bar", {}}; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +} // namespace nix +#endif diff --git a/tests/unit/libstore/local-store.cc b/tests/unit/libstore/local-store.cc new file mode 100644 index 000000000..abc3ea796 --- /dev/null +++ b/tests/unit/libstore/local-store.cc @@ -0,0 +1,40 @@ +// FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 +# include + +# include "local-store.hh" + +// Needed for template specialisations. This is not good! When we +// overhaul how store configs work, this should be fixed. +# include "args.hh" +# include "config-impl.hh" +# include "abstract-setting-to-json.hh" + +namespace nix { + +TEST(LocalStore, constructConfig_rootQueryParam) +{ + LocalStoreConfig config{ + "local", + "", + { + { + "root", + "/foo/bar", + }, + }, + }; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +TEST(LocalStore, constructConfig_rootPath) +{ + LocalStoreConfig config{"local", "/foo/bar", {}}; + + EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); +} + +} // namespace nix +#endif diff --git a/tests/unit/libstore/local.mk b/tests/unit/libstore/local.mk index b8f895fad..8d3d6b0af 100644 --- a/tests/unit/libstore/local.mk +++ b/tests/unit/libstore/local.mk @@ -4,7 +4,7 @@ programs += libstore-tests libstore-tests_NAME = libnixstore-tests -libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data +libstore-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libstore-tests.xml libstore-tests_DIR := $(d) @@ -31,3 +31,8 @@ libstore-tests_LIBS = \ libstore libstorec libutil libutilc libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) + +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + libstore-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif diff --git a/tests/unit/libstore/machines.cc b/tests/unit/libstore/machines.cc index 9fd7fda54..2d66e9534 100644 --- a/tests/unit/libstore/machines.cc +++ b/tests/unit/libstore/machines.cc @@ -1,46 +1,31 @@ #include "machines.hh" -#include "globals.hh" #include "file-system.hh" #include "util.hh" +#include "tests/characterization.hh" + +#include #include using testing::Contains; using testing::ElementsAre; -using testing::EndsWith; using testing::Eq; using testing::Field; using testing::SizeIs; -using nix::absPath; -using nix::FormatError; -using nix::UsageError; -using nix::getMachines; -using nix::Machine; -using nix::Machines; -using nix::pathExists; -using nix::Settings; -using nix::settings; +namespace nix::fs { using namespace std::filesystem; } -class Environment : public ::testing::Environment { - public: - void SetUp() override { settings.thisSystem = "TEST_ARCH-TEST_OS"; } -}; - -testing::Environment* const foo_env = - testing::AddGlobalTestEnvironment(new Environment); +using namespace nix; TEST(machines, getMachinesWithEmptyBuilders) { - settings.builders = ""; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({}, ""); ASSERT_THAT(actual, SizeIs(0)); } TEST(machines, getMachinesUriOnly) { - settings.builders = "nix@scratchy.labs.cs.uu.nl"; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({"TEST_ARCH-TEST_OS"}, "nix@scratchy.labs.cs.uu.nl"); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); @@ -51,10 +36,9 @@ TEST(machines, getMachinesUriOnly) { } TEST(machines, getMachinesDefaults) { - settings.builders = "nix@scratchy.labs.cs.uu.nl - - - - - - -"; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({"TEST_ARCH-TEST_OS"}, "nix@scratchy.labs.cs.uu.nl - - - - - - -"); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq("ssh://nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, Eq(StoreReference::parse("ssh://nix@scratchy.labs.cs.uu.nl")))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("TEST_ARCH-TEST_OS"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, SizeIs(0))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(1))); @@ -64,29 +48,38 @@ TEST(machines, getMachinesDefaults) { EXPECT_THAT(actual[0], Field(&Machine::sshPublicHostKey, SizeIs(0))); } +MATCHER_P(AuthorityMatches, authority, "") { + *result_listener + << "where the authority of " + << arg.render() + << " is " + << authority; + auto * generic = std::get_if(&arg.variant); + if (!generic) return false; + return generic->authority == authority; +} + TEST(machines, getMachinesWithNewLineSeparator) { - settings.builders = "nix@scratchy.labs.cs.uu.nl\nnix@itchy.labs.cs.uu.nl"; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({}, "nix@scratchy.labs.cs.uu.nl\nnix@itchy.labs.cs.uu.nl"); ASSERT_THAT(actual, SizeIs(2)); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl")))); } TEST(machines, getMachinesWithSemicolonSeparator) { - settings.builders = "nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl"; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({}, "nix@scratchy.labs.cs.uu.nl ; nix@itchy.labs.cs.uu.nl"); EXPECT_THAT(actual, SizeIs(2)); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl")))); } TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) { - settings.builders = "nix@scratchy.labs.cs.uu.nl i686-linux " - "/home/nix/.ssh/id_scratchy_auto 8 3 kvm " - "benchmark SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({}, + "nix@scratchy.labs.cs.uu.nl i686-linux " + "/home/nix/.ssh/id_scratchy_auto 8 3 kvm " + "benchmark SSH+HOST+PUBLIC+KEY+BASE64+ENCODED=="); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); @@ -98,13 +91,12 @@ TEST(machines, getMachinesWithCorrectCompleteSingleBuilder) { TEST(machines, getMachinesWithCorrectCompleteSingleBuilderWithTabColumnDelimiter) { - settings.builders = + auto actual = Machine::parseConfig({}, "nix@scratchy.labs.cs.uu.nl\ti686-linux\t/home/nix/.ssh/" "id_scratchy_auto\t8\t3\tkvm\tbenchmark\tSSH+HOST+PUBLIC+" - "KEY+BASE64+ENCODED=="; - Machines actual = getMachines(); + "KEY+BASE64+ENCODED=="); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("i686-linux"))); EXPECT_THAT(actual[0], Field(&Machine::sshKey, Eq("/home/nix/.ssh/id_scratchy_auto"))); EXPECT_THAT(actual[0], Field(&Machine::maxJobs, Eq(8))); @@ -115,58 +107,63 @@ TEST(machines, } TEST(machines, getMachinesWithMultiOptions) { - settings.builders = "nix@scratchy.labs.cs.uu.nl Arch1,Arch2 - - - " - "SupportedFeature1,SupportedFeature2 " - "MandatoryFeature1,MandatoryFeature2"; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({}, + "nix@scratchy.labs.cs.uu.nl Arch1,Arch2 - - - " + "SupportedFeature1,SupportedFeature2 " + "MandatoryFeature1,MandatoryFeature2"); ASSERT_THAT(actual, SizeIs(1)); - EXPECT_THAT(actual[0], Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl"))); + EXPECT_THAT(actual[0], Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl"))); EXPECT_THAT(actual[0], Field(&Machine::systemTypes, ElementsAre("Arch1", "Arch2"))); EXPECT_THAT(actual[0], Field(&Machine::supportedFeatures, ElementsAre("SupportedFeature1", "SupportedFeature2"))); EXPECT_THAT(actual[0], Field(&Machine::mandatoryFeatures, ElementsAre("MandatoryFeature1", "MandatoryFeature2"))); } TEST(machines, getMachinesWithIncorrectFormat) { - settings.builders = "nix@scratchy.labs.cs.uu.nl - - eight"; - EXPECT_THROW(getMachines(), FormatError); - settings.builders = "nix@scratchy.labs.cs.uu.nl - - -1"; - EXPECT_THROW(getMachines(), FormatError); - settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 three"; - EXPECT_THROW(getMachines(), FormatError); - settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 -3"; - EXPECT_THROW(getMachines(), UsageError); - settings.builders = "nix@scratchy.labs.cs.uu.nl - - 8 3 - - BAD_BASE64"; - EXPECT_THROW(getMachines(), FormatError); + EXPECT_THROW( + Machine::parseConfig({}, "nix@scratchy.labs.cs.uu.nl - - eight"), + FormatError); + EXPECT_THROW( + Machine::parseConfig({}, "nix@scratchy.labs.cs.uu.nl - - -1"), + FormatError); + EXPECT_THROW( + Machine::parseConfig({}, "nix@scratchy.labs.cs.uu.nl - - 8 three"), + FormatError); + EXPECT_THROW( + Machine::parseConfig({}, "nix@scratchy.labs.cs.uu.nl - - 8 -3"), + UsageError); + EXPECT_THROW( + Machine::parseConfig({}, "nix@scratchy.labs.cs.uu.nl - - 8 3 - - BAD_BASE64"), + FormatError); } TEST(machines, getMachinesWithCorrectFileReference) { - auto path = absPath("tests/unit/libstore/test-data/machines.valid"); - ASSERT_TRUE(pathExists(path)); + auto path = fs::weakly_canonical(getUnitTestData() / "machines/valid"); + ASSERT_TRUE(fs::exists(path)); - settings.builders = std::string("@") + path; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({}, "@" + path.string()); ASSERT_THAT(actual, SizeIs(3)); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@scratchy.labs.cs.uu.nl")))); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@itchy.labs.cs.uu.nl")))); - EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, EndsWith("nix@poochie.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@scratchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@itchy.labs.cs.uu.nl")))); + EXPECT_THAT(actual, Contains(Field(&Machine::storeUri, AuthorityMatches("nix@poochie.labs.cs.uu.nl")))); } TEST(machines, getMachinesWithCorrectFileReferenceToEmptyFile) { - auto path = "/dev/null"; - ASSERT_TRUE(pathExists(path)); + fs::path path = "/dev/null"; + ASSERT_TRUE(fs::exists(path)); - settings.builders = std::string("@") + path; - Machines actual = getMachines(); + auto actual = Machine::parseConfig({}, "@" + path.string()); ASSERT_THAT(actual, SizeIs(0)); } TEST(machines, getMachinesWithIncorrectFileReference) { - settings.builders = std::string("@") + absPath("/not/a/file"); - Machines actual = getMachines(); + auto path = fs::weakly_canonical("/not/a/file"); + ASSERT_TRUE(!fs::exists(path)); + auto actual = Machine::parseConfig({}, "@" + path.string()); ASSERT_THAT(actual, SizeIs(0)); } TEST(machines, getMachinesWithCorrectFileReferenceToIncorrectFile) { - settings.builders = std::string("@") + absPath("tests/unit/libstore/test-data/machines.bad_format"); - EXPECT_THROW(getMachines(), FormatError); + EXPECT_THROW( + Machine::parseConfig({}, "@" + fs::weakly_canonical(getUnitTestData() / "machines" / "bad_format").string()), + FormatError); } diff --git a/tests/unit/libstore/meson.build b/tests/unit/libstore/meson.build new file mode 100644 index 000000000..3b36cd62f --- /dev/null +++ b/tests/unit/libstore/meson.build @@ -0,0 +1,105 @@ +project('nix-store-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-store'), + dependency('nix-store-c'), + dependency('nix-store-test-support'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +subdir('build-utils-meson/export-all-symbols') + +sqlite = dependency('sqlite3', 'sqlite', version : '>=3.6.19') +deps_private += sqlite + +rapidcheck = dependency('rapidcheck') +deps_private += rapidcheck + +gtest = dependency('gtest', main : true) +deps_private += gtest + +gtest = dependency('gmock') +deps_private += gtest + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-store.hh', + '-include', 'config-util.h', + '-include', 'config-store.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'common-protocol.cc', + 'content-address.cc', + 'derivation-advanced-attrs.cc', + 'derivation.cc', + 'derived-path.cc', + 'downstream-placeholder.cc', + 'http-binary-cache-store.cc', + 'legacy-ssh-store.cc', + 'local-binary-cache-store.cc', + 'local-overlay-store.cc', + 'local-store.cc', + 'machines.cc', + 'nar-info-disk-cache.cc', + 'nar-info.cc', + 'nix_api_store.cc', + 'outputs-spec.cc', + 'path-info.cc', + 'path.cc', + 'references.cc', + 's3-binary-cache-store.cc', + 'serve-protocol.cc', + 'ssh-store.cc', + 'store-reference.cc', + 'uds-remote-store.cc', + 'worker-protocol.cc', +) + +include_dirs = [include_directories('.')] + + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + # TODO: -lrapidcheck, see ../libutil-support/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + # get main from gtest + install : true, +) + +test( + meson.project_name(), + this_exe, + env : { + '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + }, + protocol : 'gtest', +) diff --git a/tests/unit/libstore/nar-info.cc b/tests/unit/libstore/nar-info.cc index bd10602e7..0d155743d 100644 --- a/tests/unit/libstore/nar-info.cc +++ b/tests/unit/libstore/nar-info.cc @@ -13,10 +13,10 @@ using nlohmann::json; class NarInfoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/nar-info"; + std::filesystem::path unitTestData = getUnitTestData() / "nar-info"; - Path goldenMaster(PathView testStem) const override { - return unitTestData + "/" + testStem + ".json"; + std::filesystem::path goldenMaster(PathView testStem) const override { + return unitTestData / (testStem + ".json"); } }; @@ -25,7 +25,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { store, "foo", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { diff --git a/tests/unit/libstore/package.nix b/tests/unit/libstore/package.nix new file mode 100644 index 000000000..7560a5b79 --- /dev/null +++ b/tests/unit/libstore/package.nix @@ -0,0 +1,110 @@ +{ lib +, buildPackages +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-store +, nix-store-c +, nix-store-test-support +, sqlite + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +, filesetToSource +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-store-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-store + nix-store-c + nix-store-test-support + sqlite + rapidcheck + gtest + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru = { + tests = { + run = let + # Some data is shared with the functional tests: they create it, + # we consume it. + data = filesetToSource { + root = ../..; + fileset = lib.fileset.unions [ + ./data + ../../functional/derivation + ]; + }; + in runCommand "${finalAttrs.pname}-run" { + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' + export _NIX_TEST_UNIT_DATA=${data + "/unit/libstore/data"} + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} + touch $out + ''); + }; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; + }; + +}) diff --git a/tests/unit/libstore/path-info.cc b/tests/unit/libstore/path-info.cc index 80d6fcfed..d6c4c2a7f 100644 --- a/tests/unit/libstore/path-info.cc +++ b/tests/unit/libstore/path-info.cc @@ -12,19 +12,27 @@ using nlohmann::json; class PathInfoTest : public CharacterizationTest, public LibStoreTest { - Path unitTestData = getUnitTestData() + "/path-info"; + std::filesystem::path unitTestData = getUnitTestData() / "path-info"; - Path goldenMaster(PathView testStem) const override { - return unitTestData + "/" + testStem + ".json"; + std::filesystem::path goldenMaster(PathView testStem) const override { + return unitTestData / (testStem + ".json"); } }; -static UnkeyedValidPathInfo makePathInfo(const Store & store, bool includeImpureInfo) { - UnkeyedValidPathInfo info = ValidPathInfo { +static UnkeyedValidPathInfo makeEmpty() +{ + return { + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), + }; +} + +static ValidPathInfo makeFullKeyed(const Store & store, bool includeImpureInfo) +{ + ValidPathInfo info = ValidPathInfo { store, "foo", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { @@ -49,23 +57,25 @@ static UnkeyedValidPathInfo makePathInfo(const Store & store, bool includeImpure } return info; } +static UnkeyedValidPathInfo makeFull(const Store & store, bool includeImpureInfo) { + return makeFullKeyed(store, includeImpureInfo); +} -#define JSON_TEST(STEM, PURE) \ +#define JSON_TEST(STEM, OBJ, PURE) \ TEST_F(PathInfoTest, PathInfo_ ## STEM ## _from_json) { \ readTest(#STEM, [&](const auto & encoded_) { \ auto encoded = json::parse(encoded_); \ UnkeyedValidPathInfo got = UnkeyedValidPathInfo::fromJSON( \ *store, \ encoded); \ - auto expected = makePathInfo(*store, PURE); \ + auto expected = OBJ; \ ASSERT_EQ(got, expected); \ }); \ } \ \ TEST_F(PathInfoTest, PathInfo_ ## STEM ## _to_json) { \ writeTest(#STEM, [&]() -> json { \ - return makePathInfo(*store, PURE) \ - .toJSON(*store, PURE, HashFormat::SRI); \ + return OBJ.toJSON(*store, PURE, HashFormat::SRI); \ }, [](const auto & file) { \ return json::parse(readFile(file)); \ }, [](const auto & file, const auto & got) { \ @@ -73,7 +83,19 @@ static UnkeyedValidPathInfo makePathInfo(const Store & store, bool includeImpure }); \ } -JSON_TEST(pure, false) -JSON_TEST(impure, true) +JSON_TEST(empty_pure, makeEmpty(), false) +JSON_TEST(empty_impure, makeEmpty(), true) +JSON_TEST(pure, makeFull(*store, false), false) +JSON_TEST(impure, makeFull(*store, true), true) + +TEST_F(PathInfoTest, PathInfo_full_shortRefs) { + ValidPathInfo it = makeFullKeyed(*store, true); + // it.references = unkeyed.references; + auto refs = it.shortRefs(); + ASSERT_EQ(refs.size(), 2); + ASSERT_EQ(*refs.begin(), "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"); + ASSERT_EQ(*++refs.begin(), "n5wkd9frr45pa74if5gpz9j7mifg27fh-foo"); } + +} // namespace nix diff --git a/tests/unit/libstore/path.cc b/tests/unit/libstore/path.cc index 213b6e95f..c4c055abf 100644 --- a/tests/unit/libstore/path.cc +++ b/tests/unit/libstore/path.cc @@ -26,10 +26,19 @@ static std::regex nameRegex { std::string { nameRegexStr } }; TEST_F(StorePathTest, bad_ ## NAME) { \ std::string_view str = \ STORE_DIR HASH_PART "-" STR; \ - ASSERT_THROW( \ - store->parseStorePath(str), \ - BadStorePath); \ + /* ASSERT_THROW generates a duplicate goto label */ \ + /* A lambda isolates those labels. */ \ + [&](){ \ + ASSERT_THROW( \ + store->parseStorePath(str), \ + BadStorePath); \ + }(); \ std::string name { STR }; \ + [&](){ \ + ASSERT_THROW( \ + nix::checkName(name), \ + BadStorePathName); \ + }(); \ EXPECT_FALSE(std::regex_match(name, nameRegex)); \ } @@ -54,6 +63,7 @@ TEST_DONT_PARSE(dot_dash_a, ".-a") STORE_DIR HASH_PART "-" STR; \ auto p = store->parseStorePath(str); \ std::string name { p.name() }; \ + EXPECT_EQ(p.name(), STR); \ EXPECT_TRUE(std::regex_match(name, nameRegex)); \ } diff --git a/tests/unit/libstore/s3-binary-cache-store.cc b/tests/unit/libstore/s3-binary-cache-store.cc new file mode 100644 index 000000000..7aa5f2f2c --- /dev/null +++ b/tests/unit/libstore/s3-binary-cache-store.cc @@ -0,0 +1,18 @@ +#if ENABLE_S3 + +# include + +# include "s3-binary-cache-store.hh" + +namespace nix { + +TEST(S3BinaryCacheStore, constructConfig) +{ + S3BinaryCacheStoreConfig config{"s3", "foobar", {}}; + + EXPECT_EQ(config.bucketName, "foobar"); +} + +} // namespace nix + +#endif diff --git a/tests/unit/libstore/serve-protocol.cc b/tests/unit/libstore/serve-protocol.cc index b2fd0fb82..2505c5a9a 100644 --- a/tests/unit/libstore/serve-protocol.cc +++ b/tests/unit/libstore/serve-protocol.cc @@ -6,6 +6,7 @@ #include "serve-protocol.hh" #include "serve-protocol-impl.hh" +#include "serve-protocol-connection.hh" #include "build-result.hh" #include "file-descriptor.hh" #include "tests/protocol.hh" @@ -54,15 +55,15 @@ VERSIONED_CHARACTERIZATION_TEST( defaultVersion, (std::tuple { ContentAddress { - .method = TextIngestionMethod {}, + .method = ContentAddressMethod::Raw::Text, .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -279,7 +280,7 @@ VERSIONED_CHARACTERIZATION_TEST( *LibStoreTest::store, "foo", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { .others = { @@ -397,7 +398,7 @@ VERSIONED_CHARACTERIZATION_TEST( std::nullopt, std::optional { ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, @@ -505,7 +506,8 @@ TEST_F(ServeProtoTest, handshake_client_corrupted_throws) } else { auto ver = ServeProto::BasicClientConnection::handshake( nullSink, in, defaultVersion, "blah"); - EXPECT_NE(ver, defaultVersion); + // `std::min` of this and the other version saves us + EXPECT_EQ(ver, defaultVersion); } } }); diff --git a/tests/unit/libstore/ssh-store.cc b/tests/unit/libstore/ssh-store.cc new file mode 100644 index 000000000..b853a5f1f --- /dev/null +++ b/tests/unit/libstore/ssh-store.cc @@ -0,0 +1,55 @@ +// FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 +# include + +# include "ssh-store.hh" + +namespace nix { + +TEST(SSHStore, constructConfig) +{ + SSHStoreConfig config{ + "ssh", + "localhost", + StoreConfig::Params{ + { + "remote-program", + // TODO #11106, no more split on space + "foo bar", + }, + }, + }; + + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); +} + +TEST(MountedSSHStore, constructConfig) +{ + MountedSSHStoreConfig config{ + "mounted-ssh", + "localhost", + StoreConfig::Params{ + { + "remote-program", + // TODO #11106, no more split on space + "foo bar", + }, + }, + }; + + EXPECT_EQ( + config.remoteProgram.get(), + (Strings{ + "foo", + "bar", + })); +} + +} +#endif diff --git a/tests/unit/libstore/store-reference.cc b/tests/unit/libstore/store-reference.cc new file mode 100644 index 000000000..d4c42f0fd --- /dev/null +++ b/tests/unit/libstore/store-reference.cc @@ -0,0 +1,123 @@ +#include +#include + +#include "file-system.hh" +#include "store-reference.hh" + +#include "tests/characterization.hh" +#include "tests/libstore.hh" + +namespace nix { + +using nlohmann::json; + +class StoreReferenceTest : public CharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "store-reference"; + + std::filesystem::path goldenMaster(PathView testStem) const override + { + return unitTestData / (testStem + ".txt"); + } +}; + +#define URI_TEST_READ(STEM, OBJ) \ + TEST_F(StoreReferenceTest, PathInfo_##STEM##_from_uri) \ + { \ + readTest(#STEM, ([&](const auto & encoded) { \ + StoreReference expected = OBJ; \ + auto got = StoreReference::parse(encoded); \ + ASSERT_EQ(got, expected); \ + })); \ + } + +#define URI_TEST_WRITE(STEM, OBJ) \ + TEST_F(StoreReferenceTest, PathInfo_##STEM##_to_uri) \ + { \ + writeTest( \ + #STEM, \ + [&]() -> StoreReference { return OBJ; }, \ + [](const auto & file) { return StoreReference::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.render()); }); \ + } + +#define URI_TEST(STEM, OBJ) \ + URI_TEST_READ(STEM, OBJ) \ + URI_TEST_WRITE(STEM, OBJ) + +URI_TEST( + auto, + (StoreReference{ + .variant = StoreReference::Auto{}, + .params = {}, + })) + +URI_TEST( + auto_param, + (StoreReference{ + .variant = StoreReference::Auto{}, + .params = + { + {"root", "/foo/bar/baz"}, + }, + })) + +static StoreReference localExample_1{ + .variant = + StoreReference::Specified{ + .scheme = "local", + }, + .params = + { + {"root", "/foo/bar/baz"}, + }, +}; + +static StoreReference localExample_2{ + .variant = + StoreReference::Specified{ + .scheme = "local", + .authority = "/foo/bar/baz", + }, + .params = + { + {"trusted", "true"}, + }, +}; + +URI_TEST(local_1, localExample_1) + +URI_TEST(local_2, localExample_2) + +URI_TEST_READ(local_shorthand_1, localExample_1) + +URI_TEST_READ(local_shorthand_2, localExample_2) + +static StoreReference unixExample{ + .variant = + StoreReference::Specified{ + .scheme = "unix", + }, + .params = + { + {"max-connections", "7"}, + {"trusted", "true"}, + }, +}; + +URI_TEST(unix, unixExample) + +URI_TEST_READ(unix_shorthand, unixExample) + +URI_TEST( + ssh, + (StoreReference{ + .variant = + StoreReference::Specified{ + .scheme = "ssh", + .authority = "localhost", + }, + .params = {}, + })) + +} diff --git a/tests/unit/libstore/uds-remote-store.cc b/tests/unit/libstore/uds-remote-store.cc new file mode 100644 index 000000000..5ccb20871 --- /dev/null +++ b/tests/unit/libstore/uds-remote-store.cc @@ -0,0 +1,23 @@ +// FIXME: Odd failures for templates that are causing the PR to break +// for now with discussion with @Ericson2314 to comment out. +#if 0 +# include + +# include "uds-remote-store.hh" + +namespace nix { + +TEST(UDSRemoteStore, constructConfig) +{ + UDSRemoteStoreConfig config{"unix", "/tmp/socket", {}}; + + EXPECT_EQ(config.path, "/tmp/socket"); +} + +TEST(UDSRemoteStore, constructConfigWrongScheme) +{ + EXPECT_THROW(UDSRemoteStoreConfig("http", "/tmp/socket", {}), UsageError); +} + +} // namespace nix +#endif diff --git a/tests/unit/libstore/worker-protocol.cc b/tests/unit/libstore/worker-protocol.cc index 2b2e559a9..bbea9ed75 100644 --- a/tests/unit/libstore/worker-protocol.cc +++ b/tests/unit/libstore/worker-protocol.cc @@ -1,9 +1,11 @@ #include +#include #include #include #include "worker-protocol.hh" +#include "worker-protocol-connection.hh" #include "worker-protocol-impl.hh" #include "derived-path.hh" #include "build-result.hh" @@ -18,9 +20,9 @@ struct WorkerProtoTest : VersionedProtoTest { /** * For serializers that don't care about the minimum version, we - * used the oldest one: 1.0. + * used the oldest one: 1.10. */ - WorkerProto::Version defaultVersion = 1 << 8 | 0; + WorkerProto::Version defaultVersion = 1 << 8 | 10; }; @@ -54,15 +56,15 @@ VERSIONED_CHARACTERIZATION_TEST( defaultVersion, (std::tuple { ContentAddress { - .method = TextIngestionMethod {}, + .method = ContentAddressMethod::Raw::Text, .hash = hashString(HashAlgorithm::SHA256, "Derive(...)"), }, ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, ContentAddress { - .method = FileIngestionMethod::Recursive, + .method = ContentAddressMethod::Raw::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), }, })) @@ -510,7 +512,7 @@ VERSIONED_CHARACTERIZATION_TEST( *LibStoreTest::store, "foo", FixedOutputInfo { - .method = FileIngestionMethod::Recursive, + .method = FileIngestionMethod::NixArchive, .hash = hashString(HashAlgorithm::SHA256, "(...)"), .references = { .others = { @@ -529,6 +531,17 @@ VERSIONED_CHARACTERIZATION_TEST( }), })) +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + buildMode, + "build-mode", + defaultVersion, + (std::tuple { + bmNormal, + bmRepair, + bmCheck, + })) + VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, optionalTrustedFlag, @@ -585,10 +598,185 @@ VERSIONED_CHARACTERIZATION_TEST( std::nullopt, std::optional { ContentAddress { - .method = FileIngestionMethod::Flat, + .method = ContentAddressMethod::Raw::Flat, .hash = hashString(HashAlgorithm::SHA1, "blob blob..."), }, }, })) +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + clientHandshakeInfo_1_30, + "client-handshake-info_1_30", + 1 << 8 | 30, + (std::tuple { + {}, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + clientHandshakeInfo_1_33, + "client-handshake-info_1_33", + 1 << 8 | 33, + (std::tuple { + { + .daemonNixVersion = std::optional { "foo" }, + }, + { + .daemonNixVersion = std::optional { "bar" }, + }, + })) + +VERSIONED_CHARACTERIZATION_TEST( + WorkerProtoTest, + clientHandshakeInfo_1_35, + "client-handshake-info_1_35", + 1 << 8 | 35, + (std::tuple { + { + .daemonNixVersion = std::optional { "foo" }, + .remoteTrustsUs = std::optional { NotTrusted }, + }, + { + .daemonNixVersion = std::optional { "bar" }, + .remoteTrustsUs = std::optional { Trusted }, + }, + })) + +TEST_F(WorkerProtoTest, handshake_log) +{ + CharacterizationTest::writeTest("handshake-to-client", [&]() -> std::string { + StringSink toClientLog; + + Pipe toClient, toServer; + toClient.create(); + toServer.create(); + + WorkerProto::Version clientResult; + + auto thread = std::thread([&]() { + FdSink out { toServer.writeSide.get() }; + FdSource in0 { toClient.readSide.get() }; + TeeSource in { in0, toClientLog }; + clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake( + out, in, defaultVersion, {})); + }); + + { + FdSink out { toClient.writeSide.get() }; + FdSource in { toServer.readSide.get() }; + WorkerProto::BasicServerConnection::handshake( + out, in, defaultVersion, {}); + }; + + thread.join(); + + return std::move(toClientLog.s); + }); +} + +TEST_F(WorkerProtoTest, handshake_features) +{ + Pipe toClient, toServer; + toClient.create(); + toServer.create(); + + std::tuple> clientResult; + + auto clientThread = std::thread([&]() { + FdSink out { toServer.writeSide.get() }; + FdSource in { toClient.readSide.get() }; + clientResult = WorkerProto::BasicClientConnection::handshake( + out, in, 123, {"bar", "aap", "mies", "xyzzy"}); + }); + + FdSink out { toClient.writeSide.get() }; + FdSource in { toServer.readSide.get() }; + auto daemonResult = WorkerProto::BasicServerConnection::handshake( + out, in, 456, {"foo", "bar", "xyzzy"}); + + clientThread.join(); + + EXPECT_EQ(clientResult, daemonResult); + EXPECT_EQ(std::get<0>(clientResult), 123); + EXPECT_EQ(std::get<1>(clientResult), std::set({"bar", "xyzzy"})); +} + +/// Has to be a `BufferedSink` for handshake. +struct NullBufferedSink : BufferedSink { + void writeUnbuffered(std::string_view data) override { } +}; + +TEST_F(WorkerProtoTest, handshake_client_replay) +{ + CharacterizationTest::readTest("handshake-to-client", [&](std::string toClientLog) { + NullBufferedSink nullSink; + + StringSource in { toClientLog }; + auto clientResult = std::get<0>(WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {})); + + EXPECT_EQ(clientResult, defaultVersion); + }); +} + +TEST_F(WorkerProtoTest, handshake_client_truncated_replay_throws) +{ + CharacterizationTest::readTest("handshake-to-client", [&](std::string toClientLog) { + for (size_t len = 0; len < toClientLog.size(); ++len) { + NullBufferedSink nullSink; + StringSource in { + // truncate + toClientLog.substr(0, len) + }; + if (len < 8) { + EXPECT_THROW( + WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {}), + EndOfFile); + } else { + // Not sure why cannot keep on checking for `EndOfFile`. + EXPECT_THROW( + WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {}), + Error); + } + } + }); +} + +TEST_F(WorkerProtoTest, handshake_client_corrupted_throws) +{ + CharacterizationTest::readTest("handshake-to-client", [&](const std::string toClientLog) { + for (size_t idx = 0; idx < toClientLog.size(); ++idx) { + // corrupt a copy + std::string toClientLogCorrupt = toClientLog; + toClientLogCorrupt[idx] *= 4; + ++toClientLogCorrupt[idx]; + + NullBufferedSink nullSink; + StringSource in { toClientLogCorrupt }; + + if (idx < 4 || idx == 9) { + // magic bytes don't match + EXPECT_THROW( + WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {}), + Error); + } else if (idx < 8 || idx >= 12) { + // Number out of bounds + EXPECT_THROW( + WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {}), + SerialisationError); + } else { + auto ver = std::get<0>(WorkerProto::BasicClientConnection::handshake( + nullSink, in, defaultVersion, {})); + // `std::min` of this and the other version saves us + EXPECT_EQ(ver, defaultVersion); + } + } + }); +} + } diff --git a/tests/unit/libutil-support/.version b/tests/unit/libutil-support/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libutil-support/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libutil-support/build-utils-meson b/tests/unit/libutil-support/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libutil-support/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libutil-support/meson.build b/tests/unit/libutil-support/meson.build new file mode 100644 index 000000000..42b49a6a0 --- /dev/null +++ b/tests/unit/libutil-support/meson.build @@ -0,0 +1,73 @@ +project('nix-util-test-support', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ +] +deps_public_maybe_subproject = [ + dependency('nix-util'), +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +rapidcheck = dependency('rapidcheck') +deps_public += rapidcheck + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'tests/hash.cc', + 'tests/string_callback.cc', +) + +include_dirs = [include_directories('.')] + +headers = files( + 'tests/characterization.hh', + 'tests/gtest-with-params.hh', + 'tests/hash.hh', + 'tests/nix_api_util.hh', + 'tests/string_callback.hh', +) + +subdir('build-utils-meson/export-all-symbols') + +this_library = library( + 'nix-util-test-support', + sources, + dependencies : deps_public + deps_private + deps_other, + include_directories : include_dirs, + # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326 + # is available. See also ../libutil/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + prelink : true, # For C++ static initializers + install : true, +) + +install_headers(headers, subdir : 'nix', preserve_path : true) + +libraries_private = [] + +subdir('build-utils-meson/export') diff --git a/tests/unit/libutil-support/package.nix b/tests/unit/libutil-support/package.nix new file mode 100644 index 000000000..1665804cb --- /dev/null +++ b/tests/unit/libutil-support/package.nix @@ -0,0 +1,75 @@ +{ lib +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util + +, rapidcheck + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-util-test-support"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + outputs = [ "out" "dev" ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + propagatedBuildInputs = [ + nix-util + rapidcheck + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + }; + +}) diff --git a/tests/unit/libutil-support/tests/characterization.hh b/tests/unit/libutil-support/tests/characterization.hh index 9d6c850f0..5e790e75b 100644 --- a/tests/unit/libutil-support/tests/characterization.hh +++ b/tests/unit/libutil-support/tests/characterization.hh @@ -5,6 +5,7 @@ #include "types.hh" #include "environment-variables.hh" +#include "file-system.hh" namespace nix { @@ -12,7 +13,7 @@ namespace nix { * The path to the unit test data directory. See the contributing guide * in the manual for further details. */ -static Path getUnitTestData() { +static inline std::filesystem::path getUnitTestData() { return getEnv("_NIX_TEST_UNIT_DATA").value(); } @@ -21,7 +22,7 @@ static Path getUnitTestData() { * against them. See the contributing guide in the manual for further * details. */ -static bool testAccept() { +static inline bool testAccept() { return getEnv("_NIX_TEST_ACCEPT") == "1"; } @@ -35,7 +36,7 @@ protected: * While the "golden master" for this characterization test is * located. It should not be shared with any other test. */ - virtual Path goldenMaster(PathView testStem) const = 0; + virtual std::filesystem::path goldenMaster(PathView testStem) const = 0; public: /** @@ -76,7 +77,7 @@ public: if (testAccept()) { - createDirs(dirOf(file)); + std::filesystem::create_directories(file.parent_path()); writeFile2(file, got); GTEST_SKIP() << "Updating golden master " @@ -96,10 +97,10 @@ public: { writeTest( testStem, test, - [](const Path & f) -> std::string { + [](const std::filesystem::path & f) -> std::string { return readFile(f); }, - [](const Path & f, const std::string & c) { + [](const std::filesystem::path & f, const std::string & c) { return writeFile(f, c); }); } diff --git a/tests/unit/libutil-support/tests/gtest-with-params.hh b/tests/unit/libutil-support/tests/gtest-with-params.hh new file mode 100644 index 000000000..323a083fe --- /dev/null +++ b/tests/unit/libutil-support/tests/gtest-with-params.hh @@ -0,0 +1,54 @@ +#pragma once +// SPDX-FileCopyrightText: 2014 Emil Eriksson +// +// SPDX-License-Identifier: BSD-2-Clause +// +// The lion's share of this code is copy pasted directly out of RapidCheck +// headers, so the copyright is set accordingly. +/** + * @file Implements the ability to run a RapidCheck test under gtest with changed + * test parameters such as the number of tests to run. This is useful for + * running very large numbers of the extremely cheap property tests. + */ + +#include +#include +#include + +namespace rc::detail { + +using MakeTestParams = TestParams (*)(); + +template +void checkGTestWith(Testable && testable, MakeTestParams makeTestParams) +{ + const auto testInfo = ::testing::UnitTest::GetInstance()->current_test_info(); + detail::TestMetadata metadata; + metadata.id = std::string(testInfo->test_case_name()) + "/" + std::string(testInfo->name()); + metadata.description = std::string(testInfo->name()); + + const auto result = checkTestable(std::forward(testable), metadata, makeTestParams()); + + if (result.template is()) { + const auto success = result.template get(); + if (!success.distribution.empty()) { + printResultMessage(result, std::cout); + std::cout << std::endl; + } + } else { + std::ostringstream ss; + printResultMessage(result, ss); + FAIL() << ss.str() << std::endl; + } +} +} + +#define RC_GTEST_PROP_WITH_PARAMS(TestCase, Name, MakeParams, ArgList) \ + void rapidCheck_propImpl_##TestCase##_##Name ArgList; \ + \ + TEST(TestCase, Name) \ + { \ + ::rc::detail::checkGTestWith(&rapidCheck_propImpl_##TestCase##_##Name, MakeParams); \ + } \ + \ + void rapidCheck_propImpl_##TestCase##_##Name ArgList diff --git a/tests/unit/libutil-support/tests/string_callback.cc b/tests/unit/libutil-support/tests/string_callback.cc index 2d0e0dad0..7a13bd4ff 100644 --- a/tests/unit/libutil-support/tests/string_callback.cc +++ b/tests/unit/libutil-support/tests/string_callback.cc @@ -2,9 +2,10 @@ namespace nix::testing { -void observe_string_cb(const char * start, unsigned int n, std::string * user_data) +void observe_string_cb(const char * start, unsigned int n, void * user_data) { - *user_data = std::string(start); + auto user_data_casted = reinterpret_cast(user_data); + *user_data_casted = std::string(start); } } diff --git a/tests/unit/libutil-support/tests/string_callback.hh b/tests/unit/libutil-support/tests/string_callback.hh index 3a3e545e9..9a7e8d85d 100644 --- a/tests/unit/libutil-support/tests/string_callback.hh +++ b/tests/unit/libutil-support/tests/string_callback.hh @@ -3,14 +3,13 @@ namespace nix::testing { -void observe_string_cb(const char * start, unsigned int n, std::string * user_data); +void observe_string_cb(const char * start, unsigned int n, void * user_data); inline void * observe_string_cb_data(std::string & out) { return (void *) &out; }; -#define OBSERVE_STRING(str) \ - (nix_get_string_callback) nix::testing::observe_string_cb, nix::testing::observe_string_cb_data(str) +#define OBSERVE_STRING(str) nix::testing::observe_string_cb, nix::testing::observe_string_cb_data(str) } diff --git a/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc new file mode 100644 index 000000000..122a09dcb --- /dev/null +++ b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.cc @@ -0,0 +1,34 @@ +#include +#include "tracing-file-system-object-sink.hh" + +namespace nix::test { + +void TracingFileSystemObjectSink::createDirectory(const CanonPath & path) +{ + std::cerr << "createDirectory(" << path << ")\n"; + sink.createDirectory(path); +} + +void TracingFileSystemObjectSink::createRegularFile( + const CanonPath & path, std::function fn) +{ + std::cerr << "createRegularFile(" << path << ")\n"; + sink.createRegularFile(path, [&](CreateRegularFileSink & crf) { + // We could wrap this and trace about the chunks of data and such + fn(crf); + }); +} + +void TracingFileSystemObjectSink::createSymlink(const CanonPath & path, const std::string & target) +{ + std::cerr << "createSymlink(" << path << ", target: " << target << ")\n"; + sink.createSymlink(path, target); +} + +void TracingExtendedFileSystemObjectSink::createHardlink(const CanonPath & path, const CanonPath & target) +{ + std::cerr << "createHardlink(" << path << ", target: " << target << ")\n"; + sink.createHardlink(path, target); +} + +} // namespace nix::test diff --git a/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh new file mode 100644 index 000000000..895ac3664 --- /dev/null +++ b/tests/unit/libutil-support/tests/tracing-file-system-object-sink.hh @@ -0,0 +1,41 @@ +#pragma once +#include "fs-sink.hh" + +namespace nix::test { + +/** + * A `FileSystemObjectSink` that traces calls, writing to stderr. + */ +class TracingFileSystemObjectSink : public virtual FileSystemObjectSink +{ + FileSystemObjectSink & sink; +public: + TracingFileSystemObjectSink(FileSystemObjectSink & sink) + : sink(sink) + { + } + + void createDirectory(const CanonPath & path) override; + + void createRegularFile(const CanonPath & path, std::function fn) override; + + void createSymlink(const CanonPath & path, const std::string & target) override; +}; + +/** + * A `ExtendedFileSystemObjectSink` that traces calls, writing to stderr. + */ +class TracingExtendedFileSystemObjectSink : public TracingFileSystemObjectSink, public ExtendedFileSystemObjectSink +{ + ExtendedFileSystemObjectSink & sink; +public: + TracingExtendedFileSystemObjectSink(ExtendedFileSystemObjectSink & sink) + : TracingFileSystemObjectSink(sink) + , sink(sink) + { + } + + void createHardlink(const CanonPath & path, const CanonPath & target) override; +}; + +} diff --git a/tests/unit/libutil/.version b/tests/unit/libutil/.version new file mode 120000 index 000000000..0df9915bf --- /dev/null +++ b/tests/unit/libutil/.version @@ -0,0 +1 @@ +../../../.version \ No newline at end of file diff --git a/tests/unit/libutil/build-utils-meson b/tests/unit/libutil/build-utils-meson new file mode 120000 index 000000000..f2d8e8a50 --- /dev/null +++ b/tests/unit/libutil/build-utils-meson @@ -0,0 +1 @@ +../../../build-utils-meson/ \ No newline at end of file diff --git a/tests/unit/libutil/checked-arithmetic.cc b/tests/unit/libutil/checked-arithmetic.cc new file mode 100644 index 000000000..75018660d --- /dev/null +++ b/tests/unit/libutil/checked-arithmetic.cc @@ -0,0 +1,158 @@ +#include +#include +#include +#include +#include +#include + +#include + +#include "tests/gtest-with-params.hh" + +namespace rc { +using namespace nix; + +template +struct Arbitrary> +{ + static Gen> arbitrary() + { + return gen::arbitrary(); + } +}; + +} + +namespace nix::checked { + +// Pointer to member function! Mildly gross. +template +using Oper = Checked::Result (Checked::*)(T const other) const; + +template +using ReferenceOper = T (*)(T a, T b); + +/** + * Checks that performing an operation that overflows into an inaccurate result + * has the desired behaviour. + * + * TBig is a type large enough to represent all results of TSmall operations. + */ +template +void checkType(TSmall a_, TSmall b, Oper oper, ReferenceOper reference) +{ + // Sufficient to fit all values + TBig referenceResult = reference(a_, b); + constexpr const TSmall minV = std::numeric_limits::min(); + constexpr const TSmall maxV = std::numeric_limits::max(); + + Checked a{a_}; + auto result = (a.*(oper))(b); + + // Just truncate it to get the in-range result + RC_ASSERT(result.valueWrapping() == static_cast(referenceResult)); + + if (referenceResult > maxV || referenceResult < minV) { + RC_ASSERT(result.overflowed()); + RC_ASSERT(!result.valueChecked().has_value()); + } else { + RC_ASSERT(!result.overflowed()); + RC_ASSERT(result.valueChecked().has_value()); + RC_ASSERT(*result.valueChecked() == referenceResult); + } +} + +/** + * Checks that performing an operation that overflows into an inaccurate result + * has the desired behaviour. + * + * TBig is a type large enough to represent all results of TSmall operations. + */ +template +void checkDivision(TSmall a_, TSmall b) +{ + // Sufficient to fit all values + constexpr const TSmall minV = std::numeric_limits::min(); + + Checked a{a_}; + auto result = a / b; + + if (std::is_signed() && a_ == minV && b == -1) { + // This is the only possible overflow condition + RC_ASSERT(result.valueWrapping() == minV); + RC_ASSERT(result.overflowed()); + } else if (b == 0) { + RC_ASSERT(result.divideByZero()); + RC_ASSERT_THROWS_AS(result.valueWrapping(), nix::checked::DivideByZero); + RC_ASSERT(result.valueChecked() == std::nullopt); + } else { + TBig referenceResult = a_ / b; + auto result_ = result.valueChecked(); + RC_ASSERT(result_.has_value()); + RC_ASSERT(*result_ == referenceResult); + RC_ASSERT(result.valueWrapping() == referenceResult); + } +} + +/** Creates parameters that perform a more adequate number of checks to validate + * extremely cheap tests such as arithmetic tests */ +static rc::detail::TestParams makeParams() +{ + auto const & conf = rc::detail::configuration(); + auto newParams = conf.testParams; + newParams.maxSuccess = 10000; + return newParams; +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, add_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkType(a, b, &Checked::operator+, [](int32_t a, int32_t b) { return a + b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, add_signed, makeParams, (int16_t a, int16_t b)) +{ + checkType(a, b, &Checked::operator+, [](int32_t a, int32_t b) { return a + b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, sub_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkType(a, b, &Checked::operator-, [](int32_t a, int32_t b) { return a - b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, sub_signed, makeParams, (int16_t a, int16_t b)) +{ + checkType(a, b, &Checked::operator-, [](int32_t a, int32_t b) { return a - b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, mul_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkType(a, b, &Checked::operator*, [](int64_t a, int64_t b) { return a * b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, mul_signed, makeParams, (int16_t a, int16_t b)) +{ + checkType(a, b, &Checked::operator*, [](int64_t a, int64_t b) { return a * b; }); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, div_unsigned, makeParams, (uint16_t a, uint16_t b)) +{ + checkDivision(a, b); +} + +RC_GTEST_PROP_WITH_PARAMS(Checked, div_signed, makeParams, (int16_t a, int16_t b)) +{ + checkDivision(a, b); +} + +// Make absolutely sure that we check the special cases if the proptest +// generator does not come up with them. This one is especially important +// because it has very specific pairs required for the edge cases unlike the +// others. +TEST(Checked, div_signed_special_cases) +{ + checkDivision(std::numeric_limits::min(), -1); + checkDivision(std::numeric_limits::min(), 0); + checkDivision(0, 0); +} + +} diff --git a/tests/unit/libutil/executable-path.cc b/tests/unit/libutil/executable-path.cc new file mode 100644 index 000000000..8d182357d --- /dev/null +++ b/tests/unit/libutil/executable-path.cc @@ -0,0 +1,64 @@ +#include + +#include "executable-path.hh" + +namespace nix { + +#ifdef WIN32 +# define PATH_VAR_SEP L";" +#else +# define PATH_VAR_SEP ":" +#endif + +#define PATH_ENV_ROUND_TRIP(NAME, STRING_LIT, CXX_LIT) \ + TEST(ExecutablePath, NAME) \ + { \ + OsString s = STRING_LIT; \ + auto v = ExecutablePath::parse(s); \ + EXPECT_EQ(v, (ExecutablePath CXX_LIT)); \ + auto s2 = v.render(); \ + EXPECT_EQ(s2, s); \ + } + +PATH_ENV_ROUND_TRIP(emptyRoundTrip, OS_STR(""), ({})) + +PATH_ENV_ROUND_TRIP( + oneElemRoundTrip, + OS_STR("/foo"), + ({ + OS_STR("/foo"), + })) + +PATH_ENV_ROUND_TRIP( + twoElemsRoundTrip, + OS_STR("/foo" PATH_VAR_SEP "/bar"), + ({ + OS_STR("/foo"), + OS_STR("/bar"), + })) + +PATH_ENV_ROUND_TRIP( + threeElemsRoundTrip, + OS_STR("/foo" PATH_VAR_SEP "." PATH_VAR_SEP "/bar"), + ({ + OS_STR("/foo"), + OS_STR("."), + OS_STR("/bar"), + })) + +TEST(ExecutablePath, elementyElemNormalize) +{ + auto v = ExecutablePath::parse(PATH_VAR_SEP PATH_VAR_SEP PATH_VAR_SEP); + EXPECT_EQ( + v, + (ExecutablePath{{ + OS_STR("."), + OS_STR("."), + OS_STR("."), + OS_STR("."), + }})); + auto s2 = v.render(); + EXPECT_EQ(s2, OS_STR("." PATH_VAR_SEP "." PATH_VAR_SEP "." PATH_VAR_SEP ".")); +} + +} diff --git a/tests/unit/libutil/file-content-address.cc b/tests/unit/libutil/file-content-address.cc index 294e39806..27d926a87 100644 --- a/tests/unit/libutil/file-content-address.cc +++ b/tests/unit/libutil/file-content-address.cc @@ -11,7 +11,7 @@ namespace nix { TEST(FileSerialisationMethod, testRoundTripPrintParse_1) { for (const FileSerialisationMethod fim : { FileSerialisationMethod::Flat, - FileSerialisationMethod::Recursive, + FileSerialisationMethod::NixArchive, }) { EXPECT_EQ(parseFileSerialisationMethod(renderFileSerialisationMethod(fim)), fim); } @@ -37,7 +37,7 @@ TEST(FileSerialisationMethod, testParseFileSerialisationMethodOptException) { TEST(FileIngestionMethod, testRoundTripPrintParse_1) { for (const FileIngestionMethod fim : { FileIngestionMethod::Flat, - FileIngestionMethod::Recursive, + FileIngestionMethod::NixArchive, FileIngestionMethod::Git, }) { EXPECT_EQ(parseFileIngestionMethod(renderFileIngestionMethod(fim)), fim); diff --git a/tests/unit/libutil/file-system.cc b/tests/unit/libutil/file-system.cc new file mode 100644 index 000000000..7ef804f34 --- /dev/null +++ b/tests/unit/libutil/file-system.cc @@ -0,0 +1,264 @@ +#include "util.hh" +#include "types.hh" +#include "file-system.hh" +#include "processes.hh" +#include "terminal.hh" +#include "strings.hh" + +#include +#include +#include + +#include + +#ifdef _WIN32 +# define FS_SEP L"\\" +# define FS_ROOT L"C:" FS_SEP // Need a mounted one, C drive is likely +#else +# define FS_SEP "/" +# define FS_ROOT FS_SEP +#endif + +#ifndef PATH_MAX +# define PATH_MAX 4096 +#endif + +#ifdef _WIN32 +# define GET_CWD _wgetcwd +#else +# define GET_CWD getcwd +#endif + +namespace nix { + +/* ----------- tests for file-system.hh -------------------------------------*/ + +/* ---------------------------------------------------------------------------- + * absPath + * --------------------------------------------------------------------------*/ + +TEST(absPath, doesntChangeRoot) +{ + auto p = absPath(std::filesystem::path{FS_ROOT}); + + ASSERT_EQ(p, FS_ROOT); +} + +TEST(absPath, turnsEmptyPathIntoCWD) +{ + OsChar cwd[PATH_MAX + 1]; + auto p = absPath(std::filesystem::path{""}); + + ASSERT_EQ(p, GET_CWD((OsChar *) &cwd, PATH_MAX)); +} + +TEST(absPath, usesOptionalBasePathWhenGiven) +{ + OsChar _cwd[PATH_MAX + 1]; + OsChar * cwd = GET_CWD((OsChar *) &_cwd, PATH_MAX); + + auto p = absPath(std::filesystem::path{""}.string(), std::filesystem::path{cwd}.string()); + + ASSERT_EQ(p, std::filesystem::path{cwd}.string()); +} + +TEST(absPath, isIdempotent) +{ + OsChar _cwd[PATH_MAX + 1]; + OsChar * cwd = GET_CWD((OsChar *) &_cwd, PATH_MAX); + auto p1 = absPath(std::filesystem::path{cwd}); + auto p2 = absPath(p1); + + ASSERT_EQ(p1, p2); +} + +TEST(absPath, pathIsCanonicalised) +{ + auto path = FS_ROOT OS_STR("some/path/with/trailing/dot/."); + auto p1 = absPath(std::filesystem::path{path}); + auto p2 = absPath(p1); + + ASSERT_EQ(p1, FS_ROOT "some" FS_SEP "path" FS_SEP "with" FS_SEP "trailing" FS_SEP "dot"); + ASSERT_EQ(p1, p2); +} + +/* ---------------------------------------------------------------------------- + * canonPath + * --------------------------------------------------------------------------*/ + +TEST(canonPath, removesTrailingSlashes) +{ + std::filesystem::path path = FS_ROOT "this/is/a/path//"; + auto p = canonPath(path.string()); + + ASSERT_EQ(p, std::filesystem::path{FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"}.string()); +} + +TEST(canonPath, removesDots) +{ + std::filesystem::path path = FS_ROOT "this/./is/a/path/./"; + auto p = canonPath(path.string()); + + ASSERT_EQ(p, std::filesystem::path{FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"}.string()); +} + +TEST(canonPath, removesDots2) +{ + std::filesystem::path path = FS_ROOT "this/a/../is/a////path/foo/.."; + auto p = canonPath(path.string()); + + ASSERT_EQ(p, std::filesystem::path{FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"}.string()); +} + +TEST(canonPath, requiresAbsolutePath) +{ + ASSERT_ANY_THROW(canonPath(".")); + ASSERT_ANY_THROW(canonPath("..")); + ASSERT_ANY_THROW(canonPath("../")); + ASSERT_DEATH({ canonPath(""); }, "path != \"\""); +} + +/* ---------------------------------------------------------------------------- + * dirOf + * --------------------------------------------------------------------------*/ + +TEST(dirOf, returnsEmptyStringForRoot) +{ + auto p = dirOf("/"); + + ASSERT_EQ(p, "/"); +} + +TEST(dirOf, returnsFirstPathComponent) +{ + auto p1 = dirOf("/dir/"); + ASSERT_EQ(p1, "/dir"); + auto p2 = dirOf("/dir"); + ASSERT_EQ(p2, "/"); + auto p3 = dirOf("/dir/.."); + ASSERT_EQ(p3, "/dir"); + auto p4 = dirOf("/dir/../"); + ASSERT_EQ(p4, "/dir/.."); +} + +/* ---------------------------------------------------------------------------- + * baseNameOf + * --------------------------------------------------------------------------*/ + +TEST(baseNameOf, emptyPath) +{ + auto p1 = baseNameOf(""); + ASSERT_EQ(p1, ""); +} + +TEST(baseNameOf, pathOnRoot) +{ + auto p1 = baseNameOf("/dir"); + ASSERT_EQ(p1, "dir"); +} + +TEST(baseNameOf, relativePath) +{ + auto p1 = baseNameOf("dir/foo"); + ASSERT_EQ(p1, "foo"); +} + +TEST(baseNameOf, pathWithTrailingSlashRoot) +{ + auto p1 = baseNameOf("/"); + ASSERT_EQ(p1, ""); +} + +TEST(baseNameOf, trailingSlash) +{ + auto p1 = baseNameOf("/dir/"); + ASSERT_EQ(p1, "dir"); +} + +TEST(baseNameOf, trailingSlashes) +{ + auto p1 = baseNameOf("/dir//"); + ASSERT_EQ(p1, "dir"); +} + +TEST(baseNameOf, absoluteNothingSlashNothing) +{ + auto p1 = baseNameOf("//"); + ASSERT_EQ(p1, ""); +} + +/* ---------------------------------------------------------------------------- + * isInDir + * --------------------------------------------------------------------------*/ + +TEST(isInDir, trivialCase) +{ + auto p1 = isInDir("/foo/bar", "/foo"); + ASSERT_EQ(p1, true); +} + +TEST(isInDir, notInDir) +{ + auto p1 = isInDir("/zes/foo/bar", "/foo"); + ASSERT_EQ(p1, false); +} + +// XXX: hm, bug or feature? :) Looking at the implementation +// this might be problematic. +TEST(isInDir, emptyDir) +{ + auto p1 = isInDir("/zes/foo/bar", ""); + ASSERT_EQ(p1, true); +} + +/* ---------------------------------------------------------------------------- + * isDirOrInDir + * --------------------------------------------------------------------------*/ + +TEST(isDirOrInDir, trueForSameDirectory) +{ + ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true); + ASSERT_EQ(isDirOrInDir("/", "/"), true); +} + +TEST(isDirOrInDir, trueForEmptyPaths) +{ + ASSERT_EQ(isDirOrInDir("", ""), true); +} + +TEST(isDirOrInDir, falseForDisjunctPaths) +{ + ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false); +} + +TEST(isDirOrInDir, relativePaths) +{ + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true); +} + +// XXX: while it is possible to use "." or ".." in the +// first argument this doesn't seem to work in the second. +TEST(isDirOrInDir, DISABLED_shouldWork) +{ + ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true); +} + +/* ---------------------------------------------------------------------------- + * pathExists + * --------------------------------------------------------------------------*/ + +TEST(pathExists, rootExists) +{ + ASSERT_TRUE(pathExists(std::filesystem::path{FS_ROOT}.string())); +} + +TEST(pathExists, cwdExists) +{ + ASSERT_TRUE(pathExists(".")); +} + +TEST(pathExists, bogusPathDoesNotExist) +{ + ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); +} +} diff --git a/tests/unit/libutil/git.cc b/tests/unit/libutil/git.cc index ff934c117..9232de5b9 100644 --- a/tests/unit/libutil/git.cc +++ b/tests/unit/libutil/git.cc @@ -11,12 +11,12 @@ using namespace git; class GitTest : public CharacterizationTest { - Path unitTestData = getUnitTestData() + "/git"; + std::filesystem::path unitTestData = getUnitTestData() / "git"; public: - Path goldenMaster(std::string_view testStem) const override { - return unitTestData + "/" + testStem; + std::filesystem::path goldenMaster(std::string_view testStem) const override { + return unitTestData / std::string(testStem); } /** @@ -67,7 +67,7 @@ TEST_F(GitTest, blob_read) { StringSink out; RegularFileSink out2 { out }; ASSERT_EQ(parseObjectType(in, mockXpSettings), ObjectType::Blob); - parseBlob(out2, "", in, BlobMode::Regular, mockXpSettings); + parseBlob(out2, CanonPath::root, in, BlobMode::Regular, mockXpSettings); auto expected = readFile(goldenMaster("hello-world.bin")); @@ -132,8 +132,8 @@ TEST_F(GitTest, tree_read) { NullFileSystemObjectSink out; Tree got; ASSERT_EQ(parseObjectType(in, mockXpSettings), ObjectType::Tree); - parseTree(out, "", in, [&](auto & name, auto entry) { - auto name2 = name; + parseTree(out, CanonPath::root, in, [&](auto & name, auto entry) { + auto name2 = std::string{name.rel()}; if (entry.mode == Mode::Directory) name2 += '/'; got.insert_or_assign(name2, std::move(entry)); @@ -210,14 +210,14 @@ TEST_F(GitTest, both_roundrip) { MemorySink sinkFiles2 { *files2 }; - std::function mkSinkHook; + std::function mkSinkHook; mkSinkHook = [&](auto prefix, auto & hash, auto blobMode) { StringSource in { cas[hash] }; parse( sinkFiles2, prefix, in, blobMode, - [&](const Path & name, const auto & entry) { + [&](const CanonPath & name, const auto & entry) { mkSinkHook( - prefix + "/" + name, + prefix / name, entry.hash, // N.B. this cast would not be acceptable in real // code, because it would make an assert reachable, @@ -227,9 +227,9 @@ TEST_F(GitTest, both_roundrip) { mockXpSettings); }; - mkSinkHook("", root.hash, BlobMode::Regular); + mkSinkHook(CanonPath::root, root.hash, BlobMode::Regular); - ASSERT_EQ(*files, *files2); + ASSERT_EQ(files->root, files2->root); } TEST(GitLsRemote, parseSymrefLineWithReference) { diff --git a/tests/unit/libutil/json-utils.cc b/tests/unit/libutil/json-utils.cc index c9370a74b..704a4acb0 100644 --- a/tests/unit/libutil/json-utils.cc +++ b/tests/unit/libutil/json-utils.cc @@ -175,13 +175,16 @@ TEST(optionalValueAt, empty) { TEST(getNullable, null) { auto json = R"(null)"_json; - ASSERT_EQ(getNullable(json), std::nullopt); + ASSERT_EQ(getNullable(json), nullptr); } TEST(getNullable, empty) { auto json = R"({})"_json; - ASSERT_EQ(getNullable(json), std::optional { R"({})"_json }); + auto * p = getNullable(json); + + ASSERT_NE(p, nullptr); + ASSERT_EQ(*p, R"({})"_json); } } /* namespace nix */ diff --git a/tests/unit/libutil/local.mk b/tests/unit/libutil/local.mk index 39b4c0782..404f35cf1 100644 --- a/tests/unit/libutil/local.mk +++ b/tests/unit/libutil/local.mk @@ -4,7 +4,7 @@ programs += libutil-tests libutil-tests_NAME = libnixutil-tests -libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data +libutil-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data GTEST_OUTPUT=xml:$$testresults/libutil-tests.xml libutil-tests_DIR := $(d) @@ -27,6 +27,11 @@ libutil-tests_LIBS = libutil-test-support libutil libutilc libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) +ifdef HOST_WINDOWS + # Increase the default reserved stack size to 65 MB so Nix doesn't run out of space + libutil-tests_LDFLAGS += -Wl,--stack,$(shell echo $$((65 * 1024 * 1024))) +endif + check: $(d)/data/git/check-data.sh.test $(eval $(call run-test,$(d)/data/git/check-data.sh)) diff --git a/tests/unit/libutil/meson.build b/tests/unit/libutil/meson.build new file mode 100644 index 000000000..c39db8cda --- /dev/null +++ b/tests/unit/libutil/meson.build @@ -0,0 +1,100 @@ +project('nix-util-tests', 'cpp', + version : files('.version'), + default_options : [ + 'cpp_std=c++2a', + # TODO(Qyriad): increase the warning level + 'warning_level=1', + 'debug=true', + 'optimization=2', + 'errorlogs=true', # Please print logs for tests that fail + ], + meson_version : '>= 1.1', + license : 'LGPL-2.1-or-later', +) + +cxx = meson.get_compiler('cpp') + +subdir('build-utils-meson/deps-lists') + +deps_private_maybe_subproject = [ + dependency('nix-util'), + dependency('nix-util-c'), + dependency('nix-util-test-support'), +] +deps_public_maybe_subproject = [ +] +subdir('build-utils-meson/subprojects') + +subdir('build-utils-meson/threads') + +subdir('build-utils-meson/export-all-symbols') + +rapidcheck = dependency('rapidcheck') +deps_private += rapidcheck + +gtest = dependency('gtest', main : true) +deps_private += gtest + +add_project_arguments( + # TODO(Qyriad): Yes this is how the autoconf+Make system did it. + # It would be nice for our headers to be idempotent instead. + '-include', 'config-util.hh', + '-include', 'config-util.h', + language : 'cpp', +) + +subdir('build-utils-meson/diagnostics') + +sources = files( + 'args.cc', + 'canon-path.cc', + 'checked-arithmetic.cc', + 'chunked-vector.cc', + 'closure.cc', + 'compression.cc', + 'config.cc', + 'executable-path.cc', + 'file-content-address.cc', + 'file-system.cc', + 'git.cc', + 'hash.cc', + 'hilite.cc', + 'json-utils.cc', + 'logging.cc', + 'lru-cache.cc', + 'nix_api_util.cc', + 'pool.cc', + 'position.cc', + 'processes.cc', + 'references.cc', + 'spawn.cc', + 'strings.cc', + 'suggestions.cc', + 'terminal.cc', + 'url.cc', + 'util.cc', + 'xml-writer.cc', +) + +include_dirs = [include_directories('.')] + + +this_exe = executable( + meson.project_name(), + sources, + dependencies : deps_private_subproject + deps_private + deps_other, + include_directories : include_dirs, + # TODO: -lrapidcheck, see ../libutil-support/build.meson + link_args: linker_export_flags + ['-lrapidcheck'], + # get main from gtest + install : true, +) + +test( + meson.project_name(), + this_exe, + env : { + '_NIX_TEST_UNIT_DATA': meson.current_source_dir() / 'data', + }, + protocol : 'gtest', +) diff --git a/tests/unit/libutil/nix_api_util.cc b/tests/unit/libutil/nix_api_util.cc index d2999f55b..0f1cbe5dd 100644 --- a/tests/unit/libutil/nix_api_util.cc +++ b/tests/unit/libutil/nix_api_util.cc @@ -1,4 +1,4 @@ -#include "config.hh" +#include "config-global.hh" #include "args.hh" #include "nix_api_util.h" #include "nix_api_util_internal.h" @@ -31,6 +31,9 @@ TEST_F(nix_api_util_context, nix_context_error) } ASSERT_EQ(ctx->last_err_code, NIX_ERR_UNKNOWN); ASSERT_EQ(*ctx->last_err, err_msg_ref); + + nix_clear_err(ctx); + ASSERT_EQ(ctx->last_err_code, NIX_OK); } TEST_F(nix_api_util_context, nix_set_err_msg) diff --git a/tests/unit/libutil/package.nix b/tests/unit/libutil/package.nix new file mode 100644 index 000000000..2fce5bfa8 --- /dev/null +++ b/tests/unit/libutil/package.nix @@ -0,0 +1,97 @@ +{ lib +, buildPackages +, stdenv +, mkMesonDerivation +, releaseTools + +, meson +, ninja +, pkg-config + +, nix-util +, nix-util-c +, nix-util-test-support + +, rapidcheck +, gtest +, runCommand + +# Configuration Options + +, version +}: + +let + inherit (lib) fileset; +in + +mkMesonDerivation (finalAttrs: { + pname = "nix-util-tests"; + inherit version; + + workDir = ./.; + fileset = fileset.unions [ + ../../../build-utils-meson + ./build-utils-meson + ../../../.version + ./.version + ./meson.build + # ./meson.options + (fileset.fileFilter (file: file.hasExt "cc") ./.) + (fileset.fileFilter (file: file.hasExt "hh") ./.) + ]; + + nativeBuildInputs = [ + meson + ninja + pkg-config + ]; + + buildInputs = [ + nix-util + nix-util-c + nix-util-test-support + rapidcheck + gtest + ]; + + preConfigure = + # "Inline" .version so it's not a symlink, and includes the suffix. + # Do the meson utils, without modification. + '' + chmod u+w ./.version + echo ${version} > ../../../.version + ''; + + mesonFlags = [ + ]; + + env = lib.optionalAttrs (stdenv.isLinux && !(stdenv.hostPlatform.isStatic && stdenv.system == "aarch64-linux")) { + LDFLAGS = "-fuse-ld=gold"; + }; + + separateDebugInfo = !stdenv.hostPlatform.isStatic; + + hardeningDisable = lib.optional stdenv.hostPlatform.isStatic "pie"; + + passthru = { + tests = { + run = runCommand "${finalAttrs.pname}-run" { + meta.broken = !stdenv.hostPlatform.emulatorAvailable buildPackages; + } (lib.optionalString stdenv.hostPlatform.isWindows '' + export HOME="$PWD/home-dir" + mkdir -p "$HOME" + '' + '' + export _NIX_TEST_UNIT_DATA=${./data} + ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} + touch $out + ''); + }; + }; + + meta = { + platforms = lib.platforms.unix ++ lib.platforms.windows; + mainProgram = finalAttrs.pname + stdenv.hostPlatform.extensions.executable; + }; + +}) diff --git a/tests/unit/libutil/position.cc b/tests/unit/libutil/position.cc new file mode 100644 index 000000000..484ecc247 --- /dev/null +++ b/tests/unit/libutil/position.cc @@ -0,0 +1,122 @@ +#include + +#include "position.hh" + +namespace nix { + +inline Pos::Origin makeStdin(std::string s) +{ + return Pos::Stdin{make_ref(s)}; +} + +TEST(Position, getSnippetUpTo_0) +{ + Pos::Origin o = makeStdin(""); + Pos p(1, 1, o); + ASSERT_EQ(p.getSnippetUpTo(p), ""); +} +TEST(Position, getSnippetUpTo_1) +{ + Pos::Origin o = makeStdin("x"); + { + // NOTE: line and column are actually 1-based indexes + Pos start(0, 0, o); + Pos end(99, 99, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "x"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), std::nullopt); + } + { + // NOTE: line and column are actually 1-based indexes + Pos start(0, 99, o); + Pos end(99, 0, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + + // "x" might be preferable, but we only care about not crashing for invalid inputs + ASSERT_EQ(start.getSnippetUpTo(end), ""); + + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), std::nullopt); + } + { + Pos start(1, 1, o); + Pos end(1, 99, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "x"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), ""); + } + { + Pos start(1, 1, o); + Pos end(99, 99, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "x"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + ASSERT_EQ(end.getSnippetUpTo(start), std::nullopt); + } +} +TEST(Position, getSnippetUpTo_2) +{ + Pos::Origin o = makeStdin("asdf\njkl\nqwer"); + { + Pos start(1, 1, o); + Pos end(1, 2, o); + ASSERT_EQ(start.getSnippetUpTo(start), ""); + ASSERT_EQ(start.getSnippetUpTo(end), "a"); + ASSERT_EQ(end.getSnippetUpTo(end), ""); + + // nullopt? I feel like changing the column handling would just make it more fragile + ASSERT_EQ(end.getSnippetUpTo(start), ""); + } + { + Pos start(1, 2, o); + Pos end(1, 3, o); + ASSERT_EQ(start.getSnippetUpTo(end), "s"); + } + { + Pos start(1, 2, o); + Pos end(2, 2, o); + ASSERT_EQ(start.getSnippetUpTo(end), "sdf\nj"); + } + { + Pos start(1, 2, o); + Pos end(3, 2, o); + ASSERT_EQ(start.getSnippetUpTo(end), "sdf\njkl\nq"); + } + { + Pos start(1, 2, o); + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "sdf\njkl"); + } + { + Pos start(1, 4, o); + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "f\njkl"); + } + { + Pos start(1, 5, o); + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "\njkl"); + } + { + Pos start(1, 6, o); // invalid: starting column past last "line character", ie at the newline + Pos end(2, 99, o); + ASSERT_EQ(start.getSnippetUpTo(end), "\njkl"); // jkl might be acceptable for this invalid start position + } + { + Pos start(1, 1, o); + Pos end(2, 0, o); // invalid + ASSERT_EQ(start.getSnippetUpTo(end), "asdf\n"); + } +} + +TEST(Position, example_1) +{ + Pos::Origin o = makeStdin(" unambiguous = \n /** Very close */\n x: x;\n# ok\n"); + Pos start(2, 5, o); + Pos end(2, 22, o); + ASSERT_EQ(start.getSnippetUpTo(end), "/** Very close */"); +} + +} // namespace nix diff --git a/tests/unit/libutil/processes.cc b/tests/unit/libutil/processes.cc new file mode 100644 index 000000000..9033595e8 --- /dev/null +++ b/tests/unit/libutil/processes.cc @@ -0,0 +1,17 @@ +#include "processes.hh" + +#include + +namespace nix { + +/* ---------------------------------------------------------------------------- + * statusOk + * --------------------------------------------------------------------------*/ + +TEST(statusOk, zeroIsOk) +{ + ASSERT_EQ(statusOk(0), true); + ASSERT_EQ(statusOk(1), false); +} + +} // namespace nix diff --git a/tests/unit/libutil/references.cc b/tests/unit/libutil/references.cc index a517d9aa1..c3efa6d51 100644 --- a/tests/unit/libutil/references.cc +++ b/tests/unit/libutil/references.cc @@ -15,7 +15,7 @@ struct RewriteParams { strRewrites.insert(from + "->" + to); return os << "OriginalString: " << bar.originalString << std::endl << - "Rewrites: " << concatStringsSep(",", strRewrites) << std::endl << + "Rewrites: " << dropEmptyInitThenConcatStringsSep(",", strRewrites) << std::endl << "Expected result: " << bar.finalString; } }; diff --git a/tests/unit/libutil/spawn.cc b/tests/unit/libutil/spawn.cc new file mode 100644 index 000000000..c617acae0 --- /dev/null +++ b/tests/unit/libutil/spawn.cc @@ -0,0 +1,36 @@ +#include + +#include "processes.hh" + +namespace nix { + +#ifdef _WIN32 +TEST(SpawnTest, spawnEcho) +{ + auto output = runProgram(RunOptions{.program = "cmd.exe", .args = {"/C", "echo", "hello world"}}); + ASSERT_EQ(output.first, 0); + ASSERT_EQ(output.second, "\"hello world\"\r\n"); +} + +std::string windowsEscape(const std::string & str, bool cmd); + +TEST(SpawnTest, windowsEscape) +{ + auto empty = windowsEscape("", false); + ASSERT_EQ(empty, R"("")"); + // There's no quotes in this argument so the input should equal the output + auto backslashStr = R"(\\\\)"; + auto backslashes = windowsEscape(backslashStr, false); + ASSERT_EQ(backslashes, backslashStr); + + auto nestedQuotes = windowsEscape(R"(he said: "hello there")", false); + ASSERT_EQ(nestedQuotes, R"("he said: \"hello there\"")"); + + auto middleQuote = windowsEscape(R"( \\\" )", false); + ASSERT_EQ(middleQuote, R"(" \\\\\\\" ")"); + + auto space = windowsEscape("hello world", false); + ASSERT_EQ(space, R"("hello world")"); +} +#endif +} diff --git a/tests/unit/libutil/strings.cc b/tests/unit/libutil/strings.cc new file mode 100644 index 000000000..8ceb16767 --- /dev/null +++ b/tests/unit/libutil/strings.cc @@ -0,0 +1,348 @@ +#include +#include + +#include "strings.hh" + +namespace nix { + +using Strings = std::vector; + +/* ---------------------------------------------------------------------------- + * concatStringsSep + * --------------------------------------------------------------------------*/ + +TEST(concatStringsSep, empty) +{ + Strings strings; + + ASSERT_EQ(concatStringsSep(",", strings), ""); +} + +TEST(concatStringsSep, justOne) +{ + Strings strings; + strings.push_back("this"); + + ASSERT_EQ(concatStringsSep(",", strings), "this"); +} + +TEST(concatStringsSep, emptyString) +{ + Strings strings; + strings.push_back(""); + + ASSERT_EQ(concatStringsSep(",", strings), ""); +} + +TEST(concatStringsSep, emptyStrings) +{ + Strings strings; + strings.push_back(""); + strings.push_back(""); + + ASSERT_EQ(concatStringsSep(",", strings), ","); +} + +TEST(concatStringsSep, threeEmptyStrings) +{ + Strings strings; + strings.push_back(""); + strings.push_back(""); + strings.push_back(""); + + ASSERT_EQ(concatStringsSep(",", strings), ",,"); +} + +TEST(concatStringsSep, buildCommaSeparatedString) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep(",", strings), "this,is,great"); +} + +TEST(concatStringsSep, buildStringWithEmptySeparator) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(concatStringsSep("", strings), "thisisgreat"); +} + +TEST(concatStringsSep, buildSingleString) +{ + Strings strings; + strings.push_back("this"); + + ASSERT_EQ(concatStringsSep(",", strings), "this"); +} + +/* ---------------------------------------------------------------------------- + * dropEmptyInitThenConcatStringsSep + * --------------------------------------------------------------------------*/ + +TEST(dropEmptyInitThenConcatStringsSep, empty) +{ + Strings strings; + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), ""); +} + +TEST(dropEmptyInitThenConcatStringsSep, buildCommaSeparatedString) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this,is,great"); +} + +TEST(dropEmptyInitThenConcatStringsSep, buildStringWithEmptySeparator) +{ + Strings strings; + strings.push_back("this"); + strings.push_back("is"); + strings.push_back("great"); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep("", strings), "thisisgreat"); +} + +TEST(dropEmptyInitThenConcatStringsSep, buildSingleString) +{ + Strings strings; + strings.push_back("this"); + strings.push_back(""); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), "this,"); +} + +TEST(dropEmptyInitThenConcatStringsSep, emptyStrings) +{ + Strings strings; + strings.push_back(""); + strings.push_back(""); + + ASSERT_EQ(dropEmptyInitThenConcatStringsSep(",", strings), ""); +} + +/* ---------------------------------------------------------------------------- + * tokenizeString + * --------------------------------------------------------------------------*/ + +TEST(tokenizeString, empty) +{ + Strings expected = {}; + + ASSERT_EQ(tokenizeString(""), expected); +} + +TEST(tokenizeString, oneSep) +{ + Strings expected = {}; + + ASSERT_EQ(tokenizeString(" "), expected); +} + +TEST(tokenizeString, twoSep) +{ + Strings expected = {}; + + ASSERT_EQ(tokenizeString(" \n"), expected); +} + +TEST(tokenizeString, tokenizeSpacesWithDefaults) +{ + auto s = "foo bar baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsWithDefaults) +{ + auto s = "foo\tbar\tbaz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsSpacesWithDefaults) +{ + auto s = "foo\t bar\t baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsSpacesNewlineWithDefaults) +{ + auto s = "foo\t\n bar\t\n baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); +} + +TEST(tokenizeString, tokenizeTabsSpacesNewlineRetWithDefaults) +{ + auto s = "foo\t\n\r bar\t\n\r baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s), expected); + + auto s2 = "foo \t\n\r bar \t\n\r baz"; + Strings expected2 = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s2), expected2); +} + +TEST(tokenizeString, tokenizeWithCustomSep) +{ + auto s = "foo\n,bar\n,baz\n"; + Strings expected = {"foo\n", "bar\n", "baz\n"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +TEST(tokenizeString, tokenizeSepAtStart) +{ + auto s = ",foo,bar,baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +TEST(tokenizeString, tokenizeSepAtEnd) +{ + auto s = "foo,bar,baz,"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +TEST(tokenizeString, tokenizeSepEmpty) +{ + auto s = "foo,,baz"; + Strings expected = {"foo", "baz"}; + + ASSERT_EQ(tokenizeString(s, ","), expected); +} + +/* ---------------------------------------------------------------------------- + * splitString + * --------------------------------------------------------------------------*/ + +TEST(splitString, empty) +{ + Strings expected = {""}; + + ASSERT_EQ(splitString("", " \t\n\r"), expected); +} + +TEST(splitString, oneSep) +{ + Strings expected = {"", ""}; + + ASSERT_EQ(splitString(" ", " \t\n\r"), expected); +} + +TEST(splitString, twoSep) +{ + Strings expected = {"", "", ""}; + + ASSERT_EQ(splitString(" \n", " \t\n\r"), expected); +} + +TEST(splitString, tokenizeSpacesWithSpaces) +{ + auto s = "foo bar baz"; + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsWithDefaults) +{ + auto s = "foo\tbar\tbaz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "bar", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsSpacesWithDefaults) +{ + auto s = "foo\t bar\t baz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "", "bar", "", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsSpacesNewlineWithDefaults) +{ + auto s = "foo\t\n bar\t\n baz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "", "", "bar", "", "", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); +} + +TEST(splitString, tokenizeTabsSpacesNewlineRetWithDefaults) +{ + auto s = "foo\t\n\r bar\t\n\r baz"; + // Using it like this is weird, but shows the difference with tokenizeString, which also has this test + Strings expected = {"foo", "", "", "", "bar", "", "", "", "baz"}; + + ASSERT_EQ(splitString(s, " \t\n\r"), expected); + + auto s2 = "foo \t\n\r bar \t\n\r baz"; + Strings expected2 = {"foo", "", "", "", "", "bar", "", "", "", "", "baz"}; + + ASSERT_EQ(splitString(s2, " \t\n\r"), expected2); +} + +TEST(splitString, tokenizeWithCustomSep) +{ + auto s = "foo\n,bar\n,baz\n"; + Strings expected = {"foo\n", "bar\n", "baz\n"}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +TEST(splitString, tokenizeSepAtStart) +{ + auto s = ",foo,bar,baz"; + Strings expected = {"", "foo", "bar", "baz"}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +TEST(splitString, tokenizeSepAtEnd) +{ + auto s = "foo,bar,baz,"; + Strings expected = {"foo", "bar", "baz", ""}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +TEST(splitString, tokenizeSepEmpty) +{ + auto s = "foo,,baz"; + Strings expected = {"foo", "", "baz"}; + + ASSERT_EQ(splitString(s, ","), expected); +} + +// concatStringsSep sep . splitString sep = id if sep is 1 char +RC_GTEST_PROP(splitString, recoveredByConcatStringsSep, (const std::string & s)) +{ + RC_ASSERT(concatStringsSep("/", splitString(s, "/")) == s); + RC_ASSERT(concatStringsSep("a", splitString(s, "a")) == s); +} + +} // namespace nix diff --git a/tests/unit/libutil/terminal.cc b/tests/unit/libutil/terminal.cc new file mode 100644 index 000000000..cdeb9fd94 --- /dev/null +++ b/tests/unit/libutil/terminal.cc @@ -0,0 +1,60 @@ +#include "util.hh" +#include "types.hh" +#include "terminal.hh" +#include "strings.hh" + +#include +#include + +#include + +namespace nix { + +/* ---------------------------------------------------------------------------- + * filterANSIEscapes + * --------------------------------------------------------------------------*/ + +TEST(filterANSIEscapes, emptyString) +{ + auto s = ""; + auto expected = ""; + + ASSERT_EQ(filterANSIEscapes(s), expected); +} + +TEST(filterANSIEscapes, doesntChangePrintableChars) +{ + auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f"; + + ASSERT_EQ(filterANSIEscapes(s), s); +} + +TEST(filterANSIEscapes, filtersColorCodes) +{ + auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m"; + + ASSERT_EQ(filterANSIEscapes(s, true, 2), " A"); + ASSERT_EQ(filterANSIEscapes(s, true, 3), " A "); + ASSERT_EQ(filterANSIEscapes(s, true, 4), " A "); + ASSERT_EQ(filterANSIEscapes(s, true, 5), " A B"); + ASSERT_EQ(filterANSIEscapes(s, true, 8), " A B C"); +} + +TEST(filterANSIEscapes, expandsTabs) +{ + auto s = "foo\tbar\tbaz"; + + ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz"); +} + +TEST(filterANSIEscapes, utf8) +{ + ASSERT_EQ(filterANSIEscapes("foobar", true, 5), "fooba"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 6), "fóóbär"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 5), "fóóbä"); + ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó"); + ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b"); + ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b"); +} + +} // namespace nix diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc deleted file mode 100644 index b66872a6e..000000000 --- a/tests/unit/libutil/tests.cc +++ /dev/null @@ -1,682 +0,0 @@ -#include "util.hh" -#include "types.hh" -#include "file-system.hh" -#include "processes.hh" -#include "terminal.hh" - -#include -#include - -#include - -#ifdef _WIN32 -# define FS_SEP "\\" -# define FS_ROOT "C:" FS_SEP // Need a mounted one, C drive is likely -#else -# define FS_SEP "/" -# define FS_ROOT FS_SEP -#endif - -namespace nix { - -/* ----------- tests for util.hh ------------------------------------------------*/ - - /* ---------------------------------------------------------------------------- - * absPath - * --------------------------------------------------------------------------*/ - - TEST(absPath, doesntChangeRoot) { - auto p = absPath(FS_ROOT); - - ASSERT_EQ(p, FS_ROOT); - } - - - - - TEST(absPath, turnsEmptyPathIntoCWD) { - char cwd[PATH_MAX+1]; - auto p = absPath(""); - - ASSERT_EQ(p, getcwd((char*)&cwd, PATH_MAX)); - } - - TEST(absPath, usesOptionalBasePathWhenGiven) { - char _cwd[PATH_MAX+1]; - char* cwd = getcwd((char*)&_cwd, PATH_MAX); - - auto p = absPath("", cwd); - - ASSERT_EQ(p, cwd); - } - - TEST(absPath, isIdempotent) { - char _cwd[PATH_MAX+1]; - char* cwd = getcwd((char*)&_cwd, PATH_MAX); - auto p1 = absPath(cwd); - auto p2 = absPath(p1); - - ASSERT_EQ(p1, p2); - } - - - TEST(absPath, pathIsCanonicalised) { - auto path = FS_ROOT "some/path/with/trailing/dot/."; - auto p1 = absPath(path); - auto p2 = absPath(p1); - - ASSERT_EQ(p1, FS_ROOT "some" FS_SEP "path" FS_SEP "with" FS_SEP "trailing" FS_SEP "dot"); - ASSERT_EQ(p1, p2); - } - - /* ---------------------------------------------------------------------------- - * canonPath - * --------------------------------------------------------------------------*/ - - TEST(canonPath, removesTrailingSlashes) { - auto path = FS_ROOT "this/is/a/path//"; - auto p = canonPath(path); - - ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); - } - - TEST(canonPath, removesDots) { - auto path = FS_ROOT "this/./is/a/path/./"; - auto p = canonPath(path); - - ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); - } - - TEST(canonPath, removesDots2) { - auto path = FS_ROOT "this/a/../is/a////path/foo/.."; - auto p = canonPath(path); - - ASSERT_EQ(p, FS_ROOT "this" FS_SEP "is" FS_SEP "a" FS_SEP "path"); - } - - TEST(canonPath, requiresAbsolutePath) { - ASSERT_ANY_THROW(canonPath(".")); - ASSERT_ANY_THROW(canonPath("..")); - ASSERT_ANY_THROW(canonPath("../")); - ASSERT_DEATH({ canonPath(""); }, "path != \"\""); - } - - /* ---------------------------------------------------------------------------- - * dirOf - * --------------------------------------------------------------------------*/ - - TEST(dirOf, returnsEmptyStringForRoot) { - auto p = dirOf("/"); - - ASSERT_EQ(p, "/"); - } - - TEST(dirOf, returnsFirstPathComponent) { - auto p1 = dirOf("/dir/"); - ASSERT_EQ(p1, "/dir"); - auto p2 = dirOf("/dir"); - ASSERT_EQ(p2, "/"); - auto p3 = dirOf("/dir/.."); - ASSERT_EQ(p3, "/dir"); - auto p4 = dirOf("/dir/../"); - ASSERT_EQ(p4, "/dir/.."); - } - - /* ---------------------------------------------------------------------------- - * baseNameOf - * --------------------------------------------------------------------------*/ - - TEST(baseNameOf, emptyPath) { - auto p1 = baseNameOf(""); - ASSERT_EQ(p1, ""); - } - - TEST(baseNameOf, pathOnRoot) { - auto p1 = baseNameOf("/dir"); - ASSERT_EQ(p1, "dir"); - } - - TEST(baseNameOf, relativePath) { - auto p1 = baseNameOf("dir/foo"); - ASSERT_EQ(p1, "foo"); - } - - TEST(baseNameOf, pathWithTrailingSlashRoot) { - auto p1 = baseNameOf("/"); - ASSERT_EQ(p1, ""); - } - - TEST(baseNameOf, trailingSlash) { - auto p1 = baseNameOf("/dir/"); - ASSERT_EQ(p1, "dir"); - } - - TEST(baseNameOf, trailingSlashes) { - auto p1 = baseNameOf("/dir//"); - ASSERT_EQ(p1, "dir"); - } - - TEST(baseNameOf, absoluteNothingSlashNothing) { - auto p1 = baseNameOf("//"); - ASSERT_EQ(p1, ""); - } - - /* ---------------------------------------------------------------------------- - * isInDir - * --------------------------------------------------------------------------*/ - - TEST(isInDir, trivialCase) { - auto p1 = isInDir("/foo/bar", "/foo"); - ASSERT_EQ(p1, true); - } - - TEST(isInDir, notInDir) { - auto p1 = isInDir("/zes/foo/bar", "/foo"); - ASSERT_EQ(p1, false); - } - - // XXX: hm, bug or feature? :) Looking at the implementation - // this might be problematic. - TEST(isInDir, emptyDir) { - auto p1 = isInDir("/zes/foo/bar", ""); - ASSERT_EQ(p1, true); - } - - /* ---------------------------------------------------------------------------- - * isDirOrInDir - * --------------------------------------------------------------------------*/ - - TEST(isDirOrInDir, trueForSameDirectory) { - ASSERT_EQ(isDirOrInDir("/nix", "/nix"), true); - ASSERT_EQ(isDirOrInDir("/", "/"), true); - } - - TEST(isDirOrInDir, trueForEmptyPaths) { - ASSERT_EQ(isDirOrInDir("", ""), true); - } - - TEST(isDirOrInDir, falseForDisjunctPaths) { - ASSERT_EQ(isDirOrInDir("/foo", "/bar"), false); - } - - TEST(isDirOrInDir, relativePaths) { - ASSERT_EQ(isDirOrInDir("/foo/..", "/foo"), true); - } - - // XXX: while it is possible to use "." or ".." in the - // first argument this doesn't seem to work in the second. - TEST(isDirOrInDir, DISABLED_shouldWork) { - ASSERT_EQ(isDirOrInDir("/foo/..", "/foo/."), true); - - } - - /* ---------------------------------------------------------------------------- - * pathExists - * --------------------------------------------------------------------------*/ - - TEST(pathExists, rootExists) { - ASSERT_TRUE(pathExists(FS_ROOT)); - } - - TEST(pathExists, cwdExists) { - ASSERT_TRUE(pathExists(".")); - } - - TEST(pathExists, bogusPathDoesNotExist) { - ASSERT_FALSE(pathExists("/schnitzel/darmstadt/pommes")); - } - - /* ---------------------------------------------------------------------------- - * concatStringsSep - * --------------------------------------------------------------------------*/ - - TEST(concatStringsSep, buildCommaSeparatedString) { - Strings strings; - strings.push_back("this"); - strings.push_back("is"); - strings.push_back("great"); - - ASSERT_EQ(concatStringsSep(",", strings), "this,is,great"); - } - - TEST(concatStringsSep, buildStringWithEmptySeparator) { - Strings strings; - strings.push_back("this"); - strings.push_back("is"); - strings.push_back("great"); - - ASSERT_EQ(concatStringsSep("", strings), "thisisgreat"); - } - - TEST(concatStringsSep, buildSingleString) { - Strings strings; - strings.push_back("this"); - - ASSERT_EQ(concatStringsSep(",", strings), "this"); - } - - /* ---------------------------------------------------------------------------- - * hasPrefix - * --------------------------------------------------------------------------*/ - - TEST(hasPrefix, emptyStringHasNoPrefix) { - ASSERT_FALSE(hasPrefix("", "foo")); - } - - TEST(hasPrefix, emptyStringIsAlwaysPrefix) { - ASSERT_TRUE(hasPrefix("foo", "")); - ASSERT_TRUE(hasPrefix("jshjkfhsadf", "")); - } - - TEST(hasPrefix, trivialCase) { - ASSERT_TRUE(hasPrefix("foobar", "foo")); - } - - /* ---------------------------------------------------------------------------- - * hasSuffix - * --------------------------------------------------------------------------*/ - - TEST(hasSuffix, emptyStringHasNoSuffix) { - ASSERT_FALSE(hasSuffix("", "foo")); - } - - TEST(hasSuffix, trivialCase) { - ASSERT_TRUE(hasSuffix("foo", "foo")); - ASSERT_TRUE(hasSuffix("foobar", "bar")); - } - - /* ---------------------------------------------------------------------------- - * base64Encode - * --------------------------------------------------------------------------*/ - - TEST(base64Encode, emptyString) { - ASSERT_EQ(base64Encode(""), ""); - } - - TEST(base64Encode, encodesAString) { - ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); - } - - TEST(base64Encode, encodeAndDecode) { - auto s = "quod erat demonstrandum"; - auto encoded = base64Encode(s); - auto decoded = base64Decode(encoded); - - ASSERT_EQ(decoded, s); - } - - TEST(base64Encode, encodeAndDecodeNonPrintable) { - char s[256]; - std::iota(std::rbegin(s), std::rend(s), 0); - - auto encoded = base64Encode(s); - auto decoded = base64Decode(encoded); - - EXPECT_EQ(decoded.length(), 255); - ASSERT_EQ(decoded, s); - } - - /* ---------------------------------------------------------------------------- - * base64Decode - * --------------------------------------------------------------------------*/ - - TEST(base64Decode, emptyString) { - ASSERT_EQ(base64Decode(""), ""); - } - - TEST(base64Decode, decodeAString) { - ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); - } - - TEST(base64Decode, decodeThrowsOnInvalidChar) { - ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error); - } - - /* ---------------------------------------------------------------------------- - * getLine - * --------------------------------------------------------------------------*/ - - TEST(getLine, all) { - { - auto [line, rest] = getLine("foo\nbar\nxyzzy"); - ASSERT_EQ(line, "foo"); - ASSERT_EQ(rest, "bar\nxyzzy"); - } - - { - auto [line, rest] = getLine("foo\r\nbar\r\nxyzzy"); - ASSERT_EQ(line, "foo"); - ASSERT_EQ(rest, "bar\r\nxyzzy"); - } - - { - auto [line, rest] = getLine("foo\n"); - ASSERT_EQ(line, "foo"); - ASSERT_EQ(rest, ""); - } - - { - auto [line, rest] = getLine("foo"); - ASSERT_EQ(line, "foo"); - ASSERT_EQ(rest, ""); - } - - { - auto [line, rest] = getLine(""); - ASSERT_EQ(line, ""); - ASSERT_EQ(rest, ""); - } - } - - /* ---------------------------------------------------------------------------- - * toLower - * --------------------------------------------------------------------------*/ - - TEST(toLower, emptyString) { - ASSERT_EQ(toLower(""), ""); - } - - TEST(toLower, nonLetters) { - auto s = "!@(*$#)(@#=\\234_"; - ASSERT_EQ(toLower(s), s); - } - - // std::tolower() doesn't handle unicode characters. In the context of - // store paths this isn't relevant but doesn't hurt to record this behavior - // here. - TEST(toLower, umlauts) { - auto s = "ÄÖÜ"; - ASSERT_EQ(toLower(s), "ÄÖÜ"); - } - - /* ---------------------------------------------------------------------------- - * string2Float - * --------------------------------------------------------------------------*/ - - TEST(string2Float, emptyString) { - ASSERT_EQ(string2Float(""), std::nullopt); - } - - TEST(string2Float, trivialConversions) { - ASSERT_EQ(string2Float("1.0"), 1.0); - - ASSERT_EQ(string2Float("0.0"), 0.0); - - ASSERT_EQ(string2Float("-100.25"), -100.25); - } - - /* ---------------------------------------------------------------------------- - * string2Int - * --------------------------------------------------------------------------*/ - - TEST(string2Int, emptyString) { - ASSERT_EQ(string2Int(""), std::nullopt); - } - - TEST(string2Int, trivialConversions) { - ASSERT_EQ(string2Int("1"), 1); - - ASSERT_EQ(string2Int("0"), 0); - - ASSERT_EQ(string2Int("-100"), -100); - } - -#ifndef _WIN32 // TODO re-enable on Windows, once we can start processes - /* ---------------------------------------------------------------------------- - * statusOk - * --------------------------------------------------------------------------*/ - - TEST(statusOk, zeroIsOk) { - ASSERT_EQ(statusOk(0), true); - ASSERT_EQ(statusOk(1), false); - } -#endif - - - /* ---------------------------------------------------------------------------- - * rewriteStrings - * --------------------------------------------------------------------------*/ - - TEST(rewriteStrings, emptyString) { - StringMap rewrites; - rewrites["this"] = "that"; - - ASSERT_EQ(rewriteStrings("", rewrites), ""); - } - - TEST(rewriteStrings, emptyRewrites) { - StringMap rewrites; - - ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); - } - - TEST(rewriteStrings, successfulRewrite) { - StringMap rewrites; - rewrites["this"] = "that"; - - ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that"); - } - - TEST(rewriteStrings, doesntOccur) { - StringMap rewrites; - rewrites["foo"] = "bar"; - - ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); - } - - /* ---------------------------------------------------------------------------- - * replaceStrings - * --------------------------------------------------------------------------*/ - - TEST(replaceStrings, emptyString) { - ASSERT_EQ(replaceStrings("", "this", "that"), ""); - ASSERT_EQ(replaceStrings("this and that", "", ""), "this and that"); - } - - TEST(replaceStrings, successfulReplace) { - ASSERT_EQ(replaceStrings("this and that", "this", "that"), "that and that"); - } - - TEST(replaceStrings, doesntOccur) { - ASSERT_EQ(replaceStrings("this and that", "foo", "bar"), "this and that"); - } - - /* ---------------------------------------------------------------------------- - * trim - * --------------------------------------------------------------------------*/ - - TEST(trim, emptyString) { - ASSERT_EQ(trim(""), ""); - } - - TEST(trim, removesWhitespace) { - ASSERT_EQ(trim("foo"), "foo"); - ASSERT_EQ(trim(" foo "), "foo"); - ASSERT_EQ(trim(" foo bar baz"), "foo bar baz"); - ASSERT_EQ(trim(" \t foo bar baz\n"), "foo bar baz"); - } - - /* ---------------------------------------------------------------------------- - * chomp - * --------------------------------------------------------------------------*/ - - TEST(chomp, emptyString) { - ASSERT_EQ(chomp(""), ""); - } - - TEST(chomp, removesWhitespace) { - ASSERT_EQ(chomp("foo"), "foo"); - ASSERT_EQ(chomp("foo "), "foo"); - ASSERT_EQ(chomp(" foo "), " foo"); - ASSERT_EQ(chomp(" foo bar baz "), " foo bar baz"); - ASSERT_EQ(chomp("\t foo bar baz\n"), "\t foo bar baz"); - } - - /* ---------------------------------------------------------------------------- - * quoteStrings - * --------------------------------------------------------------------------*/ - - TEST(quoteStrings, empty) { - Strings s = { }; - Strings expected = { }; - - ASSERT_EQ(quoteStrings(s), expected); - } - - TEST(quoteStrings, emptyStrings) { - Strings s = { "", "", "" }; - Strings expected = { "''", "''", "''" }; - ASSERT_EQ(quoteStrings(s), expected); - - } - - TEST(quoteStrings, trivialQuote) { - Strings s = { "foo", "bar", "baz" }; - Strings expected = { "'foo'", "'bar'", "'baz'" }; - - ASSERT_EQ(quoteStrings(s), expected); - } - - TEST(quoteStrings, quotedStrings) { - Strings s = { "'foo'", "'bar'", "'baz'" }; - Strings expected = { "''foo''", "''bar''", "''baz''" }; - - ASSERT_EQ(quoteStrings(s), expected); - } - - /* ---------------------------------------------------------------------------- - * tokenizeString - * --------------------------------------------------------------------------*/ - - TEST(tokenizeString, empty) { - Strings expected = { }; - - ASSERT_EQ(tokenizeString(""), expected); - } - - TEST(tokenizeString, tokenizeSpacesWithDefaults) { - auto s = "foo bar baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsWithDefaults) { - auto s = "foo\tbar\tbaz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsSpacesWithDefaults) { - auto s = "foo\t bar\t baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsSpacesNewlineWithDefaults) { - auto s = "foo\t\n bar\t\n baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - } - - TEST(tokenizeString, tokenizeTabsSpacesNewlineRetWithDefaults) { - auto s = "foo\t\n\r bar\t\n\r baz"; - Strings expected = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s), expected); - - auto s2 = "foo \t\n\r bar \t\n\r baz"; - Strings expected2 = { "foo", "bar", "baz" }; - - ASSERT_EQ(tokenizeString(s2), expected2); - } - - TEST(tokenizeString, tokenizeWithCustomSep) { - auto s = "foo\n,bar\n,baz\n"; - Strings expected = { "foo\n", "bar\n", "baz\n" }; - - ASSERT_EQ(tokenizeString(s, ","), expected); - } - - /* ---------------------------------------------------------------------------- - * get - * --------------------------------------------------------------------------*/ - - TEST(get, emptyContainer) { - StringMap s = { }; - auto expected = nullptr; - - ASSERT_EQ(get(s, "one"), expected); - } - - TEST(get, getFromContainer) { - StringMap s; - s["one"] = "yi"; - s["two"] = "er"; - auto expected = "yi"; - - ASSERT_EQ(*get(s, "one"), expected); - } - - TEST(getOr, emptyContainer) { - StringMap s = { }; - auto expected = "yi"; - - ASSERT_EQ(getOr(s, "one", "yi"), expected); - } - - TEST(getOr, getFromContainer) { - StringMap s; - s["one"] = "yi"; - s["two"] = "er"; - auto expected = "yi"; - - ASSERT_EQ(getOr(s, "one", "nope"), expected); - } - - /* ---------------------------------------------------------------------------- - * filterANSIEscapes - * --------------------------------------------------------------------------*/ - - TEST(filterANSIEscapes, emptyString) { - auto s = ""; - auto expected = ""; - - ASSERT_EQ(filterANSIEscapes(s), expected); - } - - TEST(filterANSIEscapes, doesntChangePrintableChars) { - auto s = "09 2q304ruyhr slk2-19024 kjsadh sar f"; - - ASSERT_EQ(filterANSIEscapes(s), s); - } - - TEST(filterANSIEscapes, filtersColorCodes) { - auto s = "\u001b[30m A \u001b[31m B \u001b[32m C \u001b[33m D \u001b[0m"; - - ASSERT_EQ(filterANSIEscapes(s, true, 2), " A" ); - ASSERT_EQ(filterANSIEscapes(s, true, 3), " A " ); - ASSERT_EQ(filterANSIEscapes(s, true, 4), " A " ); - ASSERT_EQ(filterANSIEscapes(s, true, 5), " A B" ); - ASSERT_EQ(filterANSIEscapes(s, true, 8), " A B C" ); - } - - TEST(filterANSIEscapes, expandsTabs) { - auto s = "foo\tbar\tbaz"; - - ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz" ); - } - - TEST(filterANSIEscapes, utf8) { - ASSERT_EQ(filterANSIEscapes("foobar", true, 5), "fooba"); - ASSERT_EQ(filterANSIEscapes("fóóbär", true, 6), "fóóbär"); - ASSERT_EQ(filterANSIEscapes("fóóbär", true, 5), "fóóbä"); - ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó"); - ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b"); - ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b"); - } - -} diff --git a/tests/unit/libutil/util.cc b/tests/unit/libutil/util.cc new file mode 100644 index 000000000..a3f7c720a --- /dev/null +++ b/tests/unit/libutil/util.cc @@ -0,0 +1,385 @@ +#include "util.hh" +#include "types.hh" +#include "file-system.hh" +#include "terminal.hh" +#include "strings.hh" + +#include +#include + +#include + +namespace nix { + +/* ----------- tests for util.hh --------------------------------------------*/ + +/* ---------------------------------------------------------------------------- + * hasPrefix + * --------------------------------------------------------------------------*/ + +TEST(hasPrefix, emptyStringHasNoPrefix) +{ + ASSERT_FALSE(hasPrefix("", "foo")); +} + +TEST(hasPrefix, emptyStringIsAlwaysPrefix) +{ + ASSERT_TRUE(hasPrefix("foo", "")); + ASSERT_TRUE(hasPrefix("jshjkfhsadf", "")); +} + +TEST(hasPrefix, trivialCase) +{ + ASSERT_TRUE(hasPrefix("foobar", "foo")); +} + +/* ---------------------------------------------------------------------------- + * hasSuffix + * --------------------------------------------------------------------------*/ + +TEST(hasSuffix, emptyStringHasNoSuffix) +{ + ASSERT_FALSE(hasSuffix("", "foo")); +} + +TEST(hasSuffix, trivialCase) +{ + ASSERT_TRUE(hasSuffix("foo", "foo")); + ASSERT_TRUE(hasSuffix("foobar", "bar")); +} + +/* ---------------------------------------------------------------------------- + * base64Encode + * --------------------------------------------------------------------------*/ + +TEST(base64Encode, emptyString) +{ + ASSERT_EQ(base64Encode(""), ""); +} + +TEST(base64Encode, encodesAString) +{ + ASSERT_EQ(base64Encode("quod erat demonstrandum"), "cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="); +} + +TEST(base64Encode, encodeAndDecode) +{ + auto s = "quod erat demonstrandum"; + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + ASSERT_EQ(decoded, s); +} + +TEST(base64Encode, encodeAndDecodeNonPrintable) +{ + char s[256]; + std::iota(std::rbegin(s), std::rend(s), 0); + + auto encoded = base64Encode(s); + auto decoded = base64Decode(encoded); + + EXPECT_EQ(decoded.length(), 255); + ASSERT_EQ(decoded, s); +} + +/* ---------------------------------------------------------------------------- + * base64Decode + * --------------------------------------------------------------------------*/ + +TEST(base64Decode, emptyString) +{ + ASSERT_EQ(base64Decode(""), ""); +} + +TEST(base64Decode, decodeAString) +{ + ASSERT_EQ(base64Decode("cXVvZCBlcmF0IGRlbW9uc3RyYW5kdW0="), "quod erat demonstrandum"); +} + +TEST(base64Decode, decodeThrowsOnInvalidChar) +{ + ASSERT_THROW(base64Decode("cXVvZCBlcm_0IGRlbW9uc3RyYW5kdW0="), Error); +} + +/* ---------------------------------------------------------------------------- + * getLine + * --------------------------------------------------------------------------*/ + +TEST(getLine, all) +{ + { + auto [line, rest] = getLine("foo\nbar\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\r\nbar\r\nxyzzy"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, "bar\r\nxyzzy"); + } + + { + auto [line, rest] = getLine("foo\n"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine("foo"); + ASSERT_EQ(line, "foo"); + ASSERT_EQ(rest, ""); + } + + { + auto [line, rest] = getLine(""); + ASSERT_EQ(line, ""); + ASSERT_EQ(rest, ""); + } +} + +/* ---------------------------------------------------------------------------- + * toLower + * --------------------------------------------------------------------------*/ + +TEST(toLower, emptyString) +{ + ASSERT_EQ(toLower(""), ""); +} + +TEST(toLower, nonLetters) +{ + auto s = "!@(*$#)(@#=\\234_"; + ASSERT_EQ(toLower(s), s); +} + +// std::tolower() doesn't handle unicode characters. In the context of +// store paths this isn't relevant but doesn't hurt to record this behavior +// here. +TEST(toLower, umlauts) +{ + auto s = "ÄÖÜ"; + ASSERT_EQ(toLower(s), "ÄÖÜ"); +} + +/* ---------------------------------------------------------------------------- + * string2Float + * --------------------------------------------------------------------------*/ + +TEST(string2Float, emptyString) +{ + ASSERT_EQ(string2Float(""), std::nullopt); +} + +TEST(string2Float, trivialConversions) +{ + ASSERT_EQ(string2Float("1.0"), 1.0); + + ASSERT_EQ(string2Float("0.0"), 0.0); + + ASSERT_EQ(string2Float("-100.25"), -100.25); +} + +/* ---------------------------------------------------------------------------- + * string2Int + * --------------------------------------------------------------------------*/ + +TEST(string2Int, emptyString) +{ + ASSERT_EQ(string2Int(""), std::nullopt); +} + +TEST(string2Int, trivialConversions) +{ + ASSERT_EQ(string2Int("1"), 1); + + ASSERT_EQ(string2Int("0"), 0); + + ASSERT_EQ(string2Int("-100"), -100); +} + +/* ---------------------------------------------------------------------------- + * renderSize + * --------------------------------------------------------------------------*/ + +TEST(renderSize, misc) +{ + ASSERT_EQ(renderSize(0, true), " 0.0 KiB"); + ASSERT_EQ(renderSize(100, true), " 0.1 KiB"); + ASSERT_EQ(renderSize(100), "0.1 KiB"); + ASSERT_EQ(renderSize(972, true), " 0.9 KiB"); + ASSERT_EQ(renderSize(973, true), " 1.0 KiB"); // FIXME: should round down + ASSERT_EQ(renderSize(1024, true), " 1.0 KiB"); + ASSERT_EQ(renderSize(1024 * 1024, true), "1024.0 KiB"); + ASSERT_EQ(renderSize(1100 * 1024, true), " 1.1 MiB"); + ASSERT_EQ(renderSize(2ULL * 1024 * 1024 * 1024, true), " 2.0 GiB"); + ASSERT_EQ(renderSize(2100ULL * 1024 * 1024 * 1024, true), " 2.1 TiB"); +} + +/* ---------------------------------------------------------------------------- + * rewriteStrings + * --------------------------------------------------------------------------*/ + +TEST(rewriteStrings, emptyString) +{ + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_EQ(rewriteStrings("", rewrites), ""); +} + +TEST(rewriteStrings, emptyRewrites) +{ + StringMap rewrites; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); +} + +TEST(rewriteStrings, successfulRewrite) +{ + StringMap rewrites; + rewrites["this"] = "that"; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "that and that"); +} + +TEST(rewriteStrings, doesntOccur) +{ + StringMap rewrites; + rewrites["foo"] = "bar"; + + ASSERT_EQ(rewriteStrings("this and that", rewrites), "this and that"); +} + +/* ---------------------------------------------------------------------------- + * replaceStrings + * --------------------------------------------------------------------------*/ + +TEST(replaceStrings, emptyString) +{ + ASSERT_EQ(replaceStrings("", "this", "that"), ""); + ASSERT_EQ(replaceStrings("this and that", "", ""), "this and that"); +} + +TEST(replaceStrings, successfulReplace) +{ + ASSERT_EQ(replaceStrings("this and that", "this", "that"), "that and that"); +} + +TEST(replaceStrings, doesntOccur) +{ + ASSERT_EQ(replaceStrings("this and that", "foo", "bar"), "this and that"); +} + +/* ---------------------------------------------------------------------------- + * trim + * --------------------------------------------------------------------------*/ + +TEST(trim, emptyString) +{ + ASSERT_EQ(trim(""), ""); +} + +TEST(trim, removesWhitespace) +{ + ASSERT_EQ(trim("foo"), "foo"); + ASSERT_EQ(trim(" foo "), "foo"); + ASSERT_EQ(trim(" foo bar baz"), "foo bar baz"); + ASSERT_EQ(trim(" \t foo bar baz\n"), "foo bar baz"); +} + +/* ---------------------------------------------------------------------------- + * chomp + * --------------------------------------------------------------------------*/ + +TEST(chomp, emptyString) +{ + ASSERT_EQ(chomp(""), ""); +} + +TEST(chomp, removesWhitespace) +{ + ASSERT_EQ(chomp("foo"), "foo"); + ASSERT_EQ(chomp("foo "), "foo"); + ASSERT_EQ(chomp(" foo "), " foo"); + ASSERT_EQ(chomp(" foo bar baz "), " foo bar baz"); + ASSERT_EQ(chomp("\t foo bar baz\n"), "\t foo bar baz"); +} + +/* ---------------------------------------------------------------------------- + * quoteStrings + * --------------------------------------------------------------------------*/ + +TEST(quoteStrings, empty) +{ + Strings s = {}; + Strings expected = {}; + + ASSERT_EQ(quoteStrings(s), expected); +} + +TEST(quoteStrings, emptyStrings) +{ + Strings s = {"", "", ""}; + Strings expected = {"''", "''", "''"}; + ASSERT_EQ(quoteStrings(s), expected); +} + +TEST(quoteStrings, trivialQuote) +{ + Strings s = {"foo", "bar", "baz"}; + Strings expected = {"'foo'", "'bar'", "'baz'"}; + + ASSERT_EQ(quoteStrings(s), expected); +} + +TEST(quoteStrings, quotedStrings) +{ + Strings s = {"'foo'", "'bar'", "'baz'"}; + Strings expected = {"''foo''", "''bar''", "''baz''"}; + + ASSERT_EQ(quoteStrings(s), expected); +} + +/* ---------------------------------------------------------------------------- + * get + * --------------------------------------------------------------------------*/ + +TEST(get, emptyContainer) +{ + StringMap s = {}; + auto expected = nullptr; + + ASSERT_EQ(get(s, "one"), expected); +} + +TEST(get, getFromContainer) +{ + StringMap s; + s["one"] = "yi"; + s["two"] = "er"; + auto expected = "yi"; + + ASSERT_EQ(*get(s, "one"), expected); +} + +TEST(getOr, emptyContainer) +{ + StringMap s = {}; + auto expected = "yi"; + + ASSERT_EQ(getOr(s, "one", "yi"), expected); +} + +TEST(getOr, getFromContainer) +{ + StringMap s; + s["one"] = "yi"; + s["two"] = "er"; + auto expected = "yi"; + + ASSERT_EQ(getOr(s, "one", "nope"), expected); +} + +} // namespace nix