diff --git a/.clang-format b/.clang-format index 9c0c0946a..73eac7ef6 100644 --- a/.clang-format +++ b/.clang-format @@ -28,3 +28,5 @@ EmptyLineBeforeAccessModifier: Leave #PackConstructorInitializers: BinPack BreakBeforeBinaryOperators: NonAssignment AlwaysBreakBeforeMultilineStrings: true +IndentPPDirectives: AfterHash +PPIndentWidth: 2 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 526fecabf..59db217d9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -14,4 +14,4 @@ src/libexpr/primops.cc @roberth # Libstore layer -/src/libstore @thufschmitt +/src/libstore @thufschmitt @ericson2314 diff --git a/.github/labeler.yml b/.github/labeler.yml index b1b18c488..280265dbe 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,3 +1,10 @@ +"contributor-experience": + - changed-files: + - any-glob-to-any-file: "CONTRIBUTING.md" + - any-glob-to-any-file: ".github/ISSUE_TEMPLATE/*" + - any-glob-to-any-file: ".github/PULL_REQUEST_TEMPLATE.md" + - any-glob-to-any-file: "doc/manual/src/contributing/**" + "documentation": - changed-files: - any-glob-to-any-file: "doc/manual/*" diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index 5b75704b5..8f83b913c 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -21,7 +21,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v2.4.1 + uses: zeebe-io/backport-action@v2.5.0 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8bd355cca..2b8eac49d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,3 +159,11 @@ jobs: # deprecated 2024-02-24 docker tag nix:$NIX_VERSION $IMAGE_ID:master docker push $IMAGE_ID:master + + vm_tests: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/nix-installer-action@main + - uses: DeterminateSystems/magic-nix-cache-action@main + - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes diff --git a/.gitignore b/.gitignore index 01fafa5a9..6996ca484 100644 --- a/.gitignore +++ b/.gitignore @@ -49,6 +49,9 @@ perl/Makefile.config /src/libexpr/tests /tests/unit/libexpr/libnixexpr-tests +# /src/libfetchers +/tests/unit/libfetchers/libnixfetchers-tests + # /src/libstore/ *.gen.* /src/libstore/tests @@ -115,8 +118,6 @@ perl/Makefile.config /misc/systemd/nix-daemon.conf /misc/upstart/nix-daemon.conf -/src/resolve-system-dependencies/resolve-system-dependencies - outputs/ *.a diff --git a/Makefile b/Makefile index c3dc83c77..0028c957a 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,9 @@ makefiles = \ src/libexpr/local.mk \ src/libcmd/local.mk \ src/nix/local.mk \ - src/resolve-system-dependencies/local.mk \ + src/libutil-c/local.mk \ + src/libstore-c/local.mk \ + src/libexpr-c/local.mk \ scripts/local.mk \ misc/bash/local.mk \ misc/fish/local.mk \ @@ -34,6 +36,7 @@ makefiles += \ tests/unit/libutil-support/local.mk \ tests/unit/libstore/local.mk \ tests/unit/libstore-support/local.mk \ + tests/unit/libfetchers/local.mk \ tests/unit/libexpr/local.mk \ tests/unit/libexpr-support/local.mk endif @@ -44,6 +47,7 @@ makefiles += \ tests/functional/ca/local.mk \ tests/functional/git-hashing/local.mk \ tests/functional/dyn-drv/local.mk \ + tests/functional/local-overlay-store/local.mk \ tests/functional/test-libstoreconsumer/local.mk \ tests/functional/plugins/local.mk endif @@ -59,6 +63,10 @@ ifeq ($(ENABLE_INTERNAL_API_DOCS), yes) makefiles-late += doc/internal-api/local.mk endif +ifeq ($(ENABLE_EXTERNAL_API_DOCS), yes) +makefiles-late += doc/external-api/local.mk +endif + # Miscellaneous global Flags OPTIMIZE = 1 @@ -123,3 +131,10 @@ internal-api-html: @echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'." @exit 1 endif + +ifneq ($(ENABLE_EXTERNAL_API_DOCS), yes) +.PHONY: external-api-html +external-api-html: + @echo "External API docs are disabled. Configure with '--enable-external-api-docs', or avoid calling 'make external-api-html'." + @exit 1 +endif diff --git a/Makefile.config.in b/Makefile.config.in index d5c382630..7f517898c 100644 --- a/Makefile.config.in +++ b/Makefile.config.in @@ -12,6 +12,7 @@ ENABLE_BUILD = @ENABLE_BUILD@ ENABLE_DOC_GEN = @ENABLE_DOC_GEN@ ENABLE_FUNCTIONAL_TESTS = @ENABLE_FUNCTIONAL_TESTS@ ENABLE_INTERNAL_API_DOCS = @ENABLE_INTERNAL_API_DOCS@ +ENABLE_EXTERNAL_API_DOCS = @ENABLE_EXTERNAL_API_DOCS@ ENABLE_S3 = @ENABLE_S3@ ENABLE_UNIT_TESTS = @ENABLE_UNIT_TESTS@ GTEST_LIBS = @GTEST_LIBS@ diff --git a/configure.ac b/configure.ac index 676b145a5..1d327d51d 100644 --- a/configure.ac +++ b/configure.ac @@ -150,6 +150,11 @@ AC_ARG_ENABLE(unit-tests, AS_HELP_STRING([--disable-unit-tests],[Do not build th ENABLE_UNIT_TESTS=$enableval, ENABLE_UNIT_TESTS=$ENABLE_BUILD) AC_SUBST(ENABLE_UNIT_TESTS) +# Build external API docs by default +AC_ARG_ENABLE(external_api_docs, AS_HELP_STRING([--enable-external-api-docs],[Build API docs for Nix's C interface]), + external_api_docs=$enableval, external_api_docs=yes) +AC_SUBST(external_api_docs) + AS_IF( [test "$ENABLE_BUILD" == "no" && test "$ENABLE_UNIT_TESTS" == "yes"], [AC_MSG_ERROR([Cannot enable unit tests when building overall is disabled. Please do not pass '--enable-unit-tests' or do not pass '--disable-build'.])]) @@ -172,6 +177,10 @@ AC_ARG_ENABLE(internal-api-docs, AS_HELP_STRING([--enable-internal-api-docs],[Bu ENABLE_INTERNAL_API_DOCS=$enableval, ENABLE_INTERNAL_API_DOCS=no) AC_SUBST(ENABLE_INTERNAL_API_DOCS) +AC_ARG_ENABLE(external-api-docs, AS_HELP_STRING([--enable-external-api-docs],[Build API docs for Nix's external unstable C interfaces]), + ENABLE_EXTERNAL_API_DOCS=$enableval, ENABLE_EXTERNAL_API_DOCS=no) +AC_SUBST(ENABLE_EXTERNAL_API_DOCS) + AS_IF( [test "$ENABLE_FUNCTIONAL_TESTS" == "yes" || test "$ENABLE_DOC_GEN" == "yes"], [NEED_PROG(jq, jq)]) diff --git a/doc/external-api/.gitignore b/doc/external-api/.gitignore new file mode 100644 index 000000000..dab28b6b0 --- /dev/null +++ b/doc/external-api/.gitignore @@ -0,0 +1,3 @@ +/doxygen.cfg +/html +/latex diff --git a/doc/external-api/README.md b/doc/external-api/README.md new file mode 100644 index 000000000..167c02199 --- /dev/null +++ b/doc/external-api/README.md @@ -0,0 +1,121 @@ +# Getting started + +> **Warning** These bindings are **experimental**, which means they can change +> at any time or be removed outright; nevertheless the plan is to provide a +> stable external C API to the Nix language and the Nix store. + +The language library allows evaluating Nix expressions and interacting with Nix +language values. The Nix store API is still rudimentary, and only allows +initialising and connecting to a store for the Nix language evaluator to +interact with. + +Currently there are two ways to interface with the Nix language evaluator +programmatically: + +1. Embedding the evaluator +2. Writing language plug-ins + +Embedding means you link the Nix C libraries in your program and use them from +there. Adding a plug-in means you make a library that gets loaded by the Nix +language evaluator, specified through a configuration option. + +Many of the components and mechanisms involved are not yet documented, therefore +please refer to the [Nix source code](https://github.com/NixOS/nix/) for +details. Additions to in-code documentation and the reference manual are highly +appreciated. + +The following examples, for simplicity, don't include error handling. See the +[Handling errors](@ref errors) section for more information. + +# Embedding the Nix Evaluator + +In this example we programmatically start the Nix language evaluator with a +dummy store (that has no store paths and cannot be written to), and evaluate the +Nix expression `builtins.nixVersion`. + +**main.c:** + +```C +#include +#include +#include +#include +#include +#include + +// NOTE: This example lacks all error handling. Production code must check for +// errors, as some return values will be undefined. + +void my_get_string_cb(const char * start, unsigned int n, char ** user_data) +{ + *user_data = strdup(start); +} + +int main() +{ + nix_libexpr_init(NULL); + + Store * store = nix_store_open(NULL, "dummy://", NULL); + EvalState * state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH) + Value * value = nix_alloc_value(NULL, state); + + nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value); + nix_value_force(NULL, state, value); + + char * version; + nix_get_string(NULL, value, my_get_string_cb, version); + printf("Nix version: %s\n", version); + + free(version); + nix_gc_decref(NULL, value); + nix_state_free(state); + nix_store_free(store); + return 0; +} +``` + +**Usage:** + +```ShellSession +$ gcc main.c $(pkg-config nix-expr-c --libs --cflags) -o main +$ ./main +Nix version: 2.17 +``` + +# Writing a Nix language plug-in + +In this example we add a custom primitive operation (_primop_) to `builtins`. It +will increment the argument if it is an integer and throw an error otherwise. + +**plugin.c:** + +```C +#include +#include +#include + +void increment(void* user_data, nix_c_context* ctx, EvalState* state, Value** args, Value* v) { + nix_value_force(NULL, state, args[0]); + if (nix_get_type(NULL, args[0]) == NIX_TYPE_INT) { + nix_init_int(NULL, v, nix_get_int(NULL, args[0]) + 1); + } else { + nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "First argument should be an integer."); + } +} + +void nix_plugin_entry() { + const char* args[] = {"n", NULL}; + PrimOp *p = nix_alloc_primop(NULL, increment, 1, "increment", args, "Example custom built-in function: increments an integer", NULL); + nix_register_primop(NULL, p); + nix_gc_decref(NULL, p); +} +``` + +**Usage:** + +```ShellSession +$ gcc plugin.c $(pkg-config nix-expr-c --libs --cflags) -shared -o plugin.so +$ nix --plugin-files ./plugin.so repl +nix-repl> builtins.increment 1 +2 +``` diff --git a/doc/external-api/doxygen.cfg.in b/doc/external-api/doxygen.cfg.in new file mode 100644 index 000000000..cd8b4989b --- /dev/null +++ b/doc/external-api/doxygen.cfg.in @@ -0,0 +1,57 @@ +# Doxyfile 1.9.5 + +# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by +# double-quotes, unless you are using Doxywizard) that should identify the +# project for which the documentation is generated. This name is used in the +# title of most generated pages and in a few other places. +# The default value is: My Project. + +PROJECT_NAME = "Nix" + +# The PROJECT_NUMBER tag can be used to enter a project or revision number. This +# could be handy for archiving the generated documentation or if some version +# control system is used. + +PROJECT_NUMBER = @PACKAGE_VERSION@ + +# Using the PROJECT_BRIEF tag one can provide an optional one line description +# for a project that appears at the top of each page and should give viewer a +# quick idea about the purpose of the project. Keep the description short. + +PROJECT_BRIEF = "Nix, the purely functional package manager: C API (experimental)" + +# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. +# The default value is: YES. + +GENERATE_LATEX = NO + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +# FIXME Make this list more maintainable somehow. We could maybe generate this +# in the Makefile, but we would need to change how `.in` files are preprocessed +# so they can expand variables despite configure variables. + +INPUT = \ + src/libutil-c \ + src/libexpr-c \ + src/libstore-c \ + doc/external-api/README.md + +FILE_PATTERNS = nix_api_*.h *.md + +# The INCLUDE_PATH tag can be used to specify one or more directories that +# contain include files that are not input files but should be processed by the +# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of +# RECURSIVE has no effect here. +# This tag requires that the tag SEARCH_INCLUDES is set to YES. + +INCLUDE_PATH = @RAPIDCHECK_HEADERS@ +EXCLUDE_PATTERNS = *_internal.h +GENERATE_TREEVIEW = YES +OPTIMIZE_OUTPUT_FOR_C = YES + +USE_MDFILE_AS_MAINPAGE = doc/external-api/README.md diff --git a/doc/external-api/local.mk b/doc/external-api/local.mk new file mode 100644 index 000000000..c739bdaf0 --- /dev/null +++ b/doc/external-api/local.mk @@ -0,0 +1,7 @@ +$(docdir)/external-api/html/index.html $(docdir)/external-api/latex: $(d)/doxygen.cfg + mkdir -p $(docdir)/external-api + { cat $< ; echo "OUTPUT_DIRECTORY=$(docdir)/external-api" ; } | doxygen - + +# Generate the HTML API docs for Nix's unstable C bindings +.PHONY: external-api-html +external-api-html: $(docdir)/external-api/html/index.html diff --git a/doc/manual/custom.css b/doc/manual/custom.css index b90f5423f..9e8e3886f 100644 --- a/doc/manual/custom.css +++ b/doc/manual/custom.css @@ -1,3 +1,25 @@ +:root { + --sidebar-width: 23em; +} + +h1.menu-title::before { + content: ""; + background-image: url("./favicon.svg"); + padding: 1.25em; + background-position: center center; + background-size: 2em; + background-repeat: no-repeat; +} + + +h1.menu-title { + padding: 0.5em; +} + +.sidebar .sidebar-scrollbox { + padding: 1em; +} + h1:not(:first-of-type) { margin-top: 1.3em; } diff --git a/doc/manual/local.mk b/doc/manual/local.mk index b77168885..71ad5c8e6 100644 --- a/doc/manual/local.mk +++ b/doc/manual/local.mk @@ -175,6 +175,16 @@ $(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md # Generate the HTML manual. .PHONY: manual-html manual-html: $(docdir)/manual/index.html + +# Open the built HTML manual in the default browser. +manual-html-open: $(docdir)/manual/index.html + @echo " OPEN " $<; \ + xdg-open $< \ + || open $< \ + || { \ + echo "Could not open the manual in a browser. Please open '$<'" >&2; \ + false; \ + } install: $(docdir)/manual/index.html # Generate 'nix' manpages. @@ -207,7 +217,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli # `@docroot@` is to be preserved for documenting the mechanism # FIXME: maybe contributing guides should live right next to the code # instead of in the manual -$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md +$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg $(trace-gen) \ tmp="$$(mktemp -d)"; \ cp -r doc/manual "$$tmp"; \ diff --git a/doc/manual/rl-next/remove-repl-flake.md b/doc/manual/rl-next/remove-repl-flake.md new file mode 100644 index 000000000..23298e2ed --- /dev/null +++ b/doc/manual/rl-next/remove-repl-flake.md @@ -0,0 +1,8 @@ +--- +synopsis: Remove experimental repl-flake +significance: significant +issues: 10103 +prs: 10299 +--- + +The `repl-flake` experimental feature has been removed. The `nix repl` command now works like the rest of the new CLI in that `nix repl {path}` now tries to load a flake at `{path}` (or fails if the `flakes` experimental feature isn't enabled).* diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 1149fc7b4..d9044fbda 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -110,6 +110,7 @@ - [Derivation](protocols/json/derivation.md) - [Serving Tarball Flakes](protocols/tarball-fetcher.md) - [Store Path Specification](protocols/store-path.md) + - [Nix Archive (NAR) Format](protocols/nix-archive.md) - [Derivation "ATerm" file format](protocols/derivation-aterm.md) - [Glossary](glossary.md) - [Contributing](contributing/index.md) @@ -119,7 +120,7 @@ - [Experimental Features](contributing/experimental-features.md) - [CLI guideline](contributing/cli-guideline.md) - [C++ style guide](contributing/cxx.md) -- [Release Notes](release-notes/index.md) +- [Releases](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} - [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md) - [Release 2.20 (2024-01-29)](release-notes/rl-2.20.md) diff --git a/doc/manual/src/architecture/architecture.md b/doc/manual/src/architecture/architecture.md index 2fec4ed20..867a9c992 100644 --- a/doc/manual/src/architecture/architecture.md +++ b/doc/manual/src/architecture/architecture.md @@ -69,7 +69,7 @@ It can also execute build plans to produce new data, which are made available to A build plan itself is a series of *build tasks*, together with their build inputs. > **Important** -> A build task in Nix is called [derivation](../glossary.md#gloss-derivation). +> A build task in Nix is called [derivation](@docroot@/glossary.md#gloss-derivation). Each build task has a special build input executed as *build instructions* in order to perform the build. The result of a build task can be input to another build task. diff --git a/doc/manual/src/command-ref/nix-build.md b/doc/manual/src/command-ref/nix-build.md index b548edf82..e4223b542 100644 --- a/doc/manual/src/command-ref/nix-build.md +++ b/doc/manual/src/command-ref/nix-build.md @@ -41,7 +41,7 @@ expression to a low-level [store derivation]) and [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md) (to build the store derivation). -[store derivation]: ../glossary.md#gloss-store-derivation +[store derivation]: @docroot@/glossary.md#gloss-store-derivation > **Warning** > diff --git a/doc/manual/src/command-ref/nix-copy-closure.md b/doc/manual/src/command-ref/nix-copy-closure.md index fbf6828da..eb1693e1e 100644 --- a/doc/manual/src/command-ref/nix-copy-closure.md +++ b/doc/manual/src/command-ref/nix-copy-closure.md @@ -49,7 +49,7 @@ authentication, you can avoid typing the passphrase with `ssh-agent`. - `--include-outputs`\ Also copy the outputs of [store derivation]s included in the closure. - [store derivation]: ../glossary.md#gloss-store-derivation + [store derivation]: @docroot@/glossary.md#gloss-store-derivation - `--use-substitutes` / `-s`\ Attempt to download missing paths on the target machine using Nix’s diff --git a/doc/manual/src/command-ref/nix-instantiate.md b/doc/manual/src/command-ref/nix-instantiate.md index 479c9abcf..dffbb2d70 100644 --- a/doc/manual/src/command-ref/nix-instantiate.md +++ b/doc/manual/src/command-ref/nix-instantiate.md @@ -23,7 +23,7 @@ It evaluates the Nix expressions in each of *files* (which defaults to derivation, a list of derivations, or a set of derivations. The paths of the resulting store derivations are printed on standard output. -[store derivation]: ../glossary.md#gloss-store-derivation +[store derivation]: @docroot@/glossary.md#gloss-store-derivation If *files* is the character `-`, then a Nix expression will be read from standard input. diff --git a/doc/manual/src/command-ref/nix-store/query.md b/doc/manual/src/command-ref/nix-store/query.md index a158c76aa..0bcacfe0c 100644 --- a/doc/manual/src/command-ref/nix-store/query.md +++ b/doc/manual/src/command-ref/nix-store/query.md @@ -40,12 +40,12 @@ symlink. derivations *paths*. These are the paths that will be produced when the derivation is built. - [output paths]: ../../glossary.md#gloss-output-path + [output paths]: @docroot@/glossary.md#gloss-output-path - `--requisites`; `-R`\ Prints out the [closure] of the store path *paths*. - [closure]: ../../glossary.md#gloss-closure + [closure]: @docroot@/glossary.md#gloss-closure This query has one option: @@ -66,7 +66,7 @@ symlink. *paths*, that is, their immediate dependencies. (For *all* dependencies, use `--requisites`.) - [references]: ../../glossary.md#gloss-reference + [references]: @docroot@/glossary.md#gloss-reference - `--referrers`\ Prints the set of *referrers* of the store paths *paths*, that is, @@ -90,7 +90,7 @@ symlink. example when *paths* were substituted from a binary cache. Use `--valid-derivers` instead to obtain valid paths only. - [deriver]: ../../glossary.md#gloss-deriver + [deriver]: @docroot@/glossary.md#gloss-deriver - `--valid-derivers`\ Prints a set of derivation files (`.drv`) which are supposed produce diff --git a/doc/manual/src/contributing/documentation.md b/doc/manual/src/contributing/documentation.md index 46cca759d..e7f94ab8c 100644 --- a/doc/manual/src/contributing/documentation.md +++ b/doc/manual/src/contributing/documentation.md @@ -27,11 +27,9 @@ and open `./result-doc/share/doc/nix/manual/index.html`. To build the manual incrementally, [enter the development shell](./hacking.md) and run: ```console -make manual-html -j $NIX_BUILD_CORES +make manual-html-open -j $NIX_BUILD_CORES ``` -and open `./outputs/doc/share/doc/nix/manual/language/index.html`. - In order to reflect changes to the [Makefile for the manual], clear all generated files before re-building: [Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk @@ -208,3 +206,22 @@ or inside `nix-shell` or `nix develop`: # make internal-api-html # xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html ``` + +## C API documentation (experimental) + +[C API documentation] is available online. +You can also build and view it yourself: + +[C API documentation]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs + +```console +# nix build .#hydraJobs.external-api-docs +# xdg-open ./result/share/doc/nix/external-api/html/index.html +``` + +or inside `nix-shell` or `nix develop`: + +``` +# make external-api-html +# xdg-open ./outputs/doc/share/doc/nix/external-api/html/index.html +``` diff --git a/doc/manual/src/contributing/hacking.md b/doc/manual/src/contributing/hacking.md index 2ff70f500..d56ac29a4 100644 --- a/doc/manual/src/contributing/hacking.md +++ b/doc/manual/src/contributing/hacking.md @@ -196,7 +196,7 @@ In order to facilitate this, Nix has some support for being built out of tree ## System type -Nix uses a string with he following format to identify the *system type* or *platform* it runs on: +Nix uses a string with the following format to identify the *system type* or *platform* it runs on: ``` -[-] diff --git a/doc/manual/src/favicon.png b/doc/manual/src/favicon.png new file mode 100644 index 000000000..1ed2b5fe0 Binary files /dev/null and b/doc/manual/src/favicon.png differ diff --git a/doc/manual/src/favicon.svg b/doc/manual/src/favicon.svg new file mode 100644 index 000000000..1d2a6e835 --- /dev/null +++ b/doc/manual/src/favicon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/doc/manual/src/installation/upgrading.md b/doc/manual/src/installation/upgrading.md index 38edcdbc5..a433f1d30 100644 --- a/doc/manual/src/installation/upgrading.md +++ b/doc/manual/src/installation/upgrading.md @@ -28,7 +28,7 @@ $ sudo su ## macOS multi-user ```console -$ sudo nix-env --install --file '' --attr nix -I nixpkgs=channel:nixpkgs-unstable +$ sudo nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable $ sudo launchctl remove org.nixos.nix-daemon $ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist ``` diff --git a/doc/manual/src/language/advanced-attributes.md b/doc/manual/src/language/advanced-attributes.md index b3e3afe3b..1fcc5a95b 100644 --- a/doc/manual/src/language/advanced-attributes.md +++ b/doc/manual/src/language/advanced-attributes.md @@ -207,12 +207,17 @@ Derivations can declare some infrequently used optional attributes. This is the default. - - `"recursive"`\ - The hash is computed over the NAR archive dump of the output + - `"recursive"` or `"nar"`\ + The hash is computed over the [NAR archive](@docroot@/glossary.md#gloss-nar) dump of the output (i.e., the result of [`nix-store --dump`](@docroot@/command-ref/nix-store/dump.md)). In this case, the output can be anything, including a directory tree. + `"recursive"` is the traditional way of indicating this, + and is supported since 2005 (virtually the entire history of Nix). + `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI), + however support for it is only added in Nix version 2.21. + - [`__contentAddressed`]{#adv-attr-__contentAddressed} > **Warning** > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md). @@ -298,7 +303,7 @@ Derivations can declare some infrequently used optional attributes. [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites), the following attributes are available: - - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object). + - `maxSize` defines the maximum size of the resulting [store object](@docroot@/glossary.md#gloss-store-object). - `maxClosureSize` defines the maximum size of the output's closure. - `ignoreSelfRefs` controls whether self-references should be considered when checking for allowed references/requisites. diff --git a/doc/manual/src/language/operators.md b/doc/manual/src/language/operators.md index 6fd66864b..698fed47e 100644 --- a/doc/manual/src/language/operators.md +++ b/doc/manual/src/language/operators.md @@ -128,8 +128,8 @@ The result is a string. > The file or directory at *path* must exist and is copied to the [store]. > The path appears in the result as the corresponding [store path]. -[store path]: ../glossary.md#gloss-store-path -[store]: ../glossary.md#gloss-store +[store path]: @docroot@/glossary.md#gloss-store-path +[store]: @docroot@/glossary.md#gloss-store [String and path concatenation]: #string-and-path-concatenation diff --git a/doc/manual/src/language/string-interpolation.md b/doc/manual/src/language/string-interpolation.md index 7d81c2020..1f8fecca8 100644 --- a/doc/manual/src/language/string-interpolation.md +++ b/doc/manual/src/language/string-interpolation.md @@ -20,7 +20,7 @@ Rather than writing (where `freetype` is a [derivation]), you can instead write -[derivation]: ../glossary.md#gloss-derivation +[derivation]: @docroot@/glossary.md#gloss-derivation ```nix "--with-freetype2-library=${freetype}/lib" @@ -107,9 +107,9 @@ An expression that is interpolated must evaluate to one of the following: A string interpolates to itself. -A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](../glossary.md#gloss-store-object). +A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](@docroot@/glossary.md#gloss-store-object). -[store path]: ../glossary.md#gloss-store-path +[store path]: @docroot@/glossary.md#gloss-store-path > **Example** > diff --git a/doc/manual/src/language/values.md b/doc/manual/src/language/values.md index 74ffc7070..568542c0b 100644 --- a/doc/manual/src/language/values.md +++ b/doc/manual/src/language/values.md @@ -113,7 +113,7 @@ For example, assume you used a file path in an interpolated string during a `nix repl` session. Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents. - [store path]: ../glossary.md#gloss-store-path + [store path]: @docroot@/glossary.md#gloss-store-path Paths can include [string interpolation] and can themselves be [interpolated in other expressions]. diff --git a/doc/manual/src/protocols/json/store-object-info.md b/doc/manual/src/protocols/json/store-object-info.md index ba4ab098f..179cafbb4 100644 --- a/doc/manual/src/protocols/json/store-object-info.md +++ b/doc/manual/src/protocols/json/store-object-info.md @@ -83,7 +83,7 @@ This information is not intrinsic to the store object, but about how it is store ## Computed closure fields -These fields are not stored at all, but computed by traverising the other other fields across all the store objects in a [closure]. +These fields are not stored at all, but computed by traversing the other fields across all the store objects in a [closure]. * `closureSize`: diff --git a/doc/manual/src/protocols/nix-archive.md b/doc/manual/src/protocols/nix-archive.md new file mode 100644 index 000000000..4fb6282ee --- /dev/null +++ b/doc/manual/src/protocols/nix-archive.md @@ -0,0 +1,42 @@ +# Nix Archive (NAR) format + +This is the complete specification of the Nix Archive format. +The Nix Archive format closely follows the abstract specification of a [file system object] tree, +because it is designed to serialize exactly that data structure. + +[file system object]: @docroot@/store/file-system-object.md + +The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings. +This makes the resulting binary format easier to parse. + +Regular users do *not* need to know this information. +But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful. + +```ebnf +nar = str("nix-archive-1"), nar-obj; + +nar-obj = str("("), nar-obj-inner, str(")"); + +nar-obj-inner + = str("type"), str("regular") regular + | str("type"), str("symlink") symlink + | str("type"), str("directory") directory + ; + +regular = [ str("executable"), str("") ], str("contents"), str(contents); + +symlink = str("target"), str(target); + +(* side condition: directory entries must be ordered by their names *) +directory = str("type"), str("directory") { directory-entry }; + +directory-entry = str("entry"), str("("), str("name"), str(name), str("node"), nar-obj, str(")"); +``` + +The `str` function / parameterized rule is defined as follows: + +- `str(s)` = `int(|s|), pad(s);` + +- `int(n)` = the 64-bit little endian representation of the number `n` + +- `pad(s)` = the byte sequence `s`, padded with 0s to a multiple of 8 byte diff --git a/doc/manual/src/release-notes/index.md b/doc/manual/src/release-notes/index.md index cc805e631..d4e6292a6 100644 --- a/doc/manual/src/release-notes/index.md +++ b/doc/manual/src/release-notes/index.md @@ -1,12 +1,13 @@ # Nix Release Notes +The Nix release cycle is calendar-based as follows: + Nix has a release cycle of roughly 6 weeks. Notable changes and additions are announced in the release notes for each version. -Bugfixes can be backported on request to previous Nix releases. -We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). - -Backports never skip releases. -If a feature is backported to version `x.y`, it must also be available in version `x.(y+1)`. -This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing. +The supported Nix versions are: +- The latest release +- The version used in the stable NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes). +Bugfixes and security issues are backported to every supported version. +Patch releases are published as needed. diff --git a/doc/manual/src/release-notes/rl-2.15.md b/doc/manual/src/release-notes/rl-2.15.md index 133121999..e7e52631b 100644 --- a/doc/manual/src/release-notes/rl-2.15.md +++ b/doc/manual/src/release-notes/rl-2.15.md @@ -11,7 +11,7 @@ As the choice of hash formats is no longer binary, the `--base16` flag is also added to explicitly specify the Base16 format, which is still the default. -* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. +* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](@docroot@/glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents. The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation. Using this is better and more clear than relying on the now-removed `.drv` special handling. diff --git a/doc/manual/src/release-notes/rl-2.20.md b/doc/manual/src/release-notes/rl-2.20.md index 8ede168a4..eb724f600 100644 --- a/doc/manual/src/release-notes/rl-2.20.md +++ b/doc/manual/src/release-notes/rl-2.20.md @@ -200,3 +200,9 @@ while performing various operations (including `nix develop`, `nix flake update`, and so on). With several fixes to Nix's signal handlers, Nix commands will now exit quickly after Ctrl-C is pressed. + +- `nix copy` to a `ssh-ng` store now needs `--substitute-on-destination` (a.k.a. `-s`) + in order to substitute paths on the remote store instead of copying them. + The behavior is consistent with `nix copy` to a different kind of remote store. + Previously this behavior was controlled by the + `builders-use-substitutes` setting and `--substitute-on-destination` was ignored. diff --git a/doc/manual/src/store/store-path.md b/doc/manual/src/store/store-path.md index b5ad0c654..085aead51 100644 --- a/doc/manual/src/store/store-path.md +++ b/doc/manual/src/store/store-path.md @@ -46,7 +46,7 @@ But if the store has a file system representation, the store directory contains [file system objects]: ./file-system-object.md -This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in. +This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in. > **Note** > diff --git a/flake.nix b/flake.nix index 89b928e83..be4e68783 100644 --- a/flake.nix +++ b/flake.nix @@ -285,6 +285,13 @@ enableInternalAPIDocs = true; }; + # API docs for Nix's C bindings. + external-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix { + inherit fileset; + doBuild = false; + enableExternalAPIDocs = true; + }; + # System tests. tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // { diff --git a/local.mk b/local.mk index 3f3abb9f0..67ec35dcd 100644 --- a/local.mk +++ b/local.mk @@ -2,9 +2,14 @@ GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers. ERROR_SWITCH_ENUM = -Werror=switch-enum -$(foreach i, config.h $(wildcard src/lib*/*.hh), \ +$(foreach i, config.h $(wildcard src/lib*/*.hh) $(wildcard src/lib*/*.h $(filter-out %_internal.h, $(wildcard src/lib*c/*.h))), \ $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +ifdef HOST_UNIX + $(foreach i, $(wildcard src/lib*/unix/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644))) +endif + $(GCH): src/libutil/util.hh config.h -GCH_CXXFLAGS = -I src/libutil +GCH_CXXFLAGS = $(INCLUDE_libutil) diff --git a/package.nix b/package.nix index 7d9a39771..1e5b9e449 100644 --- a/package.nix +++ b/package.nix @@ -5,6 +5,7 @@ , autoreconfHook , aws-sdk-cpp , boehmgc +, buildPackages , nlohmann_json , bison , boost @@ -75,7 +76,10 @@ # sounds so long as evaluation just takes places within short-lived # processes. (When the process exits, the memory is reclaimed; it is # only leaked *within* the process.) -, enableGC ? true +# +# Temporarily disabled on Windows because the `GC_throw_bad_alloc` +# symbol is missing during linking. +, enableGC ? !stdenv.hostPlatform.isWindows # Whether to enable Markdown rendering in the Nix binary. , enableMarkdown ? !stdenv.hostPlatform.isWindows @@ -88,9 +92,10 @@ # - readline , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline" -# Whether to build the internal API docs, can be done separately from +# Whether to build the internal/external API docs, can be done separately from # everything else. -, enableInternalAPIDocs ? false +, enableInternalAPIDocs ? forDevShell +, enableExternalAPIDocs ? forDevShell # Whether to install unit tests. This is useful when cross compiling # since we cannot run them natively during the build, but can do so @@ -179,6 +184,9 @@ in { ./doc/manual ] ++ lib.optionals enableInternalAPIDocs [ ./doc/internal-api + ] ++ lib.optionals enableExternalAPIDocs [ + ./doc/external-api + ] ++ lib.optionals (enableInternalAPIDocs || enableExternalAPIDocs) [ # Source might not be compiled, but still must be available # for Doxygen to gather comments. ./src @@ -196,7 +204,7 @@ in { ++ lib.optional doBuild "dev" # If we are doing just build or just docs, the one thing will use # "out". We only need additional outputs if we are doing both. - ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc" + ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs || enableExternalAPIDocs)) "doc" ++ lib.optional installUnitTests "check"; nativeBuildInputs = [ @@ -218,7 +226,7 @@ in { ] ++ lib.optionals (doInstallCheck || enableManual) [ jq # Also for custom mdBook preprocessor. ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux - ++ lib.optional enableInternalAPIDocs doxygen + ++ lib.optional (enableInternalAPIDocs || enableExternalAPIDocs) doxygen ; buildInputs = lib.optionals doBuild [ @@ -282,6 +290,7 @@ in { (lib.enableFeature buildUnitTests "unit-tests") (lib.enableFeature doInstallCheck "functional-tests") (lib.enableFeature enableInternalAPIDocs "internal-api-docs") + (lib.enableFeature enableExternalAPIDocs "external-api-docs") (lib.enableFeature enableManual "doc-gen") (lib.enableFeature enableGC "gc") (lib.enableFeature enableMarkdown "markdown") @@ -306,7 +315,8 @@ in { makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1"; installTargets = lib.optional doBuild "install" - ++ lib.optional enableInternalAPIDocs "internal-api-html"; + ++ lib.optional enableInternalAPIDocs "internal-api-html" + ++ lib.optional enableExternalAPIDocs "external-api-html"; installFlags = "sysconfdir=$(out)/etc"; @@ -333,6 +343,16 @@ in { '' + lib.optionalString enableInternalAPIDocs '' mkdir -p ''${!outputDoc}/nix-support echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + '' + + lib.optionalString enableExternalAPIDocs '' + mkdir -p ''${!outputDoc}/nix-support + echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products + ''; + + # So the check output gets links for DLLs in the out output. + preFixup = lib.optionalString (stdenv.hostPlatform.isWindows && builtins.elem "check" finalAttrs.outputs) '' + ln -s "$check/lib/"*.dll "$check/bin" + ln -s "$out/bin/"*.dll "$check/bin" ''; doInstallCheck = attrs.doInstallCheck; diff --git a/scripts/nix-profile-daemon.sh.in b/scripts/nix-profile-daemon.sh.in index d256b24ed..0ec72e797 100644 --- a/scripts/nix-profile-daemon.sh.in +++ b/scripts/nix-profile-daemon.sh.in @@ -69,4 +69,4 @@ else fi export PATH="$NIX_LINK/bin:@localstatedir@/nix/profiles/default/bin:$PATH" -unset NIX_LINK +unset NIX_LINK NIX_LINK_NEW diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 118468477..18eee830b 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -202,7 +202,7 @@ static int main_build_remote(int argc, char * * argv) else drvstr = ""; - auto error = HintFmt(errorText); + auto error = HintFmt::fromFormatString(errorText); error % drvstr % neededSystem diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 369fa6004..220a90cf6 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -148,7 +148,7 @@ MixOperateOnOptions::MixOperateOnOptions() { addFlag({ .longName = "derivation", - .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.", + .description = "Operate on the [store derivation](@docroot@/glossary.md#gloss-store-derivation) rather than its outputs.", .category = installablesCategory, .handler = {&operateOn, OperateOn::Derivation}, }); diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk index abb7459a7..9aa33a9d3 100644 --- a/src/libcmd/local.mk +++ b/src/libcmd/local.mk @@ -6,7 +6,7 @@ libcmd_DIR := $(d) libcmd_SOURCES := $(wildcard $(d)/*.cc) -libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers +libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libmain) libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS) diff --git a/src/libcmd/markdown.cc b/src/libcmd/markdown.cc index a4e3c5a77..d62ff0d96 100644 --- a/src/libcmd/markdown.cc +++ b/src/libcmd/markdown.cc @@ -50,7 +50,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown) if (!rndr_res) throw Error("allocation error while rendering Markdown"); - return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI()); + return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY()); #else return std::string(markdown); #endif diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index a97d8aaf4..ffbb43a69 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -357,7 +357,7 @@ ProcessLineResult NixRepl::processLine(std::string line) if (line.empty()) return ProcessLineResult::PromptAgain; - _isInterrupted = false; + setInterrupted(false); std::string command, arg; diff --git a/src/libexpr-c/local.mk b/src/libexpr-c/local.mk new file mode 100644 index 000000000..51b02562e --- /dev/null +++ b/src/libexpr-c/local.mk @@ -0,0 +1,25 @@ +libraries += libexprc + +libexprc_NAME = libnixexprc + +libexprc_DIR := $(d) + +libexprc_SOURCES := \ + $(wildcard $(d)/*.cc) \ + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libexprc := -I $(d) +libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) $(INCLUDE_libstorec) \ + $(INCLUDE_libexpr) $(INCLUDE_libexprc) + +libexprc_LIBS = libutil libutilc libstore libstorec libexpr + +libexprc_LDFLAGS += $(THREAD_LDFLAGS) + +$(eval $(call install-file-in, $(d)/nix-expr-c.pc, $(libdir)/pkgconfig, 0644)) + +libexprc_FORCE_INSTALL := 1 + diff --git a/src/libexpr-c/nix-expr-c.pc.in b/src/libexpr-c/nix-expr-c.pc.in new file mode 100644 index 000000000..06897064d --- /dev/null +++ b/src/libexpr-c/nix-expr-c.pc.in @@ -0,0 +1,10 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Language Evaluator - C API +Version: @PACKAGE_VERSION@ +Requires: nix-store-c +Libs: -L${libdir} -lnixexprc +Cflags: -I${includedir}/nix diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc new file mode 100644 index 000000000..a5c03d5aa --- /dev/null +++ b/src/libexpr-c/nix_api_expr.cc @@ -0,0 +1,178 @@ +#include +#include +#include +#include + +#include "config.hh" +#include "eval.hh" +#include "globals.hh" +#include "util.hh" + +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +#ifdef HAVE_BOEHMGC +#include +#define GC_INCLUDE_NEW 1 +#include "gc_cpp.h" +#endif + +nix_err nix_libexpr_init(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + { + auto ret = nix_libutil_init(context); + if (ret != NIX_OK) + return ret; + } + { + auto ret = nix_libstore_init(context); + if (ret != NIX_OK) + return ret; + } + try { + nix::initGC(); + } + NIXC_CATCH_ERRS +} + +nix_err nix_expr_eval_from_string( + nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path))); + state->state.eval(parsedExpr, *(nix::Value *) value); + state->state.forceValue(*(nix::Value *) value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.callFunction(*(nix::Value *) fn, *(nix::Value *) arg, *(nix::Value *) value, nix::noPos); + state->state.forceValue(*(nix::Value *) value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValue(*(nix::Value *) value, nix::noPos); + } + NIXC_CATCH_ERRS +} + +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + state->state.forceValueDeep(*(nix::Value *) value); + } + NIXC_CATCH_ERRS +} + +EvalState * nix_state_create(nix_c_context * context, const char ** searchPath_c, Store * store) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Strings searchPath; + if (searchPath_c != nullptr) + for (size_t i = 0; searchPath_c[i] != nullptr; i++) + searchPath.push_back(searchPath_c[i]); + + return new EvalState{nix::EvalState(nix::SearchPath::parse(searchPath), store->ptr)}; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_state_free(EvalState * state) +{ + delete state; +} + +#ifdef HAVE_BOEHMGC +std::unordered_map< + const void *, + unsigned int, + std::hash, + std::equal_to, + traceable_allocator>> + nix_refcounts; + +std::mutex nix_refcount_lock; + +nix_err nix_gc_incref(nix_c_context * context, const void * p) +{ + if (context) + context->last_err_code = NIX_OK; + try { + std::scoped_lock lock(nix_refcount_lock); + auto f = nix_refcounts.find(p); + if (f != nix_refcounts.end()) { + f->second++; + } else { + nix_refcounts[p] = 1; + } + } + NIXC_CATCH_ERRS +} + +nix_err nix_gc_decref(nix_c_context * context, const void * p) +{ + + if (context) + context->last_err_code = NIX_OK; + try { + std::scoped_lock lock(nix_refcount_lock); + auto f = nix_refcounts.find(p); + if (f != nix_refcounts.end()) { + if (--f->second == 0) + nix_refcounts.erase(f); + } else + throw std::runtime_error("nix_gc_decref: object was not referenced"); + } + NIXC_CATCH_ERRS +} + +void nix_gc_now() +{ + GC_gcollect(); +} + +#else +nix_err nix_gc_incref(nix_c_context * context, const void *) +{ + if (context) + context->last_err_code = NIX_OK; + return NIX_OK; +} +nix_err nix_gc_decref(nix_c_context * context, const void *) +{ + if (context) + context->last_err_code = NIX_OK; + return NIX_OK; +} +void nix_gc_now() {} +#endif + +void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd)) +{ +#ifdef HAVE_BOEHMGC + GC_REGISTER_FINALIZER(obj, finalizer, cd, 0, 0); +#endif +} diff --git a/src/libexpr-c/nix_api_expr.h b/src/libexpr-c/nix_api_expr.h new file mode 100644 index 000000000..7504b5d7a --- /dev/null +++ b/src/libexpr-c/nix_api_expr.h @@ -0,0 +1,213 @@ +#ifndef NIX_API_EXPR_H +#define NIX_API_EXPR_H +/** @defgroup libexpr libexpr + * @brief Bindings to the Nix language evaluator + * + * Example (without error handling): + * @code{.c} + * int main() { + * nix_libexpr_init(NULL); + * + * Store* store = nix_store_open(NULL, "dummy", NULL); + * EvalState* state = nix_state_create(NULL, NULL, store); // empty nix path + * Value *value = nix_alloc_value(NULL, state); + * + * nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value); + * nix_value_force(NULL, state, value); + * printf("nix version: %s\n", nix_get_string(NULL, value)); + * + * nix_gc_decref(NULL, value); + * nix_state_free(state); + * nix_store_free(store); + * return 0; + * } + * @endcode + * @{ + */ +/** @file + * @brief Main entry for the libexpr C bindings + */ + +#include "nix_api_store.h" +#include "nix_api_util.h" + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +// Type definitions +/** + * @brief Represents a state of the Nix language evaluator. + * + * Multiple states can be created for multi-threaded + * operation. + * @struct EvalState + * @see nix_state_create + */ +typedef struct EvalState EvalState; // nix::EvalState +/** + * @brief Represents a value in the Nix language. + * + * Owned by the garbage collector. + * @struct Value + * @see value_manip + */ +typedef void Value; // nix::Value + +// Function prototypes +/** + * @brief Initialize the Nix language evaluator. + * + * This function must be called at least once, + * at some point before constructing a EvalState for the first time. + * This function can be called multiple times, and is idempotent. + * + * @param[out] context Optional, stores error information + * @return NIX_OK if the initialization was successful, an error code otherwise. + */ +nix_err nix_libexpr_init(nix_c_context * context); + +/** + * @brief Parses and evaluates a Nix expression from a string. + * + * @param[out] context Optional, stores error information + * @param[in] state The state of the evaluation. + * @param[in] expr The Nix expression to parse. + * @param[in] path The file path to associate with the expression. + * This is required for expressions that contain relative paths (such as `./.`) that are resolved relative to the given + * directory. + * @param[out] value The result of the evaluation. You must allocate this + * yourself. + * @return NIX_OK if the evaluation was successful, an error code otherwise. + */ +nix_err nix_expr_eval_from_string( + nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value); + +/** + * @brief Calls a Nix function with an argument. + * + * @param[out] context Optional, stores error information + * @param[in] state The state of the evaluation. + * @param[in] fn The Nix function to call. + * @param[in] arg The argument to pass to the function. + * @param[out] value The result of the function call. + * @return NIX_OK if the function call was successful, an error code otherwise. + */ +nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value); + +/** + * @brief Forces the evaluation of a Nix value. + * + * The Nix interpreter is lazy, and not-yet-evaluated Values can be + * of type NIX_TYPE_THUNK instead of their actual value. + * + * This function converts these Values into their final type. + * + * @note You don't need this function for basic API usage, since all functions + * that return a value call it for you. The only place you will see a + * NIX_TYPE_THUNK is in the arguments that are passed to a PrimOp function + * you supplied to nix_alloc_primop. + * + * @param[out] context Optional, stores error information + * @param[in] state The state of the evaluation. + * @param[in,out] value The Nix value to force. + * @post value is not of type NIX_TYPE_THUNK + * @return NIX_OK if the force operation was successful, an error code + * otherwise. + */ +nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value); + +/** + * @brief Forces the deep evaluation of a Nix value. + * + * Recursively calls nix_value_force + * + * @see nix_value_force + * @warning Calling this function on a recursive data structure will cause a + * stack overflow. + * @param[out] context Optional, stores error information + * @param[in] state The state of the evaluation. + * @param[in,out] value The Nix value to force. + * @return NIX_OK if the deep force operation was successful, an error code + * otherwise. + */ +nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value); + +/** + * @brief Create a new Nix language evaluator state. + * + * @param[out] context Optional, stores error information + * @param[in] searchPath Array of strings corresponding to entries in NIX_PATH. + * @param[in] store The Nix store to use. + * @return A new Nix state or NULL on failure. + */ +EvalState * nix_state_create(nix_c_context * context, const char ** searchPath, Store * store); + +/** + * @brief Frees a Nix state. + * + * Does not fail. + * + * @param[in] state The state to free. + */ +void nix_state_free(EvalState * state); + +/** @addtogroup GC + * @brief Reference counting and garbage collector operations + * + * The Nix language evaluator uses a garbage collector. To ease C interop, we implement + * a reference counting scheme, where objects will be deallocated + * when there are no references from the Nix side, and the reference count kept + * by the C API reaches `0`. + * + * Functions returning a garbage-collected object will automatically increase + * the refcount for you. You should make sure to call `nix_gc_decref` when + * you're done with a value returned by the evaluator. + * @{ + */ +/** + * @brief Increment the garbage collector reference counter for the given object. + * + * The Nix language evaluator C API keeps track of alive objects by reference counting. + * When you're done with a refcounted pointer, call nix_gc_decref(). + * + * @param[out] context Optional, stores error information + * @param[in] object The object to keep alive + */ +nix_err nix_gc_incref(nix_c_context * context, const void * object); +/** + * @brief Decrement the garbage collector reference counter for the given object + * + * @param[out] context Optional, stores error information + * @param[in] object The object to stop referencing + */ +nix_err nix_gc_decref(nix_c_context * context, const void * object); + +/** + * @brief Trigger the garbage collector manually + * + * You should not need to do this, but it can be useful for debugging. + */ +void nix_gc_now(); + +/** + * @brief Register a callback that gets called when the object is garbage + * collected. + * @note Objects can only have a single finalizer. This function overwrites existing values + * silently. + * @param[in] obj the object to watch + * @param[in] cd the data to pass to the finalizer + * @param[in] finalizer the callback function, called with obj and cd + */ +void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd)); + +/** @} */ +// cffi end +#ifdef __cplusplus +} +#endif + +/** @} */ + +#endif // NIX_API_EXPR_H diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h new file mode 100644 index 000000000..7743849fd --- /dev/null +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -0,0 +1,44 @@ +#ifndef NIX_API_EXPR_INTERNAL_H +#define NIX_API_EXPR_INTERNAL_H + +#include "eval.hh" +#include "attr-set.hh" +#include "nix_api_value.h" + +struct EvalState +{ + nix::EvalState state; +}; + +struct BindingsBuilder +{ + nix::BindingsBuilder builder; +}; + +struct ListBuilder +{ + nix::ListBuilder builder; +}; + +struct nix_string_return +{ + std::string str; +}; + +struct nix_printer +{ + std::ostream & s; +}; + +struct nix_string_context +{ + nix::NixStringContext & ctx; +}; + +struct nix_realised_string +{ + std::string str; + std::vector storePaths; +}; + +#endif // NIX_API_EXPR_INTERNAL_H diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc new file mode 100644 index 000000000..3c3dd6ca9 --- /dev/null +++ b/src/libexpr-c/nix_api_external.cc @@ -0,0 +1,198 @@ +#include "attr-set.hh" +#include "config.hh" +#include "eval.hh" +#include "globals.hh" +#include "value.hh" + +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_external.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "nix_api_value.h" +#include "value/context.hh" + +#include + +#ifdef HAVE_BOEHMGC +# include "gc/gc.h" +# define GC_INCLUDE_NEW 1 +# include "gc_cpp.h" +#endif + +void nix_set_string_return(nix_string_return * str, const char * c) +{ + str->str = c; +} + +nix_err nix_external_print(nix_c_context * context, nix_printer * printer, const char * c) +{ + if (context) + context->last_err_code = NIX_OK; + try { + printer->s << c; + } + NIXC_CATCH_ERRS +} + +nix_err nix_external_add_string_context(nix_c_context * context, nix_string_context * ctx, const char * c) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto r = nix::NixStringContextElem::parse(c); + ctx->ctx.insert(r); + } + NIXC_CATCH_ERRS +} + +class NixCExternalValue : public nix::ExternalValueBase +{ + NixCExternalValueDesc & desc; + void * v; + +public: + NixCExternalValue(NixCExternalValueDesc & desc, void * v) + : desc(desc) + , v(v){}; + void * get_ptr() + { + return v; + } + /** + * Print out the value + */ + virtual std::ostream & print(std::ostream & str) const override + { + nix_printer p{str}; + desc.print(v, &p); + return str; + } + + /** + * Return a simple string describing the type + */ + virtual std::string showType() const override + { + nix_string_return res; + desc.showType(v, &res); + return std::move(res.str); + } + + /** + * Return a string to be used in builtins.typeOf + */ + virtual std::string typeOf() const override + { + nix_string_return res; + desc.typeOf(v, &res); + return std::move(res.str); + } + + /** + * Coerce the value to a string. + */ + virtual std::string coerceToString( + nix::EvalState & state, + const nix::PosIdx & pos, + nix::NixStringContext & context, + bool copyMore, + bool copyToStore) const override + { + if (!desc.coerceToString) { + return nix::ExternalValueBase::coerceToString(state, pos, context, copyMore, copyToStore); + } + nix_string_context ctx{context}; + nix_string_return res{""}; + // todo: pos, errors + desc.coerceToString(v, &ctx, copyMore, copyToStore, &res); + if (res.str.empty()) { + return nix::ExternalValueBase::coerceToString(state, pos, context, copyMore, copyToStore); + } + return std::move(res.str); + } + + /** + * Compare to another value of the same type. + */ + virtual bool operator==(const ExternalValueBase & b) const override + { + if (!desc.equal) { + return false; + } + auto r = dynamic_cast(&b); + if (!r) + return false; + return desc.equal(v, r->v); + } + + /** + * Print the value as JSON. + */ + virtual nlohmann::json printValueAsJSON( + nix::EvalState & state, bool strict, nix::NixStringContext & context, bool copyToStore = true) const override + { + if (!desc.printValueAsJSON) { + return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore); + } + nix_string_context ctx{context}; + nix_string_return res{""}; + desc.printValueAsJSON(v, (EvalState *) &state, strict, &ctx, copyToStore, &res); + if (res.str.empty()) { + return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore); + } + return nlohmann::json::parse(res.str); + } + + /** + * Print the value as XML. + */ + virtual void printValueAsXML( + nix::EvalState & state, + bool strict, + bool location, + nix::XMLWriter & doc, + nix::NixStringContext & context, + nix::PathSet & drvsSeen, + const nix::PosIdx pos) const override + { + if (!desc.printValueAsXML) { + return nix::ExternalValueBase::printValueAsXML(state, strict, location, doc, context, drvsSeen, pos); + } + nix_string_context ctx{context}; + desc.printValueAsXML( + v, (EvalState *) &state, strict, location, &doc, &ctx, &drvsSeen, + *reinterpret_cast(&pos)); + } + + virtual ~NixCExternalValue() override{}; +}; + +ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto ret = new +#ifdef HAVE_BOEHMGC + (GC) +#endif + NixCExternalValue(*desc, v); + nix_gc_incref(nullptr, ret); + return (ExternalValue *) ret; + } + NIXC_CATCH_ERRS_NULL +} + +void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto r = dynamic_cast((nix::ExternalValueBase *) b); + if (r) + return r->get_ptr(); + return nullptr; + } + NIXC_CATCH_ERRS_NULL +} diff --git a/src/libexpr-c/nix_api_external.h b/src/libexpr-c/nix_api_external.h new file mode 100644 index 000000000..12ea00407 --- /dev/null +++ b/src/libexpr-c/nix_api_external.h @@ -0,0 +1,196 @@ +#ifndef NIX_API_EXTERNAL_H +#define NIX_API_EXTERNAL_H +/** @ingroup libexpr + * @addtogroup Externals + * @brief Deal with external values + * @{ + */ +/** @file + * @brief libexpr C bindings dealing with external values + */ + +#include "nix_api_expr.h" +#include "nix_api_util.h" +#include "nix_api_value.h" +#include "stdbool.h" +#include "stddef.h" +#include "stdint.h" + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +/** + * @brief Represents a string owned by the Nix language evaluator. + * @see nix_set_owned_string + */ +typedef struct nix_string_return nix_string_return; +/** + * @brief Wraps a stream that can output multiple string pieces. + */ +typedef struct nix_printer nix_printer; +/** + * @brief A list of string context items + */ +typedef struct nix_string_context nix_string_context; + +/** + * @brief Sets the contents of a nix_string_return + * + * Copies the passed string. + * @param[out] str the nix_string_return to write to + * @param[in] c The string to copy + */ +void nix_set_string_return(nix_string_return * str, const char * c); + +/** + * Print to the nix_printer + * + * @param[out] context Optional, stores error information + * @param printer The nix_printer to print to + * @param[in] str The string to print + * @returns NIX_OK if everything worked + */ +nix_err nix_external_print(nix_c_context * context, nix_printer * printer, const char * str); + +/** + * Add string context to the nix_string_context object + * @param[out] context Optional, stores error information + * @param[out] string_context The nix_string_context to add to + * @param[in] c The context string to add + * @returns NIX_OK if everything worked + */ +nix_err nix_external_add_string_context(nix_c_context * context, nix_string_context * string_context, const char * c); + +/** + * @brief Definition for a class of external values + * + * Create and implement one of these, then pass it to nix_create_external_value + * Make sure to keep it alive while the external value lives. + * + * Optional functions can be set to NULL + * + * @see nix_create_external_value + */ +typedef struct NixCExternalValueDesc +{ + /** + * @brief Called when printing the external value + * + * @param[in] self the void* passed to nix_create_external_value + * @param[out] printer The printer to print to, pass to nix_external_print + */ + void (*print)(void * self, nix_printer * printer); + /** + * @brief Called on :t + * @param[in] self the void* passed to nix_create_external_value + * @param[out] res the return value + */ + void (*showType)(void * self, nix_string_return * res); + /** + * @brief Called on `builtins.typeOf` + * @param self the void* passed to nix_create_external_value + * @param[out] res the return value + */ + void (*typeOf)(void * self, nix_string_return * res); + /** + * @brief Called on "${str}" and builtins.toString. + * + * The latter with coerceMore=true + * Optional, the default is to throw an error. + * @param[in] self the void* passed to nix_create_external_value + * @param[out] c writable string context for the resulting string + * @param[in] coerceMore boolean, try to coerce to strings in more cases + * instead of throwing an error + * @param[in] copyToStore boolean, whether to copy referenced paths to store + * or keep them as-is + * @param[out] res the return value. Not touching this, or setting it to the + * empty string, will make the conversion throw an error. + */ + void (*coerceToString)( + void * self, nix_string_context * c, int coerceMore, int copyToStore, nix_string_return * res); + /** + * @brief Try to compare two external values + * + * Optional, the default is always false. + * If the other object was not a Nix C external value, this comparison will + * also return false + * @param[in] self the void* passed to nix_create_external_value + * @param[in] other the void* passed to the other object's + * nix_create_external_value + * @returns true if the objects are deemed to be equal + */ + int (*equal)(void * self, void * other); + /** + * @brief Convert the external value to json + * + * Optional, the default is to throw an error + * @param[in] self the void* passed to nix_create_external_value + * @param[in] state The evaluator state + * @param[in] strict boolean Whether to force the value before printing + * @param[out] c writable string context for the resulting string + * @param[in] copyToStore whether to copy referenced paths to store or keep + * them as-is + * @param[out] res the return value. Gets parsed as JSON. Not touching this, + * or setting it to the empty string, will make the conversion throw an error. + */ + void (*printValueAsJSON)( + void * self, EvalState *, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res); + /** + * @brief Convert the external value to XML + * + * Optional, the default is to throw an error + * @todo The mechanisms for this call are incomplete. There are no C + * bindings to work with XML, pathsets and positions. + * @param[in] self the void* passed to nix_create_external_value + * @param[in] state The evaluator state + * @param[in] strict boolean Whether to force the value before printing + * @param[in] location boolean Whether to include position information in the + * xml + * @param[out] doc XML document to output to + * @param[out] c writable string context for the resulting string + * @param[in,out] drvsSeen a path set to avoid duplicating derivations + * @param[in] pos The position of the call. + */ + void (*printValueAsXML)( + void * self, + EvalState *, + int strict, + int location, + void * doc, + nix_string_context * c, + void * drvsSeen, + int pos); +} NixCExternalValueDesc; + +/** + * @brief Create an external value, that can be given to nix_init_external + * + * Owned by the GC. Use nix_gc_decref when you're done with the pointer. + * + * @param[out] context Optional, stores error information + * @param[in] desc a NixCExternalValueDesc, you should keep this alive as long + * as the ExternalValue lives + * @param[in] v the value to store + * @returns external value, owned by the garbage collector + * @see nix_init_external + */ +ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v); + +/** + * @brief Extract the pointer from a nix c external value. + * @param[out] context Optional, stores error information + * @param[in] b The external value + * @returns The pointer, or null if the external value was not from nix c. + * @see nix_get_external + */ +void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b); + +// cffi end +#ifdef __cplusplus +} +#endif +/** @} */ + +#endif // NIX_API_EXTERNAL_H diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc new file mode 100644 index 000000000..79e62a1d2 --- /dev/null +++ b/src/libexpr-c/nix_api_value.cc @@ -0,0 +1,582 @@ +#include "attr-set.hh" +#include "config.hh" +#include "eval.hh" +#include "globals.hh" +#include "path.hh" +#include "primops.hh" +#include "value.hh" + +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "nix_api_store_internal.h" +#include "nix_api_value.h" +#include "value/context.hh" + +#ifdef HAVE_BOEHMGC +# include "gc/gc.h" +# define GC_INCLUDE_NEW 1 +# include "gc_cpp.h" +#endif + +// Helper function to throw an exception if value is null +static const nix::Value & check_value_not_null(const Value * value) +{ + if (!value) { + throw std::runtime_error("Value is null"); + } + return *((const nix::Value *) value); +} + +static nix::Value & check_value_not_null(Value * value) +{ + if (!value) { + throw std::runtime_error("Value is null"); + } + return *((nix::Value *) value); +} + +/** + * Helper function to convert calls from nix into C API. + * + * Deals with errors and converts arguments from C++ into C types. + */ +static void nix_c_primop_wrapper( + PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v) +{ + nix_c_context ctx; + f(userdata, &ctx, (EvalState *) &state, (Value **) args, (Value *) &v); + /* TODO: In the future, this should throw different errors depending on the error code */ + if (ctx.last_err_code != NIX_OK) + state.error("Error from builtin function: %s", *ctx.last_err).atPos(pos).debugThrow(); +} + +PrimOp * nix_alloc_primop( + nix_c_context * context, + PrimOpFun fun, + int arity, + const char * name, + const char ** args, + const char * doc, + void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + using namespace std::placeholders; + auto p = new +#ifdef HAVE_BOEHMGC + (GC) +#endif + nix::PrimOp{ + .name = name, + .args = {}, + .arity = (size_t) arity, + .doc = doc, + .fun = std::bind(nix_c_primop_wrapper, fun, user_data, _1, _2, _3, _4)}; + if (args) + for (size_t i = 0; args[i]; i++) + p->args.emplace_back(*args); + nix_gc_incref(nullptr, p); + return (PrimOp *) p; + } + NIXC_CATCH_ERRS_NULL +} + +nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::RegisterPrimOp r(std::move(*((nix::PrimOp *) primOp))); + } + NIXC_CATCH_ERRS +} + +Value * nix_alloc_value(nix_c_context * context, EvalState * state) +{ + if (context) + context->last_err_code = NIX_OK; + try { + Value * res = state->state.allocValue(); + nix_gc_incref(nullptr, res); + return res; + } + NIXC_CATCH_ERRS_NULL +} + +ValueType nix_get_type(nix_c_context * context, const Value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + using namespace nix; + switch (v.type()) { + case nThunk: + return NIX_TYPE_THUNK; + case nInt: + return NIX_TYPE_INT; + case nFloat: + return NIX_TYPE_FLOAT; + case nBool: + return NIX_TYPE_BOOL; + case nString: + return NIX_TYPE_STRING; + case nPath: + return NIX_TYPE_PATH; + case nNull: + return NIX_TYPE_NULL; + case nAttrs: + return NIX_TYPE_ATTRS; + case nList: + return NIX_TYPE_LIST; + case nFunction: + return NIX_TYPE_FUNCTION; + case nExternal: + return NIX_TYPE_EXTERNAL; + } + return NIX_TYPE_NULL; + } + NIXC_CATCH_ERRS_RES(NIX_TYPE_NULL); +} + +const char * nix_get_typename(nix_c_context * context, const Value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + auto s = nix::showType(v); + return strdup(s.c_str()); + } + NIXC_CATCH_ERRS_NULL +} + +bool nix_get_bool(nix_c_context * context, const Value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + assert(v.type() == nix::nBool); + return v.boolean(); + } + NIXC_CATCH_ERRS_RES(false); +} + +nix_err nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + assert(v.type() == nix::nString); + call_nix_get_string_callback(v.c_str(), callback, user_data); + } + NIXC_CATCH_ERRS +} + +const char * nix_get_path_string(nix_c_context * context, const Value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + assert(v.type() == nix::nPath); + // NOTE (from @yorickvP) + // v._path.path should work but may not be how Eelco intended it. + // Long-term this function should be rewritten to copy some data into a + // user-allocated string. + // We could use v.path().to_string().c_str(), but I'm concerned this + // crashes. Looks like .path() allocates a CanonPath with a copy of the + // string, then it gets the underlying data from that. + return v.payload.path.path; + } + NIXC_CATCH_ERRS_NULL +} + +unsigned int nix_get_list_size(nix_c_context * context, const Value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + assert(v.type() == nix::nList); + return v.listSize(); + } + NIXC_CATCH_ERRS_RES(0); +} + +unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + assert(v.type() == nix::nAttrs); + return v.attrs()->size(); + } + NIXC_CATCH_ERRS_RES(0); +} + +double nix_get_float(nix_c_context * context, const Value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + assert(v.type() == nix::nFloat); + return v.fpoint(); + } + NIXC_CATCH_ERRS_RES(0.0); +} + +int64_t nix_get_int(nix_c_context * context, const Value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + assert(v.type() == nix::nInt); + return v.integer(); + } + NIXC_CATCH_ERRS_RES(0); +} + +ExternalValue * nix_get_external(nix_c_context * context, Value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + assert(v.type() == nix::nExternal); + return (ExternalValue *) v.external(); + } + NIXC_CATCH_ERRS_NULL; +} + +Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + assert(v.type() == nix::nList); + auto * p = v.listElems()[ix]; + nix_gc_incref(nullptr, p); + if (p != nullptr) + state->state.forceValue(*p, nix::noPos); + return (Value *) p; + } + NIXC_CATCH_ERRS_NULL +} + +Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + assert(v.type() == nix::nAttrs); + nix::Symbol s = state->state.symbols.create(name); + auto attr = v.attrs()->get(s); + if (attr) { + nix_gc_incref(nullptr, attr->value); + state->state.forceValue(*attr->value, nix::noPos); + return attr->value; + } + nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute"); + return nullptr; + } + NIXC_CATCH_ERRS_NULL +} + +bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + assert(v.type() == nix::nAttrs); + nix::Symbol s = state->state.symbols.create(name); + auto attr = v.attrs()->get(s); + if (attr) + return true; + return false; + } + NIXC_CATCH_ERRS_RES(false); +} + +Value * +nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + const nix::Attr & a = (*v.attrs())[i]; + *name = ((const std::string &) (state->state.symbols[a.name])).c_str(); + nix_gc_incref(nullptr, a.value); + state->state.forceValue(*a.value, nix::noPos); + return a.value; + } + NIXC_CATCH_ERRS_NULL +} + +const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + const nix::Attr & a = (*v.attrs())[i]; + return ((const std::string &) (state->state.symbols[a.name])).c_str(); + } + NIXC_CATCH_ERRS_NULL +} + +nix_err nix_init_bool(nix_c_context * context, Value * value, bool b) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + v.mkBool(b); + } + NIXC_CATCH_ERRS +} + +// todo string context +nix_err nix_init_string(nix_c_context * context, Value * value, const char * str) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + v.mkString(std::string_view(str)); + } + NIXC_CATCH_ERRS +} + +nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + v.mkPath(s->state.rootPath(nix::CanonPath(str))); + } + NIXC_CATCH_ERRS +} + +nix_err nix_init_float(nix_c_context * context, Value * value, double d) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + v.mkFloat(d); + } + NIXC_CATCH_ERRS +} + +nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + v.mkInt(i); + } + NIXC_CATCH_ERRS +} + +nix_err nix_init_null(nix_c_context * context, Value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + v.mkNull(); + } + NIXC_CATCH_ERRS +} + +nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + auto r = (nix::ExternalValueBase *) val; + v.mkExternal(r); + } + NIXC_CATCH_ERRS +} + +ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto builder = state->state.buildList(capacity); + return new +#if HAVE_BOEHMGC + (NoGC) +#endif + ListBuilder{std::move(builder)}; + } + NIXC_CATCH_ERRS_NULL +} + +nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, Value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & e = check_value_not_null(value); + list_builder->builder[index] = &e; + } + NIXC_CATCH_ERRS +} + +void nix_list_builder_free(ListBuilder * list_builder) +{ +#if HAVE_BOEHMGC + GC_FREE(list_builder); +#else + delete list_builder; +#endif +} + +nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + v.mkList(list_builder->builder); + } + NIXC_CATCH_ERRS +} + +nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * p) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + v.mkPrimOp((nix::PrimOp *) p); + } + NIXC_CATCH_ERRS +} + +nix_err nix_copy_value(nix_c_context * context, Value * value, Value * source) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + auto & s = check_value_not_null(source); + v = s; + } + NIXC_CATCH_ERRS +} + +nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + v.mkAttrs(b->builder); + } + NIXC_CATCH_ERRS +} + +BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto bb = state->state.buildBindings(capacity); + return new +#if HAVE_BOEHMGC + (NoGC) +#endif + BindingsBuilder{std::move(bb)}; + } + NIXC_CATCH_ERRS_NULL +} + +nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * bb, const char * name, Value * value) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + nix::Symbol s = bb->builder.state.symbols.create(name); + bb->builder.insert(s, &v); + } + NIXC_CATCH_ERRS +} + +void nix_bindings_builder_free(BindingsBuilder * bb) +{ +#if HAVE_BOEHMGC + GC_FREE((nix::BindingsBuilder *) bb); +#else + delete (nix::BindingsBuilder *) bb; +#endif +} + +nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_not_null(value); + nix::NixStringContext stringContext; + auto rawStr = state->state.coerceToString(nix::noPos, v, stringContext, "while realising a string").toOwned(); + nix::StorePathSet storePaths; + auto rewrites = state->state.realiseContext(stringContext, &storePaths); + + auto s = nix::rewriteStrings(rawStr, rewrites); + + // Convert to the C API StorePath type and convert to vector for index-based access + std::vector vec; + for (auto & sp : storePaths) { + vec.push_back(StorePath{sp}); + } + + return new nix_realised_string{.str = s, .storePaths = vec}; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_realised_string_free(nix_realised_string * s) +{ + delete s; +} + +size_t nix_realised_string_get_buffer_size(nix_realised_string * s) +{ + return s->str.size(); +} + +const char * nix_realised_string_get_buffer_start(nix_realised_string * s) +{ + return s->str.data(); +} + +size_t nix_realised_string_get_store_path_count(nix_realised_string * s) +{ + return s->storePaths.size(); +} + +const StorePath * nix_realised_string_get_store_path(nix_realised_string * s, size_t i) +{ + return &s->storePaths[i]; +} diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h new file mode 100644 index 000000000..e6744e610 --- /dev/null +++ b/src/libexpr-c/nix_api_value.h @@ -0,0 +1,494 @@ +#ifndef NIX_API_VALUE_H +#define NIX_API_VALUE_H + +/** @addtogroup libexpr + * @{ + */ +/** @file + * @brief libexpr C bindings dealing with values + */ + +#include "nix_api_util.h" +#include "nix_api_store.h" +#include "stdbool.h" +#include "stddef.h" +#include "stdint.h" + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +// Type definitions +typedef enum { + NIX_TYPE_THUNK, + NIX_TYPE_INT, + NIX_TYPE_FLOAT, + NIX_TYPE_BOOL, + NIX_TYPE_STRING, + NIX_TYPE_PATH, + NIX_TYPE_NULL, + NIX_TYPE_ATTRS, + NIX_TYPE_LIST, + NIX_TYPE_FUNCTION, + NIX_TYPE_EXTERNAL +} ValueType; + +// forward declarations +typedef void Value; +typedef struct EvalState EvalState; +// type defs +/** @brief Stores an under-construction set of bindings + * @ingroup value_manip + * + * Do not reuse. + * @see nix_make_bindings_builder, nix_bindings_builder_free, nix_make_attrs + * @see nix_bindings_builder_insert + */ +typedef struct BindingsBuilder BindingsBuilder; + +/** @brief Stores an under-construction list + * @ingroup value_manip + * + * Do not reuse. + * @see nix_make_list_builder, nix_list_builder_free, nix_make_list + * @see nix_list_builder_insert + */ +typedef struct ListBuilder ListBuilder; + +/** @brief PrimOp function + * @ingroup primops + * + * Owned by the GC + * @see nix_alloc_primop, nix_init_primop + */ +typedef struct PrimOp PrimOp; +/** @brief External Value + * @ingroup Externals + * + * Owned by the GC + */ +typedef struct ExternalValue ExternalValue; + +/** @brief String without placeholders, and realised store paths + */ +typedef struct nix_realised_string nix_realised_string; + +/** @defgroup primops + * @brief Create your own primops + * @{ + */ +/** @brief Function pointer for primops + * When you want to return an error, call nix_set_err_msg(context, NIX_ERR_UNKNOWN, "your error message here"). + * + * @param[in] user_data Arbitrary data that was initially supplied to nix_alloc_primop + * @param[out] context Stores error information. + * @param[in] state Evaluator state + * @param[in] args list of arguments. Note that these can be thunks and should be forced using nix_value_force before + * use. + * @param[out] ret return value + * @see nix_alloc_primop, nix_init_primop + */ +typedef void (*PrimOpFun)(void * user_data, nix_c_context * context, EvalState * state, Value ** args, Value * ret); + +/** @brief Allocate a PrimOp + * + * Owned by the garbage collector. + * Use nix_gc_decref() when you're done with the returned PrimOp. + * + * @param[out] context Optional, stores error information + * @param[in] fun callback + * @param[in] arity expected number of function arguments + * @param[in] name function name + * @param[in] args array of argument names, NULL-terminated + * @param[in] doc optional, documentation for this primop + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called + * @return primop, or null in case of errors + * @see nix_init_primop + */ +PrimOp * nix_alloc_primop( + nix_c_context * context, + PrimOpFun fun, + int arity, + const char * name, + const char ** args, + const char * doc, + void * user_data); + +/** @brief add a primop to the `builtins` attribute set + * + * Only applies to States created after this call. + * + * Moves your PrimOp content into the global evaluator + * registry, meaning your input PrimOp pointer is no longer usable. + * You are free to remove your references to it, + * after which it will be garbage collected. + * + * @param[out] context Optional, stores error information + * @return primop, or null in case of errors + * + */ +nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp); +/** @} */ + +// Function prototypes + +/** @brief Allocate a Nix value + * + * Owned by the GC. Use nix_gc_decref() when you're done with the pointer + * @param[out] context Optional, stores error information + * @param[in] state nix evaluator state + * @return value, or null in case of errors + * + */ +Value * nix_alloc_value(nix_c_context * context, EvalState * state); + +/** @addtogroup value_manip Manipulating values + * @brief Functions to inspect and change Nix language values, represented by Value. + * @{ + */ +/** @name Getters + */ +/**@{*/ +/** @brief Get value type + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return type of nix value + */ +ValueType nix_get_type(nix_c_context * context, const Value * value); + +/** @brief Get type name of value as defined in the evaluator + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return type name, owned string + * @todo way to free the result + */ +const char * nix_get_typename(nix_c_context * context, const Value * value); + +/** @brief Get boolean value + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return true or false, error info via context + */ +bool nix_get_bool(nix_c_context * context, const Value * value); + +/** @brief Get the raw string + * + * This may contain placeholders. + * + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @param[in] callback Called with the string value. + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. + * @return string + * @return error code, NIX_OK on success. + */ +nix_err +nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data); + +/** @brief Get path as string + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return string + * @return NULL in case of error. + */ +const char * nix_get_path_string(nix_c_context * context, const Value * value); + +/** @brief Get the length of a list + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return length of list, error info via context + */ +unsigned int nix_get_list_size(nix_c_context * context, const Value * value); + +/** @brief Get the element count of an attrset + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return attrset element count, error info via context + */ +unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value); + +/** @brief Get float value in 64 bits + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return float contents, error info via context + */ +double nix_get_float(nix_c_context * context, const Value * value); + +/** @brief Get int value + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return int contents, error info via context + */ +int64_t nix_get_int(nix_c_context * context, const Value * value); + +/** @brief Get external reference + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @return reference to external, NULL in case of error + */ +ExternalValue * nix_get_external(nix_c_context * context, Value *); + +/** @brief Get the ix'th element of a list + * + * Owned by the GC. Use nix_gc_decref when you're done with the pointer + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @param[in] state nix evaluator state + * @param[in] ix list element to get + * @return value, NULL in case of errors + */ +Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix); + +/** @brief Get an attr by name + * + * Owned by the GC. Use nix_gc_decref when you're done with the pointer + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @param[in] state nix evaluator state + * @param[in] name attribute name + * @return value, NULL in case of errors + */ +Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name); + +/** @brief Check if an attribute name exists on a value + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @param[in] state nix evaluator state + * @param[in] name attribute name + * @return value, error info via context + */ +bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name); + +/** @brief Get an attribute by index in the sorted bindings + * + * Also gives you the name. + * + * Owned by the GC. Use nix_gc_decref when you're done with the pointer + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @param[in] state nix evaluator state + * @param[in] i attribute index + * @param[out] name will store a pointer to the attribute name + * @return value, NULL in case of errors + */ +Value * +nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name); + +/** @brief Get an attribute name by index in the sorted bindings + * + * Useful when you want the name but want to avoid evaluation. + * + * Owned by the nix EvalState + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect + * @param[in] state nix evaluator state + * @param[in] i attribute index + * @return name, NULL in case of errors + */ +const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i); + +/**@}*/ +/** @name Initializers + * + * Values are typically "returned" by initializing already allocated memory that serves as the return value. + * For this reason, the construction of values is not tied their allocation. + * Nix is a language with immutable values. Respect this property by only initializing Values once; and only initialize + * Values that are meant to be initialized by you. Failing to adhere to these rules may lead to undefined behavior. + */ +/**@{*/ +/** @brief Set boolean value + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] b the boolean value + * @return error code, NIX_OK on success. + */ +nix_err nix_init_bool(nix_c_context * context, Value * value, bool b); + +/** @brief Set a string + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] str the string, copied + * @return error code, NIX_OK on success. + */ +nix_err nix_init_string(nix_c_context * context, Value * value, const char * str); + +/** @brief Set a path + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] str the path string, copied + * @return error code, NIX_OK on success. + */ +nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str); + +/** @brief Set a float + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] d the float, 64-bits + * @return error code, NIX_OK on success. + */ +nix_err nix_init_float(nix_c_context * context, Value * value, double d); + +/** @brief Set an int + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] i the int + * @return error code, NIX_OK on success. + */ + +nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i); +/** @brief Set null + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @return error code, NIX_OK on success. + */ + +nix_err nix_init_null(nix_c_context * context, Value * value); +/** @brief Set an external value + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] val the external value to set. Will be GC-referenced by the value. + * @return error code, NIX_OK on success. + */ +nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val); + +/** @brief Create a list from a list builder + * @param[out] context Optional, stores error information + * @param[in] list_builder list builder to use. Make sure to unref this afterwards. + * @param[out] value Nix value to modify + * @return error code, NIX_OK on success. + */ +nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value * value); + +/** @brief Create a list builder + * @param[out] context Optional, stores error information + * @param[in] state nix evaluator state + * @param[in] capacity how many bindings you'll add. Don't exceed. + * @return owned reference to a list builder. Make sure to unref when you're done. + */ +ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity); + +/** @brief Insert bindings into a builder + * @param[out] context Optional, stores error information + * @param[in] list_builder ListBuilder to insert into + * @param[in] index index to manipulate + * @param[in] value value to insert + * @return error code, NIX_OK on success. + */ +nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, Value * value); + +/** @brief Free a list builder + * + * Does not fail. + * @param[in] builder the builder to free + */ +void nix_list_builder_free(ListBuilder * list_builder); + +/** @brief Create an attribute set from a bindings builder + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] b bindings builder to use. Make sure to unref this afterwards. + * @return error code, NIX_OK on success. + */ +nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b); + +/** @brief Set primop + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] op primop, will be gc-referenced by the value + * @see nix_alloc_primop + * @return error code, NIX_OK on success. + */ +nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * op); +/** @brief Copy from another value + * @param[out] context Optional, stores error information + * @param[out] value Nix value to modify + * @param[in] source value to copy from + * @return error code, NIX_OK on success. + */ +nix_err nix_copy_value(nix_c_context * context, Value * value, Value * source); +/**@}*/ + +/** @brief Create a bindings builder +* @param[out] context Optional, stores error information +* @param[in] state nix evaluator state +* @param[in] capacity how many bindings you'll add. Don't exceed. +* @return owned reference to a bindings builder. Make sure to unref when you're +done. +*/ +BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity); + +/** @brief Insert bindings into a builder + * @param[out] context Optional, stores error information + * @param[in] builder BindingsBuilder to insert into + * @param[in] name attribute name, copied into the symbol store + * @param[in] value value to give the binding + * @return error code, NIX_OK on success. + */ +nix_err +nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, const char * name, Value * value); + +/** @brief Free a bindings builder + * + * Does not fail. + * @param[in] builder the builder to free + */ +void nix_bindings_builder_free(BindingsBuilder * builder); +/**@}*/ + +/** @brief Realise a string context. + * + * This will + * - realise the store paths referenced by the string's context, and + * - perform the replacement of placeholders. + * - create temporary garbage collection roots for the store paths, for + * the lifetime of the current process. + * - log to stderr + * + * @param[out] context Optional, stores error information + * @param[in] value Nix value, which must be a string + * @param[in] state Nix evaluator state + * @param[in] isIFD If true, disallow derivation outputs if setting `allow-import-from-derivation` is false. + You should set this to true when this call is part of a primop. + You should set this to false when building for your application's purpose. + * @return NULL if failed, are a new nix_realised_string, which must be freed with nix_realised_string_free + */ +nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD); + +/** @brief Start of the string + * @param[in] realised_string + * @return pointer to the start of the string. It may not be null-terminated. + */ +const char * nix_realised_string_get_buffer_start(nix_realised_string * realised_string); + +/** @brief Length of the string + * @param[in] realised_string + * @return length of the string in bytes + */ +size_t nix_realised_string_get_buffer_size(nix_realised_string * realised_string); + +/** @brief Number of realised store paths + * @param[in] realised_string + * @return number of realised store paths that were referenced by the string via its context + */ +size_t nix_realised_string_get_store_path_count(nix_realised_string * realised_string); + +/** @brief Get a store path. The store paths are stored in an arbitrary order. + * @param[in] realised_string + * @param[in] index index of the store path, must be less than the count + * @return store path + */ +const StorePath * nix_realised_string_get_store_path(nix_realised_string * realised_string, size_t index); + +/** @brief Free a realised string + * @param[in] realised_string + */ +void nix_realised_string_free(nix_realised_string * realised_string); + +// cffi end +#ifdef __cplusplus +} +#endif + +/** @} */ +#endif // NIX_API_VALUE_H diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 7147e7997..a21e1d75e 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -892,7 +892,6 @@ void Value::mkStringMove(const char * s, const NixStringContext & context) mkString(s, encodeContext(context)); } - void Value::mkPath(const SourcePath & path) { mkPath(&*path.accessor, makeImmutableString(path.path.abs())); @@ -1660,7 +1659,8 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & try { fn->fun(*this, vCur.determinePos(noPos), args, vCur); } catch (Error & e) { - addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); + if (fn->addTrace) + addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); throw; } @@ -1708,7 +1708,8 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & // so the debugger allows to inspect the wrong parameters passed to the builtin. fn->fun(*this, vCur.determinePos(noPos), vArgs, vCur); } catch (Error & e) { - addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); + if (fn->addTrace) + addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); throw; } diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0e569de26..df388d93e 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -17,6 +17,7 @@ #include #include #include +#include namespace nix { @@ -69,10 +70,17 @@ struct PrimOp */ const char * doc = nullptr; + /** + * Add a trace item, `while calling the '' builtin` + * + * This is used to remove the redundant item for `builtins.addErrorContext`. + */ + bool addTrace = true; + /** * Implementation of the primop. */ - PrimOpFun fun; + std::function::type> fun; /** * Optional experimental for this to be gated on. @@ -722,10 +730,12 @@ public: bool fullGC(); /** - * Realise the given context, and return a mapping from the placeholders - * used to construct the associated value to their final store path + * Realise the given context + * @param[in] context the context to realise + * @param[out] maybePaths if not nullptr, all built or referenced store paths will be added to this set + * @return a mapping from the placeholders used to construct the associated value to their final store path. */ - [[nodiscard]] StringMap realiseContext(const NixStringContext & context); + [[nodiscard]] StringMap realiseContext(const NixStringContext & context, StorePathSet * maybePaths = nullptr, bool isIFD = true); /* Call the binary path filter predicate used builtins.path etc. */ bool callPathFilter( diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 1f0326eb5..3af9ef14e 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -10,6 +10,7 @@ #include "finally.hh" #include "fetch-settings.hh" #include "value-to-json.hh" +#include "local-fs-store.hh" namespace nix { @@ -755,7 +756,17 @@ void callFlake(EvalState & state, auto lockedNode = node.dynamic_pointer_cast(); - auto [storePath, subdir] = state.store->toStorePath(sourcePath.path.abs()); + // FIXME: This is a hack to support chroot stores. Remove this + // once we can pass a sourcePath rather than a storePath to + // call-flake.nix. + auto path = sourcePath.path.abs(); + if (auto store = state.store.dynamic_pointer_cast()) { + auto realStoreDir = store->getRealStoreDir(); + if (isInDir(path, realStoreDir)) + path = store->storeDir + path.substr(realStoreDir.size()); + } + + auto [storePath, subdir] = state.store->toStorePath(path); emitTreeAttrs( state, @@ -857,10 +868,13 @@ static RegisterPrimOp r3({ Parse a flake reference, and return its exploded form. For example: + ```nix builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib" ``` + evaluates to: + ```nix { dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } ``` @@ -909,12 +923,15 @@ static RegisterPrimOp r4({ Convert a flake reference from attribute set format to URL format. For example: + ```nix builtins.flakeRefToString { dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; } ``` + evaluates to + ```nix "github:NixOS/nixpkgs/23.05?dir=lib" ``` diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 0c3e36750..ecadc5e5d 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -11,8 +11,11 @@ libexpr_SOURCES := \ $(wildcard $(d)/flake/*.cc) \ $(d)/lexer-tab.cc \ $(d)/parser-tab.cc +# Not just for this library itself, but also for downstream libraries using this library -libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr +INCLUDE_libexpr := -I $(d) + +libexpr_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libmain) $(INCLUDE_libexpr) libexpr_LIBS = libutil libstore libfetchers diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 630ee8071..7e7952735 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -39,7 +39,7 @@ namespace nix { * Miscellaneous *************************************************************/ -StringMap EvalState::realiseContext(const NixStringContext & context) +StringMap EvalState::realiseContext(const NixStringContext & context, StorePathSet * maybePathsOut, bool isIFD) { std::vector drvs; StringMap res; @@ -59,21 +59,23 @@ StringMap EvalState::realiseContext(const NixStringContext & context) }, [&](const NixStringContextElem::Opaque & o) { auto ctxS = store->printStorePath(o.path); - res.insert_or_assign(ctxS, ctxS); ensureValid(o.path); + if (maybePathsOut) + maybePathsOut->emplace(o.path); }, [&](const NixStringContextElem::DrvDeep & d) { /* Treat same as Opaque */ auto ctxS = store->printStorePath(d.drvPath); - res.insert_or_assign(ctxS, ctxS); ensureValid(d.drvPath); + if (maybePathsOut) + maybePathsOut->emplace(d.drvPath); }, }, c.raw); } if (drvs.empty()) return {}; - if (!evalSettings.enableImportFromDerivation) + if (isIFD && !evalSettings.enableImportFromDerivation) error( "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled", drvs.begin()->to_string(*store) @@ -90,6 +92,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context) auto outputs = resolveDerivedPath(*buildStore, drv, &*store); for (auto & [outputName, outputPath] : outputs) { outputsToCopyAndAllow.insert(outputPath); + if (maybePathsOut) + maybePathsOut->emplace(outputPath); /* Get all the output paths corresponding to the placeholders we had */ if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) { @@ -106,10 +110,13 @@ StringMap EvalState::realiseContext(const NixStringContext & context) } if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow); - for (auto & outputPath : outputsToCopyAndAllow) { - /* Add the output of this derivations to the allowed - paths. */ - allowPath(outputPath); + + if (isIFD) { + for (auto & outputPath : outputsToCopyAndAllow) { + /* Add the output of this derivations to the allowed + paths. */ + allowPath(outputPath); + } } return res; @@ -826,7 +833,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * auto message = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.addErrorContext", false, false).toOwned(); - e.addTrace(nullptr, HintFmt(message)); + e.addTrace(nullptr, HintFmt(message), TracePrint::Always); throw; } } @@ -834,6 +841,8 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * static RegisterPrimOp primop_addErrorContext(PrimOp { .name = "__addErrorContext", .arity = 2, + // The normal trace item is redundant + .addTrace = false, .fun = prim_addErrorContext, }); @@ -1132,7 +1141,7 @@ static void derivationStrictInternal( bool contentAddressed = false; bool isImpure = false; std::optional outputHash; - std::string outputHashAlgo; + std::optional outputHashAlgo; std::optional ingestionMethod; StringSet outputs; @@ -1144,18 +1153,20 @@ static void derivationStrictInternal( vomit("processing attribute '%1%'", key); auto handleHashMode = [&](const std::string_view s) { - if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; - else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; - else if (s == "git") { - experimentalFeatureSettings.require(Xp::GitHashing); - ingestionMethod = FileIngestionMethod::Git; - } else if (s == "text") { - experimentalFeatureSettings.require(Xp::DynamicDerivations); - ingestionMethod = TextIngestionMethod {}; - } else + if (s == "recursive") { + // back compat, new name is "nar" + ingestionMethod = FileIngestionMethod::Recursive; + } else try { + ingestionMethod = ContentAddressMethod::parse(s); + } catch (UsageError &) { state.error( "invalid value '%s' for 'outputHashMode' attribute", s ).atPos(v).debugThrow(); + } + if (ingestionMethod == TextIngestionMethod {}) + experimentalFeatureSettings.require(Xp::DynamicDerivations); + if (ingestionMethod == FileIngestionMethod::Git) + experimentalFeatureSettings.require(Xp::GitHashing); }; auto handleOutputs = [&](const Strings & ss) { @@ -1231,7 +1242,7 @@ static void derivationStrictInternal( else if (i->name == state.sOutputHash) outputHash = state.forceStringNoCtx(*i->value, pos, context_below); else if (i->name == state.sOutputHashAlgo) - outputHashAlgo = state.forceStringNoCtx(*i->value, pos, context_below); + outputHashAlgo = parseHashAlgoOpt(state.forceStringNoCtx(*i->value, pos, context_below)); else if (i->name == state.sOutputHashMode) handleHashMode(state.forceStringNoCtx(*i->value, pos, context_below)); else if (i->name == state.sOutputs) { @@ -1249,7 +1260,7 @@ static void derivationStrictInternal( if (i->name == state.sBuilder) drv.builder = std::move(s); else if (i->name == state.sSystem) drv.platform = std::move(s); else if (i->name == state.sOutputHash) outputHash = std::move(s); - else if (i->name == state.sOutputHashAlgo) outputHashAlgo = std::move(s); + else if (i->name == state.sOutputHashAlgo) outputHashAlgo = parseHashAlgoOpt(s); else if (i->name == state.sOutputHashMode) handleHashMode(s); else if (i->name == state.sOutputs) handleOutputs(tokenizeString(s)); @@ -1332,7 +1343,7 @@ static void derivationStrictInternal( "multiple outputs are not supported in fixed-output derivations" ).atPos(v).debugThrow(); - auto h = newHashAllowEmpty(*outputHash, parseHashAlgoOpt(outputHashAlgo)); + auto h = newHashAllowEmpty(*outputHash, outputHashAlgo); auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); @@ -1352,7 +1363,7 @@ static void derivationStrictInternal( state.error("derivation cannot be both content-addressed and impure") .atPos(v).debugThrow(); - auto ha = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256); + auto ha = outputHashAlgo.value_or(HashAlgorithm::SHA256); auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive); for (auto & i : outputs) { @@ -1575,23 +1586,50 @@ static RegisterPrimOp primop_pathExists({ .fun = prim_pathExists, }); +// Ideally, all trailing slashes should have been removed, but it's been like this for +// almost a decade as of writing. Changing it will affect reproducibility. +static std::string_view legacyBaseNameOf(std::string_view path) +{ + if (path.empty()) + return ""; + + auto last = path.size() - 1; + if (path[last] == '/' && last > 0) + last -= 1; + + auto pos = path.rfind('/', last); + if (pos == path.npos) + pos = 0; + else + pos += 1; + + return path.substr(pos, last - pos + 1); +} + /* Return the base name of the given string, i.e., everything following the last slash. */ static void prim_baseNameOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { NixStringContext context; - v.mkString(baseNameOf(*state.coerceToString(pos, *args[0], context, + v.mkString(legacyBaseNameOf(*state.coerceToString(pos, *args[0], context, "while evaluating the first argument passed to builtins.baseNameOf", false, false)), context); } static RegisterPrimOp primop_baseNameOf({ .name = "baseNameOf", - .args = {"s"}, + .args = {"x"}, .doc = R"( - Return the *base name* of the string *s*, that is, everything - following the final slash in the string. This is similar to the GNU - `basename` command. + Return the *base name* of either a [path value](@docroot@/language/values.md#type-path) *x* or a string *x*, depending on which type is passed, and according to the following rules. + + For a path value, the *base name* is considered to be the part of the path after the last directory separator, including any file extensions. + This is the simple case, as path values don't have trailing slashes. + + When the argument is a string, a more involved logic applies. If the string ends with a `/`, only this one final slash is removed. + + After this, the *base name* is returned as previously described, assuming `/` as the directory separator. (Note that evaluation must be platform independent.) + + This is somewhat similar to the [GNU `basename`](https://www.gnu.org/software/coreutils/manual/html_node/basename-invocation.html) command, but GNU `basename` will strip any number of trailing slashes. )", .fun = prim_baseNameOf, }); @@ -1893,11 +1931,13 @@ static RegisterPrimOp primop_outputOf({ *`derivation reference`* must be a string that may contain a regular store path to a derivation, or may be a placeholder reference. If the derivation is produced by a derivation, you must explicitly select `drv.outPath`. This primop can be chained arbitrarily deeply. For instance, + ```nix builtins.outputOf (builtins.outputOf myDrv "out") "out" ``` + will return a placeholder for the output of the output of `myDrv`. This primop corresponds to the `^` sigil for derivable paths, e.g. as part of installable syntax on the command line. @@ -3379,10 +3419,11 @@ static void prim_sort(EvalState & state, const PosIdx pos, Value * * args, Value auto comparator = [&](Value * a, Value * b) { /* Optimization: if the comparator is lessThan, bypass callFunction. */ - /* TODO: (layus) this is absurd. An optimisation like this - should be outside the lambda creation */ - if (args[0]->isPrimOp() && args[0]->primOp()->fun == prim_lessThan) - return CompareValues(state, noPos, "while evaluating the ordering function passed to builtins.sort")(a, b); + if (args[0]->isPrimOp()) { + auto ptr = args[0]->primOp()->fun.target(); + if (ptr && *ptr == prim_lessThan) + return CompareValues(state, noPos, "while evaluating the ordering function passed to builtins.sort")(a, b); + } Value * vs[] = {a, b}; Value vBool; @@ -3838,7 +3879,7 @@ static RegisterPrimOp primop_stringLength({ .name = "__stringLength", .args = {"e"}, .doc = R"( - Return the length of the string *e*. If *e* is not a string, + Return the number of bytes of the string *e*. If *e* is not a string, evaluation is aborted. )", .fun = prim_stringLength, diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 409b36d81..e27f30512 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -473,7 +473,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v auto storePath = unpack ? fetchToStore(*state.store, fetchers::downloadTarball(*url).accessor, FetchMode::Copy, name) - : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath; + : fetchers::downloadFile(state.store, *url, name).storePath; if (expectedHash) { auto hash = unpack @@ -650,12 +650,14 @@ static RegisterPrimOp primop_fetchGit({ The public keys against which `rev` is verified if `verifyCommit` is enabled. Must be given as a list of attribute sets with the following form: + ```nix { key = ""; type = ""; # optional, default: "ssh-ed25519" } ``` + Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches). diff --git a/src/libexpr/search-path.cc b/src/libexpr/search-path.cc index a25767496..e2c3e050a 100644 --- a/src/libexpr/search-path.cc +++ b/src/libexpr/search-path.cc @@ -44,7 +44,7 @@ SearchPath::Elem SearchPath::Elem::parse(std::string_view rawElem) } -SearchPath parseSearchPath(const Strings & rawElems) +SearchPath SearchPath::parse(const Strings & rawElems) { SearchPath res; for (auto & rawElem : rawElems) diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 21eb71d5c..7ed3fa5a9 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -321,6 +321,7 @@ public: } void mkPath(const SourcePath & path); + void mkPath(std::string_view path); inline void mkPath(InputAccessor * accessor, const char * path) { diff --git a/src/libfetchers/fetch-settings.hh b/src/libfetchers/fetch-settings.hh index f095963a8..d085f0d82 100644 --- a/src/libfetchers/fetch-settings.hh +++ b/src/libfetchers/fetch-settings.hh @@ -78,7 +78,6 @@ struct FetchSettings : public Config )", {}, true, Xp::Flakes}; - Setting useRegistries{this, true, "use-registries", "Whether to use flake registries to resolve flake references.", {}, true, Xp::Flakes}; @@ -94,6 +93,22 @@ struct FetchSettings : public Config empty, the summary is generated based on the action performed. )", {}, true, Xp::Flakes}; + + Setting trustTarballsFromGitForges{ + this, true, "trust-tarballs-from-git-forges", + R"( + If enabled (the default), Nix will consider tarballs from + GitHub and similar Git forges to be locked if a Git revision + is specified, + e.g. `github:NixOS/patchelf/7c2f768bf9601268a4e71c2ebe91e2011918a70f`. + This requires Nix to trust that the provider will return the + correct contents for the specified Git revision. + + If disabled, such tarballs are only considered locked if a + `narHash` attribute is specified, + e.g. `github:NixOS/patchelf/7c2f768bf9601268a4e71c2ebe91e2011918a70f?narHash=sha256-PPXqKY2hJng4DBVE0I4xshv/vGLUskL7jl53roB8UdU%3D`. + )"}; + }; // FIXME: don't use a global variable. diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 483796f0b..a06d931db 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -3,6 +3,7 @@ #include "input-accessor.hh" #include "source-path.hh" #include "fetch-to-store.hh" +#include "json-utils.hh" #include @@ -412,3 +413,20 @@ std::string publicKeys_to_string(const std::vector& publicKeys) } } + +namespace nlohmann { + +using namespace nix; + +fetchers::PublicKey adl_serializer::from_json(const json & json) { + auto type = optionalValueAt(json, "type").value_or("ssh-ed25519"); + auto key = valueAt(json, "key"); + return fetchers::PublicKey { getString(type), getString(key) }; +} + +void adl_serializer::to_json(json & json, fetchers::PublicKey p) { + json["type"] = p.type; + json["key"] = p.key; +} + +} diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index cd11f9eae..bb21c68cc 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -4,6 +4,7 @@ #include "types.hh" #include "hash.hh" #include "canon-path.hh" +#include "json-impls.hh" #include "attrs.hh" #include "url.hh" @@ -230,8 +231,9 @@ struct PublicKey std::string type = "ssh-ed25519"; std::string key; }; -NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(PublicKey, type, key) std::string publicKeys_to_string(const std::vector&); } + +JSON_IMPL(fetchers::PublicKey) diff --git a/src/libfetchers/filtering-input-accessor.cc b/src/libfetchers/filtering-input-accessor.cc index 32343abc4..e0cbfd905 100644 --- a/src/libfetchers/filtering-input-accessor.cc +++ b/src/libfetchers/filtering-input-accessor.cc @@ -38,7 +38,7 @@ std::string FilteringInputAccessor::readLink(const CanonPath & path) std::string FilteringInputAccessor::showPath(const CanonPath & path) { - return next->showPath(prefix / path); + return displayPrefix + next->showPath(prefix / path) + displaySuffix; } void FilteringInputAccessor::checkAccess(const CanonPath & path) diff --git a/src/libfetchers/filtering-input-accessor.hh b/src/libfetchers/filtering-input-accessor.hh index 8111a72c5..133a6cee3 100644 --- a/src/libfetchers/filtering-input-accessor.hh +++ b/src/libfetchers/filtering-input-accessor.hh @@ -27,7 +27,9 @@ struct FilteringInputAccessor : InputAccessor : next(src.accessor) , prefix(src.path) , makeNotAllowedError(std::move(makeNotAllowedError)) - { } + { + displayPrefix.clear(); + } std::string readFile(const CanonPath & path) override; diff --git a/src/libfetchers/fs-input-accessor.cc b/src/libfetchers/fs-input-accessor.cc index ee24c621a..d85363808 100644 --- a/src/libfetchers/fs-input-accessor.cc +++ b/src/libfetchers/fs-input-accessor.cc @@ -24,7 +24,10 @@ ref makeStorePathAccessor( const StorePath & storePath) { // FIXME: should use `store->getFSAccessor()` - return makeFSInputAccessor(std::filesystem::path { store->toRealPath(storePath) }); + auto root = std::filesystem::path { store->toRealPath(storePath) }; + auto accessor = makeFSInputAccessor(root); + accessor->setPathDisplay(root); + return accessor; } SourcePath getUnfilteredRootPath(CanonPath path) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index b723554cc..5ecd825b7 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -198,6 +198,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this return git_repository_is_shallow(*this); } + void setRemote(const std::string & name, const std::string & url) override + { + if (git_remote_set_url(*this, name.c_str(), url.c_str())) + throw Error("setting remote '%s' URL to '%s': %s", name, url, git_error_last()->message); + } + Hash resolveRef(std::string ref) override { Object object; @@ -302,9 +308,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this std::vector> getSubmodules(const Hash & rev, bool exportIgnore) override; - std::string resolveSubmoduleUrl( - const std::string & url, - const std::string & base) override + std::string resolveSubmoduleUrl(const std::string & url) override { git_buf buf = GIT_BUF_INIT; if (git_submodule_resolve_url(&buf, *this, url.c_str())) @@ -312,10 +316,6 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this Finally cleanup = [&]() { git_buf_dispose(&buf); }; std::string res(buf.ptr); - - if (!hasPrefix(res, "/") && res.find("://") == res.npos) - res = parseURL(base + "/" + res).canonicalise().to_string(); - return res; } @@ -348,7 +348,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this { auto act = (Activity *) payload; act->result(resFetchStatus, trim(std::string_view(str, len))); - return _isInterrupted ? -1 : 0; + return getInterrupted() ? -1 : 0; } static int transferProgressCallback(const git_indexer_progress * stats, void * payload) @@ -361,7 +361,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this stats->indexed_deltas, stats->total_deltas, stats->received_bytes / (1024.0 * 1024.0))); - return _isInterrupted ? -1 : 0; + return getInterrupted() ? -1 : 0; } void fetch( diff --git a/src/libfetchers/git-utils.hh b/src/libfetchers/git-utils.hh index fbb2d947b..600a42da0 100644 --- a/src/libfetchers/git-utils.hh +++ b/src/libfetchers/git-utils.hh @@ -32,6 +32,8 @@ struct GitRepo /* Return the commit hash to which a ref points. */ virtual Hash resolveRef(std::string ref) = 0; + virtual void setRemote(const std::string & name, const std::string & url) = 0; + /** * Info about a submodule. */ @@ -69,9 +71,7 @@ struct GitRepo */ virtual std::vector> getSubmodules(const Hash & rev, bool exportIgnore) = 0; - virtual std::string resolveSubmoduleUrl( - const std::string & url, - const std::string & base) = 0; + virtual std::string resolveSubmoduleUrl(const std::string & url) = 0; virtual bool hasObject(const Hash & oid) = 0; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 8100afe4d..985f2e479 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -294,7 +294,9 @@ struct GitArchiveInputScheme : InputScheme Git revision alone, we also require a NAR hash for locking. FIXME: in the future, we may want to require a Git tree hash instead of a NAR hash. */ - return input.getRev().has_value() && input.getNarHash().has_value(); + return input.getRev().has_value() + && (fetchSettings.trustTarballsFromGitForges || + input.getNarHash().has_value()); } std::optional experimentalFeature() const override @@ -355,7 +357,7 @@ struct GitHubInputScheme : GitArchiveInputScheme auto json = nlohmann::json::parse( readFile( store->toRealPath( - downloadFile(store, url, "source", false, headers).storePath))); + downloadFile(store, url, "source", headers).storePath))); return RefInfo { .rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1), @@ -429,7 +431,7 @@ struct GitLabInputScheme : GitArchiveInputScheme auto json = nlohmann::json::parse( readFile( store->toRealPath( - downloadFile(store, url, "source", false, headers).storePath))); + downloadFile(store, url, "source", headers).storePath))); return RefInfo { .rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1) @@ -493,7 +495,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::string refUri; if (ref == "HEAD") { auto file = store->toRealPath( - downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers).storePath); + downloadFile(store, fmt("%s/HEAD", base_url), "source", headers).storePath); std::ifstream is(file); std::string line; getline(is, line); @@ -509,7 +511,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme std::regex refRegex(refUri); auto file = store->toRealPath( - downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath); + downloadFile(store, fmt("%s/info/refs", base_url), "source", headers).storePath); std::ifstream is(file); std::string line; diff --git a/src/libfetchers/local.mk b/src/libfetchers/local.mk index e54db4937..0fef1466b 100644 --- a/src/libfetchers/local.mk +++ b/src/libfetchers/local.mk @@ -5,8 +5,18 @@ libfetchers_NAME = libnixfetchers libfetchers_DIR := $(d) libfetchers_SOURCES := $(wildcard $(d)/*.cc) +ifdef HOST_UNIX + libfetchers_SOURCES += $(wildcard $(d)/unix/*.cc) +endif -libfetchers_CXXFLAGS += -I src/libutil -I src/libstore +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libfetchers := -I $(d) +ifdef HOST_UNIX + INCLUDE_libfetchers += -I $(d)/unix +endif + +libfetchers_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) libfetchers_LDFLAGS += $(THREAD_LDFLAGS) $(LIBGIT2_LIBS) -larchive diff --git a/src/libfetchers/mounted-input-accessor.cc b/src/libfetchers/mounted-input-accessor.cc index 6f397eb17..b1eeaa97d 100644 --- a/src/libfetchers/mounted-input-accessor.cc +++ b/src/libfetchers/mounted-input-accessor.cc @@ -9,6 +9,8 @@ struct MountedInputAccessor : InputAccessor MountedInputAccessor(std::map> _mounts) : mounts(std::move(_mounts)) { + displayPrefix.clear(); + // Currently we require a root filesystem. This could be relaxed. assert(mounts.contains(CanonPath::root)); @@ -48,7 +50,7 @@ struct MountedInputAccessor : InputAccessor std::string showPath(const CanonPath & path) override { auto [accessor, subpath] = resolve(path); - return accessor->showPath(subpath); + return displayPrefix + accessor->showPath(subpath) + displaySuffix; } std::pair, CanonPath> resolve(CanonPath path) diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 9c7bc0cfe..e00b9de46 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -158,7 +158,7 @@ static std::shared_ptr getGlobalRegistry(ref store) } if (!hasPrefix(path, "/")) { - auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; + auto storePath = downloadFile(store, path, "flake-registry.json").storePath; if (auto store2 = store.dynamic_pointer_cast()) store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json"); path = store->toRealPath(storePath); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index f08509cb7..a1f934c35 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -19,7 +19,6 @@ DownloadFileResult downloadFile( ref store, const std::string & url, const std::string & name, - bool locked, const Headers & headers) { // FIXME: check store @@ -101,7 +100,7 @@ DownloadFileResult downloadFile( inAttrs, infoAttrs, *storePath, - locked); + false); } return { @@ -306,7 +305,7 @@ struct FileInputScheme : CurlInputScheme the Nix store directly, since there is little deduplication benefit in using the Git cache for single big files like tarballs. */ - auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false); + auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName()); auto narHash = store->queryPathInfo(file.storePath)->narHash; input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); diff --git a/src/libfetchers/tarball.hh b/src/libfetchers/tarball.hh index 77ad3bf09..bcb5dcc5e 100644 --- a/src/libfetchers/tarball.hh +++ b/src/libfetchers/tarball.hh @@ -25,7 +25,6 @@ DownloadFileResult downloadFile( ref store, const std::string & url, const std::string & name, - bool locked, const Headers & headers = {}); struct DownloadTarballResult diff --git a/src/libfetchers/git.cc b/src/libfetchers/unix/git.cc similarity index 97% rename from src/libfetchers/git.cc rename to src/libfetchers/unix/git.cc index 34cfd3f5b..18915c0a7 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/unix/git.cc @@ -147,9 +147,12 @@ std::vector getPublicKeys(const Attrs & attrs) { std::vector publicKeys; if (attrs.contains("publicKeys")) { - nlohmann::json publicKeysJson = nlohmann::json::parse(getStrAttr(attrs, "publicKeys")); - ensureType(publicKeysJson, nlohmann::json::value_t::array); - publicKeys = publicKeysJson.get>(); + auto pubKeysJson = nlohmann::json::parse(getStrAttr(attrs, "publicKeys")); + auto & pubKeys = getArray(pubKeysJson); + + for (auto & key : pubKeys) { + publicKeys.push_back(key); + } } if (attrs.contains("publicKey")) publicKeys.push_back(PublicKey{maybeGetStrAttr(attrs, "keytype").value_or("ssh-ed25519"),getStrAttr(attrs, "publicKey")}); @@ -523,6 +526,9 @@ struct GitInputScheme : InputScheme auto repo = GitRepo::openRepo(cacheDir, true, true); + // We need to set the origin so resolving submodule URLs works + repo->setRemote("origin", repoInfo.url); + Path localRefFile = ref.compare(0, 5, "refs/") == 0 ? cacheDir + "/" + ref @@ -626,7 +632,7 @@ struct GitInputScheme : InputScheme std::map> mounts; for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev, exportIgnore)) { - auto resolved = repo->resolveSubmoduleUrl(submodule.url, repoInfo.url); + auto resolved = repo->resolveSubmoduleUrl(submodule.url); debug("Git submodule %s: %s %s %s -> %s", submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved); fetchers::Attrs attrs; @@ -639,6 +645,7 @@ struct GitInputScheme : InputScheme auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store); + submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); mounts.insert_or_assign(submodule.path, submoduleAccessor); } @@ -675,6 +682,8 @@ struct GitInputScheme : InputScheme exportIgnore, makeNotAllowedError(repoInfo.url)); + accessor->setPathDisplay(repoInfo.url); + /* If the repo has submodules, return a mounted input accessor consisting of the accessor for the top-level repo and the accessors for the submodule workdirs. */ @@ -691,6 +700,7 @@ struct GitInputScheme : InputScheme auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs)); auto [submoduleAccessor, submoduleInput2] = submoduleInput.getAccessor(store); + submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»"); /* If the submodule is dirty, mark this repo dirty as well. */ diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/unix/mercurial.cc similarity index 98% rename from src/libfetchers/mercurial.cc rename to src/libfetchers/unix/mercurial.cc index a2702338f..4e0b26274 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/unix/mercurial.cc @@ -352,7 +352,11 @@ struct MercurialInputScheme : InputScheme auto storePath = fetchToStore(store, input); - return {makeStorePathAccessor(store, storePath), input}; + auto accessor = makeStorePathAccessor(store, storePath); + + accessor->setPathDisplay("«" + input.to_string() + "»"); + + return {accessor, input}; } bool isLocked(const Input & input) const override diff --git a/src/libmain/local.mk b/src/libmain/local.mk index 5c7061863..d41c49dd7 100644 --- a/src/libmain/local.mk +++ b/src/libmain/local.mk @@ -5,8 +5,13 @@ libmain_NAME = libnixmain libmain_DIR := $(d) libmain_SOURCES := $(wildcard $(d)/*.cc) +ifdef HOST_UNIX + libmain_SOURCES += $(wildcard $(d)/unix/*.cc) +endif -libmain_CXXFLAGS += -I src/libutil -I src/libstore +INCLUDE_libmain := -I $(d) + +libmain_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libmain) libmain_LDFLAGS += $(OPENSSL_LIBS) diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 3aa012ee1..ce45eae2b 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -123,14 +123,18 @@ public: } void pause() override { - state_.lock()->paused = true; - writeToStderr("\r\e[K"); + auto state (state_.lock()); + state->paused = true; + if (state->active) + writeToStderr("\r\e[K"); } void resume() override { - state_.lock()->paused = false; - writeToStderr("\r\e[K"); - state_.lock()->haveUpdate = true; + auto state (state_.lock()); + state->paused = false; + if (state->active) + writeToStderr("\r\e[K"); + state->haveUpdate = true; updateCV.notify_one(); } @@ -162,9 +166,7 @@ public: writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n"); draw(state); } else { - auto s2 = s + ANSI_NORMAL "\n"; - if (!isTTY) s2 = filterANSIEscapes(s2, true); - writeToStderr(s2); + writeToStderr(filterANSIEscapes(s, !isTTY) + "\n"); } } @@ -519,7 +521,7 @@ public: std::optional ask(std::string_view msg) override { auto state(state_.lock()); - if (!state->active || !isatty(STDIN_FILENO)) return {}; + if (!state->active) return {}; std::cerr << fmt("\r\e[K%s ", msg); auto s = trim(readLine(STDIN_FILENO)); if (s.size() != 1) return {}; @@ -535,7 +537,7 @@ public: Logger * makeProgressBar() { - return new ProgressBar(shouldANSI()); + return new ProgressBar(isTTY()); } void startProgressBar() diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 7bced0aa4..4c9051d3b 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -121,7 +121,7 @@ void initNix() initLibStore(); - startSignalHandlerThread(); + unix::startSignalHandlerThread(); /* Reset SIGCHLD to its default. */ struct sigaction act; @@ -308,7 +308,7 @@ void printVersion(const std::string & programName) void showManPage(const std::string & name) { restoreProcessContext(); - setenv("MANPATH", settings.nixManDir.c_str(), 1); + setEnv("MANPATH", settings.nixManDir.c_str()); execlp("man", "man", name.c_str(), nullptr); throw SysError("command 'man %1%' failed", name.c_str()); } @@ -369,7 +369,7 @@ RunPager::RunPager() if (dup2(toPager.readSide.get(), STDIN_FILENO) == -1) throw SysError("dupping stdin"); if (!getenv("LESS")) - setenv("LESS", "FRSXMK", 1); + setEnv("LESS", "FRSXMK"); restoreProcessContext(); if (pager) execl("/bin/sh", "sh", "-c", pager, nullptr); diff --git a/src/libstore-c/local.mk b/src/libstore-c/local.mk new file mode 100644 index 000000000..5e3eff06a --- /dev/null +++ b/src/libstore-c/local.mk @@ -0,0 +1,21 @@ +libraries += libstorec + +libstorec_NAME = libnixstorec + +libstorec_DIR := $(d) + +libstorec_SOURCES := $(wildcard $(d)/*.cc) + +libstorec_LIBS = libutil libstore libutilc + +libstorec_LDFLAGS += $(THREAD_LDFLAGS) + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libstorec := -I $(d) +libstorec_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \ + $(INCLUDE_libstore) $(INCLUDE_libstorec) + +$(eval $(call install-file-in, $(d)/nix-store-c.pc, $(libdir)/pkgconfig, 0644)) + +libstorec_FORCE_INSTALL := 1 diff --git a/src/libstore-c/nix-store-c.pc.in b/src/libstore-c/nix-store-c.pc.in new file mode 100644 index 000000000..de3c7b4c6 --- /dev/null +++ b/src/libstore-c/nix-store-c.pc.in @@ -0,0 +1,9 @@ +prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: Nix +Description: Nix Store - C API +Version: @PACKAGE_VERSION@ +Libs: -L${libdir} -lnixstorec -lnixutilc +Cflags: -I${includedir}/nix diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc new file mode 100644 index 000000000..6ce4d01bb --- /dev/null +++ b/src/libstore-c/nix_api_store.cc @@ -0,0 +1,146 @@ +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" + +#include "path.hh" +#include "store-api.hh" +#include "build-result.hh" + +#include "globals.hh" + +nix_err nix_libstore_init(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::initLibStore(); + } + NIXC_CATCH_ERRS +} + +nix_err nix_init_plugins(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::initPlugins(); + } + NIXC_CATCH_ERRS +} + +Store * nix_store_open(nix_c_context * context, const char * uri, const char *** params) +{ + if (context) + context->last_err_code = NIX_OK; + try { + std::string uri_str = uri ? uri : ""; + + if (uri_str.empty()) + return new Store{nix::openStore()}; + + if (!params) + return new Store{nix::openStore(uri_str)}; + + nix::Store::Params params_map; + for (size_t i = 0; params[i] != nullptr; i++) { + params_map[params[i][0]] = params[i][1]; + } + return new Store{nix::openStore(uri_str, params_map)}; + } + NIXC_CATCH_ERRS_NULL +} + +void nix_store_free(Store * store) +{ + delete store; +} + +nix_err nix_store_get_uri(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto res = store->ptr->getUri(); + return call_nix_get_string_callback(res, callback, user_data); + } + NIXC_CATCH_ERRS +} + +nix_err +nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto res = store->ptr->getVersion(); + return call_nix_get_string_callback(res.value_or(""), callback, user_data); + } + NIXC_CATCH_ERRS +} + +bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * path) +{ + if (context) + context->last_err_code = NIX_OK; + try { + return store->ptr->isValidPath(path->path); + } + NIXC_CATCH_ERRS_RES(false); +} + +StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::StorePath s = store->ptr->parseStorePath(path); + return new StorePath{std::move(s)}; + } + NIXC_CATCH_ERRS_NULL +} + +nix_err nix_store_realise( + nix_c_context * context, + Store * store, + StorePath * path, + void * userdata, + void (*callback)(void * userdata, const char *, const char *)) +{ + if (context) + context->last_err_code = NIX_OK; + try { + + const std::vector paths{nix::DerivedPath::Built{ + .drvPath = nix::makeConstantStorePathRef(path->path), .outputs = nix::OutputsSpec::All{}}}; + + const auto nixStore = store->ptr; + auto results = nixStore->buildPathsWithResults(paths, nix::bmNormal, nixStore); + + if (callback) { + for (const auto & result : results) { + for (const auto & [outputName, realisation] : result.builtOutputs) { + auto op = store->ptr->printStorePath(realisation.outPath); + callback(userdata, outputName.c_str(), op.c_str()); + } + } + } + } + NIXC_CATCH_ERRS +} + +void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data) +{ + std::string_view name = store_path->path.name(); + callback(name.data(), name.size(), user_data); +} + +void nix_store_path_free(StorePath * sp) +{ + delete sp; +} + +StorePath * nix_store_path_clone(const StorePath * p) +{ + return new StorePath{p->path}; +} diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h new file mode 100644 index 000000000..c83aca3f7 --- /dev/null +++ b/src/libstore-c/nix_api_store.h @@ -0,0 +1,169 @@ +#ifndef NIX_API_STORE_H +#define NIX_API_STORE_H +/** + * @defgroup libstore libstore + * @brief C bindings for nix libstore + * + * libstore is used for talking to a Nix store + * @{ + */ +/** @file + * @brief Main entry for the libstore C bindings + */ + +#include "nix_api_util.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +/** @brief Reference to a Nix store */ +typedef struct Store Store; +/** @brief Nix store path */ +typedef struct StorePath StorePath; + +/** + * @brief Initializes the Nix store library + * + * This function should be called before creating a Store + * This function can be called multiple times. + * + * @param[out] context Optional, stores error information + * @return NIX_OK if the initialization was successful, an error code otherwise. + */ +nix_err nix_libstore_init(nix_c_context * context); + +/** + * @brief Loads the plugins specified in Nix's plugin-files setting. + * + * Call this once, after calling your desired init functions and setting + * relevant settings. + * + * @param[out] context Optional, stores error information + * @return NIX_OK if the initialization was successful, an error code otherwise. + */ +nix_err nix_init_plugins(nix_c_context * context); + +/** + * @brief Open a nix store + * Store instances may share state and resources behind the scenes. + * @param[out] context Optional, stores error information + * @param[in] uri URI of the nix store, copied + * @param[in] params optional, array of key-value pairs, {{"endpoint", + * "https://s3.local"}} + * @return a Store pointer, NULL in case of errors + * @see nix_store_free + */ +Store * nix_store_open(nix_c_context *, const char * uri, const char *** params); + +/** + * @brief Deallocate a nix store and free any resources if not also held by other Store instances. + * + * Does not fail. + * + * @param[in] store the store to free + */ +void nix_store_free(Store * store); + +/** + * @brief get the URI of a nix store + * @param[out] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] callback Called with the URI. + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. + * @see nix_get_string_callback + * @return error code, NIX_OK on success. + */ +nix_err nix_store_get_uri(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data); + +// returns: owned StorePath* +/** + * @brief Parse a Nix store path into a StorePath + * + * @note Don't forget to free this path using nix_store_path_free()! + * @param[out] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] path Path string to parse, copied + * @return owned store path, NULL on error + */ +StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path); + +/** + * @brief Get the path name (e.g. "name" in /nix/store/...-name) + * + * @param[in] store_path the path to get the name from + * @param[in] callback called with the name + * @param[in] user_data arbitrary data, passed to the callback when it's called. + */ +void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data); + +/** + * @brief Copy a StorePath + * + * @param[in] p the path to copy + * @return a new StorePath + */ +StorePath * nix_store_path_clone(const StorePath * p); + +/** @brief Deallocate a StorePath + * + * Does not fail. + * @param[in] p the path to free + */ +void nix_store_path_free(StorePath * p); + +/** + * @brief Check if a StorePath is valid (i.e. that corresponding store object and its closure of references exists in + * the store) + * @param[out] context Optional, stores error information + * @param[in] store Nix Store reference + * @param[in] path Path to check + * @return true or false, error info in context + */ +bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * path); +// nix_err nix_store_ensure(Store*, const char*); +// nix_err nix_store_build_paths(Store*); +/** + * @brief Realise a Nix store path + * + * Blocking, calls callback once for each realised output. + * + * @note When working with expressions, consider using e.g. nix_string_realise to get the output. `.drvPath` may not be + * accurate or available in the future. See https://github.com/NixOS/nix/issues/6507 + * + * @param[out] context Optional, stores error information + * @param[in] store Nix Store reference + * @param[in] path Path to build + * @param[in] userdata data to pass to every callback invocation + * @param[in] callback called for every realised output + */ +nix_err nix_store_realise( + nix_c_context * context, + Store * store, + StorePath * path, + void * userdata, + void (*callback)(void * userdata, const char * outname, const char * out)); + +/** + * @brief get the version of a nix store. + * If the store doesn't have a version (like the dummy store), returns an empty string. + * @param[out] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] callback Called with the version. + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. + * @see nix_get_string_callback + * @return error code, NIX_OK on success. + */ +nix_err +nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data); + +// cffi end +#ifdef __cplusplus +} +#endif +/** + * @} + */ +#endif // NIX_API_STORE_H diff --git a/src/libstore-c/nix_api_store_internal.h b/src/libstore-c/nix_api_store_internal.h new file mode 100644 index 000000000..13db0c07c --- /dev/null +++ b/src/libstore-c/nix_api_store_internal.h @@ -0,0 +1,15 @@ +#ifndef NIX_API_STORE_INTERNAL_H +#define NIX_API_STORE_INTERNAL_H +#include "store-api.hh" + +struct Store +{ + nix::ref ptr; +}; + +struct StorePath +{ + nix::StorePath path; +}; + +#endif diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index bea2bb370..97b6ec052 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -124,14 +124,6 @@ void BinaryCacheStore::writeNarInfo(ref narInfo) diskCache->upsertNarInfo(getUri(), std::string(narInfo->path.hashPart()), std::shared_ptr(narInfo)); } -AutoCloseFD openFile(const Path & path) -{ - auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) - throw SysError("opening file '%1%'", path); - return fd; -} - ref BinaryCacheStore::addToStoreCommon( Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs, std::function mkInfo) diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 29bf2852f..4d4342996 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -1138,7 +1138,7 @@ void DerivationGoal::resolvedFinished() HookReply DerivationGoal::tryBuildHook() { - if (!worker.tryBuildHook || !useDerivation) return rpDecline; + if (settings.buildHook.get().empty() || !worker.tryBuildHook || !useDerivation) return rpDecline; if (!worker.hook) worker.hook = std::make_unique(); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 612434e4d..99a91a7f1 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -14,10 +14,8 @@ #include "topo-sort.hh" #include "callback.hh" #include "json-utils.hh" -#include "cgroup.hh" #include "personality.hh" #include "current-process.hh" -#include "namespaces.hh" #include "child.hh" #include "unix-domain-socket.hh" #include "posix-fs-canonicalise.hh" @@ -40,18 +38,20 @@ /* Includes required for chroot support. */ #if __linux__ -#include -#include -#include -#include -#include -#include -#include -#include -#if HAVE_SECCOMP -#include -#endif -#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) +# include +# include +# include +# include +# include +# include +# include +# include +# include "namespaces.hh" +# if HAVE_SECCOMP +# include +# endif +# define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) +# include "cgroup.hh" #endif #if __APPLE__ @@ -395,21 +395,33 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() #if __linux__ static void doBind(const Path & source, const Path & target, bool optional = false) { debug("bind mounting '%1%' to '%2%'", source, target); - struct stat st; - if (stat(source.c_str(), &st) == -1) { - if (optional && errno == ENOENT) + + auto bindMount = [&]() { + if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("bind mount from '%1%' to '%2%' failed", source, target); + }; + + auto maybeSt = maybeLstat(source); + if (!maybeSt) { + if (optional) return; else throw SysError("getting attributes of path '%1%'", source); } - if (S_ISDIR(st.st_mode)) + auto st = *maybeSt; + + if (S_ISDIR(st.st_mode)) { createDirs(target); - else { + bindMount(); + } else if (S_ISLNK(st.st_mode)) { + // Symlinks can (apparently) not be bind-mounted, so just copy it + createDirs(dirOf(target)); + copyFile(source, target, /* andDelete */ false); + } else { createDirs(dirOf(target)); writeFile(target, ""); + bindMount(); } - if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("bind mount from '%1%' to '%2%' failed", source, target); }; #endif @@ -488,7 +500,7 @@ void LocalDerivationGoal::startBuilder() /* Create a temporary directory where the build will take place. */ - tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700); + tmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700); chownToBuilder(tmpDir); @@ -1811,11 +1823,18 @@ void LocalDerivationGoal::runChild() if (pathExists(path)) ss.push_back(path); - if (settings.caFile != "") - pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true); + if (settings.caFile != "" && pathExists(settings.caFile)) { + Path caFile = settings.caFile; + pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", canonPath(caFile, true), true); + } } - for (auto & i : ss) pathsInChroot.emplace(i, i); + for (auto & i : ss) { + // For backwards-compatibiliy, resolve all the symlinks in the + // chroot paths + auto canonicalPath = canonPath(i, true); + pathsInChroot.emplace(i, canonicalPath); + } /* Bind-mount all the directories from the "host" filesystem that we want in the chroot @@ -2053,13 +2072,13 @@ void LocalDerivationGoal::runChild() i.first, i.second.source); std::string path = i.first; - struct stat st; - if (lstat(path.c_str(), &st)) { - if (i.second.optional && errno == ENOENT) + auto optSt = maybeLstat(path.c_str()); + if (!optSt) { + if (i.second.optional) continue; - throw SysError("getting attributes of path '%s", path); + throw SysError("getting attributes of required path '%s", path); } - if (S_ISDIR(st.st_mode)) + if (S_ISDIR(optSt->st_mode)) sandboxProfile += fmt("\t(subpath \"%s\")\n", path); else sandboxProfile += fmt("\t(literal \"%s\")\n", path); @@ -2089,7 +2108,7 @@ void LocalDerivationGoal::runChild() bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking"); /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms - to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */ + to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */ Path globalTmpDir = canonPath(defaultTempDir(), true); /* They don't like trailing slashes on subpath directives */ @@ -2271,14 +2290,12 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs() continue; } - struct stat st; - if (lstat(actualPath.c_str(), &st) == -1) { - if (errno == ENOENT) - throw BuildError( - "builder for '%s' failed to produce output path for output '%s' at '%s'", - worker.store.printStorePath(drvPath), outputName, actualPath); - throw SysError("getting attributes of path '%s'", actualPath); - } + auto optSt = maybeLstat(actualPath.c_str()); + if (!optSt) + throw BuildError( + "builder for '%s' failed to produce output path for output '%s' at '%s'", + worker.store.printStorePath(drvPath), outputName, actualPath); + struct stat & st = *optSt; #ifndef __CYGWIN__ /* Check that the output is not group or world writable, as @@ -2950,16 +2967,25 @@ bool LocalDerivationGoal::isReadDesc(int fd) StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName) { + // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path + // See doc/manual/src/protocols/store-path.md for details + // TODO: We may want to separate the responsibilities of constructing the path fingerprint and of actually doing the hashing + auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName); return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName), + pathType, + // pass an all-zeroes hash Hash(HashAlgorithm::SHA256), outputPathName(drv->name, outputName)); } StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path) { + // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path + // See doc/manual/src/protocols/store-path.md for details + auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()); return worker.store.makeStorePath( - "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()), + pathType, + // pass an all-zeroes hash Hash(HashAlgorithm::SHA256), path.name()); } diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc index 1ed7b39cc..31a6b32f1 100644 --- a/src/libstore/builtins/buildenv.cc +++ b/src/libstore/builtins/buildenv.cc @@ -64,9 +64,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, continue; else if (S_ISDIR(srcSt.st_mode)) { - struct stat dstSt; - auto res = lstat(dstFile.c_str(), &dstSt); - if (res == 0) { + auto dstStOpt = maybeLstat(dstFile.c_str()); + if (dstStOpt) { + auto & dstSt = *dstStOpt; if (S_ISDIR(dstSt.st_mode)) { createLinks(state, srcFile, dstFile, priority); continue; @@ -82,14 +82,13 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, createLinks(state, srcFile, dstFile, priority); continue; } - } else if (errno != ENOENT) - throw SysError("getting status of '%1%'", dstFile); + } } else { - struct stat dstSt; - auto res = lstat(dstFile.c_str(), &dstSt); - if (res == 0) { + auto dstStOpt = maybeLstat(dstFile.c_str()); + if (dstStOpt) { + auto & dstSt = *dstStOpt; if (S_ISLNK(dstSt.st_mode)) { auto prevPriority = state.priorities[dstFile]; if (prevPriority == priority) @@ -104,8 +103,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir, throw SysError("unlinking '%1%'", dstFile); } else if (S_ISDIR(dstSt.st_mode)) throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile); - } else if (errno != ENOENT) - throw SysError("getting status of '%1%'", dstFile); + } } createSymlink(srcFile, dstFile); diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 2c808015d..def2c80b2 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -1,5 +1,6 @@ #include "daemon.hh" #include "monitor-fd.hh" +#include "signals.hh" #include "worker-protocol.hh" #include "worker-protocol-impl.hh" #include "build-result.hh" @@ -1038,7 +1039,7 @@ void processConnection( unsigned int opCount = 0; Finally finally([&]() { - _isInterrupted = false; + setInterrupted(false); printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount); }); diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index df14e979f..fcf813a37 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -1239,16 +1239,14 @@ DerivationOutput DerivationOutput::fromJSON( const ExperimentalFeatureSettings & xpSettings) { std::set keys; - ensureType(_json, nlohmann::detail::value_t::object); - auto json = (std::map) _json; + auto & json = getObject(_json); for (const auto & [key, _] : json) keys.insert(key); auto methodAlgo = [&]() -> std::pair { - std::string hashAlgoStr = json["hashAlgo"]; - // remaining to parse, will be mutated by parsers - std::string_view s = hashAlgoStr; + auto & str = getString(valueAt(json, "hashAlgo")); + std::string_view s = str; ContentAddressMethod method = ContentAddressMethod::parsePrefix(s); if (method == TextIngestionMethod {}) xpSettings.require(Xp::DynamicDerivations); @@ -1258,7 +1256,7 @@ DerivationOutput DerivationOutput::fromJSON( if (keys == (std::set { "path" })) { return DerivationOutput::InputAddressed { - .path = store.parseStorePath((std::string) json["path"]), + .path = store.parseStorePath(getString(valueAt(json, "path"))), }; } @@ -1267,10 +1265,10 @@ DerivationOutput DerivationOutput::fromJSON( auto dof = DerivationOutput::CAFixed { .ca = ContentAddress { .method = std::move(method), - .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashAlgo), + .hash = Hash::parseNonSRIUnprefixed(getString(valueAt(json, "hash")), hashAlgo), }, }; - if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"])) + if (dof.path(store, drvName, outputName) != store.parseStorePath(getString(valueAt(json, "path")))) throw Error("Path doesn't match derivation output"); return dof; } @@ -1357,20 +1355,19 @@ nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const Derivation Derivation::fromJSON( const StoreDirConfig & store, - const nlohmann::json & json, + const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings) { using nlohmann::detail::value_t; Derivation res; - ensureType(json, value_t::object); + auto & json = getObject(_json); - res.name = ensureType(valueAt(json, "name"), value_t::string); + res.name = getString(valueAt(json, "name")); try { - auto & outputsObj = ensureType(valueAt(json, "outputs"), value_t::object); - for (auto & [outputName, output] : outputsObj.items()) { + for (auto & [outputName, output] : getObject(valueAt(json, "outputs"))) { res.outputs.insert_or_assign( outputName, DerivationOutput::fromJSON(store, res.name, outputName, output)); @@ -1381,8 +1378,7 @@ Derivation Derivation::fromJSON( } try { - auto & inputsList = ensureType(valueAt(json, "inputSrcs"), value_t::array); - for (auto & input : inputsList) + for (auto & input : getArray(valueAt(json, "inputSrcs"))) res.inputSrcs.insert(store.parseStorePath(static_cast(input))); } catch (Error & e) { e.addTrace({}, "while reading key 'inputSrcs'"); @@ -1391,18 +1387,17 @@ Derivation Derivation::fromJSON( try { std::function::ChildNode(const nlohmann::json &)> doInput; - doInput = [&](const auto & json) { + doInput = [&](const auto & _json) { + auto & json = getObject(_json); DerivedPathMap::ChildNode node; - node.value = static_cast( - ensureType(valueAt(json, "outputs"), value_t::array)); - for (auto & [outputId, childNode] : ensureType(valueAt(json, "dynamicOutputs"), value_t::object).items()) { + node.value = getStringSet(valueAt(json, "outputs")); + for (auto & [outputId, childNode] : getObject(valueAt(json, "dynamicOutputs"))) { xpSettings.require(Xp::DynamicDerivations); node.childMap[outputId] = doInput(childNode); } return node; }; - auto & inputDrvsObj = ensureType(valueAt(json, "inputDrvs"), value_t::object); - for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items()) + for (auto & [inputDrvPath, inputOutputs] : getObject(valueAt(json, "inputDrvs"))) res.inputDrvs.map[store.parseStorePath(inputDrvPath)] = doInput(inputOutputs); } catch (Error & e) { @@ -1410,10 +1405,10 @@ Derivation Derivation::fromJSON( throw; } - res.platform = ensureType(valueAt(json, "system"), value_t::string); - res.builder = ensureType(valueAt(json, "builder"), value_t::string); - res.args = ensureType(valueAt(json, "args"), value_t::array); - res.env = ensureType(valueAt(json, "env"), value_t::object); + res.platform = getString(valueAt(json, "system")); + res.builder = getString(valueAt(json, "builder")); + res.args = getStringList(valueAt(json, "args")); + res.env = getStringMap(valueAt(json, "env")); return res; } diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index bab21bf51..df89b5bd1 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -1,5 +1,4 @@ #include "filetransfer.hh" -#include "namespaces.hh" #include "globals.hh" #include "store-api.hh" #include "s3.hh" @@ -12,6 +11,10 @@ #include #endif +#if __linux__ +# include "namespaces.hh" +#endif + #include #include @@ -255,11 +258,11 @@ struct curlFileTransfer : public FileTransfer int progressCallback(double dltotal, double dlnow) { try { - act.progress(dlnow, dltotal); + act.progress(dlnow, dltotal); } catch (nix::Interrupted &) { - assert(_isInterrupted); + assert(getInterrupted()); } - return _isInterrupted; + return getInterrupted(); } static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow) @@ -463,7 +466,7 @@ struct curlFileTransfer : public FileTransfer if (errorSink) response = std::move(errorSink->s); auto exc = - code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted + code == CURLE_ABORTED_BY_CALLBACK && getInterrupted() ? FileTransferError(Interrupted, std::move(response), "%s of '%s' was interrupted", request.verb(), request.uri) : httpStatus != 0 ? FileTransferError(err, @@ -568,7 +571,9 @@ struct curlFileTransfer : public FileTransfer stopWorkerThread(); }); + #if __linux__ unshareFilesystem(); + #endif std::map> items; diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index cb820e2d5..88e943263 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -665,7 +665,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) results.paths.insert(path); uint64_t bytesFreed; - deletePath(realPath, bytesFreed); + deleteStorePath(realPath, bytesFreed); + results.bytesFreed += bytesFreed; if (results.bytesFreed > options.maxFreed) { @@ -752,7 +753,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) auto i = referrersCache.find(*path); if (i == referrersCache.end()) { StorePathSet referrers; - queryReferrers(*path, referrers); + queryGCReferrers(*path, referrers); referrersCache.emplace(*path, std::move(referrers)); i = referrersCache.find(*path); } @@ -879,7 +880,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) if (unlink(path.c_str()) == -1) throw SysError("deleting '%1%'", path); - /* Do not accound for deleted file here. Rely on deletePath() + /* Do not account for deleted file here. Rely on deletePath() accounting. */ } diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index fa0938d7b..4229fb4df 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -2,7 +2,6 @@ #include "current-process.hh" #include "archive.hh" #include "args.hh" -#include "users.hh" #include "abstract-setting-to-json.hh" #include "compute-levels.hh" @@ -57,7 +56,7 @@ Settings::Settings() , nixManDir(canonPath(NIX_MAN_DIR)) , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH))) { - buildUsersGroup = getuid() == 0 ? "nixbld" : ""; + buildUsersGroup = isRootUser() ? "nixbld" : ""; allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1"; auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or("")); @@ -346,6 +345,12 @@ void initPlugins() dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL); if (!handle) throw Error("could not dynamically open plugin file '%s': %s", file, dlerror()); + + /* Older plugins use a statically initialized object to run their code. + Newer plugins can also export nix_plugin_entry() */ + void (*nix_plugin_entry)() = (void (*)())dlsym(handle, "nix_plugin_entry"); + if (nix_plugin_entry) + nix_plugin_entry(); } } @@ -404,6 +409,7 @@ void assertLibStoreInitialized() { } void initLibStore() { + if (initLibStoreDone) return; initLibUtil(); diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index e6544976a..4bdbe3333 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -5,6 +5,7 @@ #include "config.hh" #include "environment-variables.hh" #include "experimental-features.hh" +#include "users.hh" #include #include @@ -665,7 +666,7 @@ public: Setting sandboxFallback{this, true, "sandbox-fallback", "Whether to disable sandboxing when the kernel doesn't allow it."}; - Setting requireDropSupplementaryGroups{this, getuid() == 0, "require-drop-supplementary-groups", + Setting requireDropSupplementaryGroups{this, isRootUser(), "require-drop-supplementary-groups", R"( Following the principle of least privilege, Nix will attempt to drop supplementary groups when building with sandboxing. @@ -687,16 +688,36 @@ public: Setting sandboxShmSize{ this, "50%", "sandbox-dev-shm-size", R"( - This option determines the maximum size of the `tmpfs` filesystem - mounted on `/dev/shm` in Linux sandboxes. For the format, see the - description of the `size` option of `tmpfs` in mount(8). The default - is `50%`. + *Linux only* + + This option determines the maximum size of the `tmpfs` filesystem + mounted on `/dev/shm` in Linux sandboxes. For the format, see the + description of the `size` option of `tmpfs` in mount(8). The default + is `50%`. )"}; Setting sandboxBuildDir{this, "/build", "sandbox-build-dir", - "The build directory inside the sandbox."}; + R"( + *Linux only* + + The build directory inside the sandbox. + + This directory is backed by [`build-dir`](#conf-build-dir) on the host. + )"}; #endif + Setting> buildDir{this, std::nullopt, "build-dir", + R"( + The directory on the host, in which derivations' temporary build directories are created. + + If not set, Nix will use the system temporary directory indicated by the `TMPDIR` environment variable. + Note that builds are often performed by the Nix daemon, so its `TMPDIR` is used, and not that of the Nix command line interface. + + This is also the location where [`--keep-failed`](@docroot@/command-ref/opt-common.md#opt-keep-failed) leaves its files. + + If Nix runs without sandbox, or if the platform does not support sandboxing with bind mounts (e.g. macOS), then the [`builder`](@docroot@/language/derivations.md#attr-builder)'s environment will contain this directory, instead of the virtual location [`sandbox-build-dir`](#conf-sandbox-build-dir). + )"}; + Setting allowedImpureHostPrefixes{this, {}, "allowed-impure-host-deps", "Which prefixes to allow derivations to ask for access to (primarily for Darwin)."}; @@ -1136,9 +1157,10 @@ public: this, {}, "plugin-files", R"( A list of plugin files to be loaded by Nix. Each of these files will - be dlopened by Nix, allowing them to affect execution through static - initialization. In particular, these plugins may construct static - instances of RegisterPrimOp to add new primops or constants to the + be dlopened by Nix. If they contain the symbol `nix_plugin_entry()`, + this symbol will be called. Alternatively, they can affect execution + through static initialization. In particular, these plugins may construct + static instances of RegisterPrimOp to add new primops or constants to the expression language, RegisterStoreImplementation to add new store implementations, RegisterCommand to add new subcommands to the `nix` command, and RegisterSetting to add new nix config settings. See the diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index 81c385ddb..843c0d288 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -33,6 +33,10 @@ struct LocalStoreAccessor : PosixSourceAccessor std::optional maybeLstat(const CanonPath & path) override { + /* Handle the case where `path` is (a parent of) the store. */ + if (isDirOrInDir(store->storeDir, path.abs())) + return Stat{ .type = tDirectory }; + return PosixSourceAccessor::maybeLstat(toRealPath(path)); } diff --git a/src/libstore/local-overlay-store.cc b/src/libstore/local-overlay-store.cc new file mode 100644 index 000000000..598415db8 --- /dev/null +++ b/src/libstore/local-overlay-store.cc @@ -0,0 +1,292 @@ +#include "local-overlay-store.hh" +#include "callback.hh" +#include "realisation.hh" +#include "processes.hh" +#include "url.hh" +#include + +namespace nix { + +std::string LocalOverlayStoreConfig::doc() +{ + return + #include "local-overlay-store.md" + ; +} + +Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) { + return upperLayer + "/" + path.to_string(); +} + +LocalOverlayStore::LocalOverlayStore(const Params & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , LocalStoreConfig(params) + , LocalOverlayStoreConfig(params) + , Store(params) + , LocalFSStore(params) + , LocalStore(params) + , lowerStore(openStore(percentDecode(lowerStoreUri.get())).dynamic_pointer_cast()) +{ + if (checkMount.get()) { + std::smatch match; + std::string mountInfo; + auto mounts = readFile("/proc/self/mounts"); + auto regex = std::regex(R"((^|\n)overlay )" + realStoreDir.get() + R"( .*(\n|$))"); + + // Mount points can be stacked, so there might be multiple matching entries. + // Loop until the last match, which will be the current state of the mount point. + while (std::regex_search(mounts, match, regex)) { + mountInfo = match.str(); + mounts = match.suffix(); + } + + auto checkOption = [&](std::string option, std::string value) { + return std::regex_search(mountInfo, std::regex("\\b" + option + "=" + value + "( |,)")); + }; + + auto expectedLowerDir = lowerStore->realStoreDir.get(); + if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", upperLayer)) { + debug("expected lowerdir: %s", expectedLowerDir); + debug("expected upperdir: %s", upperLayer); + debug("actual mount: %s", mountInfo); + throw Error("overlay filesystem '%s' mounted incorrectly", + realStoreDir.get()); + } + } +} + + +void LocalOverlayStore::registerDrvOutput(const Realisation & info) +{ + // First do queryRealisation on lower layer to populate DB + auto res = lowerStore->queryRealisation(info.id); + if (res) + LocalStore::registerDrvOutput(*res); + + LocalStore::registerDrvOutput(info); +} + + +void LocalOverlayStore::queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept +{ + auto callbackPtr = std::make_shared(std::move(callback)); + + LocalStore::queryPathInfoUncached(path, + {[this, path, callbackPtr](std::future> fut) { + try { + auto info = fut.get(); + if (info) + return (*callbackPtr)(std::move(info)); + } catch (...) { + return callbackPtr->rethrow(); + } + // If we don't have it, check lower store + lowerStore->queryPathInfo(path, + {[path, callbackPtr](std::future> fut) { + try { + (*callbackPtr)(fut.get().get_ptr()); + } catch (...) { + return callbackPtr->rethrow(); + } + }}); + }}); +} + + +void LocalOverlayStore::queryRealisationUncached(const DrvOutput & drvOutput, + Callback> callback) noexcept +{ + auto callbackPtr = std::make_shared(std::move(callback)); + + LocalStore::queryRealisationUncached(drvOutput, + {[this, drvOutput, callbackPtr](std::future> fut) { + try { + auto info = fut.get(); + if (info) + return (*callbackPtr)(std::move(info)); + } catch (...) { + return callbackPtr->rethrow(); + } + // If we don't have it, check lower store + lowerStore->queryRealisation(drvOutput, + {[callbackPtr](std::future> fut) { + try { + (*callbackPtr)(fut.get()); + } catch (...) { + return callbackPtr->rethrow(); + } + }}); + }}); +} + + +bool LocalOverlayStore::isValidPathUncached(const StorePath & path) +{ + auto res = LocalStore::isValidPathUncached(path); + if (res) return res; + res = lowerStore->isValidPath(path); + if (res) { + // Get path info from lower store so upper DB genuinely has it. + auto p = lowerStore->queryPathInfo(path); + // recur on references, syncing entire closure. + for (auto & r : p->references) + if (r != path) + isValidPath(r); + LocalStore::registerValidPath(*p); + } + return res; +} + + +void LocalOverlayStore::queryReferrers(const StorePath & path, StorePathSet & referrers) +{ + LocalStore::queryReferrers(path, referrers); + lowerStore->queryReferrers(path, referrers); +} + + +void LocalOverlayStore::queryGCReferrers(const StorePath & path, StorePathSet & referrers) +{ + LocalStore::queryReferrers(path, referrers); +} + + +StorePathSet LocalOverlayStore::queryValidDerivers(const StorePath & path) +{ + auto res = LocalStore::queryValidDerivers(path); + for (auto p : lowerStore->queryValidDerivers(path)) + res.insert(p); + return res; +} + + +std::optional LocalOverlayStore::queryPathFromHashPart(const std::string & hashPart) +{ + auto res = LocalStore::queryPathFromHashPart(hashPart); + if (res) + return res; + else + return lowerStore->queryPathFromHashPart(hashPart); +} + + +void LocalOverlayStore::registerValidPaths(const ValidPathInfos & infos) +{ + // First, get any from lower store so we merge + { + StorePathSet notInUpper; + for (auto & [p, _] : infos) + if (!LocalStore::isValidPathUncached(p)) // avoid divergence + notInUpper.insert(p); + auto pathsInLower = lowerStore->queryValidPaths(notInUpper); + ValidPathInfos inLower; + for (auto & p : pathsInLower) + inLower.insert_or_assign(p, *lowerStore->queryPathInfo(p)); + LocalStore::registerValidPaths(inLower); + } + // Then do original request + LocalStore::registerValidPaths(infos); +} + + +void LocalOverlayStore::collectGarbage(const GCOptions & options, GCResults & results) +{ + LocalStore::collectGarbage(options, results); + + remountIfNecessary(); +} + + +void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) +{ + auto mergedDir = realStoreDir.get() + "/"; + if (path.substr(0, mergedDir.length()) != mergedDir) { + warn("local-overlay: unexpected gc path '%s' ", path); + return; + } + + StorePath storePath = {path.substr(mergedDir.length())}; + auto upperPath = toUpperPath(storePath); + + if (pathExists(upperPath)) { + debug("upper exists: %s", path); + if (lowerStore->isValidPath(storePath)) { + debug("lower exists: %s", storePath.to_string()); + // Path also exists in lower store. + // We must delete via upper layer to avoid creating a whiteout. + deletePath(upperPath, bytesFreed); + _remountRequired = true; + } else { + // Path does not exist in lower store. + // So we can delete via overlayfs and not need to remount. + LocalStore::deleteStorePath(path, bytesFreed); + } + } +} + + +void LocalOverlayStore::optimiseStore() +{ + Activity act(*logger, actOptimiseStore); + + // Note for LocalOverlayStore, queryAllValidPaths only returns paths in upper layer + auto paths = queryAllValidPaths(); + + act.progress(0, paths.size()); + + uint64_t done = 0; + + for (auto & path : paths) { + if (lowerStore->isValidPath(path)) { + uint64_t bytesFreed = 0; + // Deduplicate store path + deleteStorePath(Store::toRealPath(path), bytesFreed); + } + done++; + act.progress(done, paths.size()); + } + + remountIfNecessary(); +} + + +LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag repair) +{ + StorePathSet done; + + auto existsInStoreDir = [&](const StorePath & storePath) { + return pathExists(realStoreDir.get() + "/" + storePath.to_string()); + }; + + bool errors = false; + StorePathSet validPaths; + + for (auto & i : queryAllValidPaths()) + verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); + + return { + .errors = errors, + .validPaths = validPaths, + }; +} + + +void LocalOverlayStore::remountIfNecessary() +{ + if (!_remountRequired) return; + + if (remountHook.get().empty()) { + warn("'%s' needs remounting, set remount-hook to do this automatically", realStoreDir.get()); + } else { + runProgram(remountHook, false, {realStoreDir}); + } + + _remountRequired = false; +} + + +static RegisterStoreImplementation regLocalOverlayStore; + +} diff --git a/src/libstore/local-overlay-store.hh b/src/libstore/local-overlay-store.hh new file mode 100644 index 000000000..2c24285dd --- /dev/null +++ b/src/libstore/local-overlay-store.hh @@ -0,0 +1,215 @@ +#include "local-store.hh" + +namespace nix { + +/** + * Configuration for `LocalOverlayStore`. + */ +struct LocalOverlayStoreConfig : virtual LocalStoreConfig +{ + LocalOverlayStoreConfig(const StringMap & params) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , LocalStoreConfig(params) + { } + + const Setting lowerStoreUri{(StoreConfig*) this, "", "lower-store", + R"( + [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format) + for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly). + + Must be a store with a store dir on the file system. + Must be used as OverlayFS lower layer for this store's store dir. + )"}; + + const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer", + R"( + Directory containing the OverlayFS upper layer for this store's store dir. + )"}; + + Setting checkMount{(StoreConfig*) this, true, "check-mount", + R"( + Check that the overlay filesystem is correctly mounted. + + Nix does not manage the overlayfs mount point itself, but the correct + functioning of the overlay store does depend on this mount point being set up + correctly. Rather than just assume this is the case, check that the lowerdir + and upperdir options are what we expect them to be. This check is on by + default, but can be disabled if needed. + )"}; + + const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook", + R"( + Script or other executable to run when overlay filesystem needs remounting. + + This is occasionally necessary when deleting a store path that exists in both upper and lower layers. + In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly + is the only way to perform the deletion without creating a "whiteout". + However this causes the OverlayFS kernel data structures to get out-of-sync, + and can lead to 'stale file handle' errors; remounting solves the problem. + + The store directory is passed as an argument to the invoked executable. + )"}; + + const std::string name() override { return "Experimental Local Overlay Store"; } + + std::optional experimentalFeature() const override + { + return ExperimentalFeature::LocalOverlayStore; + } + + std::string doc() override; + +protected: + /** + * @return The host OS path corresponding to the store path for the + * upper layer. + * + * @note The there is no guarantee a store object is actually stored + * at that file path. It might be stored in the lower layer instead, + * or it might not be part of this store at all. + */ + Path toUpperPath(const StorePath & path); +}; + +/** + * Variation of local store using OverlayFS for the store directory. + * + * Documentation on overridden methods states how they differ from their + * `LocalStore` counterparts. + */ +class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual LocalStore +{ + /** + * The store beneath us. + * + * Our store dir should be an overlay fs where the lower layer + * is that store's store dir, and the upper layer is some + * scratch storage just for us. + */ + ref lowerStore; + +public: + LocalOverlayStore(const Params & params); + + LocalOverlayStore(std::string scheme, std::string path, const Params & params) + : LocalOverlayStore(params) + { + if (!path.empty()) + throw UsageError("local-overlay:// store url doesn't support path part, only scheme and query params"); + } + + static std::set uriSchemes() + { + return { "local-overlay" }; + } + + std::string getUri() override + { + return "local-overlay://"; + } + +private: + /** + * First copy up any lower store realisation with the same key, so we + * merge rather than mask it. + */ + void registerDrvOutput(const Realisation & info) override; + + /** + * Check lower store if upper DB does not have. + */ + void queryPathInfoUncached(const StorePath & path, + Callback> callback) noexcept override; + + /** + * Check lower store if upper DB does not have. + * + * In addition, copy up metadata for lower store objects (and their + * closure). (I.e. Optimistically cache in the upper DB.) + */ + bool isValidPathUncached(const StorePath & path) override; + + /** + * Check the lower store and upper DB. + */ + void queryReferrers(const StorePath & path, StorePathSet & referrers) override; + + /** + * Check the lower store and upper DB. + */ + StorePathSet queryValidDerivers(const StorePath & path) override; + + /** + * Check lower store if upper DB does not have. + */ + std::optional queryPathFromHashPart(const std::string & hashPart) override; + + /** + * First copy up any lower store realisation with the same key, so we + * merge rather than mask it. + */ + void registerValidPaths(const ValidPathInfos & infos) override; + + /** + * Check lower store if upper DB does not have. + */ + void queryRealisationUncached(const DrvOutput&, + Callback> callback) noexcept override; + + /** + * Call `remountIfNecessary` after collecting garbage normally. + */ + void collectGarbage(const GCOptions & options, GCResults & results) override; + + /** + * Check which layers the store object exists in to try to avoid + * needing to remount. + */ + void deleteStorePath(const Path & path, uint64_t & bytesFreed) override; + + /** + * Deduplicate by removing store objects from the upper layer that + * are now in the lower layer. + * + * Operations on a layered store will not cause duplications, but addition of + * new store objects to the lower layer can instill induce them + * (there is no way to prevent that). This cleans up those + * duplications. + * + * @note We do not yet optomise the upper layer in the normal way + * (hardlink) yet. We would like to, but it requires more + * refactoring of existing code to support this sustainably. + */ + void optimiseStore() override; + + /** + * Check all paths registered in the upper DB. + * + * Note that this includes store objects that reside in either overlayfs layer; + * just enumerating the contents of the upper layer would skip them. + * + * We don't verify the contents of both layers on the assumption that the lower layer is far bigger, + * and also the observation that anything not in the upper db the overlayfs doesn't yet care about. + */ + VerificationResult verifyAllValidPaths(RepairFlag repair) override; + + /** + * Deletion only effects the upper layer, so we ignore lower-layer referrers. + */ + void queryGCReferrers(const StorePath & path, StorePathSet & referrers) override; + + /** + * Call the `remountHook` if we have done something such that the + * OverlayFS needed to be remounted. See that hook's user-facing + * documentation for further details. + */ + void remountIfNecessary(); + + /** + * State for `remountIfNecessary` + */ + std::atomic_bool _remountRequired = false; +}; + +} diff --git a/src/libstore/local-overlay-store.md b/src/libstore/local-overlay-store.md new file mode 100644 index 000000000..cc310bc7f --- /dev/null +++ b/src/libstore/local-overlay-store.md @@ -0,0 +1,123 @@ +R"( + +**Store URL format**: `local-overlay` + +This store type is a variation of the [local store] designed to leverage Linux's [Overlay Filesystem](https://docs.kernel.org/filesystems/overlayfs.html) (OverlayFS for short). +Just as OverlayFS combines a lower and upper filesystem by treating the upper one as a patch against the lower, the local overlay store combines a lower store with an upper almost-[local store]. +("almost" because while the upper fileystems for OverlayFS is valid on its own, the upper almost-store is not a valid local store on its own because some references will dangle.) +To use this store, you will first need to configure an OverlayFS mountpoint [appropriately](#example-filesystem-layout) as Nix will not do this for you (though it will verify the mountpoint is configured correctly). + +### Conceptual parts of a local overlay store + +*This is a more abstract/conceptual description of the parts of a layered store, an authoritative reference. +For more "practical" instructions, see the worked-out example in the next subsection.* + +The parts of a local overlay store are as follows: + +- **Lower store**: + + This is any store implementation that includes a store directory as part of the native operating system filesystem. + For example, this could be a [local store], [local daemon store], or even another local overlay store. + + The local overlay store never tries to modify the lower store in any way. + Something else could modify the lower store, but there are restrictions on this + Nix itself requires that this store only grow, and not change in other ways. + For example, new store objects can be added, but deleting or modifying store objects is not allowed in general, because that will confuse and corrupt any local overlay store using those objects. + (In addition, the underlying filesystem overlay mechanism may impose additional restrictions, see below.) + + The lower store must not change while it is mounted as part of an overlay store. + To ensure it does not, you might want to mount the store directory read-only (which then requires the [read-only] parameter to be set to `true`). + + Specified with the [`lower-store`](#store-experimental-local-overlay-store-lower-store) setting. + + - **Lower store directory**: + + This is the directory used/exposed by the lower store. + + Specified with `lower-store.real` setting. + + As specified above, Nix requires the local store can only grow not change in other ways. + Linux's OverlayFS in addition imposes the further requirement that this directory cannot change at all. + That means that, while any local overlay store exists that is using this store as a lower store, this directory must not change. + + - **Lower metadata source**: + + This is abstract, just some way to read the metadata of lower store [store objects][store object]. + For example it could be a SQLite database (for the [local store]), or a socket connection (for the [local daemon store]). + + This need not be writable. + As stated above a local overlay store never tries to modify its lower store. + The lower store's metadata is considered part of the lower store, just as the store's [file system objects][file system object] that appear in the store directory are. + +- **Upper almost-store**: + + This is almost but not quite just a [local store]. + That is because taken in isolation, not as part of a local overlay store, by itself, it would appear corrupted. + But combined with everything else as part of an overlay local store, it is valid. + + - **Upper layer directory**: + + This contains additional [store objects][store object] + (or, strictly speaking, their [file system objects][file system object] that the local overlay store will extend the lower store with). + + Specified with [`upper-layer`](#store-experimental-local-overlay-store-upper-layer) setting. + + - **Upper store directory**: + + This contains all the store objects from each of the two directories. + + The lower store directory and upper layer directory are combined via OverlayFS to create this directory. + Nix doesn't do this itself, because it typically wouldn't have the permissions to do so, so it is the responsibility of the user to set this up first. + Nix can, however, optionally check that that the OverlayFS mount settings appear as expected, matching Nix's own settings. + + Specified with the [`real`](#store-experimental-local-overlay-store-real) setting. + + - **Upper SQLite database**: + + This contains the metadata of all of the upper layer [store objects][store object] (everything beyond their file system objects), and also duplicate copies of some lower layer store object's metadta. + The duplication is so the metadata for the [closure](@docroot@/glossary.md#gloss-closure) of upper layer [store objects][store object] can be found entirely within the upper layer. + (This allows us to use the same SQL Schema as the [local store]'s SQLite database, as foreign keys in that schema enforce closure metadata to be self-contained in this way.) + + The location of the database is directly specified, but depends on the [`state`](#store-experimental-local-overlay-store-state) setting. + It is is always `${state}/db`. + +[file system object]: @docroot@/store/file-system-object.md +[store object]: @docroot@/store/store-object.md + + +### Example filesystem layout + +Here is a worked out example of usage, following the concepts in the previous section. + +Say we have the following paths: + +- `/mnt/example/merged-store/nix/store` + +- `/mnt/example/store-a/nix/store` + +- `/mnt/example/store-b` + +Then the following store URI can be used to access a local-overlay store at `/mnt/example/merged-store`: + +``` +local-overlay://?root=/mnt/example/merged-store&lower-store=/mnt/example/store-a&upper-layer=/mnt/example/store-b +``` + +The lower store directory is located at `/mnt/example/store-a/nix/store`, while the upper layer is at `/mnt/example/store-b`. + +Before accessing the overlay store you will need to ensure the OverlayFS mount is set up correctly: + +```shell +mount -t overlay overlay \ + -o lowerdir="/mnt/example/store-a/nix/store" \ + -o upperdir="/mnt/example/store-b" \ + -o workdir="/mnt/example/workdir" \ + "/mnt/example/merged-store/nix/store" +``` + +Note that OverlayFS requires `/mnt/example/workdir` to be on the same volume as the `upperdir`. + +By default, Nix will check that the mountpoint as been set up correctly and fail with an error if it has not. +You can override this behaviour by passing [`check-mount=false`](#store-experimental-local-overlay-store-check-mount) if you need to. + +)" diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 1bbeaa912..a32b349a1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -16,6 +16,7 @@ #include "posix-fs-canonicalise.hh" #include "posix-source-accessor.hh" #include "keys.hh" +#include "users.hh" #include #include @@ -223,7 +224,7 @@ LocalStore::LocalStore(const Params & params) /* Optionally, create directories and set permissions for a multi-user install. */ - if (getuid() == 0 && settings.buildUsersGroup != "") { + if (isRootUser() && settings.buildUsersGroup != "") { mode_t perm = 01775; struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str()); @@ -464,6 +465,12 @@ AutoCloseFD LocalStore::openGCLock() } +void LocalStore::deleteStorePath(const Path & path, uint64_t & bytesFreed) +{ + deletePath(path, bytesFreed); +} + + LocalStore::~LocalStore() { std::shared_future future; @@ -552,6 +559,19 @@ void LocalStore::openDB(State & state, bool create) sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) SQLiteError::throw_(db, "setting journal mode"); + if (mode == "wal") { + /* persist the WAL files when the db connection is closed. This allows + for read-only connections without write permissions on the + containing directory to succeed on a closed db. Setting the + journal_size_limit to 2^40 bytes results in the WAL files getting + truncated to 0 on exit and limits the on disk size of the WAL files + to 2^40 bytes following a checkpoint */ + if (sqlite3_exec(db, "pragma main.journal_size_limit = 1099511627776;", 0, 0, 0) == SQLITE_OK) { + int enable = 1; + sqlite3_file_control(db, NULL, SQLITE_FCNTL_PERSIST_WAL, &enable); + } + } + /* Increase the auto-checkpoint interval to 40000 pages. This seems enough to ensure that instantiating the NixOS system derivation is done in a single fsync(). */ @@ -573,7 +593,7 @@ void LocalStore::openDB(State & state, bool create) void LocalStore::makeStoreWritable() { #if __linux__ - if (getuid() != 0) return; + if (!isRootUser()) return; /* Check if /nix/store is on a read-only mount. */ struct statvfs stat; if (statvfs(realStoreDir.get().c_str(), &stat) != 0) @@ -1355,40 +1375,12 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) { printInfo("reading the Nix store..."); - bool errors = false; - /* Acquire the global GC lock to get a consistent snapshot of existing and valid paths. */ auto fdGCLock = openGCLock(); FdLock gcLock(fdGCLock.get(), ltRead, true, "waiting for the big garbage collector lock..."); - StorePathSet validPaths; - - { - StorePathSet storePathsInStoreDir; - /* Why aren't we using `queryAllValidPaths`? Because that would - tell us about all the paths than the database knows about. Here we - want to know about all the store paths in the store directory, - regardless of what the database thinks. - - We will end up cross-referencing these two sources of truth (the - database and the filesystem) in the loop below, in order to catch - invalid states. - */ - for (auto & i : readDirectory(realStoreDir)) { - try { - storePathsInStoreDir.insert({i.name}); - } catch (BadStorePath &) { } - } - - /* Check whether all valid paths actually exist. */ - printInfo("checking path existence..."); - - StorePathSet done; - - for (auto & i : queryAllValidPaths()) - verifyPath(i, storePathsInStoreDir, done, validPaths, repair, errors); - } + auto [errors, validPaths] = verifyAllValidPaths(repair); /* Optionally, check the content hashes (slow). */ if (checkContents) { @@ -1477,21 +1469,61 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) } -void LocalStore::verifyPath(const StorePath & path, const StorePathSet & storePathsInStoreDir, +LocalStore::VerificationResult LocalStore::verifyAllValidPaths(RepairFlag repair) +{ + StorePathSet storePathsInStoreDir; + /* Why aren't we using `queryAllValidPaths`? Because that would + tell us about all the paths than the database knows about. Here we + want to know about all the store paths in the store directory, + regardless of what the database thinks. + + We will end up cross-referencing these two sources of truth (the + database and the filesystem) in the loop below, in order to catch + invalid states. + */ + for (auto & i : readDirectory(realStoreDir)) { + try { + storePathsInStoreDir.insert({i.name}); + } catch (BadStorePath &) { } + } + + /* Check whether all valid paths actually exist. */ + printInfo("checking path existence..."); + + StorePathSet done; + + auto existsInStoreDir = [&](const StorePath & storePath) { + return storePathsInStoreDir.count(storePath); + }; + + bool errors = false; + StorePathSet validPaths; + + for (auto & i : queryAllValidPaths()) + verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); + + return { + .errors = errors, + .validPaths = validPaths, + }; +} + + +void LocalStore::verifyPath(const StorePath & path, std::function existsInStoreDir, StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors) { checkInterrupt(); if (!done.insert(path).second) return; - if (!storePathsInStoreDir.count(path)) { + if (!existsInStoreDir(path)) { /* Check any referrers first. If we can invalidate them first, then we can invalidate this path as well. */ bool canInvalidate = true; StorePathSet referrers; queryReferrers(path, referrers); for (auto & i : referrers) if (i != path) { - verifyPath(i, storePathsInStoreDir, done, validPaths, repair, errors); + verifyPath(i, existsInStoreDir, done, validPaths, repair, errors); if (validPaths.count(i)) canInvalidate = false; } @@ -1570,7 +1602,7 @@ static void makeMutable(const Path & path) /* Upgrade from schema 6 (Nix 0.15) to schema 7 (Nix >= 1.3). */ void LocalStore::upgradeStore7() { - if (getuid() != 0) return; + if (!isRootUser()) return; printInfo("removing immutable bits from the Nix store (this may take a while)..."); makeMutable(realStoreDir); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 7eff1d690..47d3c04bc 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -229,6 +229,25 @@ public: void collectGarbage(const GCOptions & options, GCResults & results) override; + /** + * Called by `collectGarbage` to trace in reverse. + * + * Using this rather than `queryReferrers` directly allows us to + * fine-tune which referrers we consider for garbage collection; + * some store implementations take advantage of this. + */ + virtual void queryGCReferrers(const StorePath & path, StorePathSet & referrers) + { + return queryReferrers(path, referrers); + } + + /** + * Called by `collectGarbage` to recursively delete a path. + * The default implementation simply calls `deletePath`, but it can be + * overridden by stores that wish to provide their own deletion behaviour. + */ + virtual void deleteStorePath(const Path & path, uint64_t & bytesFreed); + /** * Optimise the disk space usage of the Nix store by hard-linking * files with the same contents. @@ -245,6 +264,31 @@ public: bool verifyStore(bool checkContents, RepairFlag repair) override; +protected: + + /** + * Result of `verifyAllValidPaths` + */ + struct VerificationResult { + /** + * Whether any errors were encountered + */ + bool errors; + + /** + * A set of so-far valid paths. The store objects pointed to by + * those paths are suitable for further validation checking. + */ + StorePathSet validPaths; + }; + + /** + * First, unconditional step of `verifyStore` + */ + virtual VerificationResult verifyAllValidPaths(RepairFlag repair); + +public: + /** * Register the validity of a path, i.e., that `path` exists, that * the paths referenced by it exists, and in the case of an output @@ -255,7 +299,7 @@ public: */ void registerValidPath(const ValidPathInfo & info); - void registerValidPaths(const ValidPathInfos & infos); + virtual void registerValidPaths(const ValidPathInfos & infos); unsigned int getProtocol() override; @@ -290,6 +334,11 @@ public: std::optional getVersion() override; +protected: + + void verifyPath(const StorePath & path, std::function existsInStoreDir, + StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); + private: /** @@ -313,9 +362,6 @@ private: */ void invalidatePathChecked(const StorePath & path); - void verifyPath(const StorePath & path, const StorePathSet & store, - StorePathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); - std::shared_ptr queryPathInfoInternal(State & state, const StorePath & path); void updatePathInfo(State & state, const ValidPathInfo & info); diff --git a/src/libstore/local.mk b/src/libstore/local.mk index f86643849..ccb7aeee2 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -5,12 +5,15 @@ libstore_NAME = libnixstore libstore_DIR := $(d) libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc) +ifdef HOST_UNIX + libstore_SOURCES += $(wildcard $(d)/unix/*.cc) +endif libstore_LIBS = libutil libstore_LDFLAGS += $(SQLITE3_LIBS) $(LIBCURL_LIBS) $(THREAD_LDFLAGS) ifdef HOST_LINUX - libstore_LDFLAGS += -ldl + libstore_LDFLAGS += -ldl endif $(foreach file,$(libstore_FILES),$(eval $(call install-data-in,$(d)/$(file),$(datadir)/nix/sandbox))) @@ -27,8 +30,15 @@ ifeq ($(HAVE_SECCOMP), 1) libstore_LDFLAGS += $(LIBSECCOMP_LIBS) endif +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libstore := -I $(d) -I $(d)/build +ifdef HOST_UNIX + INCLUDE_libstore += -I $(d)/unix +endif + libstore_CXXFLAGS += \ - -I src/libutil -I src/libstore -I src/libstore/build \ + $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libstore) \ -DNIX_PREFIX=\"$(prefix)\" \ -DNIX_STORE_DIR=\"$(storedir)\" \ -DNIX_DATA_DIR=\"$(datadir)\" \ diff --git a/src/libstore/lock.cc b/src/libstore/lock.cc index 87f55ce49..023c74e34 100644 --- a/src/libstore/lock.cc +++ b/src/libstore/lock.cc @@ -2,6 +2,7 @@ #include "file-system.hh" #include "globals.hh" #include "pathlocks.hh" +#include "users.hh" #include #include @@ -192,10 +193,10 @@ std::unique_ptr acquireUserLock(uid_t nrIds, bool useUserNamespace) bool useBuildUsers() { #if __linux__ - static bool b = (settings.buildUsersGroup != "" || settings.autoAllocateUids) && getuid() == 0; + static bool b = (settings.buildUsersGroup != "" || settings.autoAllocateUids) && isRootUser(); return b; #elif __APPLE__ - static bool b = settings.buildUsersGroup != "" && getuid() == 0; + static bool b = settings.buildUsersGroup != "" && isRootUser(); return b; #else return false; diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index d9618d04c..0d219a489 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -172,19 +172,18 @@ NarInfo NarInfo::fromJSON( }; if (json.contains("url")) - res.url = ensureType(valueAt(json, "url"), value_t::string); + res.url = getString(valueAt(json, "url")); if (json.contains("compression")) - res.compression = ensureType(valueAt(json, "compression"), value_t::string); + res.compression = getString(valueAt(json, "compression")); if (json.contains("downloadHash")) res.fileHash = Hash::parseAny( - static_cast( - ensureType(valueAt(json, "downloadHash"), value_t::string)), + getString(valueAt(json, "downloadHash")), std::nullopt); if (json.contains("downloadSize")) - res.fileSize = ensureType(valueAt(json, "downloadSize"), value_t::number_integer); + res.fileSize = getInteger(valueAt(json, "downloadSize")); return res; } diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index d82ccd0c9..6523cb425 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -190,23 +190,18 @@ nlohmann::json UnkeyedValidPathInfo::toJSON( UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON( const Store & store, - const nlohmann::json & json) + const nlohmann::json & _json) { - using nlohmann::detail::value_t; - UnkeyedValidPathInfo res { Hash(Hash::dummy), }; - ensureType(json, value_t::object); - res.narHash = Hash::parseAny( - static_cast( - ensureType(valueAt(json, "narHash"), value_t::string)), - std::nullopt); - res.narSize = ensureType(valueAt(json, "narSize"), value_t::number_integer); + auto & json = getObject(_json); + res.narHash = Hash::parseAny(getString(valueAt(json, "narHash")), std::nullopt); + res.narSize = getInteger(valueAt(json, "narSize")); try { - auto & references = ensureType(valueAt(json, "references"), value_t::array); + auto references = getStringList(valueAt(json, "references")); for (auto & input : references) res.references.insert(store.parseStorePath(static_cast (input))); @@ -216,20 +211,16 @@ UnkeyedValidPathInfo UnkeyedValidPathInfo::fromJSON( } if (json.contains("ca")) - res.ca = ContentAddress::parse( - static_cast( - ensureType(valueAt(json, "ca"), value_t::string))); + res.ca = ContentAddress::parse(getString(valueAt(json, "ca"))); if (json.contains("deriver")) - res.deriver = store.parseStorePath( - static_cast( - ensureType(valueAt(json, "deriver"), value_t::string))); + res.deriver = store.parseStorePath(getString(valueAt(json, "deriver"))); if (json.contains("registrationTime")) - res.registrationTime = ensureType(valueAt(json, "registrationTime"), value_t::number_integer); + res.registrationTime = getInteger(valueAt(json, "registrationTime")); if (json.contains("ultimate")) - res.ultimate = ensureType(valueAt(json, "ultimate"), value_t::boolean); + res.ultimate = getBoolean(valueAt(json, "ultimate")); if (json.contains("signatures")) res.sigs = valueAt(json, "signatures"); diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index e8b88693d..73d3976f4 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -308,7 +308,7 @@ std::string optimisticLockProfile(const Path & profile) Path profilesDir() { auto profileRoot = - (getuid() == 0) + isRootUser() ? rootProfilesDir() : createNixStateDir() + "/profiles"; createDirs(profileRoot); @@ -332,7 +332,7 @@ Path getDefaultProfile() // Backwards compatibiliy measure: Make root's profile available as // `.../default` as it's what NixOS and most of the init scripts expect Path globalProfileLink = settings.nixStateDir + "/profiles/default"; - if (getuid() == 0 && !pathExists(globalProfileLink)) { + if (isRootUser() && !pathExists(globalProfileLink)) { replaceSymlink(profile, globalProfileLink); } return absPath(readLink(profileLink), dirOf(profileLink)); diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc index 06abfb90b..3175c1978 100644 --- a/src/libstore/sqlite.cc +++ b/src/libstore/sqlite.cc @@ -7,6 +7,7 @@ #include #include +#include namespace nix { @@ -256,10 +257,8 @@ void handleSQLiteBusy(const SQLiteBusy & e, time_t & nextWarning) /* Sleep for a while since retrying the transaction right away is likely to fail again. */ checkInterrupt(); - struct timespec t; - t.tv_sec = 0; - t.tv_nsec = (random() % 100) * 1000 * 1000; /* <= 0.1s */ - nanosleep(&t, 0); + /* <= 0.1s */ + std::this_thread::sleep_for(std::chrono::milliseconds { rand() % 100 }); } } diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 4356296d4..79beeebbd 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -131,12 +131,12 @@ StorePath StoreDirConfig::makeFixedOutputPath(std::string_view name, const Fixed throw Error("fixed output derivation '%s' is not allowed to refer to other store paths.\nYou may need to use the 'unsafeDiscardReferences' derivation attribute, see the manual for more details.", name); } - return makeStorePath("output:out", - hashString(HashAlgorithm::SHA256, - "fixed:out:" + // make a unique digest based on the parameters for creating this store object + auto payload = "fixed:out:" + makeFileIngestionPrefix(info.method) - + info.hash.to_string(HashFormat::Base16, true) + ":"), - name); + + info.hash.to_string(HashFormat::Base16, true) + ":"; + auto digest = hashString(HashAlgorithm::SHA256, payload); + return makeStorePath("output:out", digest, name); } } @@ -1307,7 +1307,7 @@ std::shared_ptr openFromNonUri(const std::string & uri, const Store::Para #if __linux__ else if (!pathExists(stateDir) && params.empty() - && getuid() != 0 + && !isRootUser() && !getEnv("NIX_STORE_DIR").has_value() && !getEnv("NIX_STATE_DIR").has_value()) { @@ -1402,6 +1402,7 @@ ref openStore(const std::string & uri_, params.insert(uriParams.begin(), uriParams.end()); if (auto store = openFromNonUri(uri, params)) { + experimentalFeatureSettings.require(store->experimentalFeature()); store->warnUnknownSettings(); return ref(store); } diff --git a/src/libutil-c/local.mk b/src/libutil-c/local.mk new file mode 100644 index 000000000..f2df1ef43 --- /dev/null +++ b/src/libutil-c/local.mk @@ -0,0 +1,18 @@ +libraries += libutilc + +libutilc_NAME = libnixutilc + +libutilc_DIR := $(d) + +libutilc_SOURCES := $(wildcard $(d)/*.cc) + +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libutilc := -I $(d) +libutilc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) + +libutilc_LIBS = libutil + +libutilc_LDFLAGS += $(THREAD_LDFLAGS) + +libutilc_FORCE_INSTALL := 1 diff --git a/src/libutil-c/nix_api_util.cc b/src/libutil-c/nix_api_util.cc new file mode 100644 index 000000000..0a9b49345 --- /dev/null +++ b/src/libutil-c/nix_api_util.cc @@ -0,0 +1,150 @@ +#include "nix_api_util.h" +#include "config.hh" +#include "error.hh" +#include "nix_api_util_internal.h" +#include "util.hh" + +#include +#include + +nix_c_context * nix_c_context_create() +{ + return new nix_c_context(); +} + +void nix_c_context_free(nix_c_context * context) +{ + delete context; +} + +nix_err nix_context_error(nix_c_context * context) +{ + if (context == nullptr) { + throw; + } + try { + throw; + } catch (nix::Error & e) { + /* Storing this exception is annoying, take what we need here */ + context->last_err = e.what(); + context->info = e.info(); + int status; + const char * demangled = abi::__cxa_demangle(typeid(e).name(), 0, 0, &status); + if (demangled) { + context->name = demangled; + // todo: free(demangled); + } else { + context->name = typeid(e).name(); + } + context->last_err_code = NIX_ERR_NIX_ERROR; + return context->last_err_code; + } catch (const std::exception & e) { + context->last_err = e.what(); + context->last_err_code = NIX_ERR_UNKNOWN; + return context->last_err_code; + } + // unreachable +} + +nix_err nix_set_err_msg(nix_c_context * context, nix_err err, const char * msg) +{ + if (context == nullptr) { + // todo last_err_code + throw nix::Error("Nix C api error: %s", msg); + } + context->last_err_code = err; + context->last_err = msg; + return err; +} + +const char * nix_version_get() +{ + return PACKAGE_VERSION; +} + +// Implementations + +nix_err nix_setting_get(nix_c_context * context, const char * key, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + try { + std::map settings; + nix::globalConfig.getSettings(settings); + if (settings.contains(key)) { + return call_nix_get_string_callback(settings[key].value, callback, user_data); + } else { + return nix_set_err_msg(context, NIX_ERR_KEY, "Setting not found"); + } + } + NIXC_CATCH_ERRS +} + +nix_err nix_setting_set(nix_c_context * context, const char * key, const char * value) +{ + if (context) + context->last_err_code = NIX_OK; + if (nix::globalConfig.set(key, value)) + return NIX_OK; + else { + return nix_set_err_msg(context, NIX_ERR_KEY, "Setting not found"); + } +} + +nix_err nix_libutil_init(nix_c_context * context) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::initLibUtil(); + return NIX_OK; + } + NIXC_CATCH_ERRS +} + +const char * nix_err_msg(nix_c_context * context, const nix_c_context * read_context, unsigned int * n) +{ + if (context) + context->last_err_code = NIX_OK; + if (read_context->last_err) { + if (n) + *n = read_context->last_err->size(); + return read_context->last_err->c_str(); + } + nix_set_err_msg(context, NIX_ERR_UNKNOWN, "No error message"); + return nullptr; +} + +nix_err nix_err_name( + nix_c_context * context, const nix_c_context * read_context, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + if (read_context->last_err_code != NIX_ERR_NIX_ERROR) { + return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Last error was not a nix error"); + } + return call_nix_get_string_callback(read_context->name, callback, user_data); +} + +nix_err nix_err_info_msg( + nix_c_context * context, const nix_c_context * read_context, nix_get_string_callback callback, void * user_data) +{ + if (context) + context->last_err_code = NIX_OK; + if (read_context->last_err_code != NIX_ERR_NIX_ERROR) { + return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Last error was not a nix error"); + } + return call_nix_get_string_callback(read_context->info->msg.str(), callback, user_data); +} + +nix_err nix_err_code(const nix_c_context * read_context) +{ + return read_context->last_err_code; +} + +// internal +nix_err call_nix_get_string_callback(const std::string str, nix_get_string_callback callback, void * user_data) +{ + callback(str.c_str(), str.size(), user_data); + return NIX_OK; +} diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h new file mode 100644 index 000000000..e0ca04e69 --- /dev/null +++ b/src/libutil-c/nix_api_util.h @@ -0,0 +1,302 @@ +#ifndef NIX_API_UTIL_H +#define NIX_API_UTIL_H +/** + * @defgroup libutil libutil + * @brief C bindings for nix libutil + * + * libutil is used for functionality shared between + * different Nix modules. + * @{ + */ +/** @file + * @brief Main entry for the libutil C bindings + * + * Also contains error handling utilities + */ + +#ifdef __cplusplus +extern "C" { +#endif +// cffi start + +/** @defgroup errors Handling errors + * @brief Dealing with errors from the Nix side + * + * To handle errors that can be returned from the Nix API, + * a nix_c_context can be passed to any function that potentially returns an + * error. + * + * Error information will be stored in this context, and can be retrieved + * using nix_err_code and nix_err_msg. + * + * Passing NULL instead will cause the API to throw C++ errors. + * + * Example: + * @code{.c} + * int main() { + * nix_c_context* ctx = nix_c_context_create(); + * nix_libutil_init(ctx); + * if (nix_err_code(ctx) != NIX_OK) { + * printf("error: %s\n", nix_err_msg(NULL, ctx, NULL)); + * return 1; + * } + * return 0; + * } + * @endcode + * @{ + */ +// Error codes +/** + * @brief Type for error codes in the NIX system + * + * This type can have one of several predefined constants: + * - NIX_OK: No error occurred (0) + * - NIX_ERR_UNKNOWN: An unknown error occurred (-1) + * - NIX_ERR_OVERFLOW: An overflow error occurred (-2) + * - NIX_ERR_KEY: A key error occurred (-3) + * - NIX_ERR_NIX_ERROR: A generic Nix error occurred (-4) + */ +typedef int nix_err; + +/** + * @brief No error occurred. + * + * This error code is returned when no error has occurred during the function + * execution. + */ +#define NIX_OK 0 + +/** + * @brief An unknown error occurred. + * + * This error code is returned when an unknown error occurred during the + * function execution. + */ +#define NIX_ERR_UNKNOWN -1 + +/** + * @brief An overflow error occurred. + * + * This error code is returned when an overflow error occurred during the + * function execution. + */ +#define NIX_ERR_OVERFLOW -2 + +/** + * @brief A key error occurred. + * + * This error code is returned when a key error occurred during the function + * execution. + */ +#define NIX_ERR_KEY -3 + +/** + * @brief A generic Nix error occurred. + * + * This error code is returned when a generic Nix error occurred during the + * function execution. + */ +#define NIX_ERR_NIX_ERROR -4 + +/** + * @brief This object stores error state. + * @struct nix_c_context + * + * Passed as a first parameter to functions that can fail, to store error + * information. + * + * Optional wherever it can be used, passing NULL instead will throw a C++ + * exception. + * + * The struct is laid out so that it can also be cast to nix_err* to inspect + * directly: + * @code{.c} + * assert(*(nix_err*)ctx == NIX_OK); + * @endcode + * @note These can be reused between different function calls, + * but make sure not to use them for multiple calls simultaneously (which can + * happen in callbacks). + */ +typedef struct nix_c_context nix_c_context; + +/** + * @brief Called to get the value of a string owned by Nix. + * + * @param[in] start the string to copy. + * @param[in] n the string length. + * @param[in] user_data optional, arbitrary data, passed to the nix_get_string_callback when it's called. + */ +typedef void (*nix_get_string_callback)(const char * start, unsigned int n, void * user_data); + +// Function prototypes + +/** + * @brief Allocate a new nix_c_context. + * @throws std::bad_alloc + * @return allocated nix_c_context, owned by the caller. Free using + * `nix_c_context_free`. + */ +nix_c_context * nix_c_context_create(); +/** + * @brief Free a nix_c_context. Does not fail. + * @param[out] context The context to free, mandatory. + */ +void nix_c_context_free(nix_c_context * context); +/** + * @} + */ + +/** + * @brief Initializes nix_libutil and its dependencies. + * + * This function can be called multiple times, but should be called at least + * once prior to any other nix function. + * + * @param[out] context Optional, stores error information + * @return NIX_OK if the initialization is successful, or an error code + * otherwise. + */ +nix_err nix_libutil_init(nix_c_context * context); + +/** @defgroup settings + * @{ + */ +/** + * @brief Retrieves a setting from the nix global configuration. + * + * This function requires nix_libutil_init() to be called at least once prior to + * its use. + * + * @param[out] context optional, Stores error information + * @param[in] key The key of the setting to retrieve. + * @param[in] callback Called with the setting value. + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. + * @see nix_get_string_callback + * @return NIX_ERR_KEY if the setting is unknown, or NIX_OK if the setting was retrieved + * successfully. + */ +nix_err nix_setting_get(nix_c_context * context, const char * key, nix_get_string_callback callback, void * user_data); + +/** + * @brief Sets a setting in the nix global configuration. + * + * Use "extra-" to append to the setting's value. + * + * Settings only apply for new State%s. Call nix_plugins_init() when you are + * done with the settings to load any plugins. + * + * @param[out] context optional, Stores error information + * @param[in] key The key of the setting to set. + * @param[in] value The value to set for the setting. + * @return NIX_ERR_KEY if the setting is unknown, or NIX_OK if the setting was + * set successfully. + */ +nix_err nix_setting_set(nix_c_context * context, const char * key, const char * value); + +/** + * @} + */ +// todo: nix_plugins_init() + +/** + * @brief Retrieves the nix library version. + * + * Does not fail. + * @return A static string representing the version of the nix library. + */ +const char * nix_version_get(); + +/** @addtogroup errors + * @{ + */ +/** + * @brief Retrieves the most recent error message from a context. + * + * @pre This function should only be called after a previous nix function has + * returned an error. + * + * @param[out] context optional, the context to store errors in if this function + * fails + * @param[in] ctx the context to retrieve the error message from + * @param[out] n optional: a pointer to an unsigned int that is set to the + * length of the error. + * @return nullptr if no error message was ever set, + * a borrowed pointer to the error message otherwise. + */ +const char * nix_err_msg(nix_c_context * context, const nix_c_context * ctx, unsigned int * n); + +/** + * @brief Retrieves the error message from errorInfo in a context. + * + * Used to inspect nix Error messages. + * + * @pre This function should only be called after a previous nix function has + * returned a NIX_ERR_NIX_ERROR + * + * @param[out] context optional, the context to store errors in if this function + * fails + * @param[in] read_context the context to retrieve the error message from. + * @param[in] callback Called with the error message. + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. + * @see nix_get_string_callback + * @return NIX_OK if there were no errors, an error code otherwise. + */ +nix_err nix_err_info_msg( + nix_c_context * context, const nix_c_context * read_context, nix_get_string_callback callback, void * user_data); + +/** + * @brief Retrieves the error name from a context. + * + * Used to inspect nix Error messages. + * + * @pre This function should only be called after a previous nix function has + * returned a NIX_ERR_NIX_ERROR + * + * @param context optional, the context to store errors in if this function + * fails + * @param[in] read_context the context to retrieve the error message from + * @param[in] callback Called with the error name. + * @param[in] user_data optional, arbitrary data, passed to the callback when it's called. + * @see nix_get_string_callback + * @return NIX_OK if there were no errors, an error code otherwise. + */ +nix_err nix_err_name( + nix_c_context * context, const nix_c_context * read_context, nix_get_string_callback callback, void * user_data); + +/** + * @brief Retrieves the most recent error code from a nix_c_context + * + * Equivalent to reading the first field of the context. + * + * Does not fail + * + * @param[in] read_context the context to retrieve the error message from + * @return most recent error code stored in the context. + */ +nix_err nix_err_code(const nix_c_context * read_context); + +/** + * @brief Set an error message on a nix context. + * + * This should be used when you want to throw an error from a PrimOp callback. + * + * All other use is internal to the API. + * + * @param context context to write the error message to, or NULL + * @param err The error code to set and return + * @param msg The error message to set. + * @returns the error code set + */ +nix_err nix_set_err_msg(nix_c_context * context, nix_err err, const char * msg); + +/** + * @} + */ + +// cffi end +#ifdef __cplusplus +} +#endif + +/** @} */ +#endif // NIX_API_UTIL_H diff --git a/src/libutil-c/nix_api_util_internal.h b/src/libutil-c/nix_api_util_internal.h new file mode 100644 index 000000000..aa829feaf --- /dev/null +++ b/src/libutil-c/nix_api_util_internal.h @@ -0,0 +1,49 @@ +#ifndef NIX_API_UTIL_INTERNAL_H +#define NIX_API_UTIL_INTERNAL_H + +#include +#include + +#include "error.hh" +#include "nix_api_util.h" + +struct nix_c_context +{ + nix_err last_err_code = NIX_OK; + std::optional last_err = {}; + std::optional info = {}; + std::string name = ""; +}; + +nix_err nix_context_error(nix_c_context * context); + +/** + * Internal use only. + * + * Helper to invoke nix_get_string_callback + * @param context optional, the context to store errors in if this function + * fails + * @param str The string to observe + * @param callback Called with the observed string. + * @param user_data optional, arbitrary data, passed to the callback when it's called. + * @return NIX_OK if there were no errors. + * @see nix_get_string_callback + */ +nix_err call_nix_get_string_callback(const std::string str, nix_get_string_callback callback, void * user_data); + +#define NIXC_CATCH_ERRS \ + catch (...) \ + { \ + return nix_context_error(context); \ + } \ + return NIX_OK; + +#define NIXC_CATCH_ERRS_RES(def) \ + catch (...) \ + { \ + nix_context_error(context); \ + return def; \ + } +#define NIXC_CATCH_ERRS_NULL NIXC_CATCH_ERRS_RES(nullptr) + +#endif // NIX_API_UTIL_INTERNAL_H diff --git a/src/libutil/args.cc b/src/libutil/args.cc index a981ed9fb..834fc7314 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -285,7 +285,7 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) std::string line; std::getline(stream,line); - static const std::string commentChars("#/\\%@*-"); + static const std::string commentChars("#/\\%@*-("); std::string shebangContent; while (std::getline(stream,line) && !line.empty() && commentChars.find(line[0]) != std::string::npos){ line = chomp(line); diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc index d06f1f87b..d17401f27 100644 --- a/src/libutil/compression.cc +++ b/src/libutil/compression.cc @@ -12,8 +12,6 @@ #include #include -#include - namespace nix { static const int COMPRESSION_LEVEL_DEFAULT = -1; @@ -40,20 +38,26 @@ struct ArchiveDecompressionSource : Source { std::unique_ptr archive = 0; Source & src; - ArchiveDecompressionSource(Source & src) : src(src) {} + std::optional compressionMethod; + ArchiveDecompressionSource(Source & src, std::optional compressionMethod = std::nullopt) + : src(src) + , compressionMethod(std::move(compressionMethod)) + { + } ~ArchiveDecompressionSource() override {} - size_t read(char * data, size_t len) override { + size_t read(char * data, size_t len) override + { struct archive_entry * ae; if (!archive) { - archive = std::make_unique(src, true); - this->archive->check(archive_read_next_header(this->archive->archive, &ae), - "failed to read header (%s)"); + archive = std::make_unique(src, /*raw*/ true, compressionMethod); + this->archive->check(archive_read_next_header(this->archive->archive, &ae), "failed to read header (%s)"); if (archive_filter_count(this->archive->archive) < 2) { throw CompressionError("input compression not recognized"); } } ssize_t result = archive_read_data(this->archive->archive, data, len); - if (result > 0) return result; + if (result > 0) + return result; if (result == 0) { throw EndOfFile("reached end of compressed file"); } @@ -67,16 +71,19 @@ struct ArchiveCompressionSink : CompressionSink Sink & nextSink; struct archive * archive; - ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink) + ArchiveCompressionSink(Sink & nextSink, std::string format, bool parallel, int level = COMPRESSION_LEVEL_DEFAULT) + : nextSink(nextSink) { archive = archive_write_new(); - if (!archive) throw Error("failed to initialize libarchive"); + if (!archive) + throw Error("failed to initialize libarchive"); check(archive_write_add_filter_by_name(archive, format.c_str()), "couldn't initialize compression (%s)"); check(archive_write_set_format_raw(archive)); if (parallel) check(archive_write_set_filter_option(archive, format.c_str(), "threads", "0")); if (level != COMPRESSION_LEVEL_DEFAULT) - check(archive_write_set_filter_option(archive, format.c_str(), "compression-level", std::to_string(level).c_str())); + check(archive_write_set_filter_option( + archive, format.c_str(), "compression-level", std::to_string(level).c_str())); // disable internal buffering check(archive_write_set_bytes_per_block(archive, 0)); // disable output padding @@ -86,7 +93,8 @@ struct ArchiveCompressionSink : CompressionSink ~ArchiveCompressionSink() override { - if (archive) archive_write_free(archive); + if (archive) + archive_write_free(archive); } void finish() override @@ -106,7 +114,8 @@ struct ArchiveCompressionSink : CompressionSink void writeUnbuffered(std::string_view data) override { ssize_t result = archive_write_data(archive, data.data(), data.length()); - if (result <= 0) check(result); + if (result <= 0) + check(result); } private: @@ -130,13 +139,20 @@ private: struct NoneSink : CompressionSink { Sink & nextSink; - NoneSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) : nextSink(nextSink) + NoneSink(Sink & nextSink, int level = COMPRESSION_LEVEL_DEFAULT) + : nextSink(nextSink) { if (level != COMPRESSION_LEVEL_DEFAULT) warn("requested compression level '%d' not supported by compression method 'none'", level); } - void finish() override { flush(); } - void writeUnbuffered(std::string_view data) override { nextSink(data); } + void finish() override + { + flush(); + } + void writeUnbuffered(std::string_view data) override + { + nextSink(data); + } }; struct BrotliDecompressionSink : ChunkedCompressionSink @@ -145,7 +161,8 @@ struct BrotliDecompressionSink : ChunkedCompressionSink BrotliDecoderState * state; bool finished = false; - BrotliDecompressionSink(Sink & nextSink) : nextSink(nextSink) + BrotliDecompressionSink(Sink & nextSink) + : nextSink(nextSink) { state = BrotliDecoderCreateInstance(nullptr, nullptr, nullptr); if (!state) @@ -173,10 +190,7 @@ struct BrotliDecompressionSink : ChunkedCompressionSink while (!finished && (!data.data() || avail_in)) { checkInterrupt(); - if (!BrotliDecoderDecompressStream(state, - &avail_in, &next_in, - &avail_out, &next_out, - nullptr)) + if (!BrotliDecoderDecompressStream(state, &avail_in, &next_in, &avail_out, &next_out, nullptr)) throw CompressionError("error while decompressing brotli file"); if (avail_out < sizeof(outbuf) || avail_in == 0) { @@ -206,8 +220,8 @@ std::unique_ptr makeDecompressionSink(const std::string & method, Si else if (method == "br") return std::make_unique(nextSink); else - return sourceToSink([&](Source & source) { - auto decompressionSource = std::make_unique(source); + return sourceToSink([method, &nextSink](Source & source) { + auto decompressionSource = std::make_unique(source, method); decompressionSource->drainInto(nextSink); }); } @@ -219,7 +233,8 @@ struct BrotliCompressionSink : ChunkedCompressionSink BrotliEncoderState * state; bool finished = false; - BrotliCompressionSink(Sink & nextSink) : nextSink(nextSink) + BrotliCompressionSink(Sink & nextSink) + : nextSink(nextSink) { state = BrotliEncoderCreateInstance(nullptr, nullptr, nullptr); if (!state) @@ -247,11 +262,9 @@ struct BrotliCompressionSink : ChunkedCompressionSink while (!finished && (!data.data() || avail_in)) { checkInterrupt(); - if (!BrotliEncoderCompressStream(state, - data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, - &avail_in, &next_in, - &avail_out, &next_out, - nullptr)) + if (!BrotliEncoderCompressStream( + state, data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH, &avail_in, &next_in, + &avail_out, &next_out, nullptr)) throw CompressionError("error while compressing brotli compression"); if (avail_out < sizeof(outbuf) || avail_in == 0) { @@ -267,9 +280,8 @@ struct BrotliCompressionSink : ChunkedCompressionSink ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel, int level) { - std::vector la_supports = { - "bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", "lzip", "lzma", "lzop", "xz", "zstd" - }; + std::vector la_supports = {"bzip2", "compress", "grzip", "gzip", "lrzip", "lz4", + "lzip", "lzma", "lzop", "xz", "zstd"}; if (std::find(la_supports.begin(), la_supports.end(), method) != la_supports.end()) { return make_ref(nextSink, method, parallel, level); } diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh index 4e53a7b3c..e0c531b1f 100644 --- a/src/libutil/compression.hh +++ b/src/libutil/compression.hh @@ -11,7 +11,7 @@ namespace nix { struct CompressionSink : BufferedSink, FinishSink { - using BufferedSink::operator (); + using BufferedSink::operator(); using BufferedSink::writeUnbuffered; using FinishSink::finish; }; @@ -22,7 +22,8 @@ std::unique_ptr makeDecompressionSink(const std::string & method, Si std::string compress(const std::string & method, std::string_view in, const bool parallel = false, int level = -1); -ref makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1); +ref +makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false, int level = -1); MakeError(UnknownCompressionMethod, Error); diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 617c2ec89..efde8591b 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -124,7 +124,7 @@ static void applyConfigInner(const std::string & contents, const std::string & p auto p = absPath(tokens[1], dirOf(path)); if (pathExists(p)) { try { - std::string includedContents = readFile(path); + std::string includedContents = readFile(p); applyConfigInner(includedContents, p, parsedContents); } catch (SystemError &) { // TODO: Do we actually want to ignore this? Or is it better to fail? diff --git a/src/libutil/current-process.cc b/src/libutil/current-process.cc index f80f43ef0..d33f7163a 100644 --- a/src/libutil/current-process.cc +++ b/src/libutil/current-process.cc @@ -2,7 +2,6 @@ #include #include "current-process.hh" -#include "namespaces.hh" #include "util.hh" #include "finally.hh" #include "file-system.hh" @@ -17,6 +16,7 @@ # include # include # include "cgroup.hh" +# include "namespaces.hh" #endif #include @@ -82,9 +82,11 @@ void setStackSize(rlim_t stackSize) void restoreProcessContext(bool restoreMounts) { - restoreSignals(); + unix::restoreSignals(); if (restoreMounts) { + #if __linux__ restoreMountNamespace(); + #endif } if (savedStackSize) { diff --git a/src/libutil/environment-variables.cc b/src/libutil/environment-variables.cc index 6618d7872..d43197aa0 100644 --- a/src/libutil/environment-variables.cc +++ b/src/libutil/environment-variables.cc @@ -32,7 +32,6 @@ std::map getEnv() return env; } - void clearEnv() { for (auto & name : getEnv()) @@ -43,7 +42,7 @@ void replaceEnv(const std::map & newEnv) { clearEnv(); for (auto & newEnvVar : newEnv) - setenv(newEnvVar.first.c_str(), newEnvVar.second.c_str(), 1); + setEnv(newEnvVar.first.c_str(), newEnvVar.second.c_str()); } } diff --git a/src/libutil/environment-variables.hh b/src/libutil/environment-variables.hh index 21eb4619b..21c2356a4 100644 --- a/src/libutil/environment-variables.hh +++ b/src/libutil/environment-variables.hh @@ -28,6 +28,14 @@ std::optional getEnvNonEmpty(const std::string & key); */ std::map getEnv(); +/** + * Like POSIX `setenv`, but always overrides. + * + * We don't need the non-overriding version, and this is easier to + * reimplement on Windows. + */ +int setEnv(const char * name, const char * value); + /** * Clear the environment. */ diff --git a/src/libutil/error.cc b/src/libutil/error.cc index d1e864a1a..fd4f4efd1 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -11,14 +11,15 @@ namespace nix { -void BaseError::addTrace(std::shared_ptr && e, HintFmt hint) +void BaseError::addTrace(std::shared_ptr && e, HintFmt hint, TracePrint print) { - err.traces.push_front(Trace { .pos = std::move(e), .hint = hint }); + err.traces.push_front(Trace { .pos = std::move(e), .hint = hint, .print = print }); } -void throwExceptionSelfCheck(){ +void throwExceptionSelfCheck() +{ // This is meant to be caught in initLibUtil() - throw SysError("C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded."); + throw Error("C++ exception handling is broken. This would appear to be a problem with the way Nix was compiled and/or linked and/or loaded."); } // c++ std::exception descendants must have a 'const char* what()' function. @@ -163,7 +164,7 @@ static bool printPosMaybe(std::ostream & oss, std::string_view indent, const std return hasPos; } -void printTrace( +static void printTrace( std::ostream & output, const std::string_view & indent, size_t & count, @@ -379,29 +380,39 @@ std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool s // A consecutive sequence of stack traces that are all in `tracesSeen`. std::vector skippedTraces; size_t count = 0; + bool truncate = false; for (const auto & trace : einfo.traces) { if (trace.hint.str().empty()) continue; if (!showTrace && count > 3) { - oss << "\n" << ANSI_WARNING "(stack trace truncated; use '--show-trace' to show the full trace)" ANSI_NORMAL << "\n"; - break; + truncate = true; } - if (tracesSeen.count(trace)) { - skippedTraces.push_back(trace); - continue; + if (!truncate || trace.print == TracePrint::Always) { + + if (tracesSeen.count(trace)) { + skippedTraces.push_back(trace); + continue; + } + + tracesSeen.insert(trace); + + printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen); + + count++; + + printTrace(oss, ellipsisIndent, count, trace); } - tracesSeen.insert(trace); - - printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen); - - count++; - - printTrace(oss, ellipsisIndent, count, trace); } + printSkippedTracesMaybe(oss, ellipsisIndent, count, skippedTraces, tracesSeen); + + if (truncate) { + oss << "\n" << ANSI_WARNING "(stack trace truncated; use '--show-trace' to show the full, detailed trace)" ANSI_NORMAL << "\n"; + } + oss << "\n" << prefix; } diff --git a/src/libutil/error.hh b/src/libutil/error.hh index 89f5ad021..445b1e19c 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -61,9 +61,22 @@ void printCodeLines(std::ostream & out, const Pos & errPos, const LinesOfCode & loc); +/** + * When a stack frame is printed. + */ +enum struct TracePrint { + /** + * The default behavior; always printed when `--show-trace` is set. + */ + Default, + /** Always printed. Produced by `builtins.addErrorContext`. */ + Always, +}; + struct Trace { std::shared_ptr pos; HintFmt hint; + TracePrint print = TracePrint::Default; }; inline bool operator<(const Trace& lhs, const Trace& rhs); @@ -137,6 +150,10 @@ public: : err(e) { } + std::string message() { + return err.msg.str(); + } + const char * what() const noexcept override { return calcWhat().c_str(); } const std::string & msg() const { return calcWhat(); } const ErrorInfo & info() const { calcWhat(); return err; } @@ -161,7 +178,7 @@ public: addTrace(std::move(e), HintFmt(std::string(fs), args...)); } - void addTrace(std::shared_ptr && e, HintFmt hint); + void addTrace(std::shared_ptr && e, HintFmt hint, TracePrint print = TracePrint::Default); bool hasTrace() const { return !err.traces.empty(); } diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 374e674af..1e7469cad 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -203,16 +203,6 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "https://github.com/NixOS/nix/milestone/40", }, - { - .tag = Xp::ReplFlake, - .name = "repl-flake", - .description = R"( - *Enabled with [`flakes`](#xp-feature-flakes) since 2.19* - - Allow passing [installables](@docroot@/command-ref/new-cli/nix.md#installables) to `nix repl`, making its interface consistent with the other experimental commands. - )", - .trackingUrl = "https://github.com/NixOS/nix/milestone/32", - }, { .tag = Xp::AutoAllocateUids, .name = "auto-allocate-uids", @@ -272,6 +262,14 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "https://github.com/NixOS/nix/milestone/46", }, + { + .tag = Xp::LocalOverlayStore, + .name = "local-overlay-store", + .description = R"( + Allow the use of [local overlay store](@docroot@/command-ref/new-cli/nix3-help-stores.md#local-overlay-store). + )", + .trackingUrl = "https://github.com/NixOS/nix/milestone/50", + }, { .tag = Xp::ConfigurableImpureEnv, .name = "configurable-impure-env", diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh index eae4fa9b8..1da2a3ff5 100644 --- a/src/libutil/experimental-features.hh +++ b/src/libutil/experimental-features.hh @@ -26,13 +26,13 @@ enum struct ExperimentalFeature RecursiveNix, NoUrlLiterals, FetchClosure, - ReplFlake, AutoAllocateUids, Cgroups, DaemonTrustOverride, DynamicDerivations, ParseTomlTimestamps, ReadOnlyLocalStore, + LocalOverlayStore, ConfigurableImpureEnv, MountedSSHStore, VerifiedFetches, diff --git a/src/libutil/file-descriptor.cc b/src/libutil/file-descriptor.cc index 55d57e29b..95cbb8537 100644 --- a/src/libutil/file-descriptor.cc +++ b/src/libutil/file-descriptor.cc @@ -8,74 +8,14 @@ namespace nix { -std::string readFile(int fd) -{ - struct stat st; - if (fstat(fd, &st) == -1) - throw SysError("statting file"); - - return drainFD(fd, true, st.st_size); -} - - -void readFull(int fd, char * buf, size_t count) -{ - while (count) { - checkInterrupt(); - ssize_t res = read(fd, buf, count); - if (res == -1) { - if (errno == EINTR) continue; - throw SysError("reading from file"); - } - if (res == 0) throw EndOfFile("unexpected end-of-file"); - count -= res; - buf += res; - } -} - - -void writeFull(int fd, std::string_view s, bool allowInterrupts) -{ - while (!s.empty()) { - if (allowInterrupts) checkInterrupt(); - ssize_t res = write(fd, s.data(), s.size()); - if (res == -1 && errno != EINTR) - throw SysError("writing to file"); - if (res > 0) - s.remove_prefix(res); - } -} - - -std::string readLine(int fd) -{ - std::string s; - while (1) { - checkInterrupt(); - char ch; - // FIXME: inefficient - ssize_t rd = read(fd, &ch, 1); - if (rd == -1) { - if (errno != EINTR) - throw SysError("reading a line"); - } else if (rd == 0) - throw EndOfFile("unexpected EOF reading a line"); - else { - if (ch == '\n') return s; - s += ch; - } - } -} - - -void writeLine(int fd, std::string s) +void writeLine(Descriptor fd, std::string s) { s += '\n'; writeFull(fd, s); } -std::string drainFD(int fd, bool block, const size_t reserveSize) +std::string drainFD(Descriptor fd, bool block, const size_t reserveSize) { // the parser needs two extra bytes to append terminating characters, other users will // not care very much about the extra memory. @@ -85,50 +25,18 @@ std::string drainFD(int fd, bool block, const size_t reserveSize) } -void drainFD(int fd, Sink & sink, bool block) -{ - // silence GCC maybe-uninitialized warning in finally - int saved = 0; - - if (!block) { - saved = fcntl(fd, F_GETFL); - if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) - throw SysError("making file descriptor non-blocking"); - } - - Finally finally([&] { - if (!block) { - if (fcntl(fd, F_SETFL, saved) == -1) - throw SysError("making file descriptor blocking"); - } - }); - - std::vector buf(64 * 1024); - while (1) { - checkInterrupt(); - ssize_t rd = read(fd, buf.data(), buf.size()); - if (rd == -1) { - if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) - break; - if (errno != EINTR) - throw SysError("reading from file"); - } - else if (rd == 0) break; - else sink({reinterpret_cast(buf.data()), size_t(rd)}); - } -} - ////////////////////////////////////////////////////////////////////// -AutoCloseFD::AutoCloseFD() : fd{-1} {} + +AutoCloseFD::AutoCloseFD() : fd{INVALID_DESCRIPTOR} {} -AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {} +AutoCloseFD::AutoCloseFD(Descriptor fd) : fd{fd} {} AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd} { - that.fd = -1; + that.fd = INVALID_DESCRIPTOR; } @@ -136,7 +44,7 @@ AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that) { close(); fd = that.fd; - that.fd = -1; + that.fd = INVALID_DESCRIPTOR; return *this; } @@ -151,7 +59,7 @@ AutoCloseFD::~AutoCloseFD() } -int AutoCloseFD::get() const +Descriptor AutoCloseFD::get() const { return fd; } @@ -159,56 +67,46 @@ int AutoCloseFD::get() const void AutoCloseFD::close() { - if (fd != -1) { - if (::close(fd) == -1) + if (fd != INVALID_DESCRIPTOR) { + if(::close(fd) == -1) /* This should never happen. */ throw SysError("closing file descriptor %1%", fd); - fd = -1; + fd = INVALID_DESCRIPTOR; } } void AutoCloseFD::fsync() { - if (fd != -1) { - int result; + if (fd != INVALID_DESCRIPTOR) { + int result; + result = #if __APPLE__ - result = ::fcntl(fd, F_FULLFSYNC); + ::fcntl(fd, F_FULLFSYNC) #else - result = ::fsync(fd); + ::fsync(fd) #endif - if (result == -1) - throw SysError("fsync file descriptor %1%", fd); - } + ; + if (result == -1) + throw SysError("fsync file descriptor %1%", fd); + } } AutoCloseFD::operator bool() const { - return fd != -1; + return fd != INVALID_DESCRIPTOR; } -int AutoCloseFD::release() +Descriptor AutoCloseFD::release() { - int oldFD = fd; - fd = -1; + Descriptor oldFD = fd; + fd = INVALID_DESCRIPTOR; return oldFD; } -void Pipe::create() -{ - int fds[2]; -#if HAVE_PIPE2 - if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); -#else - if (pipe(fds) != 0) throw SysError("creating pipe"); - closeOnExec(fds[0]); - closeOnExec(fds[1]); -#endif - readSide = fds[0]; - writeSide = fds[1]; -} +////////////////////////////////////////////////////////////////////// void Pipe::close() @@ -217,38 +115,4 @@ void Pipe::close() writeSide.close(); } -////////////////////////////////////////////////////////////////////// - -void closeMostFDs(const std::set & exceptions) -{ -#if __linux__ - try { - for (auto & s : readDirectory("/proc/self/fd")) { - auto fd = std::stoi(s.name); - if (!exceptions.count(fd)) { - debug("closing leaked FD %d", fd); - close(fd); - } - } - return; - } catch (SystemError &) { - } -#endif - - int maxFD = 0; - maxFD = sysconf(_SC_OPEN_MAX); - for (int fd = 0; fd < maxFD; ++fd) - if (!exceptions.count(fd)) - close(fd); /* ignore result */ -} - - -void closeOnExec(int fd) -{ - int prev; - if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || - fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) - throw SysError("setting close-on-exec flag"); -} - } diff --git a/src/libutil/file-descriptor.hh b/src/libutil/file-descriptor.hh index 80ec86135..719e1e444 100644 --- a/src/libutil/file-descriptor.hh +++ b/src/libutil/file-descriptor.hh @@ -9,53 +9,85 @@ namespace nix { struct Sink; struct Source; +/** + * Operating System capability + */ +typedef int Descriptor; + +const Descriptor INVALID_DESCRIPTOR = -1; + +/** + * Convert a native `Descriptor` to a POSIX file descriptor + * + * This is a no-op except on Windows. + */ +static inline Descriptor toDescriptor(int fd) +{ + return fd; +} + +/** + * Convert a POSIX file descriptor to a native `Descriptor` + * + * This is a no-op except on Windows. + */ +static inline int fromDescriptor(Descriptor fd, int flags) +{ + return fd; +} + /** * Read the contents of a resource into a string. */ -std::string readFile(int fd); +std::string readFile(Descriptor fd); /** * Wrappers arount read()/write() that read/write exactly the * requested number of bytes. */ -void readFull(int fd, char * buf, size_t count); +void readFull(Descriptor fd, char * buf, size_t count); -void writeFull(int fd, std::string_view s, bool allowInterrupts = true); +void writeFull(Descriptor fd, std::string_view s, bool allowInterrupts = true); /** * Read a line from a file descriptor. */ -std::string readLine(int fd); +std::string readLine(Descriptor fd); /** * Write a line to a file descriptor. */ -void writeLine(int fd, std::string s); +void writeLine(Descriptor fd, std::string s); /** * Read a file descriptor until EOF occurs. */ -std::string drainFD(int fd, bool block = true, const size_t reserveSize=0); +std::string drainFD(Descriptor fd, bool block = true, const size_t reserveSize=0); -void drainFD(int fd, Sink & sink, bool block = true); +void drainFD(Descriptor fd, Sink & sink, bool block = true); + +[[gnu::always_inline]] +inline Descriptor getStandardOut() { + return STDOUT_FILENO; +} /** * Automatic cleanup of resources. */ class AutoCloseFD { - int fd; + Descriptor fd; public: AutoCloseFD(); - AutoCloseFD(int fd); + AutoCloseFD(Descriptor fd); AutoCloseFD(const AutoCloseFD & fd) = delete; AutoCloseFD(AutoCloseFD&& fd); ~AutoCloseFD(); AutoCloseFD& operator =(const AutoCloseFD & fd) = delete; AutoCloseFD& operator =(AutoCloseFD&& fd); - int get() const; + Descriptor get() const; explicit operator bool() const; - int release(); + Descriptor release(); void close(); void fsync(); }; @@ -72,12 +104,12 @@ public: * Close all file descriptors except those listed in the given set. * Good practice in child processes. */ -void closeMostFDs(const std::set & exceptions); +void closeMostFDs(const std::set & exceptions); /** * Set the close-on-exec flag for the given file descriptor. */ -void closeOnExec(int fd); +void closeOnExec(Descriptor fd); MakeError(EndOfFile, Error); diff --git a/src/libutil/file-system.cc b/src/libutil/file-system.cc index 9f81ee452..89d309731 100644 --- a/src/libutil/file-system.cc +++ b/src/libutil/file-system.cc @@ -128,7 +128,7 @@ std::string_view baseNameOf(std::string_view path) return ""; auto last = path.size() - 1; - if (path[last] == '/' && last > 0) + while (last > 0 && path[last] == '/') last -= 1; auto pos = path.rfind('/', last); @@ -174,15 +174,23 @@ struct stat lstat(const Path & path) } +std::optional maybeLstat(const Path & path) +{ + std::optional st{std::in_place}; + if (lstat(path.c_str(), &*st)) + { + if (errno == ENOENT || errno == ENOTDIR) + st.reset(); + else + throw SysError("getting status of '%s'", path); + } + return st; +} + + bool pathExists(const Path & path) { - int res; - struct stat st; - res = lstat(path.c_str(), &st); - if (!res) return true; - if (errno != ENOENT && errno != ENOTDIR) - throw SysError("getting status of %1%", path); - return false; + return maybeLstat(path).has_value(); } bool pathAccessible(const Path & path) diff --git a/src/libutil/file-system.hh b/src/libutil/file-system.hh index 9d565c881..06a993829 100644 --- a/src/libutil/file-system.hh +++ b/src/libutil/file-system.hh @@ -84,6 +84,11 @@ bool isDirOrInDir(std::string_view path, std::string_view dir); */ struct stat stat(const Path & path); struct stat lstat(const Path & path); +/** + * `lstat` the given path if it exists. + * @return std::nullopt if the path doesn't exist, or an optional containing the result of `lstat` otherwise + */ +std::optional maybeLstat(const Path & path); /** * @return true iff the given path exists. diff --git a/src/libutil/fmt.hh b/src/libutil/fmt.hh index abbaf95b6..c178257d4 100644 --- a/src/libutil/fmt.hh +++ b/src/libutil/fmt.hh @@ -144,6 +144,10 @@ public: : HintFmt("%s", Uncolored(literal)) { } + static HintFmt fromFormatString(const std::string & format) { + return HintFmt(boost::format(format)); + } + /** * Interpolate the given arguments into the format string. */ diff --git a/src/libutil/json-utils.cc b/src/libutil/json-utils.cc index 61cef743d..1b911bf75 100644 --- a/src/libutil/json-utils.cc +++ b/src/libutil/json-utils.cc @@ -1,5 +1,8 @@ #include "json-utils.hh" #include "error.hh" +#include "types.hh" +#include +#include namespace nix { @@ -18,26 +21,113 @@ nlohmann::json * get(nlohmann::json & map, const std::string & key) } const nlohmann::json & valueAt( - const nlohmann::json & map, + const nlohmann::json::object_t & map, const std::string & key) { if (!map.contains(key)) - throw Error("Expected JSON object to contain key '%s' but it doesn't", key); + throw Error("Expected JSON object to contain key '%s' but it doesn't: %s", key, nlohmann::json(map).dump()); - return map[key]; + return map.at(key); } -const nlohmann::json & ensureType( +std::optional optionalValueAt(const nlohmann::json::object_t & map, const std::string & key) +{ + if (!map.contains(key)) + return std::nullopt; + + return std::optional { map.at(key) }; +} + + +std::optional getNullable(const nlohmann::json & value) +{ + if (value.is_null()) + return std::nullopt; + + return value.get(); +} + +/** + * Ensure the type of a JSON object is what you expect, failing with a + * ensure type if it isn't. + * + * Use before type conversions and element access to avoid ugly + * exceptions, but only part of this module to define the other `get*` + * functions. It is too cumbersome and easy to forget to expect regular + * JSON code to use it directly. + */ +static const nlohmann::json & ensureType( const nlohmann::json & value, nlohmann::json::value_type expectedType ) { if (value.type() != expectedType) throw Error( - "Expected JSON value to be of type '%s' but it is of type '%s'", + "Expected JSON value to be of type '%s' but it is of type '%s': %s", nlohmann::json(expectedType).type_name(), - value.type_name()); + value.type_name(), value.dump()); return value; } + +const nlohmann::json::object_t & getObject(const nlohmann::json & value) +{ + return ensureType(value, nlohmann::json::value_t::object).get_ref(); +} + +const nlohmann::json::array_t & getArray(const nlohmann::json & value) +{ + return ensureType(value, nlohmann::json::value_t::array).get_ref(); +} + +const nlohmann::json::string_t & getString(const nlohmann::json & value) +{ + return ensureType(value, nlohmann::json::value_t::string).get_ref(); +} + +const nlohmann::json::number_integer_t & getInteger(const nlohmann::json & value) +{ + return ensureType(value, nlohmann::json::value_t::number_integer).get_ref(); +} + +const nlohmann::json::boolean_t & getBoolean(const nlohmann::json & value) +{ + return ensureType(value, nlohmann::json::value_t::boolean).get_ref(); +} + +Strings getStringList(const nlohmann::json & value) +{ + auto & jsonArray = getArray(value); + + Strings stringList; + + for (const auto & elem : jsonArray) + stringList.push_back(getString(elem)); + + return stringList; +} + +StringMap getStringMap(const nlohmann::json & value) +{ + auto & jsonObject = getObject(value); + + StringMap stringMap; + + for (const auto & [key, value] : jsonObject) + stringMap[getString(key)] = getString(value); + + return stringMap; +} + +StringSet getStringSet(const nlohmann::json & value) +{ + auto & jsonArray = getArray(value); + + StringSet stringSet; + + for (const auto & elem : jsonArray) + stringSet.insert(getString(elem)); + + return stringSet; +} } diff --git a/src/libutil/json-utils.hh b/src/libutil/json-utils.hh index 06dd80cf7..08c98cc8c 100644 --- a/src/libutil/json-utils.hh +++ b/src/libutil/json-utils.hh @@ -3,6 +3,9 @@ #include #include +#include + +#include "types.hh" namespace nix { @@ -11,26 +14,30 @@ const nlohmann::json * get(const nlohmann::json & map, const std::string & key); nlohmann::json * get(nlohmann::json & map, const std::string & key); /** - * Get the value of a json object at a key safely, failing - * with a Nix Error if the key does not exist. + * Get the value of a json object at a key safely, failing with a nice + * error if the key does not exist. * * Use instead of nlohmann::json::at() to avoid ugly exceptions. - * - * _Does not check whether `map` is an object_, use `ensureType` for that. */ const nlohmann::json & valueAt( - const nlohmann::json & map, + const nlohmann::json::object_t & map, const std::string & key); +std::optional optionalValueAt(const nlohmann::json::object_t & value, const std::string & key); + /** - * Ensure the type of a json object is what you expect, failing - * with a Nix Error if it isn't. - * - * Use before type conversions and element access to avoid ugly exceptions. + * Downcast the json object, failing with a nice error if the conversion fails. + * See https://json.nlohmann.me/features/types/ */ -const nlohmann::json & ensureType( - const nlohmann::json & value, - nlohmann::json::value_type expectedType); +std::optional getNullable(const nlohmann::json & value); +const nlohmann::json::object_t & getObject(const nlohmann::json & value); +const nlohmann::json::array_t & getArray(const nlohmann::json & value); +const nlohmann::json::string_t & getString(const nlohmann::json & value); +const nlohmann::json::number_integer_t & getInteger(const nlohmann::json & value); +const nlohmann::json::boolean_t & getBoolean(const nlohmann::json & value); +Strings getStringList(const nlohmann::json & value); +StringMap getStringMap(const nlohmann::json & value); +StringSet getStringSet(const nlohmann::json & value); /** * For `adl_serializer>` below, we need to track what diff --git a/src/libutil/cgroup.cc b/src/libutil/linux/cgroup.cc similarity index 99% rename from src/libutil/cgroup.cc rename to src/libutil/linux/cgroup.cc index de83b5ad1..8b8942643 100644 --- a/src/libutil/cgroup.cc +++ b/src/libutil/linux/cgroup.cc @@ -1,5 +1,3 @@ -#if __linux__ - #include "cgroup.hh" #include "util.hh" #include "file-system.hh" @@ -145,5 +143,3 @@ CgroupStats destroyCgroup(const Path & cgroup) } } - -#endif diff --git a/src/libutil/cgroup.hh b/src/libutil/linux/cgroup.hh similarity index 96% rename from src/libutil/cgroup.hh rename to src/libutil/linux/cgroup.hh index 574ae8e5b..783a0ab87 100644 --- a/src/libutil/cgroup.hh +++ b/src/libutil/linux/cgroup.hh @@ -1,8 +1,6 @@ #pragma once ///@file -#if __linux__ - #include #include @@ -28,5 +26,3 @@ struct CgroupStats CgroupStats destroyCgroup(const Path & cgroup); } - -#endif diff --git a/src/libutil/namespaces.cc b/src/libutil/linux/namespaces.cc similarity index 95% rename from src/libutil/namespaces.cc rename to src/libutil/linux/namespaces.cc index a789b321e..f8289ef39 100644 --- a/src/libutil/namespaces.cc +++ b/src/libutil/linux/namespaces.cc @@ -5,18 +5,14 @@ #include "processes.hh" #include "signals.hh" -#if __linux__ -# include -# include -# include "cgroup.hh" -#endif +#include +#include +#include "cgroup.hh" #include namespace nix { -#if __linux__ - bool userNamespacesSupported() { static auto res = [&]() -> bool @@ -101,19 +97,14 @@ bool mountAndPidNamespacesSupported() return res; } -#endif - ////////////////////////////////////////////////////////////////////// -#if __linux__ static AutoCloseFD fdSavedMountNamespace; static AutoCloseFD fdSavedRoot; -#endif void saveMountNamespace() { -#if __linux__ static std::once_flag done; std::call_once(done, []() { fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); @@ -122,12 +113,10 @@ void saveMountNamespace() fdSavedRoot = open("/proc/self/root", O_RDONLY); }); -#endif } void restoreMountNamespace() { -#if __linux__ try { auto savedCwd = absPath("."); @@ -146,15 +135,12 @@ void restoreMountNamespace() } catch (Error & e) { debug(e.msg()); } -#endif } void unshareFilesystem() { -#ifdef __linux__ if (unshare(CLONE_FS) != 0 && errno != EPERM) throw SysError("unsharing filesystem state in download thread"); -#endif } } diff --git a/src/libutil/namespaces.hh b/src/libutil/linux/namespaces.hh similarity index 96% rename from src/libutil/namespaces.hh rename to src/libutil/linux/namespaces.hh index 7e4e921a8..ef3c9123f 100644 --- a/src/libutil/namespaces.hh +++ b/src/libutil/linux/namespaces.hh @@ -26,12 +26,8 @@ void restoreMountNamespace(); */ void unshareFilesystem(); -#if __linux__ - bool userNamespacesSupported(); bool mountAndPidNamespacesSupported(); -#endif - } diff --git a/src/libutil/local.mk b/src/libutil/local.mk index 200026c1e..9773ef64f 100644 --- a/src/libutil/local.mk +++ b/src/libutil/local.mk @@ -5,8 +5,23 @@ libutil_NAME = libnixutil libutil_DIR := $(d) libutil_SOURCES := $(wildcard $(d)/*.cc $(d)/signature/*.cc) +ifdef HOST_UNIX + libutil_SOURCES += $(wildcard $(d)/unix/*.cc) +endif +ifdef HOST_LINUX + libutil_SOURCES += $(wildcard $(d)/linux/*.cc) +endif -libutil_CXXFLAGS += -I src/libutil +# Not just for this library itself, but also for downstream libraries using this library + +INCLUDE_libutil := -I $(d) +ifdef HOST_UNIX + INCLUDE_libutil += -I $(d)/unix +endif +ifdef HOST_LINUX + INCLUDE_libutil += -I $(d)/linux +endif +libutil_CXXFLAGS += $(INCLUDE_libutil) libutil_LDFLAGS += $(THREAD_LDFLAGS) $(LIBCURL_LIBS) $(SODIUM_LIBS) $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 89fbd194a..5024c6081 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -37,8 +37,9 @@ void Logger::warn(const std::string & msg) void Logger::writeToStdout(std::string_view s) { - writeFull(STDOUT_FILENO, s); - writeFull(STDOUT_FILENO, "\n"); + Descriptor standard_out = getStandardOut(); + writeFull(standard_out, s); + writeFull(standard_out, "\n"); } class SimpleLogger : public Logger @@ -52,7 +53,7 @@ public: : printBuildLogs(printBuildLogs) { systemd = getEnv("IN_SYSTEMD") == "1"; - tty = shouldANSI(); + tty = isTTY(); } bool isVerbose() override { diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index 41c2db59a..8039d4b80 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -97,13 +97,7 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & pa if (i != cache->end()) return i->second; } - std::optional st{std::in_place}; - if (::lstat(absPath.c_str(), &*st)) { - if (errno == ENOENT || errno == ENOTDIR) - st.reset(); - else - throw SysError("getting status of '%s'", showPath(path)); - } + auto st = nix::maybeLstat(absPath.c_str()); auto cache(_cache.lock()); if (cache->size() >= 16384) cache->clear(); diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index d9522566f..6249ddaf5 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -119,18 +119,18 @@ protected: */ struct FdSink : BufferedSink { - int fd; + Descriptor fd; size_t written = 0; - FdSink() : fd(-1) { } - FdSink(int fd) : fd(fd) { } + FdSink() : fd(INVALID_DESCRIPTOR) { } + FdSink(Descriptor fd) : fd(fd) { } FdSink(FdSink&&) = default; FdSink & operator=(FdSink && s) { flush(); fd = s.fd; - s.fd = -1; + s.fd = INVALID_DESCRIPTOR; written = s.written; return *this; } @@ -151,18 +151,18 @@ private: */ struct FdSource : BufferedSource { - int fd; + Descriptor fd; size_t read = 0; BackedStringView endOfFileError{"unexpected end-of-file"}; - FdSource() : fd(-1) { } - FdSource(int fd) : fd(fd) { } + FdSource() : fd(INVALID_DESCRIPTOR) { } + FdSource(Descriptor fd) : fd(fd) { } FdSource(FdSource &&) = default; FdSource & operator=(FdSource && s) { fd = s.fd; - s.fd = -1; + s.fd = INVALID_DESCRIPTOR; read = s.read; return *this; } diff --git a/src/libutil/signals.hh b/src/libutil/signals.hh index 7e8beff33..8bff345c3 100644 --- a/src/libutil/signals.hh +++ b/src/libutil/signals.hh @@ -4,72 +4,39 @@ #include "types.hh" #include "error.hh" #include "logging.hh" -#include "ansicolor.hh" -#include -#include -#include -#include -#include - -#include - -#include #include -#include -#include -#include namespace nix { /* User interruption. */ -extern std::atomic _isInterrupted; +/** + * @note Does nothing on Windows + */ +static inline void setInterrupted(bool isInterrupted); -extern thread_local std::function interruptCheck; +/** + * @note Does nothing on Windows + */ +static inline bool getInterrupted(); +/** + * @note Does nothing on Windows + */ void setInterruptThrown(); -void _interrupted(); - -void inline checkInterrupt() -{ - if (_isInterrupted || (interruptCheck && interruptCheck())) - _interrupted(); -} +/** + * @note Does nothing on Windows + */ +inline void checkInterrupt(); +/** + * @note Never will happen on Windows + */ MakeError(Interrupted, BaseError); -/** - * Start a thread that handles various signals. Also block those signals - * on the current thread (and thus any threads created by it). - * Saves the signal mask before changing the mask to block those signals. - * See saveSignalMask(). - */ -void startSignalHandlerThread(); - -/** - * Saves the signal mask, which is the signal mask that nix will restore - * before creating child processes. - * See setChildSignalMask() to set an arbitrary signal mask instead of the - * current mask. - */ -void saveSignalMask(); - -/** - * To use in a process that already called `startSignalHandlerThread()` - * or `saveSignalMask()` first. - */ -void restoreSignals(); - -/** - * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't - * necessarily match the current thread's mask. - * See saveSignalMask() to set the saved mask to the current mask. - */ -void setChildSignalMask(sigset_t *sigs); - struct InterruptCallback { virtual ~InterruptCallback() { }; @@ -78,27 +45,21 @@ struct InterruptCallback /** * Register a function that gets called on SIGINT (in a non-signal * context). + * + * @note Does nothing on Windows */ std::unique_ptr createInterruptCallback( std::function callback); -void triggerInterrupt(); - /** * A RAII class that causes the current thread to receive SIGUSR1 when * the signal handler thread receives SIGINT. That is, this allows * SIGINT to be multiplexed to multiple threads. + * + * @note Does nothing on Windows */ -struct ReceiveInterrupts -{ - pthread_t target; - std::unique_ptr callback; - - ReceiveInterrupts() - : target(pthread_self()) - , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) - { } -}; - +struct ReceiveInterrupts; } + +#include "signals-impl.hh" diff --git a/src/libutil/source-accessor.cc b/src/libutil/source-accessor.cc index afbbbe1a9..66093d2cc 100644 --- a/src/libutil/source-accessor.cc +++ b/src/libutil/source-accessor.cc @@ -39,9 +39,9 @@ void SourceAccessor::readFile( } Hash SourceAccessor::hashPath( - const CanonPath & path, - PathFilter & filter, - HashAlgorithm ha) + const CanonPath & path, + PathFilter & filter, + HashAlgorithm ha) { HashSink sink(ha); dumpPath(path, sink, filter); @@ -67,4 +67,42 @@ std::string SourceAccessor::showPath(const CanonPath & path) return displayPrefix + path.abs() + displaySuffix; } +CanonPath SourceAccessor::resolveSymlinks( + const CanonPath & path, + SymlinkResolution mode) +{ + auto res = CanonPath::root; + + int linksAllowed = 1024; + + std::list todo; + for (auto & c : path) + todo.push_back(std::string(c)); + + while (!todo.empty()) { + auto c = *todo.begin(); + todo.pop_front(); + if (c == "" || c == ".") + ; + else if (c == "..") + res.pop(); + else { + res.push(c); + if (mode == SymlinkResolution::Full || !todo.empty()) { + if (auto st = maybeLstat(res); st && st->type == SourceAccessor::tSymlink) { + if (!linksAllowed--) + throw Error("infinite symlink recursion in path '%s'", showPath(path)); + auto target = readLink(res); + res.pop(); + if (hasPrefix(target, "/")) + res = CanonPath::root; + todo.splice(todo.begin(), tokenizeString>(target, "/")); + } + } + } + } + + return res; +} + } diff --git a/src/libutil/source-accessor.hh b/src/libutil/source-accessor.hh index aff7da09c..1f272327f 100644 --- a/src/libutil/source-accessor.hh +++ b/src/libutil/source-accessor.hh @@ -9,6 +9,26 @@ namespace nix { struct Sink; +/** + * Note there is a decent chance this type soon goes away because the problem is solved another way. + * See the discussion in https://github.com/NixOS/nix/pull/9985. + */ +enum class SymlinkResolution { + /** + * Resolve symlinks in the ancestors only. + * + * Only the last component of the result is possibly a symlink. + */ + Ancestors, + + /** + * Resolve symlinks fully, realpath(3)-style. + * + * No component of the result will be a symlink. + */ + Full, +}; + /** * A read-only filesystem abstraction. This is used by the Nix * evaluator and elsewhere for accessing sources in various @@ -112,9 +132,9 @@ struct SourceAccessor PathFilter & filter = defaultPathFilter); Hash hashPath( - const CanonPath & path, - PathFilter & filter = defaultPathFilter, - HashAlgorithm ha = HashAlgorithm::SHA256); + const CanonPath & path, + PathFilter & filter = defaultPathFilter, + HashAlgorithm ha = HashAlgorithm::SHA256); /** * Return a corresponding path in the root filesystem, if @@ -137,6 +157,17 @@ struct SourceAccessor void setPathDisplay(std::string displayPrefix, std::string displaySuffix = ""); virtual std::string showPath(const CanonPath & path); + + /** + * Resolve any symlinks in `path` according to the given + * resolution mode. + * + * @param mode might only be a temporary solution for this. + * See the discussion in https://github.com/NixOS/nix/pull/9985. + */ + CanonPath resolveSymlinks( + const CanonPath & path, + SymlinkResolution mode = SymlinkResolution::Full); }; } diff --git a/src/libutil/source-path.cc b/src/libutil/source-path.cc index 56ae1d699..2a5b20858 100644 --- a/src/libutil/source-path.cc +++ b/src/libutil/source-path.cc @@ -62,44 +62,6 @@ bool SourcePath::operator<(const SourcePath & x) const return std::tie(*accessor, path) < std::tie(*x.accessor, x.path); } -SourcePath SourcePath::resolveSymlinks(SymlinkResolution mode) const -{ - auto res = SourcePath(accessor); - - int linksAllowed = 1024; - - std::list todo; - for (auto & c : path) - todo.push_back(std::string(c)); - - bool resolve_last = mode == SymlinkResolution::Full; - - while (!todo.empty()) { - auto c = *todo.begin(); - todo.pop_front(); - if (c == "" || c == ".") - ; - else if (c == "..") - res.path.pop(); - else { - res.path.push(c); - if (resolve_last || !todo.empty()) { - if (auto st = res.maybeLstat(); st && st->type == InputAccessor::tSymlink) { - if (!linksAllowed--) - throw Error("infinite symlink recursion in path '%s'", path); - auto target = res.readLink(); - res.path.pop(); - if (hasPrefix(target, "/")) - res.path = CanonPath::root; - todo.splice(todo.begin(), tokenizeString>(target, "/")); - } - } - } - } - - return res; -} - std::ostream & operator<<(std::ostream & str, const SourcePath & path) { str << path.to_string(); diff --git a/src/libutil/source-path.hh b/src/libutil/source-path.hh index 59991c640..b8f69af12 100644 --- a/src/libutil/source-path.hh +++ b/src/libutil/source-path.hh @@ -11,26 +11,6 @@ namespace nix { -/** - * Note there is a decent chance this type soon goes away because the problem is solved another way. - * See the discussion in https://github.com/NixOS/nix/pull/9985. - */ -enum class SymlinkResolution { - /** - * Resolve symlinks in the ancestors only. - * - * Only the last component of the result is possibly a symlink. - */ - Ancestors, - - /** - * Resolve symlinks fully, realpath(3)-style. - * - * No component of the result will be a symlink. - */ - Full, -}; - /** * An abstraction for accessing source files during * evaluation. Currently, it's just a wrapper around `CanonPath` that @@ -123,14 +103,13 @@ struct SourcePath bool operator<(const SourcePath & x) const; /** - * Resolve any symlinks in this `SourcePath` according to the - * given resolution mode. - * - * @param mode might only be a temporary solution for this. - * See the discussion in https://github.com/NixOS/nix/pull/9985. + * Convenience wrapper around `SourceAccessor::resolveSymlinks()`. */ SourcePath resolveSymlinks( - SymlinkResolution mode = SymlinkResolution::Full) const; + SymlinkResolution mode = SymlinkResolution::Full) const + { + return {accessor, accessor->resolveSymlinks(path, mode)}; + } }; std::ostream & operator << (std::ostream & str, const SourcePath & path); diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc index 3bb6694f8..6bb2bd2f3 100644 --- a/src/libutil/tarfile.cc +++ b/src/libutil/tarfile.cc @@ -1,18 +1,21 @@ #include #include +#include "finally.hh" #include "serialise.hh" #include "tarfile.hh" #include "file-system.hh" namespace nix { -static int callback_open(struct archive *, void * self) +namespace { + +int callback_open(struct archive *, void * self) { return ARCHIVE_OK; } -static ssize_t callback_read(struct archive * archive, void * _self, const void * * buffer) +ssize_t callback_read(struct archive * archive, void * _self, const void ** buffer) { auto self = (TarArchive *) _self; *buffer = self->buffer.data(); @@ -27,41 +30,71 @@ static ssize_t callback_read(struct archive * archive, void * _self, const void } } -static int callback_close(struct archive *, void * self) +int callback_close(struct archive *, void * self) { return ARCHIVE_OK; } -void TarArchive::check(int err, const std::string & reason) +void checkLibArchive(archive * archive, int err, const std::string & reason) { if (err == ARCHIVE_EOF) throw EndOfFile("reached end of archive"); else if (err != ARCHIVE_OK) - throw Error(reason, archive_error_string(this->archive)); + throw Error(reason, archive_error_string(archive)); } -TarArchive::TarArchive(Source & source, bool raw) : buffer(65536) +constexpr auto defaultBufferSize = std::size_t{65536}; +} + +void TarArchive::check(int err, const std::string & reason) { - this->archive = archive_read_new(); - this->source = &source; + checkLibArchive(archive, err, reason); +} + +/// @brief Get filter_code from its name. +/// +/// libarchive does not provide a convenience function like archive_write_add_filter_by_name but for reading. +/// Instead it's necessary to use this kludge to convert method -> code and +/// then use archive_read_support_filter_by_code. Arguably this is better than +/// hand-rolling the equivalent function that is better implemented in libarchive. +int getArchiveFilterCodeByName(const std::string & method) +{ + auto * ar = archive_write_new(); + auto cleanup = Finally{[&ar]() { checkLibArchive(ar, archive_write_close(ar), "failed to close archive: %s"); }}; + auto err = archive_write_add_filter_by_name(ar, method.c_str()); + checkLibArchive(ar, err, "failed to get libarchive filter by name: %s"); + auto code = archive_filter_code(ar, 0); + return code; +} + +TarArchive::TarArchive(Source & source, bool raw, std::optional compression_method) + : archive{archive_read_new()} + , source{&source} + , buffer(defaultBufferSize) +{ + if (!compression_method) { + archive_read_support_filter_all(archive); + } else { + archive_read_support_filter_by_code(archive, getArchiveFilterCodeByName(*compression_method)); + } if (!raw) { - archive_read_support_filter_all(archive); archive_read_support_format_all(archive); } else { - archive_read_support_filter_all(archive); archive_read_support_format_raw(archive); archive_read_support_format_empty(archive); } + archive_read_set_option(archive, NULL, "mac-ext", NULL); - check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)"); + check( + archive_read_open(archive, (void *) this, callback_open, callback_read, callback_close), + "Failed to open archive (%s)"); } - TarArchive::TarArchive(const Path & path) + : archive{archive_read_new()} + , buffer(defaultBufferSize) { - this->archive = archive_read_new(); - archive_read_support_filter_all(archive); archive_read_support_format_all(archive); archive_read_set_option(archive, NULL, "mac-ext", NULL); @@ -75,19 +108,19 @@ void TarArchive::close() TarArchive::~TarArchive() { - if (this->archive) archive_read_free(this->archive); + if (this->archive) + archive_read_free(this->archive); } static void extract_archive(TarArchive & archive, const Path & destDir) { - int flags = ARCHIVE_EXTRACT_TIME - | ARCHIVE_EXTRACT_SECURE_SYMLINKS - | ARCHIVE_EXTRACT_SECURE_NODOTDOT; + int flags = ARCHIVE_EXTRACT_TIME | ARCHIVE_EXTRACT_SECURE_SYMLINKS | ARCHIVE_EXTRACT_SECURE_NODOTDOT; for (;;) { struct archive_entry * entry; int r = archive_read_next_header(archive.archive, &entry); - if (r == ARCHIVE_EOF) break; + if (r == ARCHIVE_EOF) + break; auto name = archive_entry_pathname(entry); if (!name) throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); @@ -96,18 +129,16 @@ static void extract_archive(TarArchive & archive, const Path & destDir) else archive.check(r); - archive_entry_copy_pathname(entry, - (destDir + "/" + name).c_str()); + archive_entry_copy_pathname(entry, (destDir + "/" + name).c_str()); // sources can and do contain dirs with no rx bits if (archive_entry_filetype(entry) == AE_IFDIR && (archive_entry_mode(entry) & 0500) != 0500) archive_entry_set_mode(entry, archive_entry_mode(entry) | 0500); // Patch hardlink path - const char *original_hardlink = archive_entry_hardlink(entry); + const char * original_hardlink = archive_entry_hardlink(entry); if (original_hardlink) { - archive_entry_copy_hardlink(entry, - (destDir + "/" + original_hardlink).c_str()); + archive_entry_copy_hardlink(entry, (destDir + "/" + original_hardlink).c_str()); } archive.check(archive_read_extract(archive.archive, entry, flags)); @@ -140,7 +171,8 @@ time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSin // FIXME: merge with extract_archive struct archive_entry * entry; int r = archive_read_next_header(archive.archive, &entry); - if (r == ARCHIVE_EOF) break; + if (r == ARCHIVE_EOF) + break; auto path = archive_entry_pathname(entry); if (!path) throw Error("cannot get archive member name: %s", archive_error_string(archive.archive)); @@ -167,8 +199,9 @@ time_t unpackTarfileToSink(TarArchive & archive, FileSystemObjectSink & parseSin auto n = archive_read_data(archive.archive, buf.data(), buf.size()); if (n < 0) throw Error("cannot read file '%s' from tarball", path); - if (n == 0) break; - crf(std::string_view { + if (n == 0) + break; + crf(std::string_view{ (const char *) buf.data(), (size_t) n, }); diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh index 6a9c42149..705d211e4 100644 --- a/src/libutil/tarfile.hh +++ b/src/libutil/tarfile.hh @@ -7,25 +7,36 @@ namespace nix { -struct TarArchive { +struct TarArchive +{ struct archive * archive; Source * source; std::vector buffer; void check(int err, const std::string & reason = "failed to extract archive (%s)"); - TarArchive(Source & source, bool raw = false); + explicit TarArchive(const Path & path); - TarArchive(const Path & path); + /// @brief Create a generic archive from source. + /// @param source - Input byte stream. + /// @param raw - Whether to enable raw file support. For more info look in docs: + /// https://manpages.debian.org/stretch/libarchive-dev/archive_read_format.3.en.html + /// @param compression_method - Primary compression method to use. std::nullopt means 'all'. + TarArchive(Source & source, bool raw = false, std::optional compression_method = std::nullopt); - /// disable copy constructor + /// Disable copy constructor. Explicitly default move assignment/constructor. TarArchive(const TarArchive &) = delete; + TarArchive & operator=(const TarArchive &) = delete; + TarArchive(TarArchive &&) = default; + TarArchive & operator=(TarArchive &&) = default; void close(); ~TarArchive(); }; +int getArchiveFilterCodeByName(const std::string & method); + void unpackTarfile(Source & source, const Path & destDir); void unpackTarfile(const Path & tarFile, const Path & destDir); diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc index 8febc8771..096252f03 100644 --- a/src/libutil/terminal.cc +++ b/src/libutil/terminal.cc @@ -7,11 +7,14 @@ namespace nix { -bool shouldANSI() +bool isTTY() { - return isatty(STDERR_FILENO) + static const bool tty = + isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb" && !(getEnv("NO_COLOR").has_value() || getEnv("NOCOLOR").has_value()); + + return tty; } std::string filterANSIEscapes(std::string_view s, bool filterAll, unsigned int width) diff --git a/src/libutil/terminal.hh b/src/libutil/terminal.hh index 9cb191308..9d8d0c743 100644 --- a/src/libutil/terminal.hh +++ b/src/libutil/terminal.hh @@ -8,7 +8,7 @@ namespace nix { * Determine whether ANSI escape sequences are appropriate for the * present output. */ -bool shouldANSI(); +bool isTTY(); /** * Truncate a string to 'width' printable characters. If 'filterAll' diff --git a/src/libutil/thread-pool.cc b/src/libutil/thread-pool.cc index 9a7dfee56..805f31d80 100644 --- a/src/libutil/thread-pool.cc +++ b/src/libutil/thread-pool.cc @@ -82,7 +82,7 @@ void ThreadPool::doWork(bool mainThread) ReceiveInterrupts receiveInterrupts; if (!mainThread) - interruptCheck = [&]() { return (bool) quit; }; + unix::interruptCheck = [&]() { return (bool) quit; }; bool didWork = false; std::exception_ptr exc; diff --git a/src/libutil/unix/environment-variables.cc b/src/libutil/unix/environment-variables.cc new file mode 100644 index 000000000..9c6fd3b18 --- /dev/null +++ b/src/libutil/unix/environment-variables.cc @@ -0,0 +1,12 @@ +#include + +#include "environment-variables.hh" + +namespace nix { + +int setEnv(const char * name, const char * value) +{ + return ::setenv(name, value, 1); +} + +} diff --git a/src/libutil/unix/file-descriptor.cc b/src/libutil/unix/file-descriptor.cc new file mode 100644 index 000000000..27c8d821b --- /dev/null +++ b/src/libutil/unix/file-descriptor.cc @@ -0,0 +1,155 @@ +#include "file-system.hh" +#include "signals.hh" +#include "finally.hh" +#include "serialise.hh" + +#include +#include + +namespace nix { + +std::string readFile(int fd) +{ + struct stat st; + if (fstat(fd, &st) == -1) + throw SysError("statting file"); + + return drainFD(fd, true, st.st_size); +} + + +void readFull(int fd, char * buf, size_t count) +{ + while (count) { + checkInterrupt(); + ssize_t res = read(fd, buf, count); + if (res == -1) { + if (errno == EINTR) continue; + throw SysError("reading from file"); + } + if (res == 0) throw EndOfFile("unexpected end-of-file"); + count -= res; + buf += res; + } +} + + +void writeFull(int fd, std::string_view s, bool allowInterrupts) +{ + while (!s.empty()) { + if (allowInterrupts) checkInterrupt(); + ssize_t res = write(fd, s.data(), s.size()); + if (res == -1 && errno != EINTR) + throw SysError("writing to file"); + if (res > 0) + s.remove_prefix(res); + } +} + + +std::string readLine(int fd) +{ + std::string s; + while (1) { + checkInterrupt(); + char ch; + // FIXME: inefficient + ssize_t rd = read(fd, &ch, 1); + if (rd == -1) { + if (errno != EINTR) + throw SysError("reading a line"); + } else if (rd == 0) + throw EndOfFile("unexpected EOF reading a line"); + else { + if (ch == '\n') return s; + s += ch; + } + } +} + + +void drainFD(int fd, Sink & sink, bool block) +{ + // silence GCC maybe-uninitialized warning in finally + int saved = 0; + + if (!block) { + saved = fcntl(fd, F_GETFL); + if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1) + throw SysError("making file descriptor non-blocking"); + } + + Finally finally([&]() { + if (!block) { + if (fcntl(fd, F_SETFL, saved) == -1) + throw SysError("making file descriptor blocking"); + } + }); + + std::vector buf(64 * 1024); + while (1) { + checkInterrupt(); + ssize_t rd = read(fd, buf.data(), buf.size()); + if (rd == -1) { + if (!block && (errno == EAGAIN || errno == EWOULDBLOCK)) + break; + if (errno != EINTR) + throw SysError("reading from file"); + } + else if (rd == 0) break; + else sink({reinterpret_cast(buf.data()), (size_t) rd}); + } +} + +////////////////////////////////////////////////////////////////////// + +void Pipe::create() +{ + int fds[2]; +#if HAVE_PIPE2 + if (pipe2(fds, O_CLOEXEC) != 0) throw SysError("creating pipe"); +#else + if (pipe(fds) != 0) throw SysError("creating pipe"); + closeOnExec(fds[0]); + closeOnExec(fds[1]); +#endif + readSide = fds[0]; + writeSide = fds[1]; +} + + +////////////////////////////////////////////////////////////////////// + +void closeMostFDs(const std::set & exceptions) +{ +#if __linux__ + try { + for (auto & s : readDirectory("/proc/self/fd")) { + auto fd = std::stoi(s.name); + if (!exceptions.count(fd)) { + debug("closing leaked FD %d", fd); + close(fd); + } + } + return; + } catch (SysError &) { + } +#endif + + int maxFD = 0; + maxFD = sysconf(_SC_OPEN_MAX); + for (int fd = 0; fd < maxFD; ++fd) + if (!exceptions.count(fd)) + close(fd); /* ignore result */ +} + + +void closeOnExec(int fd) +{ + int prev; + if ((prev = fcntl(fd, F_GETFD, 0)) == -1 || + fcntl(fd, F_SETFD, prev | FD_CLOEXEC) == -1) + throw SysError("setting close-on-exec flag"); +} + +} diff --git a/src/libutil/monitor-fd.hh b/src/libutil/unix/monitor-fd.hh similarity index 97% rename from src/libutil/monitor-fd.hh rename to src/libutil/unix/monitor-fd.hh index 228fb13f8..103894de9 100644 --- a/src/libutil/monitor-fd.hh +++ b/src/libutil/unix/monitor-fd.hh @@ -50,7 +50,7 @@ public: */ if (count == 0) continue; if (fds[0].revents & POLLHUP) { - triggerInterrupt(); + unix::triggerInterrupt(); break; } /* This will only happen on macOS. We sleep a bit to diff --git a/src/libutil/processes.cc b/src/libutil/unix/processes.cc similarity index 100% rename from src/libutil/processes.cc rename to src/libutil/unix/processes.cc diff --git a/src/libutil/processes.hh b/src/libutil/unix/processes.hh similarity index 100% rename from src/libutil/processes.hh rename to src/libutil/unix/processes.hh diff --git a/src/libutil/unix/signals-impl.hh b/src/libutil/unix/signals-impl.hh new file mode 100644 index 000000000..7ac8c914d --- /dev/null +++ b/src/libutil/unix/signals-impl.hh @@ -0,0 +1,111 @@ +#pragma once +/** + * @file + * + * Implementation of some inline definitions for Unix signals, and also + * some extra Unix-only interfaces. + * + * (The only reason everything about signals isn't Unix-only is some + * no-op definitions are provided on Windows to avoid excess CPP in + * downstream code.) + */ + +#include "types.hh" +#include "error.hh" +#include "logging.hh" +#include "ansicolor.hh" + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include + +namespace nix { + +/* User interruption. */ + +namespace unix { + +extern std::atomic _isInterrupted; + +extern thread_local std::function interruptCheck; + +void _interrupted(); + +/** + * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't + * necessarily match the current thread's mask. + * See saveSignalMask() to set the saved mask to the current mask. + */ +void setChildSignalMask(sigset_t *sigs); + +/** + * Start a thread that handles various signals. Also block those signals + * on the current thread (and thus any threads created by it). + * Saves the signal mask before changing the mask to block those signals. + * See saveSignalMask(). + */ +void startSignalHandlerThread(); + +/** + * Saves the signal mask, which is the signal mask that nix will restore + * before creating child processes. + * See setChildSignalMask() to set an arbitrary signal mask instead of the + * current mask. + */ +void saveSignalMask(); + +/** + * To use in a process that already called `startSignalHandlerThread()` + * or `saveSignalMask()` first. + */ +void restoreSignals(); + +void triggerInterrupt(); + +} + +static inline void setInterrupted(bool isInterrupted) +{ + unix::_isInterrupted = isInterrupted; +} + +static inline bool getInterrupted() +{ + return unix::_isInterrupted; +} + +void inline checkInterrupt() +{ + using namespace unix; + if (_isInterrupted || (interruptCheck && interruptCheck())) + _interrupted(); +} + +/** + * A RAII class that causes the current thread to receive SIGUSR1 when + * the signal handler thread receives SIGINT. That is, this allows + * SIGINT to be multiplexed to multiple threads. + */ +struct ReceiveInterrupts +{ + pthread_t target; + std::unique_ptr callback; + + ReceiveInterrupts() + : target(pthread_self()) + , callback(createInterruptCallback([&]() { pthread_kill(target, SIGUSR1); })) + { } +}; + + +} diff --git a/src/libutil/signals.cc b/src/libutil/unix/signals.cc similarity index 92% rename from src/libutil/signals.cc rename to src/libutil/unix/signals.cc index eaa4ea30e..7e30687d8 100644 --- a/src/libutil/signals.cc +++ b/src/libutil/unix/signals.cc @@ -8,17 +8,22 @@ namespace nix { -std::atomic _isInterrupted = false; +using namespace unix; +std::atomic unix::_isInterrupted = false; + +namespace unix { static thread_local bool interruptThrown = false; -thread_local std::function interruptCheck; +} + +thread_local std::function unix::interruptCheck; void setInterruptThrown() { - interruptThrown = true; + unix::interruptThrown = true; } -void _interrupted() +void unix::_interrupted() { /* Block user interrupts while an exception is being handled. Throwing an exception while another exception is being handled @@ -65,7 +70,7 @@ static void signalHandlerThread(sigset_t set) } } -void triggerInterrupt() +void unix::triggerInterrupt() { _isInterrupted = true; @@ -96,7 +101,7 @@ void triggerInterrupt() static sigset_t savedSignalMask; static bool savedSignalMaskIsSet = false; -void setChildSignalMask(sigset_t * sigs) +void unix::setChildSignalMask(sigset_t * sigs) { assert(sigs); // C style function, but think of sigs as a reference @@ -115,14 +120,14 @@ void setChildSignalMask(sigset_t * sigs) savedSignalMaskIsSet = true; } -void saveSignalMask() { +void unix::saveSignalMask() { if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) throw SysError("querying signal mask"); savedSignalMaskIsSet = true; } -void startSignalHandlerThread() +void unix::startSignalHandlerThread() { updateWindowSize(); @@ -141,7 +146,7 @@ void startSignalHandlerThread() std::thread(signalHandlerThread, set).detach(); } -void restoreSignals() +void unix::restoreSignals() { // If startSignalHandlerThread wasn't called, that means we're not running // in a proper libmain process, but a process that presumably manages its diff --git a/src/libutil/unix-domain-socket.cc b/src/libutil/unix/unix-domain-socket.cc similarity index 100% rename from src/libutil/unix-domain-socket.cc rename to src/libutil/unix/unix-domain-socket.cc diff --git a/src/libutil/unix-domain-socket.hh b/src/libutil/unix/unix-domain-socket.hh similarity index 100% rename from src/libutil/unix-domain-socket.hh rename to src/libutil/unix/unix-domain-socket.hh diff --git a/src/libutil/unix/users.cc b/src/libutil/unix/users.cc new file mode 100644 index 000000000..58063a953 --- /dev/null +++ b/src/libutil/unix/users.cc @@ -0,0 +1,66 @@ +#include "util.hh" +#include "users.hh" +#include "environment-variables.hh" +#include "file-system.hh" + +#include +#include +#include + +namespace nix { + +std::string getUserName() +{ + auto pw = getpwuid(geteuid()); + std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); + if (name.empty()) + throw Error("cannot figure out user name"); + return name; +} + +Path getHomeOf(uid_t userId) +{ + std::vector buf(16384); + struct passwd pwbuf; + struct passwd * pw; + if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 + || !pw || !pw->pw_dir || !pw->pw_dir[0]) + throw Error("cannot determine user's home directory"); + return pw->pw_dir; +} + +Path getHome() +{ + static Path homeDir = []() + { + std::optional unownedUserHomeDir = {}; + auto homeDir = getEnv("HOME"); + if (homeDir) { + // Only use $HOME if doesn't exist or is owned by the current user. + struct stat st; + int result = stat(homeDir->c_str(), &st); + if (result != 0) { + if (errno != ENOENT) { + warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); + homeDir.reset(); + } + } else if (st.st_uid != geteuid()) { + unownedUserHomeDir.swap(homeDir); + } + } + if (!homeDir) { + homeDir = getHomeOf(geteuid()); + if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { + warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); + } + } + return *homeDir; + }(); + return homeDir; +} + +bool isRootUser() { + return getuid() == 0; +} + +} diff --git a/src/libutil/users.cc b/src/libutil/users.cc index 95a641322..d546e364f 100644 --- a/src/libutil/users.cc +++ b/src/libutil/users.cc @@ -3,63 +3,8 @@ #include "environment-variables.hh" #include "file-system.hh" -#include -#include -#include - namespace nix { -std::string getUserName() -{ - auto pw = getpwuid(geteuid()); - std::string name = pw ? pw->pw_name : getEnv("USER").value_or(""); - if (name.empty()) - throw Error("cannot figure out user name"); - return name; -} - -Path getHomeOf(uid_t userId) -{ - std::vector buf(16384); - struct passwd pwbuf; - struct passwd * pw; - if (getpwuid_r(userId, &pwbuf, buf.data(), buf.size(), &pw) != 0 - || !pw || !pw->pw_dir || !pw->pw_dir[0]) - throw Error("cannot determine user's home directory"); - return pw->pw_dir; -} - -Path getHome() -{ - static Path homeDir = []() - { - std::optional unownedUserHomeDir = {}; - auto homeDir = getEnv("HOME"); - if (homeDir) { - // Only use $HOME if doesn't exist or is owned by the current user. - struct stat st; - int result = stat(homeDir->c_str(), &st); - if (result != 0) { - if (errno != ENOENT) { - warn("couldn't stat $HOME ('%s') for reason other than not existing ('%d'), falling back to the one defined in the 'passwd' file", *homeDir, errno); - homeDir.reset(); - } - } else if (st.st_uid != geteuid()) { - unownedUserHomeDir.swap(homeDir); - } - } - if (!homeDir) { - homeDir = getHomeOf(geteuid()); - if (unownedUserHomeDir.has_value() && unownedUserHomeDir != homeDir) { - warn("$HOME ('%s') is not owned by you, falling back to the one defined in the 'passwd' file ('%s')", *unownedUserHomeDir, *homeDir); - } - } - return *homeDir; - }(); - return homeDir; -} - - Path getCacheDir() { auto cacheDir = getEnv("XDG_CACHE_HOME"); diff --git a/src/libutil/users.hh b/src/libutil/users.hh index cecbb8bfb..449e5bbe9 100644 --- a/src/libutil/users.hh +++ b/src/libutil/users.hh @@ -55,4 +55,10 @@ Path createNixStateDir(); */ std::string expandTilde(std::string_view path); + +/** + * Is the current user UID 0 on Unix? + */ +bool isRootUser(); + } diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 06124bf15..103ce4232 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -4,7 +4,6 @@ #include #include #include -#include #include #include diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 35bc03926..198e9cda0 100644 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -24,6 +24,7 @@ #include "common-eval-args.hh" #include "attr-path.hh" #include "legacy.hh" +#include "users.hh" using namespace nix; using namespace std::string_literals; @@ -287,7 +288,7 @@ static void main_nix_build(int argc, char * * argv) } if (runEnv) - setenv("IN_NIX_SHELL", pure ? "pure" : "impure", 1); + setEnv("IN_NIX_SHELL", pure ? "pure" : "impure"); PackageInfos drvs; @@ -572,8 +573,9 @@ static void main_nix_build(int argc, char * * argv) "BASH=%5%; " "set +e; " R"s([ -n "$PS1" -a -z "$NIX_SHELL_PRESERVE_PROMPT" ] && )s" + - (getuid() == 0 ? R"s(PS1='\n\[\033[1;31m\][nix-shell:\w]\$\[\033[0m\] '; )s" - : R"s(PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s") + + (isRootUser() + ? R"s(PS1='\n\[\033[1;31m\][nix-shell:\w]\$\[\033[0m\] '; )s" + : R"s(PS1='\n\[\033[1;32m\][nix-shell:\w]\$\[\033[0m\] '; )s") + "if [ \"$(type -t runHook)\" = function ]; then runHook shellHook; fi; " "unset NIX_ENFORCE_PURITY; " "shopt -u nullglob; " diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc index 48553fa31..9f7f557b5 100644 --- a/src/nix-channel/nix-channel.cc +++ b/src/nix-channel/nix-channel.cc @@ -112,7 +112,7 @@ static void update(const StringSet & channelNames) // We want to download the url to a file to see if it's a tarball while also checking if we // got redirected in the process, so that we can grab the various parts of a nix channel // definition from a consistent location if the redirect changes mid-download. - auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url)), false); + auto result = fetchers::downloadFile(store, url, std::string(baseNameOf(url))); auto filename = store->toRealPath(result.storePath); url = result.effectiveUrl; @@ -126,9 +126,9 @@ static void update(const StringSet & channelNames) if (!unpacked) { // Download the channel tarball. try { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz", false).storePath); + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.xz", "nixexprs.tar.xz").storePath); } catch (FileTransferError & e) { - filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2", false).storePath); + filename = store->toRealPath(fetchers::downloadFile(store, url + "/nixexprs.tar.bz2", "nixexprs.tar.bz2").storePath); } } // Regardless of where it came from, add the expression representing this channel to accumulated expression diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 9aff83bb0..cf0569c9f 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -16,6 +16,7 @@ #include "xml-writer.hh" #include "legacy.hh" #include "eval-settings.hh" // for defexpr +#include "terminal.hh" #include #include @@ -108,7 +109,7 @@ static void getAllExprs(EvalState & state, const SourcePath & path, StringSet & seen, BindingsBuilder & attrs) { StringSet namesSorted; - for (auto & [name, _] : path.readDirectory()) namesSorted.insert(name); + for (auto & [name, _] : path.resolveSymlinks().readDirectory()) namesSorted.insert(name); for (auto & i : namesSorted) { /* Ignore the manifest.nix used by profiles. This is @@ -1089,7 +1090,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) return; } - bool tty = isatty(STDOUT_FILENO); + bool tty = isTTY(); RunPager pager; Table table; @@ -1411,7 +1412,7 @@ static int main_nix_env(int argc, char * * argv) replaceSymlink( defaultChannelsDir(), nixExprPath + "/channels"); - if (getuid() != 0) + if (!isRootUser()) replaceSymlink( rootChannelsDir(), nixExprPath + "/channels_root"); diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index 8d4717e15..f7c4cebec 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -145,10 +145,14 @@ struct CmdConfigCheck : StoreCommand void checkTrustedUser(ref store) { - std::string_view trusted = store->isTrustedClient() - ? "trusted" - : "not trusted"; - checkInfo(fmt("You are %s by store uri: %s", trusted, store->getUri())); + if (auto trustedMay = store->isTrustedClient()) { + std::string_view trusted = trustedMay.value() + ? "trusted" + : "not trusted"; + checkInfo(fmt("You are %s by store uri: %s", trusted, store->getUri())); + } else { + checkInfo(fmt("Store uri: %s doesn't have a notion of trusted user", store->getUri())); + } } }; diff --git a/src/nix/develop.cc b/src/nix/develop.cc index c1842f2d5..b654dc52f 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -177,6 +177,14 @@ struct BuildEnvironment throw Error("bash variable is not a string"); } + static Associative getAssociative(const Value & value) + { + if (auto assoc = std::get_if(&value)) + return *assoc; + else + throw Error("bash variable is not an associative array"); + } + static Array getStrings(const Value & value) { if (auto str = std::get_if(&value)) @@ -362,13 +370,17 @@ struct Common : InstallableCommand, MixProfile auto outputs = buildEnvironment.vars.find("outputs"); assert(outputs != buildEnvironment.vars.end()); - // FIXME: properly unquote 'outputs'. StringMap rewrites; - for (auto & outputName : BuildEnvironment::getStrings(outputs->second)) { - auto from = buildEnvironment.vars.find(outputName); - assert(from != buildEnvironment.vars.end()); - // FIXME: unquote - rewrites.insert({BuildEnvironment::getString(from->second), outputsDir + "/" + outputName}); + if (buildEnvironment.providesStructuredAttrs()) { + for (auto & [outputName, from] : BuildEnvironment::getAssociative(outputs->second)) { + rewrites.insert({from, outputsDir + "/" + outputName}); + } + } else { + for (auto & outputName : BuildEnvironment::getStrings(outputs->second)) { + auto from = buildEnvironment.vars.find(outputName); + assert(from != buildEnvironment.vars.end()); + rewrites.insert({BuildEnvironment::getString(from->second), outputsDir + "/" + outputName}); + } } /* Substitute redirects. */ @@ -603,7 +615,7 @@ struct CmdDevelop : Common, MixEnvironment setEnviron(); // prevent garbage collection until shell exits - setenv("NIX_GCROOT", gcroot.c_str(), 1); + setEnv("NIX_GCROOT", gcroot.c_str()); Path shell = "bash"; @@ -648,7 +660,7 @@ struct CmdDevelop : Common, MixEnvironment // Override SHELL with the one chosen for this environment. // This is to make sure the system shell doesn't leak into the build environment. - setenv("SHELL", shell.c_str(), 1); + setEnv("SHELL", shell.c_str()); // If running a phase or single command, don't want an interactive shell running after // Ctrl-C, so don't pass --rcfile diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index 832cc2f11..071edf9b9 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -128,20 +128,25 @@ __escapeString() { printf '"%s"' "$__s" } -# In case of `__structuredAttrs = true;` the list of outputs is an associative -# array with a format like `outname => /nix/store/hash-drvname-outname`, so `__olist` -# must contain the array's keys (hence `${!...[@]}`) in this case. -if [ -e "$NIX_ATTRS_SH_FILE" ]; then - __olist="${!outputs[@]}" -else - __olist=$outputs -fi - -for __output in $__olist; do - if [[ -z $__done ]]; then - __dumpEnv > ${!__output} +__dumpEnvToOutput() { + local __output="$1" + if [[ -z ${__done-} ]]; then + __dumpEnv > "$__output" __done=1 else - echo -n >> "${!__output}" + echo -n >> "$__output" fi -done +} + +# In case of `__structuredAttrs = true;` the list of outputs is an associative +# array with a format like `outname => /nix/store/hash-drvname-outname`. +# Otherwise it is a space-separated list of output variable names. +if [ -e "$NIX_ATTRS_SH_FILE" ]; then + for __output in "${outputs[@]}"; do + __dumpEnvToOutput "$__output" + done +else + for __outname in $outputs; do + __dumpEnvToOutput "${!__outname}" + done +fi diff --git a/src/nix/local.mk b/src/nix/local.mk index 1d6f560d6..9f6f31b3a 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -12,9 +12,19 @@ nix_SOURCES := \ $(wildcard src/nix-daemon/*.cc) \ $(wildcard src/nix-env/*.cc) \ $(wildcard src/nix-instantiate/*.cc) \ - $(wildcard src/nix-store/*.cc) \ + $(wildcard src/nix-store/*.cc) -nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain -I src/libcmd -I doc/manual +ifdef HOST_UNIX +nix_SOURCES += \ + $(wildcard $(d)/unix/*.cc) +endif + +INCLUDE_nix := -I $(d) +ifdef HOST_UNIX + INCLUDE_nix += -I $(d)/unix +endif + +nix_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libmain) -I src/libcmd -I doc/manual $(INCLUDE_nix) nix_LIBS = libexpr libmain libfetchers libstore libutil libcmd diff --git a/src/nix/main.cc b/src/nix/main.cc index 1ee9e4198..af6498218 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -2,7 +2,6 @@ #include "args/root.hh" #include "current-process.hh" -#include "namespaces.hh" #include "command.hh" #include "common-args.hh" #include "eval.hh" @@ -16,6 +15,7 @@ #include "loggers.hh" #include "markdown.hh" #include "memory-input-accessor.hh" +#include "terminal.hh" #include #include @@ -26,6 +26,10 @@ #include +#if __linux__ +# include "namespaces.hh" +#endif + extern std::string chrootHelperName; void chrootHelper(int argc, char * * argv); @@ -347,7 +351,7 @@ void mainWrapped(int argc, char * * argv) initGC(); #if __linux__ - if (getuid() == 0) { + if (isRootUser()) { try { saveMountNamespace(); if (unshare(CLONE_NEWNS) == -1) @@ -375,7 +379,9 @@ void mainWrapped(int argc, char * * argv) setLogFormat("bar"); settings.verboseBuild = false; - if (isatty(STDERR_FILENO)) { + + // If on a terminal, progress will be displayed via progress bars etc. (thus verbosity=notice) + if (nix::isTTY()) { verbosity = lvlNotice; } else { verbosity = lvlInfo; diff --git a/src/nix/nix.md b/src/nix/nix.md index 749456014..4464bef37 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -229,7 +229,7 @@ operate are determined as follows: Note that a [store derivation] (given by its `.drv` file store path) doesn't have any attributes like `meta`, and thus this case doesn't apply to it. - [store derivation]: ../../glossary.md#gloss-store-derivation + [store derivation]: @docroot@/glossary.md#gloss-store-derivation * Otherwise, Nix will use all outputs of the derivation. diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 5f10cfb61..921b25d7f 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -43,10 +43,16 @@ static json pathInfoToJSON( for (auto & storePath : storePaths) { json jsonObject; + auto printedStorePath = store.printStorePath(storePath); try { auto info = store.queryPathInfo(storePath); + // `storePath` has the representation `-x` rather than + // `-` in case of binary-cache stores & `--all` because we don't + // know the name yet until we've read the NAR info. + printedStorePath = store.printStorePath(info->path); + jsonObject = info->toJSON(store, true, HashFormat::SRI); if (showClosureSize) { @@ -74,7 +80,7 @@ static json pathInfoToJSON( jsonObject = nullptr; } - jsonAllObjects[store.printStorePath(storePath)] = std::move(jsonObject); + jsonAllObjects[printedStorePath] = std::move(jsonObject); } return jsonAllObjects; } diff --git a/src/nix/path-info.md b/src/nix/path-info.md index 4594854eb..789984559 100644 --- a/src/nix/path-info.md +++ b/src/nix/path-info.md @@ -70,7 +70,7 @@ R""( * Print the path of the [store derivation] produced by `nixpkgs#hello`: - [store derivation]: ../../glossary.md#gloss-store-derivation + [store derivation]: @docroot@/glossary.md#gloss-store-derivation ```console # nix path-info --derivation nixpkgs#hello diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index 17178c84f..d30e9d397 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -11,6 +11,7 @@ #include "legacy.hh" #include "posix-source-accessor.hh" #include "misc-store-flags.hh" +#include "terminal.hh" #include @@ -188,7 +189,7 @@ static int main_nix_prefetch_url(int argc, char * * argv) Finally f([]() { stopProgressBar(); }); - if (isatty(STDERR_FILENO)) + if (isTTY()) startProgressBar(); auto store = openStore(); diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 63fe3044b..8bbfe0f07 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -47,15 +47,6 @@ struct CmdRepl : RawInstallablesCommand void applyDefaultInstallables(std::vector & rawInstallables) override { - if (!experimentalFeatureSettings.isEnabled(Xp::Flakes) && !(file) && rawInstallables.size() >= 1) { - warn("future versions of Nix will require using `--file` to load a file"); - if (rawInstallables.size() > 1) - warn("more than one input file is not currently supported"); - auto filePath = rawInstallables[0].data(); - file = std::optional(filePath); - rawInstallables.front() = rawInstallables.back(); - rawInstallables.pop_back(); - } if (rawInstallables.empty() && (file.has_value() || expr.has_value())) { rawInstallables.push_back("."); } diff --git a/src/nix/store-copy-log.md b/src/nix/store-copy-log.md index 0937250f2..61daa75c1 100644 --- a/src/nix/store-copy-log.md +++ b/src/nix/store-copy-log.md @@ -20,7 +20,7 @@ R""( * To copy the log for a specific [store derivation] via SSH: - [store derivation]: ../../glossary.md#gloss-store-derivation + [store derivation]: @docroot@/glossary.md#gloss-store-derivation ```console # nix store copy-log --to ssh-ng://machine /nix/store/ilgm50plpmcgjhcp33z6n4qbnpqfhxym-glibc-2.33-59.drv diff --git a/src/nix/daemon.cc b/src/nix/unix/daemon.cc similarity index 100% rename from src/nix/daemon.cc rename to src/nix/unix/daemon.cc diff --git a/src/nix/daemon.md b/src/nix/unix/daemon.md similarity index 100% rename from src/nix/daemon.md rename to src/nix/unix/daemon.md diff --git a/src/nix/fmt.cc b/src/nix/unix/fmt.cc similarity index 100% rename from src/nix/fmt.cc rename to src/nix/unix/fmt.cc diff --git a/src/nix/fmt.md b/src/nix/unix/fmt.md similarity index 100% rename from src/nix/fmt.md rename to src/nix/unix/fmt.md diff --git a/src/nix/run.cc b/src/nix/unix/run.cc similarity index 97% rename from src/nix/run.cc rename to src/nix/unix/run.cc index e86837679..dfd8b643c 100644 --- a/src/nix/run.cc +++ b/src/nix/unix/run.cc @@ -124,7 +124,8 @@ struct CmdShell : InstallablesCommand, MixEnvironment if (true) pathAdditions.push_back(store->printStorePath(path) + "/bin"); - auto propPath = CanonPath(store->printStorePath(path)) / "nix-support" / "propagated-user-env-packages"; + auto propPath = accessor->resolveSymlinks( + CanonPath(store->printStorePath(path)) / "nix-support" / "propagated-user-env-packages"); if (auto st = accessor->maybeLstat(propPath); st && st->type == SourceAccessor::tRegular) { for (auto & p : tokenizeString(accessor->readFile(propPath))) todo.push(store->parseStorePath(p)); @@ -134,7 +135,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment auto unixPath = tokenizeString(getEnv("PATH").value_or(""), ":"); unixPath.insert(unixPath.begin(), pathAdditions.begin(), pathAdditions.end()); auto unixPathString = concatStringsSep(":", unixPath); - setenv("PATH", unixPathString.c_str(), 1); + setEnv("PATH", unixPathString.c_str()); Strings args; for (auto & arg : command) args.push_back(arg); diff --git a/src/nix/run.hh b/src/nix/unix/run.hh similarity index 100% rename from src/nix/run.hh rename to src/nix/unix/run.hh diff --git a/src/nix/run.md b/src/nix/unix/run.md similarity index 100% rename from src/nix/run.md rename to src/nix/unix/run.md diff --git a/src/nix/upgrade-nix.cc b/src/nix/unix/upgrade-nix.cc similarity index 100% rename from src/nix/upgrade-nix.cc rename to src/nix/unix/upgrade-nix.cc diff --git a/src/nix/upgrade-nix.md b/src/nix/unix/upgrade-nix.md similarity index 100% rename from src/nix/upgrade-nix.md rename to src/nix/unix/upgrade-nix.md diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk deleted file mode 100644 index fc48a8417..000000000 --- a/src/resolve-system-dependencies/local.mk +++ /dev/null @@ -1,13 +0,0 @@ -ifdef HOST_DARWIN - programs += resolve-system-dependencies -endif - -resolve-system-dependencies_DIR := $(d) - -resolve-system-dependencies_INSTALL_DIR := $(libexecdir)/nix - -resolve-system-dependencies_CXXFLAGS += -I src/libutil -I src/libstore -I src/libmain - -resolve-system-dependencies_LIBS := libstore libmain libutil - -resolve-system-dependencies_SOURCES := $(d)/resolve-system-dependencies.cc diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc deleted file mode 100644 index 4ea268d24..000000000 --- a/src/resolve-system-dependencies/resolve-system-dependencies.cc +++ /dev/null @@ -1,190 +0,0 @@ -#include "derivations.hh" -#include "globals.hh" -#include "shared.hh" -#include "store-api.hh" -#include -#include -#include -#include -#include -#include -#include -#include - -#define DO_SWAP(x, y) ((x) ? OSSwapInt32(y) : (y)) - -using namespace nix; - -static auto cacheDir = Path{}; - -Path resolveCacheFile(Path lib) -{ - std::replace(lib.begin(), lib.end(), '/', '%'); - return cacheDir + "/" + lib; -} - -std::set readCacheFile(const Path & file) -{ - return tokenizeString>(readFile(file), "\n"); -} - -std::set runResolver(const Path & filename) -{ - AutoCloseFD fd = open(filename.c_str(), O_RDONLY); - if (!fd) - throw SysError("opening '%s'", filename); - - struct stat st; - if (fstat(fd.get(), &st)) - throw SysError("statting '%s'", filename); - - if (!S_ISREG(st.st_mode)) { - printError("file '%s' is not a regular MACH binary", filename); - return {}; - } - - if (st.st_size < sizeof(mach_header_64)) { - printError("file '%s' is too short for a MACH binary", filename); - return {}; - } - - char* obj = (char*) mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd.get(), 0); - if (!obj) - throw SysError("mmapping '%s'", filename); - - ptrdiff_t mach64_offset = 0; - - uint32_t magic = ((mach_header_64*) obj)->magic; - if (magic == FAT_CIGAM || magic == FAT_MAGIC) { - bool should_swap = magic == FAT_CIGAM; - uint32_t narches = DO_SWAP(should_swap, ((fat_header *) obj)->nfat_arch); - for (uint32_t i = 0; i < narches; i++) { - fat_arch* arch = (fat_arch*) (obj + sizeof(fat_header) + sizeof(fat_arch) * i); - if (DO_SWAP(should_swap, arch->cputype) == CPU_TYPE_X86_64) { - mach64_offset = (ptrdiff_t) DO_SWAP(should_swap, arch->offset); - break; - } - } - if (mach64_offset == 0) { - printError("could not find any mach64 blobs in file '%1%', continuing...", filename); - return {}; - } - } else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) { - mach64_offset = 0; - } else { - printError("Object file has unknown magic number '%1%', skipping it...", magic); - return {}; - } - - mach_header_64 * m_header = (mach_header_64 *) (obj + mach64_offset); - - bool should_swap = magic == MH_CIGAM_64; - ptrdiff_t cmd_offset = mach64_offset + sizeof(mach_header_64); - - std::set libs; - for (uint32_t i = 0; i < DO_SWAP(should_swap, m_header->ncmds); i++) { - load_command * cmd = (load_command *) (obj + cmd_offset); - switch(DO_SWAP(should_swap, cmd->cmd)) { - case LC_LOAD_UPWARD_DYLIB: - case LC_LOAD_DYLIB: - case LC_REEXPORT_DYLIB: - libs.insert(std::string((char *) cmd + ((dylib_command*) cmd)->dylib.name.offset)); - break; - } - cmd_offset += DO_SWAP(should_swap, cmd->cmdsize); - } - - return libs; -} - -bool isSymlink(const Path & path) -{ - return S_ISLNK(lstat(path).st_mode); -} - -Path resolveSymlink(const Path & path) -{ - auto target = readLink(path); - return hasPrefix(target, "/") - ? target - : concatStrings(dirOf(path), "/", target); -} - -std::set resolveTree(const Path & path, PathSet & deps) -{ - std::set results; - if (!deps.insert(path).second) return {}; - for (auto & lib : runResolver(path)) { - results.insert(lib); - for (auto & p : resolveTree(lib, deps)) { - results.insert(p); - } - } - return results; -} - -std::set getPath(const Path & path) -{ - if (hasPrefix(path, "/dev")) return {}; - - Path cacheFile = resolveCacheFile(path); - if (pathExists(cacheFile)) - return readCacheFile(cacheFile); - - std::set deps, paths; - paths.insert(path); - - Path nextPath(path); - while (isSymlink(nextPath)) { - nextPath = resolveSymlink(nextPath); - paths.insert(nextPath); - } - - for (auto & t : resolveTree(nextPath, deps)) - paths.insert(t); - - writeFile(cacheFile, concatStringsSep("\n", paths)); - - return paths; -} - -int main(int argc, char ** argv) -{ - return handleExceptions(argv[0], [&]() { - initNix(); - - struct utsname _uname; - - uname(&_uname); - - auto cacheParentDir = fmt("%1%/dependency-maps", settings.nixStateDir); - - cacheDir = fmt("%1%/%2%-%3%-%4%", cacheParentDir, _uname.machine, _uname.sysname, _uname.release); - - mkdir(cacheParentDir.c_str(), 0755); - mkdir(cacheDir.c_str(), 0755); - - auto store = openStore(); - - StringSet impurePaths; - - if (std::string(argv[1]) == "--test") - impurePaths.insert(argv[2]); - else { - auto drv = store->derivationFromPath(store->parseStorePath(argv[1])); - impurePaths = tokenizeString(getOr(drv.env, "__impureHostDeps", "")); - impurePaths.insert("/usr/lib/libSystem.dylib"); - } - - std::set allPaths; - - for (auto & path : impurePaths) - for (auto & p : getPath(path)) - allPaths.insert(p); - - std::cout << "extra-chroot-dirs" << std::endl; - for (auto & path : allPaths) - std::cout << path << std::endl; - std::cout << std::endl; - }); -} diff --git a/tests/functional/binary-cache.sh b/tests/functional/binary-cache.sh index 7c64a115c..2a8d5ccdb 100644 --- a/tests/functional/binary-cache.sh +++ b/tests/functional/binary-cache.sh @@ -14,6 +14,14 @@ outPath=$(nix-build dependencies.nix --no-out-link) nix copy --to file://$cacheDir $outPath +readarray -t paths < <(nix path-info --all --json --store file://$cacheDir | jq 'keys|sort|.[]' -r) +[[ "${#paths[@]}" -eq 3 ]] +for path in "${paths[@]}"; do + [[ "$path" =~ -dependencies-input-0$ ]] \ + || [[ "$path" =~ -dependencies-input-2$ ]] \ + || [[ "$path" =~ -dependencies-top$ ]] +done + # Test copying build logs to the binary cache. expect 1 nix log --store file://$cacheDir $outPath 2>&1 | grep 'is not available' nix store copy-log --to file://$cacheDir $outPath diff --git a/tests/functional/check.sh b/tests/functional/check.sh index e13abf747..38883c5d7 100644 --- a/tests/functional/check.sh +++ b/tests/functional/check.sh @@ -34,6 +34,21 @@ nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \ [ "$status" = "100" ] if checkBuildTempDirRemoved $TEST_ROOT/log; then false; fi +test_custom_build_dir() { + local customBuildDir="$TEST_ROOT/custom-build-dir" + + # Nix does not create the parent directories, and perhaps it shouldn't try to + # decide the permissions of build-dir. + mkdir "$customBuildDir" + nix-build check.nix -A failed --argstr checkBuildId $checkBuildId \ + --no-out-link --keep-failed --option build-dir "$TEST_ROOT/custom-build-dir" 2> $TEST_ROOT/log || status=$? + [ "$status" = "100" ] + [[ 1 == "$(count "$customBuildDir/nix-build-"*)" ]] + local buildDir="$customBuildDir/nix-build-"* + grep $checkBuildId $buildDir/checkBuildId +} +test_custom_build_dir + nix-build check.nix -A deterministic --argstr checkBuildId $checkBuildId \ --no-out-link 2> $TEST_ROOT/log checkBuildTempDirRemoved $TEST_ROOT/log diff --git a/tests/functional/chroot-store.sh b/tests/functional/chroot-store.sh new file mode 100644 index 000000000..9e589d04b --- /dev/null +++ b/tests/functional/chroot-store.sh @@ -0,0 +1,45 @@ +source common.sh + +echo example > $TEST_ROOT/example.txt +mkdir -p $TEST_ROOT/x + +export NIX_STORE_DIR=/nix2/store + +CORRECT_PATH=$(cd $TEST_ROOT && nix-store --store ./x --add example.txt) + +[[ $CORRECT_PATH =~ ^/nix2/store/.*-example.txt$ ]] + +PATH1=$(cd $TEST_ROOT && nix path-info --store ./x $CORRECT_PATH) +[ $CORRECT_PATH == $PATH1 ] + +PATH2=$(nix path-info --store "$TEST_ROOT/x" $CORRECT_PATH) +[ $CORRECT_PATH == $PATH2 ] + +PATH3=$(nix path-info --store "local?root=$TEST_ROOT/x" $CORRECT_PATH) +[ $CORRECT_PATH == $PATH3 ] + +# Ensure store info trusted works with local store +nix --store $TEST_ROOT/x store info --json | jq -e '.trusted' + +# Test building in a chroot store. +if canUseSandbox; then + + flakeDir=$TEST_ROOT/flake + mkdir -p $flakeDir + + cat > $flakeDir/flake.nix < /dev/null } +# Return the number of arguments +count() { + echo $# +} + trap onError ERR fi # COMMON_VARS_AND_FUNCTIONS_SH_SOURCED diff --git a/tests/functional/config.sh b/tests/functional/config.sh index 324fe95bd..efdf2a958 100644 --- a/tests/functional/config.sh +++ b/tests/functional/config.sh @@ -43,6 +43,16 @@ export NIX_USER_CONF_FILES=$here/config/nix-with-substituters.conf var=$(nix config show | grep '^substituters =' | cut -d '=' -f 2 | xargs) [[ $var == https://example.com ]] +# Test that we can include a file. +export NIX_USER_CONF_FILES=$here/config/nix-with-include.conf +var=$(nix config show | grep '^allowed-uris =' | cut -d '=' -f 2 | xargs) +[[ $var == https://github.com/NixOS/nix ]] + +# Test that we can !include a file. +export NIX_USER_CONF_FILES=$here/config/nix-with-bang-include.conf +var=$(nix config show | grep '^experimental-features =' | cut -d '=' -f 2 | xargs) +[[ $var == nix-command ]] + # Test that it's possible to load config from the environment prev=$(nix config show | grep '^cores' | cut -d '=' -f 2 | xargs) export NIX_CONFIG="cores = 4242"$'\n'"experimental-features = nix-command flakes" @@ -56,4 +66,4 @@ exp_features=$(nix config show | grep '^experimental-features' | cut -d '=' -f 2 # Test that it's possible to retrieve a single setting's value val=$(nix config show | grep '^warn-dirty' | cut -d '=' -f 2 | xargs) val2=$(nix config show warn-dirty) -[[ $val == $val2 ]] +[[ $val == $val2 ]] \ No newline at end of file diff --git a/tests/functional/config/extra-config.conf b/tests/functional/config/extra-config.conf new file mode 100644 index 000000000..d110f06e4 --- /dev/null +++ b/tests/functional/config/extra-config.conf @@ -0,0 +1 @@ +allowed-uris = https://github.com/NixOS/nix \ No newline at end of file diff --git a/tests/functional/config/nix-with-bang-include.conf b/tests/functional/config/nix-with-bang-include.conf new file mode 100644 index 000000000..fa600e6ff --- /dev/null +++ b/tests/functional/config/nix-with-bang-include.conf @@ -0,0 +1,2 @@ +experimental-features = nix-command +!include ./missing-extra-config.conf \ No newline at end of file diff --git a/tests/functional/config/nix-with-include.conf b/tests/functional/config/nix-with-include.conf new file mode 100644 index 000000000..17b8958ba --- /dev/null +++ b/tests/functional/config/nix-with-include.conf @@ -0,0 +1,2 @@ +experimental-features = nix-command +include ./extra-config.conf \ No newline at end of file diff --git a/tests/functional/fetchMercurial.sh b/tests/functional/fetchMercurial.sh index e6f8525c6..e133df1f8 100644 --- a/tests/functional/fetchMercurial.sh +++ b/tests/functional/fetchMercurial.sh @@ -1,6 +1,6 @@ source common.sh -[[ $(type -p hq) ]] || skipTest "Mercurial not installed" +[[ $(type -p hg) ]] || skipTest "Mercurial not installed" clearStore diff --git a/tests/functional/fixed.nix b/tests/functional/fixed.nix index 5bdf79333..a920a2167 100644 --- a/tests/functional/fixed.nix +++ b/tests/functional/fixed.nix @@ -64,4 +64,6 @@ rec { (f2 "bar" ./fixed.builder2.sh "recursive" "md5" "3670af73070fa14077ad74e0f5ea4e42") ]; + # Can use "nar" instead of "recursive" now. + nar-not-recursive = f2 "foo" ./fixed.builder2.sh "nar" "md5" "3670af73070fa14077ad74e0f5ea4e42"; } diff --git a/tests/functional/fixed.sh b/tests/functional/fixed.sh index d98d4cd15..7bbecda91 100644 --- a/tests/functional/fixed.sh +++ b/tests/functional/fixed.sh @@ -61,3 +61,7 @@ out3=$(nix-store --add-fixed --recursive sha256 $TEST_ROOT/fixed) out4=$(nix-store --print-fixed-path --recursive sha256 "1ixr6yd3297ciyp9im522dfxpqbkhcw0pylkb2aab915278fqaik" fixed) [ "$out" = "$out4" ] + +# Can use `outputHashMode = "nar";` instead of `"recursive"` now. +clearStore +nix-build fixed.nix -A nar-not-recursive --no-out-link diff --git a/tests/functional/flakes/build-paths.sh b/tests/functional/flakes/build-paths.sh index ff012e1b3..4e5c68095 100644 --- a/tests/functional/flakes/build-paths.sh +++ b/tests/functional/flakes/build-paths.sh @@ -56,6 +56,23 @@ cat > $flake1Dir/flake.nix < \$foo/file + echo "out" > \$out/file + ''; + }; + in top // { + foo = top.foo // { + outputSpecified = true; + }; + }; }; } EOF @@ -94,3 +111,10 @@ nix build --json --out-link $TEST_ROOT/result $flake1Dir#a12 expectStderr 1 nix build --impure --json --out-link $TEST_ROOT/result $flake1Dir#a13 \ | grepQuiet "has 2 entries in its context. It should only have exactly one entry" + +# Test accessing output in installables with `.` (foobarbaz.) +nix build --json --no-link $flake1Dir#a14.foo | jq --exit-status ' + (.[0] | + (.drvPath | match(".*dot-installable.drv")) and + (.outputs | keys == ["foo"])) +' diff --git a/tests/functional/flakes/flakes.sh b/tests/functional/flakes/flakes.sh index 427290883..4f41cae0a 100644 --- a/tests/functional/flakes/flakes.sh +++ b/tests/functional/flakes/flakes.sh @@ -93,6 +93,24 @@ foo EOF chmod +x $nonFlakeDir/shebang-comments.sh +cat > $nonFlakeDir/shebang-different-comments.sh < $nonFlakeDir/shebang-reject.sh <&1 | grepQuiet -F 'error: unsupported unquoted character in nix shebang: *. Use double backticks to escape?' diff --git a/tests/functional/flakes/mercurial.sh b/tests/functional/flakes/mercurial.sh index 0622c79b7..7074af6f7 100644 --- a/tests/functional/flakes/mercurial.sh +++ b/tests/functional/flakes/mercurial.sh @@ -1,6 +1,6 @@ source ./common.sh -[[ $(type -p hq) ]] || skipTest "Mercurial not installed" +[[ $(type -p hg) ]] || skipTest "Mercurial not installed" flake1Dir=$TEST_ROOT/flake-hg1 mkdir -p $flake1Dir diff --git a/tests/functional/hermetic.nix b/tests/functional/hermetic.nix index 0513540f0..d1dccdff3 100644 --- a/tests/functional/hermetic.nix +++ b/tests/functional/hermetic.nix @@ -1,4 +1,9 @@ -{ busybox, seed }: +{ busybox +, seed +# If we want the final derivation output to have references to its +# dependencies. Some tests need/want this, other don't. +, withFinalRefs ? false +}: with import ./config.nix; @@ -40,7 +45,7 @@ let buildCommand = '' echo hi-input3 read x < ${input2} - echo $x BAZ > $out + echo ${input2} $x BAZ > $out ''; }; @@ -54,6 +59,6 @@ in '' read x < ${input1} read y < ${input3} - echo "$x $y" > $out + echo ${if (builtins.trace withFinalRefs withFinalRefs) then "${input1} ${input3}" else ""} "$x $y" > $out ''; } diff --git a/tests/functional/init.sh b/tests/functional/init.sh index d697b1a30..97b1b0587 100755 --- a/tests/functional/init.sh +++ b/tests/functional/init.sh @@ -3,7 +3,7 @@ source common/vars-and-functions.sh test -n "$TEST_ROOT" if test -d "$TEST_ROOT"; then - chmod -R u+w "$TEST_ROOT" + chmod -R u+rw "$TEST_ROOT" # We would delete any daemon socket, so let's stop the daemon first. killDaemon rm -rf "$TEST_ROOT" diff --git a/tests/functional/lang.sh b/tests/functional/lang.sh index 12df32c87..e35795a7a 100755 --- a/tests/functional/lang.sh +++ b/tests/functional/lang.sh @@ -68,8 +68,16 @@ done for i in lang/eval-fail-*.nix; do echo "evaluating $i (should fail)"; i=$(basename "$i" .nix) + flags="$( + if [[ -e "lang/$i.flags" ]]; then + sed -e 's/#.*//' < "lang/$i.flags" + else + # note that show-trace is also set by init.sh + echo "--eval --strict --show-trace" + fi + )" if - expectStderr 1 nix-instantiate --eval --strict --show-trace "lang/$i.nix" \ + expectStderr 1 nix-instantiate $flags "lang/$i.nix" \ | sed "s!$(pwd)!/pwd!g" > "lang/$i.err" then diffAndAccept "$i" err err.exp diff --git a/tests/functional/lang/eval-fail-addErrorContext-example.err.exp b/tests/functional/lang/eval-fail-addErrorContext-example.err.exp new file mode 100644 index 000000000..4fad8f5c8 --- /dev/null +++ b/tests/functional/lang/eval-fail-addErrorContext-example.err.exp @@ -0,0 +1,24 @@ +error: + … while counting down; n = 10 + + … while counting down; n = 9 + + … while counting down; n = 8 + + … while counting down; n = 7 + + … while counting down; n = 6 + + … while counting down; n = 5 + + … while counting down; n = 4 + + … while counting down; n = 3 + + … while counting down; n = 2 + + … while counting down; n = 1 + + (stack trace truncated; use '--show-trace' to show the full, detailed trace) + + error: kaboom diff --git a/tests/functional/lang/eval-fail-addErrorContext-example.flags b/tests/functional/lang/eval-fail-addErrorContext-example.flags new file mode 100644 index 000000000..9b1f6458f --- /dev/null +++ b/tests/functional/lang/eval-fail-addErrorContext-example.flags @@ -0,0 +1 @@ +--eval --strict --no-show-trace diff --git a/tests/functional/lang/eval-fail-addErrorContext-example.nix b/tests/functional/lang/eval-fail-addErrorContext-example.nix new file mode 100644 index 000000000..996b24688 --- /dev/null +++ b/tests/functional/lang/eval-fail-addErrorContext-example.nix @@ -0,0 +1,9 @@ +let + countDown = n: + if n == 0 + then throw "kaboom" + else + builtins.addErrorContext + "while counting down; n = ${toString n}" + ("x" + countDown (n - 1)); +in countDown 10 diff --git a/tests/functional/lang/eval-okay-baseNameOf.exp b/tests/functional/lang/eval-okay-baseNameOf.exp new file mode 100644 index 000000000..52c33a57c --- /dev/null +++ b/tests/functional/lang/eval-okay-baseNameOf.exp @@ -0,0 +1 @@ +"ok" diff --git a/tests/functional/lang/eval-okay-baseNameOf.nix b/tests/functional/lang/eval-okay-baseNameOf.nix new file mode 100644 index 000000000..a7afdd896 --- /dev/null +++ b/tests/functional/lang/eval-okay-baseNameOf.nix @@ -0,0 +1,32 @@ +assert baseNameOf "" == ""; +assert baseNameOf "." == "."; +assert baseNameOf ".." == ".."; +assert baseNameOf "a" == "a"; +assert baseNameOf "a." == "a."; +assert baseNameOf "a.." == "a.."; +assert baseNameOf "a.b" == "a.b"; +assert baseNameOf "a.b." == "a.b."; +assert baseNameOf "a.b.." == "a.b.."; +assert baseNameOf "a/" == "a"; +assert baseNameOf "a/." == "."; +assert baseNameOf "a/.." == ".."; +assert baseNameOf "a/b" == "b"; +assert baseNameOf "a/b." == "b."; +assert baseNameOf "a/b.." == "b.."; +assert baseNameOf "a/b/c" == "c"; +assert baseNameOf "a/b/c." == "c."; +assert baseNameOf "a/b/c.." == "c.."; +assert baseNameOf "a/b/c/d" == "d"; +assert baseNameOf "a/b/c/d." == "d."; +assert baseNameOf "a\\b" == "a\\b"; +assert baseNameOf "C:a" == "C:a"; +assert baseNameOf "a//b" == "b"; + +# It's been like this for close to a decade. We ought to commit to it. +# https://github.com/NixOS/nix/pull/582#issuecomment-121014450 +assert baseNameOf "a//" == ""; + +assert baseNameOf ./foo == "foo"; +assert baseNameOf ./foo/bar == "bar"; + +"ok" diff --git a/tests/functional/legacy-ssh-store.sh b/tests/functional/legacy-ssh-store.sh index 894efccd4..56b4c2d20 100644 --- a/tests/functional/legacy-ssh-store.sh +++ b/tests/functional/legacy-ssh-store.sh @@ -1,4 +1,9 @@ source common.sh +store_uri="ssh://localhost?remote-store=$TEST_ROOT/other-store" + # Check that store info trusted doesn't yet work with ssh:// -nix --store ssh://localhost?remote-store=$TEST_ROOT/other-store store info --json | jq -e 'has("trusted") | not' +nix --store "$store_uri" store info --json | jq -e 'has("trusted") | not' + +# Suppress grumpiness about multiple nixes on PATH +(nix --store "$store_uri" doctor || true) 2>&1 | grep "doesn't have a notion of trusted user" diff --git a/tests/functional/linux-sandbox.sh b/tests/functional/linux-sandbox.sh index ff7d257bd..e553791d9 100644 --- a/tests/functional/linux-sandbox.sh +++ b/tests/functional/linux-sandbox.sh @@ -60,7 +60,13 @@ testCert () { nocert=$TEST_ROOT/no-cert-file.pem cert=$TEST_ROOT/some-cert-file.pem +symlinkcert=$TEST_ROOT/symlink-cert-file.pem +transitivesymlinkcert=$TEST_ROOT/transitive-symlink-cert-file.pem +symlinkDir=$TEST_ROOT/symlink-dir echo -n "CERT_CONTENT" > $cert +ln -s $cert $symlinkcert +ln -s $symlinkcert $transitivesymlinkcert +ln -s $TEST_ROOT $symlinkDir # No cert in sandbox when not a fixed-output derivation testCert missing normal "$cert" @@ -73,3 +79,15 @@ testCert missing fixed-output "$nocert" # Cert in sandbox when ssl-cert-file is set to an existing file testCert present fixed-output "$cert" + +# Cert in sandbox when ssl-cert-file is set to a (potentially transitive) symlink to an existing file +testCert present fixed-output "$symlinkcert" +testCert present fixed-output "$transitivesymlinkcert" + +# Symlinks should be added in the sandbox directly and not followed +nix-sandbox-build symlink-derivation.nix -A depends_on_symlink +nix-sandbox-build symlink-derivation.nix -A test_sandbox_paths \ + --option extra-sandbox-paths "/file=$cert" \ + --option extra-sandbox-paths "/dir=$TEST_ROOT" \ + --option extra-sandbox-paths "/symlinkDir=$symlinkDir" \ + --option extra-sandbox-paths "/symlink=$symlinkcert" diff --git a/tests/functional/local-overlay-store/add-lower-inner.sh b/tests/functional/local-overlay-store/add-lower-inner.sh new file mode 100755 index 000000000..4efa7d088 --- /dev/null +++ b/tests/functional/local-overlay-store/add-lower-inner.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +setupStoreDirs + +initLowerStore + +mountOverlayfs + +# Add something to the overlay store +overlayPath=$(addTextToStore "$storeB" "overlay-file" "Add to overlay store") +stat "$storeBRoot/$overlayPath" + +# Now add something to the lower store +lowerPath=$(addTextToStore "$storeA" "lower-file" "Add to lower store") +stat "$storeVolume/store-a/$lowerPath" + +# Remount overlayfs to ensure synchronization +remountOverlayfs + +# Path should be accessible via overlay store +stat "$storeBRoot/$lowerPath" diff --git a/tests/functional/local-overlay-store/add-lower.sh b/tests/functional/local-overlay-store/add-lower.sh new file mode 100755 index 000000000..f0ac46a91 --- /dev/null +++ b/tests/functional/local-overlay-store/add-lower.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./add-lower-inner.sh diff --git a/tests/functional/local-overlay-store/bad-uris.sh b/tests/functional/local-overlay-store/bad-uris.sh new file mode 100644 index 000000000..2517681dd --- /dev/null +++ b/tests/functional/local-overlay-store/bad-uris.sh @@ -0,0 +1,25 @@ +source common.sh + +requireEnvironment +setupConfig +setupStoreDirs + +mkdir -p $TEST_ROOT/bad_test +badTestRoot=$TEST_ROOT/bad_test +storeBadRoot="local-overlay://?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop" +storeBadLower="local-overlay://?root=$storeBRoot&lower-store=$badTestRoot&upper-layer=$storeBTop" +storeBadUpper="local-overlay://?root=$storeBRoot&lower-store=$storeA&upper-layer=$badTestRoot" + +declare -a storesBad=( + "$storeBadRoot" "$storeBadLower" "$storeBadUpper" +) + +for i in "${storesBad[@]}"; do + echo $i + unshare --mount --map-root-user bash <> "$NIX_CONF_DIR/nix.conf" +} + +setupConfig () { + addConfig "require-drop-supplementary-groups = false" + addConfig "build-users-group = " +} + +enableFeatures "local-overlay-store" + +setupStoreDirs () { + # Attempt to create store dirs on tmpfs volume. + # This ensures lowerdir, upperdir and workdir will be on + # a consistent filesystem that fully supports OverlayFS. + storeVolume="$TEST_ROOT/stores" + mkdir -p "$storeVolume" + mount -t tmpfs tmpfs "$storeVolume" || true # But continue anyway if that fails. + + storeA="$storeVolume/store-a" + storeBTop="$storeVolume/store-b" + storeBRoot="$storeVolume/merged-store" + storeB="local-overlay://?root=$storeBRoot&lower-store=$storeA&upper-layer=$storeBTop" + # Creating testing directories + mkdir -p "$storeVolume"/{store-a/nix/store,store-b,merged-store/nix/store,workdir} +} + +# Mounting Overlay Store +mountOverlayfs () { + mount -t overlay overlay \ + -o lowerdir="$storeA/nix/store" \ + -o upperdir="$storeBTop" \ + -o workdir="$storeVolume/workdir" \ + "$storeBRoot/nix/store" \ + || skipTest "overlayfs is not supported" + + cleanupOverlay () { + umount "$storeBRoot/nix/store" + rm -r $storeVolume/workdir + } + trap cleanupOverlay EXIT +} + +remountOverlayfs () { + mount -o remount "$storeBRoot/nix/store" +} + +toRealPath () { + storeDir=$1; shift + storePath=$1; shift + echo $storeDir$(echo $storePath | sed "s^${NIX_STORE_DIR:-/nix/store}^^") +} + +initLowerStore () { + # Init lower store with some stuff + nix-store --store "$storeA" --add ../dummy + + # Build something in lower store + drvPath=$(nix-instantiate --store $storeA ../hermetic.nix --arg withFinalRefs true --arg busybox "$busybox" --arg seed 1) + pathInLowerStore=$(nix-store --store "$storeA" --realise $drvPath) +} + +execUnshare () { + exec unshare --mount --map-root-user "$SHELL" "$@" +} + +addTextToStore() { + storeDir=$1; shift + filename=$1; shift + content=$1; shift + filePath="$TEST_HOME/$filename" + echo "$content" > "$filePath" + nix-store --store "$storeDir" --add "$filePath" +} diff --git a/tests/functional/local-overlay-store/delete-duplicate-inner.sh b/tests/functional/local-overlay-store/delete-duplicate-inner.sh new file mode 100644 index 000000000..4f3ff25bd --- /dev/null +++ b/tests/functional/local-overlay-store/delete-duplicate-inner.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +setupStoreDirs + +initLowerStore + +mountOverlayfs + +# Add to overlay before lower to ensure file is duplicated +upperPath=$(nix-store --store "$storeB" --add delete-duplicate.sh) +lowerPath=$(nix-store --store "$storeA" --add delete-duplicate.sh) +[[ "$upperPath" = "$lowerPath" ]] + +# Check there really are two files with different inodes +upperInode=$(stat -c %i "$storeBRoot/$upperPath") +lowerInode=$(stat -c %i "$storeA/$lowerPath") +[[ "$upperInode" != "$lowerInode" ]] + +# Now delete file via the overlay store +nix-store --store "$storeB&remount-hook=$PWD/remount.sh" --delete "$upperPath" + +# Check there is no longer a file in upper layer +expect 1 stat "$storeBTop/${upperPath##/nix/store/}" + +# Check that overlay file is now the one in lower layer +upperInode=$(stat -c %i "$storeBRoot/$upperPath") +lowerInode=$(stat -c %i "$storeA/$lowerPath") +[[ "$upperInode" = "$lowerInode" ]] diff --git a/tests/functional/local-overlay-store/delete-duplicate.sh b/tests/functional/local-overlay-store/delete-duplicate.sh new file mode 100644 index 000000000..0c0b1a3b2 --- /dev/null +++ b/tests/functional/local-overlay-store/delete-duplicate.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./delete-duplicate-inner.sh diff --git a/tests/functional/local-overlay-store/delete-refs-inner.sh b/tests/functional/local-overlay-store/delete-refs-inner.sh new file mode 100644 index 000000000..385eeadc9 --- /dev/null +++ b/tests/functional/local-overlay-store/delete-refs-inner.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +setupStoreDirs + +initLowerStore + +mountOverlayfs + +export NIX_REMOTE="$storeB" +stateB="$storeBRoot/nix/var/nix" +hermetic=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2) +input1=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2 -A passthru.input1 -j0) +input2=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2 -A passthru.input2 -j0) +input3=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg withFinalRefs true --arg seed 2 -A passthru.input3 -j0) + +# Can't delete because referenced +expectStderr 1 nix-store --delete $input1 | grepQuiet "Cannot delete path" +expectStderr 1 nix-store --delete $input2 | grepQuiet "Cannot delete path" +expectStderr 1 nix-store --delete $input3 | grepQuiet "Cannot delete path" + +# These same paths are referenced in the lower layer (by the seed 1 +# build done in `initLowerStore`). +expectStderr 1 nix-store --store "$storeA" --delete $input2 | grepQuiet "Cannot delete path" +expectStderr 1 nix-store --store "$storeA" --delete $input3 | grepQuiet "Cannot delete path" + +# Can delete +nix-store --delete $hermetic + +# Now unreferenced in upper layer, can delete +nix-store --delete $input3 +nix-store --delete $input2 diff --git a/tests/functional/local-overlay-store/delete-refs.sh b/tests/functional/local-overlay-store/delete-refs.sh new file mode 100755 index 000000000..942d7fbdc --- /dev/null +++ b/tests/functional/local-overlay-store/delete-refs.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./delete-refs-inner.sh diff --git a/tests/functional/local-overlay-store/gc-inner.sh b/tests/functional/local-overlay-store/gc-inner.sh new file mode 100644 index 000000000..ea92154d2 --- /dev/null +++ b/tests/functional/local-overlay-store/gc-inner.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +setupStoreDirs + +initLowerStore + +mountOverlayfs + +export NIX_REMOTE="$storeB" +stateB="$storeBRoot/nix/var/nix" +outPath=$(nix-build ../hermetic.nix --no-out-link --arg busybox "$busybox" --arg seed 2) + +# Set a GC root. +mkdir -p "$stateB" +rm -f "$stateB"/gcroots/foo +ln -sf $outPath "$stateB"/gcroots/foo + +[ "$(nix-store -q --roots $outPath)" = "$stateB/gcroots/foo -> $outPath" ] + +nix-store --gc --print-roots | grep $outPath +nix-store --gc --print-live | grep $outPath +if nix-store --gc --print-dead | grep -E $outPath$; then false; fi + +nix-store --gc --print-dead + +expect 1 nix-store --delete $outPath +test -e "$storeBRoot/$outPath" + +shopt -s nullglob +for i in $storeBRoot/*; do + if [[ $i =~ /trash ]]; then continue; fi # compat with old daemon + touch $i.lock + touch $i.chroot +done + +nix-collect-garbage + +# Check that the root and its dependencies haven't been deleted. +cat "$storeBRoot/$outPath" + +rm "$stateB"/gcroots/foo + +nix-collect-garbage + +# Check that the output has been GC'd. +test ! -e $outPath + +# Check that the store is empty. +[ "$(ls -1 "$storeBTop" | wc -l)" = "0" ] diff --git a/tests/functional/local-overlay-store/gc.sh b/tests/functional/local-overlay-store/gc.sh new file mode 100755 index 000000000..1e1fb203e --- /dev/null +++ b/tests/functional/local-overlay-store/gc.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./gc-inner.sh diff --git a/tests/functional/local-overlay-store/local.mk b/tests/functional/local-overlay-store/local.mk new file mode 100644 index 000000000..6348a4423 --- /dev/null +++ b/tests/functional/local-overlay-store/local.mk @@ -0,0 +1,14 @@ +local-overlay-store-tests := \ + $(d)/check-post-init.sh \ + $(d)/redundant-add.sh \ + $(d)/build.sh \ + $(d)/bad-uris.sh \ + $(d)/add-lower.sh \ + $(d)/delete-refs.sh \ + $(d)/delete-duplicate.sh \ + $(d)/gc.sh \ + $(d)/verify.sh \ + $(d)/optimise.sh \ + $(d)/stale-file-handle.sh + +install-tests-groups += local-overlay-store diff --git a/tests/functional/local-overlay-store/optimise-inner.sh b/tests/functional/local-overlay-store/optimise-inner.sh new file mode 100755 index 000000000..eafbc77f7 --- /dev/null +++ b/tests/functional/local-overlay-store/optimise-inner.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +setupStoreDirs + +initLowerStore + +mountOverlayfs + +# Create a file to add to store +dupFilePath="$TEST_ROOT/dup-file" +echo Duplicate > "$dupFilePath" + +# Add it to the overlay store (it will be written to the upper layer) +dupFileStorePath=$(nix-store --store "$storeB" --add "$dupFilePath") + +# Now add it to the lower store so the store path is duplicated +nix-store --store "$storeA" --add "$dupFilePath" + +# Ensure overlayfs and layers and synchronised +remountOverlayfs + +dupFilename="${dupFileStorePath#/nix/store}" +lowerPath="$storeA/$dupFileStorePath" +upperPath="$storeBTop/$dupFilename" +overlayPath="$storeBRoot/nix/store/$dupFilename" + +# Check store path exists in both layers and overlay +lowerInode=$(stat -c %i "$lowerPath") +upperInode=$(stat -c %i "$upperPath") +overlayInode=$(stat -c %i "$overlayPath") +[[ $upperInode == $overlayInode ]] +[[ $upperInode != $lowerInode ]] + +# Run optimise to deduplicate store paths +nix-store --store "$storeB" --optimise +remountOverlayfs + +# Check path only exists in lower store +stat "$lowerPath" +stat "$overlayPath" +expect 1 stat "$upperPath" diff --git a/tests/functional/local-overlay-store/optimise.sh b/tests/functional/local-overlay-store/optimise.sh new file mode 100755 index 000000000..569afa248 --- /dev/null +++ b/tests/functional/local-overlay-store/optimise.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./optimise-inner.sh diff --git a/tests/functional/local-overlay-store/redundant-add-inner.sh b/tests/functional/local-overlay-store/redundant-add-inner.sh new file mode 100755 index 000000000..e37ef90e5 --- /dev/null +++ b/tests/functional/local-overlay-store/redundant-add-inner.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +setupStoreDirs + +initLowerStore + +mountOverlayfs + +### Do a redundant add + +# (Already done in `initLowerStore`, but repeated here for clarity.) +pathInLowerStore=$(nix-store --store "$storeA" --add ../dummy) + +# upper layer should not have it +expect 1 stat $(toRealPath "$storeBTop/nix/store" "$pathInLowerStore") + +pathFromB=$(nix-store --store "$storeB" --add ../dummy) + +[[ $pathInLowerStore == $pathFromB ]] + +# lower store should have it from before +stat $(toRealPath "$storeA/nix/store" "$pathInLowerStore") + +# upper layer should still not have it (no redundant copy) +expect 1 stat $(toRealPath "$storeBTop" "$pathInLowerStore") diff --git a/tests/functional/local-overlay-store/redundant-add.sh b/tests/functional/local-overlay-store/redundant-add.sh new file mode 100755 index 000000000..fbd4799e7 --- /dev/null +++ b/tests/functional/local-overlay-store/redundant-add.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./redundant-add-inner.sh diff --git a/tests/functional/local-overlay-store/remount.sh b/tests/functional/local-overlay-store/remount.sh new file mode 100755 index 000000000..0b06debb5 --- /dev/null +++ b/tests/functional/local-overlay-store/remount.sh @@ -0,0 +1,2 @@ +#!/bin/sh +mount -o remount "$1" diff --git a/tests/functional/local-overlay-store/stale-file-handle-inner.sh b/tests/functional/local-overlay-store/stale-file-handle-inner.sh new file mode 100755 index 000000000..d38f00cdc --- /dev/null +++ b/tests/functional/local-overlay-store/stale-file-handle-inner.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +setupStoreDirs + +initLowerStore + +mountOverlayfs + +buildInStore () { + nix-build --store "$1" ../hermetic.nix --arg busybox "$busybox" --arg seed 1 --no-out-link +} + +triggerStaleFileHandle () { + # Arrange it so there are duplicate paths + nix-store --store "$storeA" --gc # Clear lower store + buildInStore "$storeB" # Build into upper layer first + buildInStore "$storeA" # Then build in lower store + + # Duplicate paths mean GC will have to delete via upper layer + nix-store --store "$storeB" --gc + + # Clear lower store again to force building in upper layer + nix-store --store "$storeA" --gc + + # Now attempting to build in upper layer will fail + buildInStore "$storeB" +} + +# Without remounting, we should encounter errors +expectStderr 1 triggerStaleFileHandle | grepQuiet 'Stale file handle' + +# Configure remount-hook and reset OverlayFS +storeB="$storeB&remount-hook=$PWD/remount.sh" +remountOverlayfs + +# Now it should succeed +triggerStaleFileHandle diff --git a/tests/functional/local-overlay-store/stale-file-handle.sh b/tests/functional/local-overlay-store/stale-file-handle.sh new file mode 100755 index 000000000..5e75628ca --- /dev/null +++ b/tests/functional/local-overlay-store/stale-file-handle.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./stale-file-handle-inner.sh diff --git a/tests/functional/local-overlay-store/verify-inner.sh b/tests/functional/local-overlay-store/verify-inner.sh new file mode 100755 index 000000000..659f2ae50 --- /dev/null +++ b/tests/functional/local-overlay-store/verify-inner.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +set -eu -o pipefail + +set -x + +source common.sh + +# Avoid store dir being inside sandbox build-dir +unset NIX_STORE_DIR +unset NIX_STATE_DIR + +setupStoreDirs + +initLowerStore + +mountOverlayfs + + +## Initialise stores for test + +# Realise a derivation from the lower store to propagate paths to overlay DB +nix-store --store "$storeB" --realise $drvPath + +# Also ensure dummy file exists in overlay DB +dummyPath=$(nix-store --store "$storeB" --add ../dummy) + +# Add something to the lower store that will not be propagated to overlay DB +lowerOnlyPath=$(addTextToStore "$storeA" lower-only "Only in lower store") + +# Verify should be successful at this point +nix-store --store "$storeB" --verify --check-contents + +# Make a backup so we can repair later +backupStore="$storeVolume/backup" +mkdir "$backupStore" +cp -ar "$storeBRoot/nix" "$backupStore" + + +## Deliberately corrupt store paths + +# Delete one of the derivation inputs in the lower store +inputDrvFullPath=$(find "$storeA" -name "*-hermetic-input-1.drv") +inputDrvPath=${inputDrvFullPath/*\/nix\/store\///nix/store/} +rm -v "$inputDrvFullPath" + +# Truncate the contents of dummy file in lower store +find "$storeA" -name "*-dummy" -exec truncate -s 0 {} \; + +# Also truncate the file that only exists in lower store +truncate -s 0 "$storeA/$lowerOnlyPath" + +# Ensure overlayfs is synchronised +remountOverlayfs + + +## Now test that verify and repair work as expected + +# Verify overlay store without attempting to repair it +verifyOutput=$(expectStderr 1 nix-store --store "$storeB" --verify --check-contents) +<<<"$verifyOutput" grepQuiet "path '$inputDrvPath' disappeared, but it still has valid referrers!" +<<<"$verifyOutput" grepQuiet "path '$dummyPath' was modified! expected hash" +<<<"$verifyOutput" expectStderr 1 grepQuiet "$lowerOnlyPath" # Expect no error for corrupted lower-only path + +# Attempt to repair using backup +addConfig "substituters = $backupStore" +repairOutput=$(nix-store --store "$storeB" --verify --check-contents --repair 2>&1) +<<<"$repairOutput" grepQuiet "copying path '$inputDrvPath'" +<<<"$repairOutput" grepQuiet "copying path '$dummyPath'" diff --git a/tests/functional/local-overlay-store/verify.sh b/tests/functional/local-overlay-store/verify.sh new file mode 100755 index 000000000..8b44603ff --- /dev/null +++ b/tests/functional/local-overlay-store/verify.sh @@ -0,0 +1,5 @@ +source common.sh + +requireEnvironment +setupConfig +execUnshare ./verify-inner.sh diff --git a/tests/functional/local-store.sh b/tests/functional/local-store.sh deleted file mode 100644 index f7c8eb3f1..000000000 --- a/tests/functional/local-store.sh +++ /dev/null @@ -1,22 +0,0 @@ -source common.sh - -cd $TEST_ROOT - -echo example > example.txt -mkdir -p ./x - -NIX_STORE_DIR=$TEST_ROOT/x - -CORRECT_PATH=$(nix-store --store ./x --add example.txt) - -PATH1=$(nix path-info --store ./x $CORRECT_PATH) -[ $CORRECT_PATH == $PATH1 ] - -PATH2=$(nix path-info --store "$PWD/x" $CORRECT_PATH) -[ $CORRECT_PATH == $PATH2 ] - -PATH3=$(nix path-info --store "local?root=$PWD/x" $CORRECT_PATH) -[ $CORRECT_PATH == $PATH3 ] - -# Ensure store info trusted works with local store -nix --store ./x store info --json | jq -e '.trusted' diff --git a/tests/functional/local.mk b/tests/functional/local.mk index 8bb8e3600..ca9837d32 100644 --- a/tests/functional/local.mk +++ b/tests/functional/local.mk @@ -83,7 +83,7 @@ nix_tests = \ export.sh \ config.sh \ add.sh \ - local-store.sh \ + chroot-store.sh \ filter-source.sh \ misc.sh \ dump-db.sh \ diff --git a/tests/functional/plugins/local.mk b/tests/functional/plugins/local.mk index 40350aa96..2314e1341 100644 --- a/tests/functional/plugins/local.mk +++ b/tests/functional/plugins/local.mk @@ -8,4 +8,4 @@ libplugintest_ALLOW_UNDEFINED := 1 libplugintest_EXCLUDE_FROM_LIBRARY_LIST := 1 -libplugintest_CXXFLAGS := -I src/libutil -I src/libstore -I src/libexpr -I src/libfetchers +libplugintest_CXXFLAGS := $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libexpr) $(INCLUDE_libfetchers) diff --git a/tests/functional/remote-store.sh b/tests/functional/remote-store.sh index dc80f8b55..cc5dd1833 100644 --- a/tests/functional/remote-store.sh +++ b/tests/functional/remote-store.sh @@ -13,6 +13,8 @@ startDaemon if isDaemonNewer "2.15pre0"; then # Ensure that ping works trusted with new daemon nix store info --json | jq -e '.trusted' + # Suppress grumpiness about multiple nixes on PATH + (nix doctor || true) 2>&1 | grep 'You are trusted by' else # And the the field is absent with the old daemon nix store info --json | jq -e 'has("trusted") | not' diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index 4938c2267..f11fa7140 100644 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -47,6 +47,9 @@ testRepl () { | grep "attribute 'currentSystem' missing" nix repl "${nixArgs[@]}" 2>&1 <<< "builtins.currentSystem" \ | grep "$(nix-instantiate --eval -E 'builtins.currentSystem')" + + expectStderr 1 nix repl ${testDir}/simple.nix \ + | grepQuiet -s "error: path '$testDir/simple.nix' is not a flake" } # Simple test, try building a drv diff --git a/tests/functional/shell-hello.nix b/tests/functional/shell-hello.nix index dfe66ef93..c46fdec8a 100644 --- a/tests/functional/shell-hello.nix +++ b/tests/functional/shell-hello.nix @@ -1,6 +1,6 @@ with import ./config.nix; -{ +rec { hello = mkDerivation { name = "hello"; outputs = [ "out" "dev" ]; @@ -24,6 +24,22 @@ with import ./config.nix; ''; }; + hello-symlink = mkDerivation { + name = "hello-symlink"; + buildCommand = + '' + ln -s ${hello} $out + ''; + }; + + forbidden-symlink = mkDerivation { + name = "forbidden-symlink"; + buildCommand = + '' + ln -s /tmp/foo/bar $out + ''; + }; + salve-mundi = mkDerivation { name = "salve-mundi"; outputs = [ "out" ]; diff --git a/tests/functional/shell.nix b/tests/functional/shell.nix index 92d94fbc2..6a7dd7ad1 100644 --- a/tests/functional/shell.nix +++ b/tests/functional/shell.nix @@ -21,14 +21,6 @@ let pkgs = rec { export PATH=$PATH:$pkg/bin done - # mimic behavior of stdenv for `$out` etc. for structured attrs. - if [ -n "''${NIX_ATTRS_SH_FILE}" ]; then - for o in "''${!outputs[@]}"; do - eval "''${o}=''${outputs[$o]}" - export "''${o}" - done - fi - declare -a arr1=(1 2 "3 4" 5) declare -a arr2=(x $'\n' $'x\ny') fun() { diff --git a/tests/functional/shell.sh b/tests/functional/shell.sh index 8bbeabedf..8a3fef3e7 100644 --- a/tests/functional/shell.sh +++ b/tests/functional/shell.sh @@ -10,6 +10,11 @@ nix shell -f shell-hello.nix hello -c hello NixOS | grep 'Hello NixOS' nix shell -f shell-hello.nix hello^dev -c hello2 | grep 'Hello2' nix shell -f shell-hello.nix 'hello^*' -c hello2 | grep 'Hello2' +# Test output paths that are a symlink. +nix shell -f shell-hello.nix hello-symlink -c hello | grep 'Hello World' + +# Test that symlinks outside of the store don't work. +expect 1 nix shell -f shell-hello.nix forbidden-symlink -c hello 2>&1 | grepQuiet "is not in the Nix store" if isDaemonNewer "2.20.0pre20231220"; then # Test that command line attribute ordering is reflected in the PATH diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index f11992dcd..6711efbb4 100644 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -32,4 +32,4 @@ jsonOut="$(nix print-dev-env -f structured-attrs-shell.nix --json)" test "$(<<<"$jsonOut" jq '.structuredAttrs|keys|.[]' -r)" = "$(printf ".attrs.json\n.attrs.sh")" -test "$(<<<"$jsonOut" jq '.variables.out.value' -r)" = "$(<<<"$jsonOut" jq '.structuredAttrs.".attrs.json"' -r | jq -r '.outputs.out')" +test "$(<<<"$jsonOut" jq '.variables.outputs.value.out' -r)" = "$(<<<"$jsonOut" jq '.structuredAttrs.".attrs.json"' -r | jq -r '.outputs.out')" diff --git a/tests/functional/symlink-derivation.nix b/tests/functional/symlink-derivation.nix new file mode 100644 index 000000000..e9a74cdce --- /dev/null +++ b/tests/functional/symlink-derivation.nix @@ -0,0 +1,59 @@ +with import ./config.nix; + +let + foo_in_store = builtins.toFile "foo" "foo"; + foo_symlink = mkDerivation { + name = "foo-symlink"; + buildCommand = '' + ln -s ${foo_in_store} $out + ''; + }; + symlink_to_not_in_store = mkDerivation { + name = "symlink-to-not-in-store"; + buildCommand = '' + ln -s ${builtins.toString ./.} $out + ''; + }; +in +{ + depends_on_symlink = mkDerivation { + name = "depends-on-symlink"; + buildCommand = '' + ( + set -x + + # `foo_symlink` should be a symlink pointing to `foo_in_store` + [[ -L ${foo_symlink} ]] + [[ $(readlink ${foo_symlink}) == ${foo_in_store} ]] + + # `symlink_to_not_in_store` should be a symlink pointing to `./.`, which + # is not available in the sandbox + [[ -L ${symlink_to_not_in_store} ]] + [[ $(readlink ${symlink_to_not_in_store}) == ${builtins.toString ./.} ]] + (! ls ${symlink_to_not_in_store}/) + + # Native paths + ) + echo "Success!" > $out + ''; + }; + + test_sandbox_paths = mkDerivation { + # Depends on the caller to set a bunch of `--sandbox-path` arguments + name = "test-sandbox-paths"; + buildCommand = '' + ( + set -x + [[ -f /file ]] + [[ -d /dir ]] + + # /symlink and /symlinkDir should be available as raw symlinks + # (pointing to files outside of the sandbox) + [[ -L /symlink ]] && [[ ! -e $(readlink /symlink) ]] + [[ -L /symlinkDir ]] && [[ ! -e $(readlink /symlinkDir) ]] + ) + + touch $out + ''; + }; +} diff --git a/tests/functional/test-libstoreconsumer/local.mk b/tests/functional/test-libstoreconsumer/local.mk index a1825c405..3e8581c57 100644 --- a/tests/functional/test-libstoreconsumer/local.mk +++ b/tests/functional/test-libstoreconsumer/local.mk @@ -8,7 +8,7 @@ test-libstoreconsumer_INSTALL_DIR := test-libstoreconsumer_SOURCES := \ $(wildcard $(d)/*.cc) \ -test-libstoreconsumer_CXXFLAGS += -I src/libutil -I src/libstore +test-libstoreconsumer_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) test-libstoreconsumer_LIBS = libstore libutil diff --git a/tests/functional/user-envs.sh b/tests/functional/user-envs.sh index dcd6b1b97..7c643f355 100644 --- a/tests/functional/user-envs.sh +++ b/tests/functional/user-envs.sh @@ -189,3 +189,9 @@ nix-env --set $outPath10 [ "$(nix-store -q --resolve $profiles/test)" = $outPath10 ] nix-env --set $drvPath10 [ "$(nix-store -q --resolve $profiles/test)" = $outPath10 ] + +# Test the case where $HOME contains a symlink. +mkdir -p $TEST_ROOT/real-home/alice/.nix-defexpr/channels +ln -sfn $TEST_ROOT/real-home $TEST_ROOT/home +ln -sfn $(pwd)/user-envs.nix $TEST_ROOT/home/alice/.nix-defexpr/channels/foo +HOME=$TEST_ROOT/home/alice nix-env -i foo-0.1 diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 98de31e13..4edf40c16 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -145,6 +145,8 @@ in githubFlakes = runNixOSTestFor "x86_64-linux" ./github-flakes.nix; + gitSubmodules = runNixOSTestFor "x86_64-linux" ./git-submodules.nix; + sourcehutFlakes = runNixOSTestFor "x86_64-linux" ./sourcehut-flakes.nix; tarballFlakes = runNixOSTestFor "x86_64-linux" ./tarball-flakes.nix; @@ -158,4 +160,6 @@ in fetch-git = runNixOSTestFor "x86_64-linux" ./fetch-git; ca-fd-leak = runNixOSTestFor "x86_64-linux" ./ca-fd-leak; + + gzip-content-encoding = runNixOSTestFor "x86_64-linux" ./gzip-content-encoding.nix; } diff --git a/tests/nixos/git-submodules.nix b/tests/nixos/git-submodules.nix new file mode 100644 index 000000000..570b1822b --- /dev/null +++ b/tests/nixos/git-submodules.nix @@ -0,0 +1,70 @@ +# Test Nix's remote build feature. + +{ lib, hostPkgs, ... }: + +{ + config = { + name = lib.mkDefault "git-submodules"; + + nodes = + { + remote = + { config, pkgs, ... }: + { + services.openssh.enable = true; + environment.systemPackages = [ pkgs.git ]; + }; + + client = + { config, lib, pkgs, ... }: + { + programs.ssh.extraConfig = "ConnectTimeout 30"; + environment.systemPackages = [ pkgs.git ]; + nix.extraOptions = "experimental-features = nix-command flakes"; + }; + }; + + testScript = { nodes }: '' + # fmt: off + import subprocess + + start_all() + + # Create an SSH key on the client. + subprocess.run([ + "${hostPkgs.openssh}/bin/ssh-keygen", "-t", "ed25519", "-f", "key", "-N", "" + ], capture_output=True, check=True) + client.succeed("mkdir -p -m 700 /root/.ssh") + client.copy_from_host("key", "/root/.ssh/id_ed25519") + client.succeed("chmod 600 /root/.ssh/id_ed25519") + + # Install the SSH key on the builders. + client.wait_for_unit("network.target") + + remote.succeed("mkdir -p -m 700 /root/.ssh") + remote.copy_from_host("key.pub", "/root/.ssh/authorized_keys") + remote.wait_for_unit("sshd") + client.succeed(f"ssh -o StrictHostKeyChecking=no {remote.name} 'echo hello world'") + + remote.succeed(""" + git init bar + git -C bar config user.email foobar@example.com + git -C bar config user.name Foobar + echo test >> bar/content + git -C bar add content + git -C bar commit -m 'Initial commit' + """) + + client.succeed(f""" + git init foo + git -C foo config user.email foobar@example.com + git -C foo config user.name Foobar + git -C foo submodule add root@{remote.name}:/tmp/bar sub + git -C foo add sub + git -C foo commit -m 'Add submodule' + """) + + client.succeed("nix --flake-registry \"\" flake prefetch 'git+file:///tmp/foo?submodules=1&ref=master'") + ''; + }; +} diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index 6f8a5b9d8..221045009 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -187,9 +187,14 @@ in client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2") # Test fetchTree on a github URL. - hash = client.succeed(f"nix eval --raw --expr '(fetchTree {info['url']}).narHash'") + hash = client.succeed(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr '(fetchTree {info['url']}).narHash'") assert hash == info['locked']['narHash'] + # Fetching without a narHash should succeed if trust-github is set and fail otherwise. + client.succeed(f"nix eval --raw --expr 'builtins.fetchTree github:github:fancy-enterprise/private-flake/{info['revision']}'") + out = client.fail(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr 'builtins.fetchTree github:github:fancy-enterprise/private-flake/{info['revision']}' 2>&1") + assert "will not fetch unlocked input" in out, "--no-trust-tarballs-from-git-forges did not fail with the expected error" + # Shut down the web server. The flake should be cached on the client. github.succeed("systemctl stop httpd.service") diff --git a/tests/nixos/gzip-content-encoding.nix b/tests/nixos/gzip-content-encoding.nix new file mode 100644 index 000000000..a5a0033fd --- /dev/null +++ b/tests/nixos/gzip-content-encoding.nix @@ -0,0 +1,71 @@ +# Test that compressed files fetched from server with compressed responses +# do not get excessively decompressed. +# E.g. fetching a zstd compressed tarball from a server, +# which compresses the response with `Content-Encoding: gzip`. +# The expected result is that the fetched file is a zstd archive. + +{ lib, config, ... }: + +let + pkgs = config.nodes.machine.nixpkgs.pkgs; + + ztdCompressedFile = pkgs.stdenv.mkDerivation { + name = "dummy-zstd-compressed-archive"; + dontUnpack = true; + nativeBuildInputs = with pkgs; [ zstd ]; + buildPhase = '' + mkdir archive + for _ in {1..100}; do echo "lorem" > archive/file1; done + for _ in {1..100}; do echo "ipsum" > archive/file2; done + tar --zstd -cf archive.tar.zst archive + ''; + installPhase = '' + install -Dm 644 -T archive.tar.zst $out/share/archive + ''; + }; + + fileCmd = "${pkgs.file}/bin/file"; +in + +{ + name = "gzip-content-encoding"; + + nodes = + { machine = + { config, pkgs, ... }: + { networking.firewall.allowedTCPPorts = [ 80 ]; + + services.nginx.enable = true; + services.nginx.virtualHosts."localhost" = + { root = "${ztdCompressedFile}/share/"; + # Make sure that nginx really tries to compress the + # file on the fly with no regard to size/mime. + # http://nginx.org/en/docs/http/ngx_http_gzip_module.html + extraConfig = '' + gzip on; + gzip_types *; + gzip_proxied any; + gzip_min_length 0; + ''; + }; + virtualisation.writableStore = true; + virtualisation.additionalPaths = with pkgs; [ file ]; + nix.settings.substituters = lib.mkForce [ ]; + }; + }; + + # Check that when nix-prefetch-url is used with a zst tarball it does not get decompressed. + testScript = { nodes }: '' + # fmt: off + start_all() + + machine.wait_for_unit("nginx.service") + machine.succeed(""" + # Make sure that the file is properly compressed as the test would be meaningless otherwise + curl --compressed -v http://localhost/archive |& tr -s ' ' |& grep --ignore-case 'content-encoding: gzip' + archive_path=$(nix-prefetch-url http://localhost/archive --print-path | tail -n1) + [[ $(${fileCmd} --brief --mime-type $archive_path) == "application/zstd" ]] + tar --zstd -xf $archive_path + """) + ''; +} diff --git a/tests/unit/libexpr-support/tests/nix_api_expr.hh b/tests/unit/libexpr-support/tests/nix_api_expr.hh new file mode 100644 index 000000000..d1840d034 --- /dev/null +++ b/tests/unit/libexpr-support/tests/nix_api_expr.hh @@ -0,0 +1,31 @@ +#pragma once +///@file +#include "nix_api_expr.h" +#include "nix_api_value.h" +#include "tests/nix_api_store.hh" + +#include + +namespace nixC { + +class nix_api_expr_test : public nix_api_store_test +{ +protected: + + nix_api_expr_test() + { + nix_libexpr_init(ctx); + state = nix_state_create(nullptr, nullptr, store); + value = nix_alloc_value(nullptr, state); + } + ~nix_api_expr_test() + { + nix_gc_decref(nullptr, value); + nix_state_free(state); + } + + EvalState * state; + Value * value; +}; + +} diff --git a/tests/unit/libexpr/local.mk b/tests/unit/libexpr/local.mk index 25810ad9c..c59191db4 100644 --- a/tests/unit/libexpr/local.mk +++ b/tests/unit/libexpr/local.mk @@ -23,15 +23,18 @@ libexpr-tests_EXTRA_INCLUDES = \ -I tests/unit/libexpr-support \ -I tests/unit/libstore-support \ -I tests/unit/libutil-support \ - -I src/libexpr \ - -I src/libfetchers \ - -I src/libstore \ - -I src/libutil + $(INCLUDE_libexpr) \ + $(INCLUDE_libexprc) \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) \ + $(INCLUDE_libstorec) \ + $(INCLUDE_libutil) \ + $(INCLUDE_libutilc) libexpr-tests_CXXFLAGS += $(libexpr-tests_EXTRA_INCLUDES) libexpr-tests_LIBS = \ - libexpr-test-support libstore-test-support libutils-test-support \ - libexpr libfetchers libstore libutil + libexpr-test-support libstore-test-support libutil-test-support \ + libexpr libexprc libfetchers libstore libstorec libutil libutilc libexpr-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) -lgmock diff --git a/tests/unit/libexpr/main.cc b/tests/unit/libexpr/main.cc new file mode 100644 index 000000000..cf7fcf5a3 --- /dev/null +++ b/tests/unit/libexpr/main.cc @@ -0,0 +1,39 @@ +#include +#include +#include "globals.hh" +#include "logging.hh" + +using namespace nix; + +int main (int argc, char **argv) { + if (argc > 1 && std::string_view(argv[1]) == "__build-remote") { + printError("test-build-remote: not supported in libexpr unit tests"); + return 1; + } + + // Disable build hook. We won't be testing remote builds in these unit tests. If we do, fix the above build hook. + settings.buildHook = {}; + + #if __linux__ // should match the conditional around sandboxBuildDir declaration. + + // When building and testing nix within the host's Nix sandbox, our store dir will be located in the host's sandboxBuildDir, e.g.: + // Host + // storeDir = /nix/store + // sandboxBuildDir = /build + // This process + // storeDir = /build/foo/bar/store + // sandboxBuildDir = /build + // However, we have a rule that the store dir must not be inside the storeDir, so we need to pick a different sandboxBuildDir. + settings.sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir"; + #endif + + #if __APPLE__ + // Avoid this error, when already running in a sandbox: + // sandbox-exec: sandbox_apply: Operation not permitted + settings.sandboxMode = smDisabled; + setEnv("_NIX_TEST_NO_SANDBOX", "1"); + #endif + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/unit/libexpr/nix_api_expr.cc b/tests/unit/libexpr/nix_api_expr.cc new file mode 100644 index 000000000..0818f1cab --- /dev/null +++ b/tests/unit/libexpr/nix_api_expr.cc @@ -0,0 +1,194 @@ +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "nix_api_expr.h" +#include "nix_api_value.h" + +#include "tests/nix_api_expr.hh" +#include "tests/string_callback.hh" + +#include "gmock/gmock.h" +#include + +namespace nixC { + +TEST_F(nix_api_expr_test, nix_expr_eval_from_string) +{ + nix_expr_eval_from_string(nullptr, state, "builtins.nixVersion", ".", value); + nix_value_force(nullptr, state, value); + std::string result; + nix_get_string(nullptr, value, OBSERVE_STRING(result)); + + ASSERT_STREQ(PACKAGE_VERSION, result.c_str()); +} + +TEST_F(nix_api_expr_test, nix_expr_eval_add_numbers) +{ + nix_expr_eval_from_string(nullptr, state, "1 + 1", ".", value); + nix_value_force(nullptr, state, value); + auto result = nix_get_int(nullptr, value); + + ASSERT_EQ(2, result); +} + +TEST_F(nix_api_expr_test, nix_expr_eval_drv) +{ + auto expr = R"(derivation { name = "myname"; builder = "mybuilder"; system = "mysystem"; })"; + nix_expr_eval_from_string(nullptr, state, expr, ".", value); + ASSERT_EQ(NIX_TYPE_ATTRS, nix_get_type(nullptr, value)); + + EvalState * stateFn = nix_state_create(nullptr, nullptr, store); + Value * valueFn = nix_alloc_value(nullptr, state); + nix_expr_eval_from_string(nullptr, stateFn, "builtins.toString", ".", valueFn); + ASSERT_EQ(NIX_TYPE_FUNCTION, nix_get_type(nullptr, valueFn)); + + EvalState * stateResult = nix_state_create(nullptr, nullptr, store); + Value * valueResult = nix_alloc_value(nullptr, stateResult); + nix_value_call(ctx, stateResult, valueFn, value, valueResult); + ASSERT_EQ(NIX_TYPE_STRING, nix_get_type(nullptr, valueResult)); + + std::string p; + nix_get_string(nullptr, valueResult, OBSERVE_STRING(p)); + std::string pEnd = "-myname"; + ASSERT_EQ(pEnd, p.substr(p.size() - pEnd.size())); + + // Clean up + nix_gc_decref(nullptr, valueFn); + nix_state_free(stateFn); + + nix_gc_decref(nullptr, valueResult); + nix_state_free(stateResult); +} + +TEST_F(nix_api_expr_test, nix_build_drv) +{ + auto expr = R"(derivation { name = "myname"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "echo foo > $out" ]; + })"; + nix_expr_eval_from_string(nullptr, state, expr, ".", value); + + Value * drvPathValue = nix_get_attr_byname(nullptr, value, state, "drvPath"); + std::string drvPath; + nix_get_string(nullptr, drvPathValue, OBSERVE_STRING(drvPath)); + + std::string p = drvPath; + std::string pEnd = "-myname.drv"; + ASSERT_EQ(pEnd, p.substr(p.size() - pEnd.size())); + + // NOTE: .drvPath should be usually be ignored. Output paths are more versatile. + // See https://github.com/NixOS/nix/issues/6507 + // Use e.g. nix_string_realise to realise the output. + StorePath * drvStorePath = nix_store_parse_path(ctx, store, drvPath.c_str()); + ASSERT_EQ(true, nix_store_is_valid_path(ctx, store, drvStorePath)); + + Value * outPathValue = nix_get_attr_byname(ctx, value, state, "outPath"); + std::string outPath; + nix_get_string(ctx, outPathValue, OBSERVE_STRING(outPath)); + + p = outPath; + pEnd = "-myname"; + ASSERT_EQ(pEnd, p.substr(p.size() - pEnd.size())); + ASSERT_EQ(true, drvStorePath->path.isDerivation()); + + StorePath * outStorePath = nix_store_parse_path(ctx, store, outPath.c_str()); + ASSERT_EQ(false, nix_store_is_valid_path(ctx, store, outStorePath)); + + nix_store_realise(ctx, store, drvStorePath, nullptr, nullptr); + auto is_valid_path = nix_store_is_valid_path(ctx, store, outStorePath); + ASSERT_EQ(true, is_valid_path); + + // Clean up + nix_store_path_free(drvStorePath); + nix_store_path_free(outStorePath); +} + +TEST_F(nix_api_expr_test, nix_expr_realise_context_bad_value) +{ + auto expr = "true"; + nix_expr_eval_from_string(ctx, state, expr, ".", value); + assert_ctx_ok(); + auto r = nix_string_realise(ctx, state, value, false); + ASSERT_EQ(nullptr, r); + ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); + ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("cannot coerce"))); +} + +TEST_F(nix_api_expr_test, nix_expr_realise_context_bad_build) +{ + auto expr = R"( + derivation { name = "letsbuild"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "echo failing a build for testing purposes; exit 1;" ]; + } + )"; + nix_expr_eval_from_string(ctx, state, expr, ".", value); + assert_ctx_ok(); + auto r = nix_string_realise(ctx, state, value, false); + ASSERT_EQ(nullptr, r); + ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); + ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("failed with exit code 1"))); +} + +TEST_F(nix_api_expr_test, nix_expr_realise_context) +{ + // TODO (ca-derivations): add a content-addressed derivation output, which produces a placeholder + auto expr = R"( + '' + a derivation output: ${ + derivation { name = "letsbuild"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "echo foo > $out" ]; + }} + a path: ${builtins.toFile "just-a-file" "ooh file good"} + a derivation path by itself: ${ + builtins.unsafeDiscardOutputDependency + (derivation { + name = "not-actually-built-yet"; + system = builtins.currentSystem; + builder = "/bin/sh"; + args = [ "-c" "echo foo > $out" ]; + }).drvPath} + '' + )"; + nix_expr_eval_from_string(ctx, state, expr, ".", value); + assert_ctx_ok(); + auto r = nix_string_realise(ctx, state, value, false); + assert_ctx_ok(); + ASSERT_NE(nullptr, r); + + auto s = std::string(nix_realised_string_get_buffer_start(r), nix_realised_string_get_buffer_size(r)); + + EXPECT_THAT(s, testing::StartsWith("a derivation output:")); + EXPECT_THAT(s, testing::HasSubstr("-letsbuild\n")); + EXPECT_THAT(s, testing::Not(testing::HasSubstr("-letsbuild.drv"))); + EXPECT_THAT(s, testing::HasSubstr("a path:")); + EXPECT_THAT(s, testing::HasSubstr("-just-a-file")); + EXPECT_THAT(s, testing::Not(testing::HasSubstr("-just-a-file.drv"))); + EXPECT_THAT(s, testing::Not(testing::HasSubstr("ooh file good"))); + EXPECT_THAT(s, testing::HasSubstr("a derivation path by itself:")); + EXPECT_THAT(s, testing::EndsWith("-not-actually-built-yet.drv\n")); + + std::vector names; + size_t n = nix_realised_string_get_store_path_count(r); + for (size_t i = 0; i < n; ++i) { + const StorePath * p = nix_realised_string_get_store_path(r, i); + ASSERT_NE(nullptr, p); + std::string name; + nix_store_path_name(p, OBSERVE_STRING(name)); + names.push_back(name); + } + std::sort(names.begin(), names.end()); + ASSERT_EQ(3, names.size()); + EXPECT_THAT(names[0], testing::StrEq("just-a-file")); + EXPECT_THAT(names[1], testing::StrEq("letsbuild")); + EXPECT_THAT(names[2], testing::StrEq("not-actually-built-yet.drv")); + + nix_realised_string_free(r); +} + +} // namespace nixC diff --git a/tests/unit/libexpr/nix_api_external.cc b/tests/unit/libexpr/nix_api_external.cc new file mode 100644 index 000000000..2391f8317 --- /dev/null +++ b/tests/unit/libexpr/nix_api_external.cc @@ -0,0 +1,68 @@ +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "nix_api_expr.h" +#include "nix_api_expr_internal.h" +#include "nix_api_value.h" +#include "nix_api_external.h" + +#include "tests/nix_api_expr.hh" +#include "tests/string_callback.hh" + +#include + +namespace nixC { + +class MyExternalValueDesc : public NixCExternalValueDesc +{ +public: + MyExternalValueDesc(int x) + : _x(x) + { + print = print_function; + showType = show_type_function; + typeOf = type_of_function; + } + +private: + int _x; + static void print_function(void * self, nix_printer * printer) {} + + static void show_type_function(void * self, nix_string_return * res) {} + + static void type_of_function(void * self, nix_string_return * res) + { + MyExternalValueDesc * obj = static_cast(self); + + std::string type_string = "nix-external_x); + type_string += " )>"; + res->str = &*type_string.begin(); + } +}; + +TEST_F(nix_api_expr_test, nix_expr_eval_external) +{ + MyExternalValueDesc * external = new MyExternalValueDesc(42); + ExternalValue * val = nix_create_external_value(ctx, external, external); + nix_init_external(ctx, value, val); + + EvalState * stateResult = nix_state_create(nullptr, nullptr, store); + Value * valueResult = nix_alloc_value(nullptr, stateResult); + + EvalState * stateFn = nix_state_create(nullptr, nullptr, store); + Value * valueFn = nix_alloc_value(nullptr, stateFn); + + nix_expr_eval_from_string(nullptr, state, "builtins.typeOf", ".", valueFn); + + ASSERT_EQ(NIX_TYPE_EXTERNAL, nix_get_type(nullptr, value)); + + nix_value_call(ctx, state, valueFn, value, valueResult); + + std::string string_value; + nix_get_string(nullptr, valueResult, OBSERVE_STRING(string_value)); + ASSERT_STREQ("nix-external", string_value.c_str()); +} + +} diff --git a/tests/unit/libexpr/nix_api_value.cc b/tests/unit/libexpr/nix_api_value.cc new file mode 100644 index 000000000..7fbb2bbdc --- /dev/null +++ b/tests/unit/libexpr/nix_api_value.cc @@ -0,0 +1,190 @@ +#include "nix_api_store.h" +#include "nix_api_store_internal.h" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "nix_api_expr.h" +#include "nix_api_value.h" + +#include "tests/nix_api_expr.hh" +#include "tests/string_callback.hh" + +#include +#include + +namespace nixC { + +TEST_F(nix_api_expr_test, nix_value_set_get_int) +{ + ASSERT_EQ(0, nix_get_int(ctx, nullptr)); + ASSERT_DEATH(nix_get_int(ctx, value), ""); + + int myInt = 1; + nix_init_int(ctx, value, myInt); + + ASSERT_EQ(myInt, nix_get_int(ctx, value)); + ASSERT_STREQ("an integer", nix_get_typename(ctx, value)); + ASSERT_EQ(NIX_TYPE_INT, nix_get_type(ctx, value)); +} + +TEST_F(nix_api_expr_test, nix_value_set_get_float) +{ + ASSERT_FLOAT_EQ(0.0, nix_get_float(ctx, nullptr)); + ASSERT_DEATH(nix_get_float(ctx, value), ""); + + float myDouble = 1.0; + nix_init_float(ctx, value, myDouble); + + ASSERT_FLOAT_EQ(myDouble, nix_get_float(ctx, value)); + ASSERT_STREQ("a float", nix_get_typename(ctx, value)); + ASSERT_EQ(NIX_TYPE_FLOAT, nix_get_type(ctx, value)); +} + +TEST_F(nix_api_expr_test, nix_value_set_get_bool) +{ + ASSERT_EQ(false, nix_get_bool(ctx, nullptr)); + ASSERT_DEATH(nix_get_bool(ctx, value), ""); + + bool myBool = true; + nix_init_bool(ctx, value, myBool); + + ASSERT_EQ(myBool, nix_get_bool(ctx, value)); + ASSERT_STREQ("a Boolean", nix_get_typename(ctx, value)); + ASSERT_EQ(NIX_TYPE_BOOL, nix_get_type(ctx, value)); +} + +TEST_F(nix_api_expr_test, nix_value_set_get_string) +{ + std::string string_value; + ASSERT_EQ(NIX_ERR_UNKNOWN, nix_get_string(ctx, nullptr, OBSERVE_STRING(string_value))); + ASSERT_DEATH(nix_get_string(ctx, value, OBSERVE_STRING(string_value)), ""); + + const char * myString = "some string"; + nix_init_string(ctx, value, myString); + + nix_get_string(ctx, value, OBSERVE_STRING(string_value)); + ASSERT_STREQ(myString, string_value.c_str()); + ASSERT_STREQ("a string", nix_get_typename(ctx, value)); + ASSERT_EQ(NIX_TYPE_STRING, nix_get_type(ctx, value)); +} + +TEST_F(nix_api_expr_test, nix_value_set_get_null) +{ + ASSERT_DEATH(nix_get_typename(ctx, value), ""); + + nix_init_null(ctx, value); + + ASSERT_STREQ("null", nix_get_typename(ctx, value)); + ASSERT_EQ(NIX_TYPE_NULL, nix_get_type(ctx, value)); +} + +TEST_F(nix_api_expr_test, nix_value_set_get_path) +{ + ASSERT_EQ(nullptr, nix_get_path_string(ctx, nullptr)); + ASSERT_DEATH(nix_get_path_string(ctx, value), ""); + + const char * p = "/nix/store/40s0qmrfb45vlh6610rk29ym318dswdr-myname"; + nix_init_path_string(ctx, state, value, p); + + ASSERT_STREQ(p, nix_get_path_string(ctx, value)); + ASSERT_STREQ("a path", nix_get_typename(ctx, value)); + ASSERT_EQ(NIX_TYPE_PATH, nix_get_type(ctx, value)); +} + +TEST_F(nix_api_expr_test, nix_build_and_init_list) +{ + ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, nullptr, state, 0)); + ASSERT_EQ(0, nix_get_list_size(ctx, nullptr)); + + ASSERT_DEATH(nix_get_list_byidx(ctx, value, state, 0), ""); + ASSERT_DEATH(nix_get_list_size(ctx, value), ""); + + int size = 10; + ListBuilder * builder = nix_make_list_builder(ctx, state, size); + + Value * intValue = nix_alloc_value(ctx, state); + nix_init_int(ctx, intValue, 42); + nix_list_builder_insert(ctx, builder, 0, intValue); + nix_make_list(ctx, builder, value); + nix_list_builder_free(builder); + + ASSERT_EQ(42, nix_get_int(ctx, nix_get_list_byidx(ctx, value, state, 0))); + ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, 1)); + ASSERT_EQ(10, nix_get_list_size(ctx, value)); + + ASSERT_STREQ("a list", nix_get_typename(ctx, value)); + ASSERT_EQ(NIX_TYPE_LIST, nix_get_type(ctx, value)); + + // Clean up + nix_gc_decref(ctx, intValue); +} + +TEST_F(nix_api_expr_test, nix_build_and_init_attr) +{ + ASSERT_EQ(nullptr, nix_get_attr_byname(ctx, nullptr, state, 0)); + ASSERT_EQ(nullptr, nix_get_attr_byidx(ctx, nullptr, state, 0, nullptr)); + ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, nullptr, state, 0)); + ASSERT_EQ(0, nix_get_attrs_size(ctx, nullptr)); + ASSERT_EQ(false, nix_has_attr_byname(ctx, nullptr, state, "no-value")); + + ASSERT_DEATH(nix_get_attr_byname(ctx, value, state, 0), ""); + ASSERT_DEATH(nix_get_attr_byidx(ctx, value, state, 0, nullptr), ""); + ASSERT_DEATH(nix_get_attr_name_byidx(ctx, value, state, 0), ""); + ASSERT_DEATH(nix_get_attrs_size(ctx, value), ""); + ASSERT_DEATH(nix_has_attr_byname(ctx, value, state, "no-value"), ""); + + int size = 10; + const char ** out_name = (const char **) malloc(sizeof(char *)); + + BindingsBuilder * builder = nix_make_bindings_builder(ctx, state, size); + + Value * intValue = nix_alloc_value(ctx, state); + nix_init_int(ctx, intValue, 42); + + Value * stringValue = nix_alloc_value(ctx, state); + nix_init_string(ctx, stringValue, "foo"); + + nix_bindings_builder_insert(ctx, builder, "a", intValue); + nix_bindings_builder_insert(ctx, builder, "b", stringValue); + nix_make_attrs(ctx, value, builder); + nix_bindings_builder_free(builder); + + ASSERT_EQ(2, nix_get_attrs_size(ctx, value)); + + Value * out_value = nix_get_attr_byname(ctx, value, state, "a"); + ASSERT_EQ(42, nix_get_int(ctx, out_value)); + nix_gc_decref(ctx, out_value); + + out_value = nix_get_attr_byidx(ctx, value, state, 0, out_name); + ASSERT_EQ(42, nix_get_int(ctx, out_value)); + ASSERT_STREQ("a", *out_name); + nix_gc_decref(ctx, out_value); + + ASSERT_STREQ("a", nix_get_attr_name_byidx(ctx, value, state, 0)); + + ASSERT_EQ(true, nix_has_attr_byname(ctx, value, state, "b")); + ASSERT_EQ(false, nix_has_attr_byname(ctx, value, state, "no-value")); + + out_value = nix_get_attr_byname(ctx, value, state, "b"); + std::string string_value; + nix_get_string(ctx, out_value, OBSERVE_STRING(string_value)); + ASSERT_STREQ("foo", string_value.c_str()); + nix_gc_decref(nullptr, out_value); + + out_value = nix_get_attr_byidx(ctx, value, state, 1, out_name); + nix_get_string(ctx, out_value, OBSERVE_STRING(string_value)); + ASSERT_STREQ("foo", string_value.c_str()); + ASSERT_STREQ("b", *out_name); + nix_gc_decref(nullptr, out_value); + + ASSERT_STREQ("b", nix_get_attr_name_byidx(ctx, value, state, 1)); + + ASSERT_STREQ("a set", nix_get_typename(ctx, value)); + ASSERT_EQ(NIX_TYPE_ATTRS, nix_get_type(ctx, value)); + + // Clean up + nix_gc_decref(ctx, intValue); + nix_gc_decref(ctx, stringValue); + free(out_name); +} + +} diff --git a/tests/unit/libexpr/primops.cc b/tests/unit/libexpr/primops.cc index 7988eac70..5ddc031f7 100644 --- a/tests/unit/libexpr/primops.cc +++ b/tests/unit/libexpr/primops.cc @@ -91,7 +91,7 @@ namespace nix { } TEST_F(PrimOpTest, getEnv) { - setenv("_NIX_UNIT_TEST_ENV_VALUE", "test value", 1); + setEnv("_NIX_UNIT_TEST_ENV_VALUE", "test value"); auto v = eval("builtins.getEnv \"_NIX_UNIT_TEST_ENV_VALUE\""); ASSERT_THAT(v, IsStringEq("test value")); } diff --git a/tests/unit/libfetchers/local.mk b/tests/unit/libfetchers/local.mk new file mode 100644 index 000000000..e9f659fd7 --- /dev/null +++ b/tests/unit/libfetchers/local.mk @@ -0,0 +1,32 @@ +check: libfetchers-tests_RUN + +programs += libfetchers-tests + +libfetchers-tests_NAME = libnixfetchers-tests + +libfetchers-tests_ENV := _NIX_TEST_UNIT_DATA=$(d)/data + +libfetchers-tests_DIR := $(d) + +ifeq ($(INSTALL_UNIT_TESTS), yes) + libfetchers-tests_INSTALL_DIR := $(checkbindir) +else + libfetchers-tests_INSTALL_DIR := +endif + +libfetchers-tests_SOURCES := $(wildcard $(d)/*.cc) + +libfetchers-tests_EXTRA_INCLUDES = \ + -I tests/unit/libstore-support \ + -I tests/unit/libutil-support \ + $(INCLUDE_libfetchers) \ + $(INCLUDE_libstore) \ + $(INCLUDE_libutil) + +libfetchers-tests_CXXFLAGS += $(libfetchers-tests_EXTRA_INCLUDES) + +libfetchers-tests_LIBS = \ + libstore-test-support libutil-test-support \ + libfetchers libstore libutil + +libfetchers-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/tests/unit/libfetchers/public-key.cc b/tests/unit/libfetchers/public-key.cc new file mode 100644 index 000000000..fcd5c3af0 --- /dev/null +++ b/tests/unit/libfetchers/public-key.cc @@ -0,0 +1,18 @@ +#include +#include "fetchers.hh" +#include "json-utils.hh" + +namespace nix { + TEST(PublicKey, jsonSerialization) { + auto json = nlohmann::json(fetchers::PublicKey { .key = "ABCDE" }); + + ASSERT_EQ(json, R"({ "key": "ABCDE", "type": "ssh-ed25519" })"_json); + } + TEST(PublicKey, jsonDeserialization) { + auto pubKeyJson = R"({ "key": "ABCDE", "type": "ssh-ed25519" })"_json; + fetchers::PublicKey pubKey = pubKeyJson; + + ASSERT_EQ(pubKey.key, "ABCDE"); + ASSERT_EQ(pubKey.type, "ssh-ed25519"); + } +} diff --git a/tests/unit/libstore-support/tests/nix_api_store.hh b/tests/unit/libstore-support/tests/nix_api_store.hh new file mode 100644 index 000000000..a2d35d083 --- /dev/null +++ b/tests/unit/libstore-support/tests/nix_api_store.hh @@ -0,0 +1,66 @@ +#pragma once +///@file +#include "tests/nix_api_util.hh" + +#include "file-system.hh" + +#include "nix_api_store.h" +#include "nix_api_store_internal.h" + +#include +#include + +namespace fs = std::filesystem; + +namespace nixC { +class nix_api_store_test : public nix_api_util_context +{ +public: + nix_api_store_test() + { + nix_libstore_init(ctx); + init_local_store(); + }; + + ~nix_api_store_test() override + { + nix_store_free(store); + + for (auto & path : fs::recursive_directory_iterator(nixDir)) { + fs::permissions(path, fs::perms::owner_all); + } + fs::remove_all(nixDir); + } + + Store * store; + std::string nixDir; + std::string nixStoreDir; + +protected: + void init_local_store() + { +#ifdef _WIN32 + // no `mkdtemp` with MinGW + auto tmpl = nix::defaultTempDir() + "/tests_nix-store."; + for (size_t i = 0; true; ++i) { + nixDir = tmpl + std::string { i }; + if (fs::create_directory(nixDir)) break; + } +#else + auto tmpl = nix::defaultTempDir() + "/tests_nix-store.XXXXXX"; + nixDir = mkdtemp((char *) tmpl.c_str()); +#endif + + nixStoreDir = nixDir + "/my_nix_store"; + + // Options documented in `nix help-stores` + const char * p1[] = {"store", nixStoreDir.c_str()}; + const char * p2[] = {"state", (new std::string(nixDir + "/my_state"))->c_str()}; + const char * p3[] = {"log", (new std::string(nixDir + "/my_log"))->c_str()}; + + const char ** params[] = {p1, p2, p3, nullptr}; + + store = nix_store_open(ctx, "local", params); + } +}; +} diff --git a/tests/unit/libstore/local.mk b/tests/unit/libstore/local.mk index 63f6d011f..b8f895fad 100644 --- a/tests/unit/libstore/local.mk +++ b/tests/unit/libstore/local.mk @@ -19,13 +19,15 @@ libstore-tests_SOURCES := $(wildcard $(d)/*.cc) libstore-tests_EXTRA_INCLUDES = \ -I tests/unit/libstore-support \ -I tests/unit/libutil-support \ - -I src/libstore \ - -I src/libutil + $(INCLUDE_libstore) \ + $(INCLUDE_libstorec) \ + $(INCLUDE_libutil) \ + $(INCLUDE_libutilc) libstore-tests_CXXFLAGS += $(libstore-tests_EXTRA_INCLUDES) libstore-tests_LIBS = \ libstore-test-support libutil-test-support \ - libstore libutil + libstore libstorec libutil libutilc libstore-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/tests/unit/libstore/nix_api_store.cc b/tests/unit/libstore/nix_api_store.cc new file mode 100644 index 000000000..7c6ec0780 --- /dev/null +++ b/tests/unit/libstore/nix_api_store.cc @@ -0,0 +1,89 @@ +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "nix_api_store.h" +#include "nix_api_store_internal.h" + +#include "tests/nix_api_store.hh" +#include "tests/string_callback.hh" + +namespace nixC { + +std::string PATH_SUFFIX = "/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-name"; + +TEST_F(nix_api_util_context, nix_libstore_init) +{ + auto ret = nix_libstore_init(ctx); + ASSERT_EQ(NIX_OK, ret); +} + +TEST_F(nix_api_store_test, nix_store_get_uri) +{ + std::string str; + auto ret = nix_store_get_uri(ctx, store, OBSERVE_STRING(str)); + ASSERT_EQ(NIX_OK, ret); + ASSERT_STREQ("local", str.c_str()); +} + +TEST_F(nix_api_store_test, InvalidPathFails) +{ + nix_store_parse_path(ctx, store, "invalid-path"); + ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); +} + +TEST_F(nix_api_store_test, ReturnsValidStorePath) +{ + StorePath * result = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); + ASSERT_NE(result, nullptr); + ASSERT_STREQ("name", result->path.name().data()); + ASSERT_STREQ(PATH_SUFFIX.substr(1).c_str(), result->path.to_string().data()); +} + +TEST_F(nix_api_store_test, SetsLastErrCodeToNixOk) +{ + nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); + ASSERT_EQ(ctx->last_err_code, NIX_OK); +} + +TEST_F(nix_api_store_test, DoesNotCrashWhenContextIsNull) +{ + ASSERT_NO_THROW(nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str())); +} + +TEST_F(nix_api_store_test, get_version) +{ + std::string str; + auto ret = nix_store_get_version(ctx, store, OBSERVE_STRING(str)); + ASSERT_EQ(NIX_OK, ret); + ASSERT_STREQ(PACKAGE_VERSION, str.c_str()); +} + +TEST_F(nix_api_util_context, nix_store_open_dummy) +{ + nix_libstore_init(ctx); + Store * store = nix_store_open(ctx, "dummy://", nullptr); + ASSERT_EQ(NIX_OK, ctx->last_err_code); + ASSERT_STREQ("dummy", store->ptr->getUri().c_str()); + + std::string str; + nix_store_get_version(ctx, store, OBSERVE_STRING(str)); + ASSERT_STREQ("", str.c_str()); + + nix_store_free(store); +} + +TEST_F(nix_api_util_context, nix_store_open_invalid) +{ + nix_libstore_init(ctx); + Store * store = nix_store_open(ctx, "invalid://", nullptr); + ASSERT_EQ(NIX_ERR_NIX_ERROR, ctx->last_err_code); + ASSERT_EQ(nullptr, store); + nix_store_free(store); +} + +TEST_F(nix_api_store_test, nix_store_is_valid_path_not_in_store) +{ + StorePath * path = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); + ASSERT_EQ(false, nix_store_is_valid_path(ctx, store, path)); +} + +} diff --git a/tests/unit/libutil-support/tests/nix_api_util.hh b/tests/unit/libutil-support/tests/nix_api_util.hh new file mode 100644 index 000000000..75d302bd6 --- /dev/null +++ b/tests/unit/libutil-support/tests/nix_api_util.hh @@ -0,0 +1,37 @@ +#pragma once +///@file +#include "nix_api_util.h" + +#include + +namespace nixC { + +class nix_api_util_context : public ::testing::Test +{ +protected: + + nix_api_util_context() + { + ctx = nix_c_context_create(); + nix_libutil_init(ctx); + }; + + ~nix_api_util_context() override + { + nix_c_context_free(ctx); + ctx = nullptr; + } + + nix_c_context * ctx; + + inline void assert_ctx_ok() { + if (nix_err_code(ctx) == NIX_OK) { + return; + } + unsigned int n; + const char * p = nix_err_msg(nullptr, ctx, &n); + std::string msg(p, n); + FAIL() << "nix_err_code(ctx) != NIX_OK, message: " << msg; + } +}; +} diff --git a/tests/unit/libutil-support/tests/string_callback.cc b/tests/unit/libutil-support/tests/string_callback.cc new file mode 100644 index 000000000..2d0e0dad0 --- /dev/null +++ b/tests/unit/libutil-support/tests/string_callback.cc @@ -0,0 +1,10 @@ +#include "string_callback.hh" + +namespace nix::testing { + +void observe_string_cb(const char * start, unsigned int n, std::string * user_data) +{ + *user_data = std::string(start); +} + +} diff --git a/tests/unit/libutil-support/tests/string_callback.hh b/tests/unit/libutil-support/tests/string_callback.hh new file mode 100644 index 000000000..3a3e545e9 --- /dev/null +++ b/tests/unit/libutil-support/tests/string_callback.hh @@ -0,0 +1,16 @@ +#pragma once +#include + +namespace nix::testing { + +void observe_string_cb(const char * start, unsigned int n, std::string * user_data); + +inline void * observe_string_cb_data(std::string & out) +{ + return (void *) &out; +}; + +#define OBSERVE_STRING(str) \ + (nix_get_string_callback) nix::testing::observe_string_cb, nix::testing::observe_string_cb_data(str) + +} diff --git a/tests/unit/libutil/json-utils.cc b/tests/unit/libutil/json-utils.cc index f0ce15c93..ec653fff5 100644 --- a/tests/unit/libutil/json-utils.cc +++ b/tests/unit/libutil/json-utils.cc @@ -3,6 +3,7 @@ #include +#include "error.hh" #include "json-utils.hh" namespace nix { @@ -55,4 +56,120 @@ TEST(from_json, vectorOfOptionalInts) { ASSERT_FALSE(vals.at(1).has_value()); } +TEST(valueAt, simpleObject) { + auto simple = R"({ "hello": "world" })"_json; + + ASSERT_EQ(valueAt(getObject(simple), "hello"), "world"); + + auto nested = R"({ "hello": { "world": "" } })"_json; + + auto & nestedObject = valueAt(getObject(nested), "hello"); + + ASSERT_EQ(valueAt(nestedObject, "world"), ""); +} + +TEST(valueAt, missingKey) { + auto json = R"({ "hello": { "nested": "world" } })"_json; + + auto & obj = getObject(json); + + ASSERT_THROW(valueAt(obj, "foo"), Error); +} + +TEST(getObject, rightAssertions) { + auto simple = R"({ "object": {} })"_json; + + ASSERT_EQ(getObject(valueAt(getObject(simple), "object")), (nlohmann::json::object_t {})); + + auto nested = R"({ "object": { "object": {} } })"_json; + + auto & nestedObject = getObject(valueAt(getObject(nested), "object")); + + ASSERT_EQ(nestedObject, getObject(nlohmann::json::parse(R"({ "object": {} })"))); + ASSERT_EQ(getObject(valueAt(getObject(nestedObject), "object")), (nlohmann::json::object_t {})); +} + +TEST(getObject, wrongAssertions) { + auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json; + + auto & obj = getObject(json); + + ASSERT_THROW(getObject(valueAt(obj, "array")), Error); + ASSERT_THROW(getObject(valueAt(obj, "string")), Error); + ASSERT_THROW(getObject(valueAt(obj, "int")), Error); + ASSERT_THROW(getObject(valueAt(obj, "boolean")), Error); +} + +TEST(getArray, rightAssertions) { + auto simple = R"({ "array": [] })"_json; + + ASSERT_EQ(getArray(valueAt(getObject(simple), "array")), (nlohmann::json::array_t {})); +} + +TEST(getArray, wrongAssertions) { + auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json; + + ASSERT_THROW(getArray(valueAt(json, "object")), Error); + ASSERT_THROW(getArray(valueAt(json, "string")), Error); + ASSERT_THROW(getArray(valueAt(json, "int")), Error); + ASSERT_THROW(getArray(valueAt(json, "boolean")), Error); +} + +TEST(getString, rightAssertions) { + auto simple = R"({ "string": "" })"_json; + + ASSERT_EQ(getString(valueAt(getObject(simple), "string")), ""); +} + +TEST(getString, wrongAssertions) { + auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json; + + ASSERT_THROW(getString(valueAt(json, "object")), Error); + ASSERT_THROW(getString(valueAt(json, "array")), Error); + ASSERT_THROW(getString(valueAt(json, "int")), Error); + ASSERT_THROW(getString(valueAt(json, "boolean")), Error); +} + +TEST(getInteger, rightAssertions) { + auto simple = R"({ "int": 0 })"_json; + + ASSERT_EQ(getInteger(valueAt(getObject(simple), "int")), 0); +} + +TEST(getInteger, wrongAssertions) { + auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json; + + ASSERT_THROW(getInteger(valueAt(json, "object")), Error); + ASSERT_THROW(getInteger(valueAt(json, "array")), Error); + ASSERT_THROW(getInteger(valueAt(json, "string")), Error); + ASSERT_THROW(getInteger(valueAt(json, "boolean")), Error); +} + +TEST(getBoolean, rightAssertions) { + auto simple = R"({ "boolean": false })"_json; + + ASSERT_EQ(getBoolean(valueAt(getObject(simple), "boolean")), false); +} + +TEST(getBoolean, wrongAssertions) { + auto json = R"({ "object": {}, "array": [], "string": "", "int": 0, "boolean": false })"_json; + + ASSERT_THROW(getBoolean(valueAt(json, "object")), Error); + ASSERT_THROW(getBoolean(valueAt(json, "array")), Error); + ASSERT_THROW(getBoolean(valueAt(json, "string")), Error); + ASSERT_THROW(getBoolean(valueAt(json, "int")), Error); +} + +TEST(optionalValueAt, existing) { + auto json = R"({ "string": "ssh-rsa" })"_json; + + ASSERT_EQ(optionalValueAt(json, "string"), std::optional { "ssh-rsa" }); +} + +TEST(optionalValueAt, empty) { + auto json = R"({})"_json; + + ASSERT_EQ(optionalValueAt(json, "string2"), std::nullopt); +} + } /* namespace nix */ diff --git a/tests/unit/libutil/local.mk b/tests/unit/libutil/local.mk index 930efb90b..39b4c0782 100644 --- a/tests/unit/libutil/local.mk +++ b/tests/unit/libutil/local.mk @@ -18,11 +18,12 @@ libutil-tests_SOURCES := $(wildcard $(d)/*.cc) libutil-tests_EXTRA_INCLUDES = \ -I tests/unit/libutil-support \ - -I src/libutil + $(INCLUDE_libutil) \ + $(INCLUDE_libutilc) libutil-tests_CXXFLAGS += $(libutil-tests_EXTRA_INCLUDES) -libutil-tests_LIBS = libutil-test-support libutil +libutil-tests_LIBS = libutil-test-support libutil libutilc libutil-tests_LDFLAGS := -lrapidcheck $(GTEST_LIBS) diff --git a/tests/unit/libutil/nix_api_util.cc b/tests/unit/libutil/nix_api_util.cc new file mode 100644 index 000000000..d2999f55b --- /dev/null +++ b/tests/unit/libutil/nix_api_util.cc @@ -0,0 +1,141 @@ +#include "config.hh" +#include "args.hh" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "tests/nix_api_util.hh" +#include "tests/string_callback.hh" + +#include + +namespace nixC { + +TEST_F(nix_api_util_context, nix_context_error) +{ + std::string err_msg_ref; + try { + throw nix::Error("testing error"); + } catch (nix::Error & e) { + err_msg_ref = e.what(); + nix_context_error(ctx); + } + ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); + ASSERT_EQ(ctx->name, "nix::Error"); + ASSERT_EQ(*ctx->last_err, err_msg_ref); + ASSERT_EQ(ctx->info->msg.str(), "testing error"); + + try { + throw std::runtime_error("testing exception"); + } catch (std::exception & e) { + err_msg_ref = e.what(); + nix_context_error(ctx); + } + ASSERT_EQ(ctx->last_err_code, NIX_ERR_UNKNOWN); + ASSERT_EQ(*ctx->last_err, err_msg_ref); +} + +TEST_F(nix_api_util_context, nix_set_err_msg) +{ + ASSERT_EQ(ctx->last_err_code, NIX_OK); + nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "unknown test error"); + ASSERT_EQ(ctx->last_err_code, NIX_ERR_UNKNOWN); + ASSERT_EQ(*ctx->last_err, "unknown test error"); +} + +TEST(nix_api_util, nix_version_get) +{ + ASSERT_EQ(std::string(nix_version_get()), PACKAGE_VERSION); +} + +struct MySettings : nix::Config +{ + nix::Setting settingSet{this, "empty", "setting-name", "Description"}; +}; + +MySettings mySettings; +static nix::GlobalConfig::Register rs(&mySettings); + +TEST_F(nix_api_util_context, nix_setting_get) +{ + ASSERT_EQ(ctx->last_err_code, NIX_OK); + std::string setting_value; + nix_err result = nix_setting_get(ctx, "invalid-key", OBSERVE_STRING(setting_value)); + ASSERT_EQ(result, NIX_ERR_KEY); + + result = nix_setting_get(ctx, "setting-name", OBSERVE_STRING(setting_value)); + ASSERT_EQ(result, NIX_OK); + ASSERT_STREQ("empty", setting_value.c_str()); +} + +TEST_F(nix_api_util_context, nix_setting_set) +{ + nix_err result = nix_setting_set(ctx, "invalid-key", "new-value"); + ASSERT_EQ(result, NIX_ERR_KEY); + + result = nix_setting_set(ctx, "setting-name", "new-value"); + ASSERT_EQ(result, NIX_OK); + + std::string setting_value; + result = nix_setting_get(ctx, "setting-name", OBSERVE_STRING(setting_value)); + ASSERT_EQ(result, NIX_OK); + ASSERT_STREQ("new-value", setting_value.c_str()); +} + +TEST_F(nix_api_util_context, nix_err_msg) +{ + // no error + EXPECT_THROW(nix_err_msg(nullptr, ctx, NULL), nix::Error); + + // set error + nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "unknown test error"); + + // basic usage + std::string err_msg = nix_err_msg(NULL, ctx, NULL); + ASSERT_EQ(err_msg, "unknown test error"); + + // advanced usage + unsigned int sz; + err_msg = nix_err_msg(nix_c_context_create(), ctx, &sz); + ASSERT_EQ(sz, err_msg.size()); +} + +TEST_F(nix_api_util_context, nix_err_info_msg) +{ + std::string err_info; + + // no error + EXPECT_THROW(nix_err_info_msg(NULL, ctx, OBSERVE_STRING(err_info)), nix::Error); + + try { + throw nix::Error("testing error"); + } catch (...) { + nix_context_error(ctx); + } + nix_err_info_msg(nix_c_context_create(), ctx, OBSERVE_STRING(err_info)); + ASSERT_STREQ("testing error", err_info.c_str()); +} + +TEST_F(nix_api_util_context, nix_err_name) +{ + std::string err_name; + + // no error + EXPECT_THROW(nix_err_name(NULL, ctx, OBSERVE_STRING(err_name)), nix::Error); + + std::string err_msg_ref; + try { + throw nix::Error("testing error"); + } catch (...) { + nix_context_error(ctx); + } + nix_err_name(nix_c_context_create(), ctx, OBSERVE_STRING(err_name)); + ASSERT_EQ(std::string(err_name), "nix::Error"); +} + +TEST_F(nix_api_util_context, nix_err_code) +{ + ASSERT_EQ(nix_err_code(ctx), NIX_OK); + nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "unknown test error"); + ASSERT_EQ(nix_err_code(ctx), NIX_ERR_UNKNOWN); +} + +} diff --git a/tests/unit/libutil/tests.cc b/tests/unit/libutil/tests.cc index 4406fd184..d7e9edf0a 100644 --- a/tests/unit/libutil/tests.cc +++ b/tests/unit/libutil/tests.cc @@ -151,6 +151,16 @@ namespace nix { ASSERT_EQ(p1, "dir"); } + TEST(baseNameOf, trailingSlashes) { + auto p1 = baseNameOf("/dir//"); + ASSERT_EQ(p1, "dir"); + } + + TEST(baseNameOf, absoluteNothingSlashNothing) { + auto p1 = baseNameOf("//"); + ASSERT_EQ(p1, ""); + } + /* ---------------------------------------------------------------------------- * isInDir * --------------------------------------------------------------------------*/