Merge remote-tracking branch 'origin/master' into finish-value

.clang-format

@@ -28,3 +28,5 @@ EmptyLineBeforeAccessModifier: Leave
 #PackConstructorInitializers: BinPack
 BreakBeforeBinaryOperators: NonAssignment
 AlwaysBreakBeforeMultilineStrings: true
+IndentPPDirectives: AfterHash
+PPIndentWidth: 2

.github/CODEOWNERS

@@ -14,4 +14,4 @@
 src/libexpr/primops.cc @roberth
 
 # Libstore layer
-/src/libstore @thufschmitt
+/src/libstore @thufschmitt @ericson2314

.github/labeler.yml

@@ -1,3 +1,10 @@
+"contributor-experience":
+  - changed-files:
+    - any-glob-to-any-file: "CONTRIBUTING.md"
+    - any-glob-to-any-file: ".github/ISSUE_TEMPLATE/*"
+    - any-glob-to-any-file: ".github/PULL_REQUEST_TEMPLATE.md"
+    - any-glob-to-any-file: "doc/manual/src/contributing/**"
+
 "documentation":
   - changed-files:
     - any-glob-to-any-file: "doc/manual/*"

.github/workflows/backport.yml

@@ -21,7 +21,7 @@ jobs:
           fetch-depth: 0
       - name: Create backport PRs
         # should be kept in sync with `version`
-        uses: zeebe-io/backport-action@v2.4.1
+        uses: zeebe-io/backport-action@v2.5.0
         with:
           # Config README: https://github.com/zeebe-io/backport-action#backport-action
           github_token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci.yml

@@ -159,3 +159,11 @@ jobs:
         # deprecated 2024-02-24
         docker tag nix:$NIX_VERSION $IMAGE_ID:master
         docker push $IMAGE_ID:master
+
+  vm_tests:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: DeterminateSystems/nix-installer-action@main
+      - uses: DeterminateSystems/magic-nix-cache-action@main
+      - run: nix build -L .#hydraJobs.tests.githubFlakes .#hydraJobs.tests.tarballFlakes

.gitignore

@@ -49,6 +49,9 @@ perl/Makefile.config
 /src/libexpr/tests
 /tests/unit/libexpr/libnixexpr-tests
 
+# /src/libfetchers
+/tests/unit/libfetchers/libnixfetchers-tests
+
 # /src/libstore/
 *.gen.*
 /src/libstore/tests
@@ -115,8 +118,6 @@ perl/Makefile.config
 /misc/systemd/nix-daemon.conf
 /misc/upstart/nix-daemon.conf
 
-/src/resolve-system-dependencies/resolve-system-dependencies
-
 outputs/
 
 *.a

Makefile

@@ -18,7 +18,9 @@ makefiles = \
   src/libexpr/local.mk \
   src/libcmd/local.mk \
   src/nix/local.mk \
-  src/resolve-system-dependencies/local.mk \
+  src/libutil-c/local.mk \
+  src/libstore-c/local.mk \
+  src/libexpr-c/local.mk \
   scripts/local.mk \
   misc/bash/local.mk \
   misc/fish/local.mk \
@@ -34,6 +36,7 @@ makefiles += \
   tests/unit/libutil-support/local.mk \
   tests/unit/libstore/local.mk \
   tests/unit/libstore-support/local.mk \
+  tests/unit/libfetchers/local.mk \
   tests/unit/libexpr/local.mk \
   tests/unit/libexpr-support/local.mk
 endif
@@ -44,6 +47,7 @@ makefiles += \
   tests/functional/ca/local.mk \
   tests/functional/git-hashing/local.mk \
   tests/functional/dyn-drv/local.mk \
+  tests/functional/local-overlay-store/local.mk \
   tests/functional/test-libstoreconsumer/local.mk \
   tests/functional/plugins/local.mk
 endif
@@ -59,6 +63,10 @@ ifeq ($(ENABLE_INTERNAL_API_DOCS), yes)
 makefiles-late += doc/internal-api/local.mk
 endif
 
+ifeq ($(ENABLE_EXTERNAL_API_DOCS), yes)
+makefiles-late += doc/external-api/local.mk
+endif
+
 # Miscellaneous global Flags
 
 OPTIMIZE = 1
@@ -123,3 +131,10 @@ internal-api-html:
 	@echo "Internal API docs are disabled. Configure with '--enable-internal-api-docs', or avoid calling 'make internal-api-html'."
 	@exit 1
 endif
+
+ifneq ($(ENABLE_EXTERNAL_API_DOCS), yes)
+.PHONY: external-api-html
+external-api-html:
+	@echo "External API docs are disabled. Configure with '--enable-external-api-docs', or avoid calling 'make external-api-html'."
+	@exit 1
+endif

Makefile.config.in

@@ -12,6 +12,7 @@ ENABLE_BUILD = @ENABLE_BUILD@
 ENABLE_DOC_GEN = @ENABLE_DOC_GEN@
 ENABLE_FUNCTIONAL_TESTS = @ENABLE_FUNCTIONAL_TESTS@
 ENABLE_INTERNAL_API_DOCS = @ENABLE_INTERNAL_API_DOCS@
+ENABLE_EXTERNAL_API_DOCS = @ENABLE_EXTERNAL_API_DOCS@
 ENABLE_S3 = @ENABLE_S3@
 ENABLE_UNIT_TESTS = @ENABLE_UNIT_TESTS@
 GTEST_LIBS = @GTEST_LIBS@

configure.ac

@@ -150,6 +150,11 @@ AC_ARG_ENABLE(unit-tests, AS_HELP_STRING([--disable-unit-tests],[Do not build th
   ENABLE_UNIT_TESTS=$enableval, ENABLE_UNIT_TESTS=$ENABLE_BUILD)
 AC_SUBST(ENABLE_UNIT_TESTS)
 
+# Build external API docs by default
+AC_ARG_ENABLE(external_api_docs, AS_HELP_STRING([--enable-external-api-docs],[Build API docs for Nix's C interface]),
+  external_api_docs=$enableval, external_api_docs=yes)
+AC_SUBST(external_api_docs)
+
 AS_IF(
   [test "$ENABLE_BUILD" == "no" && test "$ENABLE_UNIT_TESTS" == "yes"],
   [AC_MSG_ERROR([Cannot enable unit tests when building overall is disabled. Please do not pass '--enable-unit-tests' or do not pass '--disable-build'.])])
@@ -172,6 +177,10 @@ AC_ARG_ENABLE(internal-api-docs, AS_HELP_STRING([--enable-internal-api-docs],[Bu
   ENABLE_INTERNAL_API_DOCS=$enableval, ENABLE_INTERNAL_API_DOCS=no)
 AC_SUBST(ENABLE_INTERNAL_API_DOCS)
 
+AC_ARG_ENABLE(external-api-docs, AS_HELP_STRING([--enable-external-api-docs],[Build API docs for Nix's external unstable C interfaces]),
+  ENABLE_EXTERNAL_API_DOCS=$enableval, ENABLE_EXTERNAL_API_DOCS=no)
+AC_SUBST(ENABLE_EXTERNAL_API_DOCS)
+
 AS_IF(
   [test "$ENABLE_FUNCTIONAL_TESTS" == "yes" || test "$ENABLE_DOC_GEN" == "yes"],
   [NEED_PROG(jq, jq)])

doc/external-api/.gitignore

@@ -0,0 +1,3 @@
+/doxygen.cfg
+/html
+/latex

doc/external-api/README.md

@@ -0,0 +1,121 @@
+# Getting started
+
+> **Warning** These bindings are **experimental**, which means they can change
+> at any time or be removed outright; nevertheless the plan is to provide a
+> stable external C API to the Nix language and the Nix store.
+
+The language library allows evaluating Nix expressions and interacting with Nix
+language values. The Nix store API is still rudimentary, and only allows
+initialising and connecting to a store for the Nix language evaluator to
+interact with.
+
+Currently there are two ways to interface with the Nix language evaluator
+programmatically:
+
+1. Embedding the evaluator
+2. Writing language plug-ins
+
+Embedding means you link the Nix C libraries in your program and use them from
+there. Adding a plug-in means you make a library that gets loaded by the Nix
+language evaluator, specified through a configuration option.
+
+Many of the components and mechanisms involved are not yet documented, therefore
+please refer to the [Nix source code](https://github.com/NixOS/nix/) for
+details. Additions to in-code documentation and the reference manual are highly
+appreciated.
+
+The following examples, for simplicity, don't include error handling. See the
+[Handling errors](@ref errors) section for more information.
+
+# Embedding the Nix Evaluator
+
+In this example we programmatically start the Nix language evaluator with a
+dummy store (that has no store paths and cannot be written to), and evaluate the
+Nix expression `builtins.nixVersion`.
+
+**main.c:**
+
+```C
+#include <nix_api_util.h>
+#include <nix_api_expr.h>
+#include <nix_api_value.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+// NOTE: This example lacks all error handling. Production code must check for
+// errors, as some return values will be undefined.
+
+void my_get_string_cb(const char * start, unsigned int n, char ** user_data)
+{
+    *user_data = strdup(start);
+}
+
+int main()
+{
+    nix_libexpr_init(NULL);
+
+    Store * store = nix_store_open(NULL, "dummy://", NULL);
+    EvalState * state = nix_state_create(NULL, NULL, store); // empty search path (NIX_PATH)
+    Value * value = nix_alloc_value(NULL, state);
+
+    nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value);
+    nix_value_force(NULL, state, value);
+
+    char * version;
+    nix_get_string(NULL, value, my_get_string_cb, version);
+    printf("Nix version: %s\n", version);
+
+    free(version);
+    nix_gc_decref(NULL, value);
+    nix_state_free(state);
+    nix_store_free(store);
+    return 0;
+}
+```
+
+**Usage:**
+
+```ShellSession
+$ gcc main.c $(pkg-config nix-expr-c --libs --cflags) -o main
+$ ./main
+Nix version: 2.17
+```
+
+# Writing a Nix language plug-in
+
+In this example we add a custom primitive operation (_primop_) to `builtins`. It
+will increment the argument if it is an integer and throw an error otherwise.
+
+**plugin.c:**
+
+```C
+#include <nix_api_util.h>
+#include <nix_api_expr.h>
+#include <nix_api_value.h>
+
+void increment(void* user_data, nix_c_context* ctx, EvalState* state, Value** args, Value* v) {
+    nix_value_force(NULL, state, args[0]);
+    if (nix_get_type(NULL, args[0]) == NIX_TYPE_INT) {
+      nix_init_int(NULL, v, nix_get_int(NULL, args[0]) + 1);
+    } else {
+      nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "First argument should be an integer.");
+    }
+}
+
+void nix_plugin_entry() {
+  const char* args[] = {"n", NULL};
+  PrimOp *p = nix_alloc_primop(NULL, increment, 1, "increment", args, "Example custom built-in function: increments an integer", NULL);
+  nix_register_primop(NULL, p);
+  nix_gc_decref(NULL, p);
+}
+```
+
+**Usage:**
+
+```ShellSession
+$ gcc plugin.c $(pkg-config nix-expr-c --libs --cflags) -shared -o plugin.so
+$ nix --plugin-files ./plugin.so repl
+nix-repl> builtins.increment 1
+2
+```

doc/external-api/doxygen.cfg.in

@@ -0,0 +1,57 @@
+# Doxyfile 1.9.5
+
+# The PROJECT_NAME tag is a single word (or a sequence of words surrounded by
+# double-quotes, unless you are using Doxywizard) that should identify the
+# project for which the documentation is generated. This name is used in the
+# title of most generated pages and in a few other places.
+# The default value is: My Project.
+
+PROJECT_NAME           = "Nix"
+
+# The PROJECT_NUMBER tag can be used to enter a project or revision number. This
+# could be handy for archiving the generated documentation or if some version
+# control system is used.
+
+PROJECT_NUMBER         = @PACKAGE_VERSION@
+
+# Using the PROJECT_BRIEF tag one can provide an optional one line description
+# for a project that appears at the top of each page and should give viewer a
+# quick idea about the purpose of the project. Keep the description short.
+
+PROJECT_BRIEF          = "Nix, the purely functional package manager: C API (experimental)"
+
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX         = NO
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
+
+# FIXME Make this list more maintainable somehow. We could maybe generate this
+# in the Makefile, but we would need to change how `.in` files are preprocessed
+# so they can expand variables despite configure variables.
+
+INPUT                  = \
+  src/libutil-c \
+  src/libexpr-c \
+  src/libstore-c \
+  doc/external-api/README.md
+
+FILE_PATTERNS = nix_api_*.h *.md
+
+# The INCLUDE_PATH tag can be used to specify one or more directories that
+# contain include files that are not input files but should be processed by the
+# preprocessor. Note that the INCLUDE_PATH is not recursive, so the setting of
+# RECURSIVE has no effect here.
+# This tag requires that the tag SEARCH_INCLUDES is set to YES.
+
+INCLUDE_PATH           = @RAPIDCHECK_HEADERS@
+EXCLUDE_PATTERNS = *_internal.h
+GENERATE_TREEVIEW = YES
+OPTIMIZE_OUTPUT_FOR_C  = YES
+
+USE_MDFILE_AS_MAINPAGE = doc/external-api/README.md

doc/external-api/local.mk

@@ -0,0 +1,7 @@
+$(docdir)/external-api/html/index.html $(docdir)/external-api/latex: $(d)/doxygen.cfg
+	mkdir -p $(docdir)/external-api
+	{ cat $< ; echo "OUTPUT_DIRECTORY=$(docdir)/external-api" ; } | doxygen -
+
+# Generate the HTML API docs for Nix's unstable C bindings
+.PHONY: external-api-html
+external-api-html: $(docdir)/external-api/html/index.html

doc/manual/custom.css

@@ -1,3 +1,25 @@
+:root {
+    --sidebar-width: 23em;
+}
+
+h1.menu-title::before {
+  content: "";
+  background-image: url("./favicon.svg");
+  padding: 1.25em;
+  background-position: center center;
+  background-size: 2em;
+  background-repeat: no-repeat;
+}
+
+
+h1.menu-title {
+  padding: 0.5em;
+}
+
+.sidebar .sidebar-scrollbox {
+  padding: 1em;
+}
+
 h1:not(:first-of-type) {
     margin-top: 1.3em;
 }

doc/manual/local.mk

@@ -175,6 +175,16 @@ $(d)/src/SUMMARY-rl-next.md: $(d)/src/release-notes/rl-next.md
 # Generate the HTML manual.
 .PHONY: manual-html
 manual-html: $(docdir)/manual/index.html
+
+# Open the built HTML manual in the default browser.
+manual-html-open: $(docdir)/manual/index.html
+	@echo "  OPEN  " $<; \
+	  xdg-open $< \
+		  || open $< \
+			|| { \
+		echo "Could not open the manual in a browser. Please open '$<'" >&2; \
+		false; \
+		}
 install: $(docdir)/manual/index.html
 
 # Generate 'nix' manpages.
@@ -207,7 +217,7 @@ doc/manual/generated/man1/nix3-manpages: $(d)/src/command-ref/new-cli
 # `@docroot@` is to be preserved for documenting the mechanism
 # FIXME: maybe contributing guides should live right next to the code
 # instead of in the manual
-$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md
+$(docdir)/manual/index.html: $(MANUAL_SRCS) $(d)/book.toml $(d)/anchors.jq $(d)/custom.css $(d)/src/SUMMARY.md $(d)/src/store/types $(d)/src/command-ref/new-cli $(d)/src/contributing/experimental-feature-descriptions.md $(d)/src/command-ref/conf-file.md $(d)/src/language/builtins.md $(d)/src/language/builtin-constants.md $(d)/src/release-notes/rl-next.md $(d)/src/figures $(d)/src/favicon.png $(d)/src/favicon.svg
 	$(trace-gen) \
 		tmp="$$(mktemp -d)"; \
 		cp -r doc/manual "$$tmp"; \

doc/manual/rl-next/remove-repl-flake.md

@@ -0,0 +1,8 @@
+---
+synopsis: Remove experimental repl-flake
+significance: significant
+issues: 10103
+prs: 10299
+---
+
+The `repl-flake` experimental feature has been removed. The `nix repl` command now works like the rest of the new CLI in that `nix repl {path}` now tries to load a flake at `{path}` (or fails if the `flakes` experimental feature isn't enabled).*

doc/manual/src/SUMMARY.md.in

@@ -110,6 +110,7 @@
     - [Derivation](protocols/json/derivation.md)
   - [Serving Tarball Flakes](protocols/tarball-fetcher.md)
   - [Store Path Specification](protocols/store-path.md)
+  - [Nix Archive (NAR) Format](protocols/nix-archive.md)
   - [Derivation "ATerm" file format](protocols/derivation-aterm.md)
 - [Glossary](glossary.md)
 - [Contributing](contributing/index.md)
@@ -119,7 +120,7 @@
   - [Experimental Features](contributing/experimental-features.md)
   - [CLI guideline](contributing/cli-guideline.md)
   - [C++ style guide](contributing/cxx.md)
-- [Release Notes](release-notes/index.md)
+- [Releases](release-notes/index.md)
 {{#include ./SUMMARY-rl-next.md}}
   - [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md)
   - [Release 2.20 (2024-01-29)](release-notes/rl-2.20.md)

doc/manual/src/architecture/architecture.md

@@ -69,7 +69,7 @@ It can also execute build plans to produce new data, which are made available to
 A build plan itself is a series of *build tasks*, together with their build inputs.
 
 > **Important**
-> A build task in Nix is called [derivation](../glossary.md#gloss-derivation).
+> A build task in Nix is called [derivation](@docroot@/glossary.md#gloss-derivation).
 
 Each build task has a special build input executed as *build instructions* in order to perform the build.
 The result of a build task can be input to another build task.

doc/manual/src/command-ref/nix-build.md

@@ -41,7 +41,7 @@ expression to a low-level [store derivation]) and [`nix-store
 --realise`](@docroot@/command-ref/nix-store/realise.md) (to build the store
 derivation).
 
-[store derivation]: ../glossary.md#gloss-store-derivation
+[store derivation]: @docroot@/glossary.md#gloss-store-derivation
 
 > **Warning**
 >

doc/manual/src/command-ref/nix-copy-closure.md

@@ -49,7 +49,7 @@ authentication, you can avoid typing the passphrase with `ssh-agent`.
   - `--include-outputs`\
     Also copy the outputs of [store derivation]s included in the closure.
 
-    [store derivation]: ../glossary.md#gloss-store-derivation
+    [store derivation]: @docroot@/glossary.md#gloss-store-derivation
 
   - `--use-substitutes` / `-s`\
     Attempt to download missing paths on the target machine using Nix’s

doc/manual/src/command-ref/nix-instantiate.md

@@ -23,7 +23,7 @@ It evaluates the Nix expressions in each of *files* (which defaults to
 derivation, a list of derivations, or a set of derivations. The paths
 of the resulting store derivations are printed on standard output.
 
-[store derivation]: ../glossary.md#gloss-store-derivation
+[store derivation]: @docroot@/glossary.md#gloss-store-derivation
 
 If *files* is the character `-`, then a Nix expression will be read from
 standard input.

doc/manual/src/command-ref/nix-store/query.md

@@ -40,12 +40,12 @@ symlink.
     derivations *paths*. These are the paths that will be produced when
     the derivation is built.
 
-    [output paths]: ../../glossary.md#gloss-output-path
+    [output paths]: @docroot@/glossary.md#gloss-output-path
 
   - `--requisites`; `-R`\
     Prints out the [closure] of the store path *paths*.
 
-    [closure]: ../../glossary.md#gloss-closure
+    [closure]: @docroot@/glossary.md#gloss-closure
 
     This query has one option:
 
@@ -66,7 +66,7 @@ symlink.
     *paths*, that is, their immediate dependencies. (For *all*
     dependencies, use `--requisites`.)
 
-    [references]: ../../glossary.md#gloss-reference
+    [references]: @docroot@/glossary.md#gloss-reference
 
   - `--referrers`\
     Prints the set of *referrers* of the store paths *paths*, that is,
@@ -90,7 +90,7 @@ symlink.
     example when *paths* were substituted from a binary cache.
     Use `--valid-derivers` instead to obtain valid paths only.
 
-    [deriver]: ../../glossary.md#gloss-deriver
+    [deriver]: @docroot@/glossary.md#gloss-deriver
 
   - `--valid-derivers`\
     Prints a set of derivation files (`.drv`) which are supposed produce

doc/manual/src/contributing/documentation.md

@@ -27,11 +27,9 @@ and open `./result-doc/share/doc/nix/manual/index.html`.
 To build the manual incrementally, [enter the development shell](./hacking.md) and run:
 
 ```console
-make manual-html -j $NIX_BUILD_CORES
+make manual-html-open -j $NIX_BUILD_CORES
 ```
 
-and open `./outputs/doc/share/doc/nix/manual/language/index.html`.
-
 In order to reflect changes to the [Makefile for the manual], clear all generated files before re-building:
 
 [Makefile for the manual]: https://github.com/NixOS/nix/blob/master/doc/manual/local.mk
@@ -208,3 +206,22 @@ or inside `nix-shell` or `nix develop`:
 # make internal-api-html
 # xdg-open ./outputs/doc/share/doc/nix/internal-api/html/index.html
 ```
+
+## C API documentation (experimental)
+
+[C API documentation] is available online.
+You can also build and view it yourself:
+
+[C API documentation]: https://hydra.nixos.org/job/nix/master/external-api-docs/latest/download-by-type/doc/external-api-docs
+
+```console
+# nix build .#hydraJobs.external-api-docs
+# xdg-open ./result/share/doc/nix/external-api/html/index.html
+```
+
+or inside `nix-shell` or `nix develop`:
+
+```
+# make external-api-html
+# xdg-open ./outputs/doc/share/doc/nix/external-api/html/index.html
+```

doc/manual/src/contributing/hacking.md

@@ -196,7 +196,7 @@ In order to facilitate this, Nix has some support for being built out of tree
 
 ## System type
 
-Nix uses a string with he following format to identify the *system type* or *platform* it runs on:
+Nix uses a string with the following format to identify the *system type* or *platform* it runs on:
 
 ```
 <cpu>-<os>[-<abi>]

doc/manual/src/favicon.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="587.11" height="516.604" viewBox="0 0 550.416 484.317"><defs><linearGradient id="a"><stop offset="0" style="stop-color:#699ad7;stop-opacity:1"/><stop offset=".243" style="stop-color:#7eb1dd;stop-opacity:1"/><stop offset="1" style="stop-color:#7ebae4;stop-opacity:1"/></linearGradient><linearGradient id="b"><stop offset="0" style="stop-color:#415e9a;stop-opacity:1"/><stop offset=".232" style="stop-color:#4a6baf;stop-opacity:1"/><stop offset="1" style="stop-color:#5277c3;stop-opacity:1"/></linearGradient><linearGradient xlink:href="#a" id="c" x1="200.597" x2="290.087" y1="351.411" y2="506.188" gradientTransform="translate(70.65 -1055.151)" gradientUnits="userSpaceOnUse"/><linearGradient xlink:href="#b" id="e" x1="-584.199" x2="-496.297" y1="782.336" y2="937.714" gradientTransform="translate(864.696 -1491.34)" gradientUnits="userSpaceOnUse"/></defs><g style="display:inline;opacity:1" transform="translate(-132.651 958.04)"><path id="d" d="m309.549-710.388 122.197 211.675-56.157.527-32.624-56.87-32.856 56.566-27.903-.011-14.29-24.69 46.81-80.49-33.23-57.826z" style="opacity:1;fill:url(#c);fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:3;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-opacity:1"/><use xlink:href="#d" width="100%" height="100%" transform="rotate(60 407.112 -715.787)"/><use xlink:href="#d" width="100%" height="100%" transform="rotate(-60 407.312 -715.7)"/><use xlink:href="#d" width="100%" height="100%" transform="rotate(180 407.419 -715.756)"/><path id="f" d="m309.549-710.388 122.197 211.675-56.157.527-32.624-56.87-32.856 56.566-27.903-.011-14.29-24.69 46.81-80.49-33.23-57.826z" style="color:#000;clip-rule:nonzero;display:inline;overflow:visible;visibility:visible;opacity:1;isolation:auto;mix-blend-mode:normal;color-interpolation:sRGB;color-interpolation-filters:linearRGB;solid-color:#000;solid-opacity:1;fill:url(#e);fill-opacity:1;fill-rule:evenodd;stroke:none;stroke-width:3;stroke-linecap:butt;stroke-linejoin:round;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1;color-rendering:auto;image-rendering:auto;shape-rendering:auto;text-rendering:auto;enable-background:accumulate"/><use xlink:href="#f" width="100%" height="100%" style="display:inline" transform="rotate(120 407.34 -716.084)"/><use xlink:href="#f" width="100%" height="100%" style="display:inline" transform="rotate(-120 407.288 -715.87)"/></g></svg>

doc/manual/src/installation/upgrading.md

@@ -28,7 +28,7 @@ $ sudo su
 ## macOS multi-user
 
 ```console
-$ sudo nix-env --install --file '<nixpkgs>' --attr nix -I nixpkgs=channel:nixpkgs-unstable
+$ sudo nix-env --install --file '<nixpkgs>' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable
 $ sudo launchctl remove org.nixos.nix-daemon
 $ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist
 ```

doc/manual/src/language/advanced-attributes.md

@@ -207,12 +207,17 @@ Derivations can declare some infrequently used optional attributes.
 
         This is the default.
 
-      - `"recursive"`\
+      - `"recursive"` or `"nar"`\
-        The hash is computed over the NAR archive dump of the output
+        The hash is computed over the [NAR archive](@docroot@/glossary.md#gloss-nar) dump of the output
         (i.e., the result of [`nix-store --dump`](@docroot@/command-ref/nix-store/dump.md)). In
         this case, the output can be anything, including a directory
         tree.
 
+    `"recursive"` is the traditional way of indicating this,
+    and is supported since 2005 (virtually the entire history of Nix).
+    `"nar"` is more clear, and consistent with other parts of Nix (such as the CLI),
+    however support for it is only added in Nix version 2.21.
+
   - [`__contentAddressed`]{#adv-attr-__contentAddressed}
     > **Warning**
     > This attribute is part of an [experimental feature](@docroot@/contributing/experimental-features.md).
@@ -298,7 +303,7 @@ Derivations can declare some infrequently used optional attributes.
     [`disallowedReferences`](#adv-attr-disallowedReferences) and [`disallowedRequisites`](#adv-attr-disallowedRequisites),
     the following attributes are available:
 
-    - `maxSize` defines the maximum size of the resulting [store object](../glossary.md#gloss-store-object).
+    - `maxSize` defines the maximum size of the resulting [store object](@docroot@/glossary.md#gloss-store-object).
     - `maxClosureSize` defines the maximum size of the output's closure.
     - `ignoreSelfRefs` controls whether self-references should be considered when
       checking for allowed references/requisites.

doc/manual/src/language/operators.md

@@ -128,8 +128,8 @@ The result is a string.
 > The file or directory at *path* must exist and is copied to the [store].
 > The path appears in the result as the corresponding [store path].
 
-[store path]: ../glossary.md#gloss-store-path
+[store path]: @docroot@/glossary.md#gloss-store-path
-[store]: ../glossary.md#gloss-store
+[store]: @docroot@/glossary.md#gloss-store
 
 [String and path concatenation]: #string-and-path-concatenation
 

doc/manual/src/language/string-interpolation.md

@@ -20,7 +20,7 @@ Rather than writing
 
 (where `freetype` is a [derivation]), you can instead write
 
-[derivation]: ../glossary.md#gloss-derivation
+[derivation]: @docroot@/glossary.md#gloss-derivation
 
 ```nix
 "--with-freetype2-library=${freetype}/lib"
@@ -107,9 +107,9 @@ An expression that is interpolated must evaluate to one of the following:
 
 A string interpolates to itself.
 
-A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](../glossary.md#gloss-store-object).
+A path in an interpolated expression is first copied into the Nix store, and the resulting string is the [store path] of the newly created [store object](@docroot@/glossary.md#gloss-store-object).
 
-[store path]: ../glossary.md#gloss-store-path
+[store path]: @docroot@/glossary.md#gloss-store-path
 
 > **Example**
 >

doc/manual/src/language/values.md

@@ -113,7 +113,7 @@
   For example, assume you used a file path in an interpolated string during a `nix repl` session.
   Later in the same session, after having changed the file contents, evaluating the interpolated string with the file path again might not return a new [store path], since Nix might not re-read the file contents.
 
-  [store path]: ../glossary.md#gloss-store-path
+  [store path]: @docroot@/glossary.md#gloss-store-path
 
   Paths can include [string interpolation] and can themselves be [interpolated in other expressions].
 

doc/manual/src/protocols/json/store-object-info.md

@@ -83,7 +83,7 @@ This information is not intrinsic to the store object, but about how it is store
 
 ## Computed closure fields
 
-These fields are not stored at all, but computed by traverising the other other fields across all the store objects in a [closure].
+These fields are not stored at all, but computed by traversing the other fields across all the store objects in a [closure].
 
 * `closureSize`:
 

doc/manual/src/protocols/nix-archive.md

@@ -0,0 +1,42 @@
+# Nix Archive (NAR) format
+
+This is the complete specification of the Nix Archive format.
+The Nix Archive format closely follows the abstract specification of a [file system object] tree,
+because it is designed to serialize exactly that data structure.
+
+[file system object]: @docroot@/store/file-system-object.md
+
+The format of this specification is close to [Extended Backus–Naur form](https://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_form), with the exception of the `str(..)` function / parameterized rule, which length-prefixes and pads strings.
+This makes the resulting binary format easier to parse.
+
+Regular users do *not* need to know this information.
+But for those interested in exactly how Nix works, e.g. if they are reimplementing it, this information can be useful.
+
+```ebnf
+nar = str("nix-archive-1"), nar-obj;
+
+nar-obj = str("("), nar-obj-inner, str(")");
+
+nar-obj-inner
+  = str("type"), str("regular") regular
+  | str("type"), str("symlink") symlink
+  | str("type"), str("directory") directory
+  ;
+
+regular = [ str("executable"), str("") ], str("contents"), str(contents);
+
+symlink = str("target"), str(target);
+
+(* side condition: directory entries must be ordered by their names *)
+directory = str("type"), str("directory") { directory-entry };
+
+directory-entry = str("entry"), str("("), str("name"), str(name), str("node"), nar-obj, str(")");
+```
+
+The `str` function / parameterized rule is defined as follows:
+
+- `str(s)` = `int(|s|), pad(s);`
+
+- `int(n)` = the 64-bit little endian representation of the number `n`
+
+- `pad(s)` = the byte sequence `s`, padded with 0s to a multiple of 8 byte

doc/manual/src/release-notes/index.md

@@ -1,12 +1,13 @@
 # Nix Release Notes
 
+The Nix release cycle is calendar-based as follows:
+
 Nix has a release cycle of roughly 6 weeks.
 Notable changes and additions are announced in the release notes for each version.
 
-Bugfixes can be backported on request to previous Nix releases.
+The supported Nix versions are:
-We typically backport only as far back as the Nix version used in the latest NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes).
+- The latest release
-
+- The version used in the stable NixOS release, which is announced in the [NixOS release notes](https://nixos.org/manual/nixos/stable/release-notes.html#ch-release-notes).
-Backports never skip releases.
-If a feature is backported to version `x.y`, it must also be available in version `x.(y+1)`.
-This ensures that upgrading from an older version with backports is still safe and no backported functionality will go missing.
 
+Bugfixes and security issues are backported to every supported version.
+Patch releases are published as needed.

doc/manual/src/release-notes/rl-2.15.md

@@ -11,7 +11,7 @@
   As the choice of hash formats is no longer binary, the `--base16` flag is also added
   to explicitly specify the Base16 format, which is still the default.
 
-* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](../glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents.
+* The special handling of an [installable](../command-ref/new-cli/nix.md#installables) with `.drv` suffix being interpreted as all of the given [store derivation](@docroot@/glossary.md#gloss-store-derivation)'s output paths is removed, and instead taken as the literal store path that it represents.
 
   The new `^` syntax for store paths introduced in Nix 2.13 allows explicitly referencing output paths of a derivation.
   Using this is better and more clear than relying on the now-removed `.drv` special handling.

doc/manual/src/release-notes/rl-2.20.md

@@ -200,3 +200,9 @@
   while performing various operations (including `nix develop`, `nix flake
   update`, and so on). With several fixes to Nix's signal handlers, Nix
   commands will now exit quickly after Ctrl-C is pressed.
+
+- `nix copy` to a `ssh-ng` store now needs `--substitute-on-destination` (a.k.a. `-s`)
+  in order to substitute paths on the remote store instead of copying them.
+  The behavior is consistent with `nix copy` to a different kind of remote store.
+  Previously this behavior was controlled by the
+  `builders-use-substitutes` setting and `--substitute-on-destination` was ignored.

doc/manual/src/store/store-path.md

@@ -46,7 +46,7 @@ But if the store has a file system representation, the store directory contains
 
 [file system objects]: ./file-system-object.md
 
-This means a store path is not just derived from the referenced store object itself, but depends on the store the store object is in.
+This means a store path is not just derived from the referenced store object itself, but depends on the store that the store object is in.
 
 > **Note**
 >

flake.nix

@@ -285,6 +285,13 @@
           enableInternalAPIDocs = true;
         };
 
+        # API docs for Nix's C bindings.
+        external-api-docs = nixpkgsFor.x86_64-linux.native.callPackage ./package.nix {
+          inherit fileset;
+          doBuild = false;
+          enableExternalAPIDocs = true;
+        };
+
         # System tests.
         tests = import ./tests/nixos { inherit lib nixpkgs nixpkgsFor; } // {
 

local.mk

@@ -2,9 +2,14 @@ GLOBAL_CXXFLAGS += -Wno-deprecated-declarations -Werror=switch
 # Allow switch-enum to be overridden for files that do not support it, usually because of dependency headers.
 ERROR_SWITCH_ENUM = -Werror=switch-enum
 
-$(foreach i, config.h $(wildcard src/lib*/*.hh), \
+$(foreach i, config.h $(wildcard src/lib*/*.hh) $(wildcard src/lib*/*.h $(filter-out %_internal.h, $(wildcard src/lib*c/*.h))), \
   $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
 
+ifdef HOST_UNIX
+  $(foreach i, $(wildcard src/lib*/unix/*.hh), \
+    $(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
+endif
+
 $(GCH): src/libutil/util.hh config.h
 
-GCH_CXXFLAGS = -I src/libutil
+GCH_CXXFLAGS = $(INCLUDE_libutil)

package.nix

@@ -5,6 +5,7 @@
 , autoreconfHook
 , aws-sdk-cpp
 , boehmgc
+, buildPackages
 , nlohmann_json
 , bison
 , boost
@@ -75,7 +76,10 @@
 # sounds so long as evaluation just takes places within short-lived
 # processes. (When the process exits, the memory is reclaimed; it is
 # only leaked *within* the process.)
-, enableGC ? true
+#
+# Temporarily disabled on Windows because the `GC_throw_bad_alloc`
+# symbol is missing during linking.
+, enableGC ? !stdenv.hostPlatform.isWindows
 
 # Whether to enable Markdown rendering in the Nix binary.
 , enableMarkdown ? !stdenv.hostPlatform.isWindows
@@ -88,9 +92,10 @@
 # - readline
 , readlineFlavor ? if stdenv.hostPlatform.isWindows then "readline" else "editline"
 
-# Whether to build the internal API docs, can be done separately from
+# Whether to build the internal/external API docs, can be done separately from
 # everything else.
-, enableInternalAPIDocs ? false
+, enableInternalAPIDocs ? forDevShell
+, enableExternalAPIDocs ? forDevShell
 
 # Whether to install unit tests. This is useful when cross compiling
 # since we cannot run them natively during the build, but can do so
@@ -179,6 +184,9 @@ in {
           ./doc/manual
         ] ++ lib.optionals enableInternalAPIDocs [
           ./doc/internal-api
+        ] ++ lib.optionals enableExternalAPIDocs [
+          ./doc/external-api
+        ] ++ lib.optionals (enableInternalAPIDocs || enableExternalAPIDocs) [
           # Source might not be compiled, but still must be available
           # for Doxygen to gather comments.
           ./src
@@ -196,7 +204,7 @@ in {
     ++ lib.optional doBuild "dev"
     # If we are doing just build or just docs, the one thing will use
     # "out". We only need additional outputs if we are doing both.
-    ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs)) "doc"
+    ++ lib.optional (doBuild && (enableManual || enableInternalAPIDocs || enableExternalAPIDocs)) "doc"
     ++ lib.optional installUnitTests "check";
 
   nativeBuildInputs = [
@@ -218,7 +226,7 @@ in {
   ] ++ lib.optionals (doInstallCheck || enableManual) [
     jq # Also for custom mdBook preprocessor.
   ] ++ lib.optional stdenv.hostPlatform.isLinux util-linux
-    ++ lib.optional enableInternalAPIDocs doxygen
+    ++ lib.optional (enableInternalAPIDocs || enableExternalAPIDocs) doxygen
   ;
 
   buildInputs = lib.optionals doBuild [
@@ -282,6 +290,7 @@ in {
     (lib.enableFeature buildUnitTests "unit-tests")
     (lib.enableFeature doInstallCheck "functional-tests")
     (lib.enableFeature enableInternalAPIDocs "internal-api-docs")
+    (lib.enableFeature enableExternalAPIDocs "external-api-docs")
     (lib.enableFeature enableManual "doc-gen")
     (lib.enableFeature enableGC "gc")
     (lib.enableFeature enableMarkdown "markdown")
@@ -306,7 +315,8 @@ in {
   makeFlags = "profiledir=$(out)/etc/profile.d PRECOMPILE_HEADERS=1";
 
   installTargets = lib.optional doBuild "install"
-    ++ lib.optional enableInternalAPIDocs "internal-api-html";
+    ++ lib.optional enableInternalAPIDocs "internal-api-html"
+    ++ lib.optional enableExternalAPIDocs "external-api-html";
 
   installFlags = "sysconfdir=$(out)/etc";
 
@@ -333,6 +343,16 @@ in {
   '' + lib.optionalString enableInternalAPIDocs ''
     mkdir -p ''${!outputDoc}/nix-support
     echo "doc internal-api-docs $out/share/doc/nix/internal-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products
+  ''
+    + lib.optionalString enableExternalAPIDocs ''
+    mkdir -p ''${!outputDoc}/nix-support
+    echo "doc external-api-docs $out/share/doc/nix/external-api/html" >> ''${!outputDoc}/nix-support/hydra-build-products
+  '';
+
+  # So the check output gets links for DLLs in the out output.
+  preFixup = lib.optionalString (stdenv.hostPlatform.isWindows && builtins.elem "check" finalAttrs.outputs) ''
+    ln -s "$check/lib/"*.dll "$check/bin"
+    ln -s "$out/bin/"*.dll "$check/bin"
   '';
 
   doInstallCheck = attrs.doInstallCheck;

scripts/nix-profile-daemon.sh.in

@@ -69,4 +69,4 @@ else
 fi
 
 export PATH="$NIX_LINK/bin:@localstatedir@/nix/profiles/default/bin:$PATH"
-unset NIX_LINK
+unset NIX_LINK NIX_LINK_NEW

src/build-remote/build-remote.cc

@@ -202,7 +202,7 @@ static int main_build_remote(int argc, char * * argv)
                         else
                             drvstr = "<unknown>";
 
-                        auto error = HintFmt(errorText);
+                        auto error = HintFmt::fromFormatString(errorText);
                         error
                             % drvstr
                             % neededSystem

src/libcmd/command.cc

@@ -148,7 +148,7 @@ MixOperateOnOptions::MixOperateOnOptions()
 {
     addFlag({
         .longName = "derivation",
-        .description = "Operate on the [store derivation](../../glossary.md#gloss-store-derivation) rather than its outputs.",
+        .description = "Operate on the [store derivation](@docroot@/glossary.md#gloss-store-derivation) rather than its outputs.",
         .category = installablesCategory,
         .handler = {&operateOn, OperateOn::Derivation},
     });

src/libcmd/local.mk

@@ -6,7 +6,7 @@ libcmd_DIR := $(d)
 
 libcmd_SOURCES := $(wildcard $(d)/*.cc)
 
-libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
+libcmd_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libexpr) $(INCLUDE_libmain)
 
 libcmd_LDFLAGS = $(EDITLINE_LIBS) $(LOWDOWN_LIBS) $(THREAD_LDFLAGS)
 

src/libcmd/markdown.cc

@@ -50,7 +50,7 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
     if (!rndr_res)
         throw Error("allocation error while rendering Markdown");
 
-    return filterANSIEscapes(std::string(buf->data, buf->size), !shouldANSI());
+    return filterANSIEscapes(std::string(buf->data, buf->size), !isTTY());
 #else
     return std::string(markdown);
 #endif

src/libcmd/repl.cc

@@ -357,7 +357,7 @@ ProcessLineResult NixRepl::processLine(std::string line)
     if (line.empty())
         return ProcessLineResult::PromptAgain;
 
-    _isInterrupted = false;
+    setInterrupted(false);
 
     std::string command, arg;
 

src/libexpr-c/local.mk

@@ -0,0 +1,25 @@
+libraries += libexprc
+
+libexprc_NAME = libnixexprc
+
+libexprc_DIR := $(d)
+
+libexprc_SOURCES := \
+  $(wildcard $(d)/*.cc) \
+
+# Not just for this library itself, but also for downstream libraries using this library
+
+INCLUDE_libexprc := -I $(d)
+libexprc_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \
+                     $(INCLUDE_libfetchers) \
+                     $(INCLUDE_libstore) $(INCLUDE_libstorec) \
+                     $(INCLUDE_libexpr) $(INCLUDE_libexprc)
+
+libexprc_LIBS = libutil libutilc libstore libstorec libexpr
+
+libexprc_LDFLAGS += $(THREAD_LDFLAGS)
+
+$(eval $(call install-file-in, $(d)/nix-expr-c.pc, $(libdir)/pkgconfig, 0644))
+
+libexprc_FORCE_INSTALL := 1
+

src/libexpr-c/nix-expr-c.pc.in

@@ -0,0 +1,10 @@
+prefix=@prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Nix
+Description: Nix Language Evaluator - C API
+Version: @PACKAGE_VERSION@
+Requires: nix-store-c
+Libs: -L${libdir} -lnixexprc
+Cflags: -I${includedir}/nix

src/libexpr-c/nix_api_expr.cc

@@ -0,0 +1,178 @@
+#include <cstring>
+#include <iostream>
+#include <stdexcept>
+#include <string>
+
+#include "config.hh"
+#include "eval.hh"
+#include "globals.hh"
+#include "util.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_store.h"
+#include "nix_api_store_internal.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+
+#ifdef HAVE_BOEHMGC
+#include <mutex>
+#define GC_INCLUDE_NEW 1
+#include "gc_cpp.h"
+#endif
+
+nix_err nix_libexpr_init(nix_c_context * context)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    {
+        auto ret = nix_libutil_init(context);
+        if (ret != NIX_OK)
+            return ret;
+    }
+    {
+        auto ret = nix_libstore_init(context);
+        if (ret != NIX_OK)
+            return ret;
+    }
+    try {
+        nix::initGC();
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_expr_eval_from_string(
+    nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path)));
+        state->state.eval(parsedExpr, *(nix::Value *) value);
+        state->state.forceValue(*(nix::Value *) value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.callFunction(*(nix::Value *) fn, *(nix::Value *) arg, *(nix::Value *) value, nix::noPos);
+        state->state.forceValue(*(nix::Value *) value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.forceValue(*(nix::Value *) value, nix::noPos);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        state->state.forceValueDeep(*(nix::Value *) value);
+    }
+    NIXC_CATCH_ERRS
+}
+
+EvalState * nix_state_create(nix_c_context * context, const char ** searchPath_c, Store * store)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::Strings searchPath;
+        if (searchPath_c != nullptr)
+            for (size_t i = 0; searchPath_c[i] != nullptr; i++)
+                searchPath.push_back(searchPath_c[i]);
+
+        return new EvalState{nix::EvalState(nix::SearchPath::parse(searchPath), store->ptr)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_state_free(EvalState * state)
+{
+    delete state;
+}
+
+#ifdef HAVE_BOEHMGC
+std::unordered_map<
+    const void *,
+    unsigned int,
+    std::hash<const void *>,
+    std::equal_to<const void *>,
+    traceable_allocator<std::pair<const void * const, unsigned int>>>
+    nix_refcounts;
+
+std::mutex nix_refcount_lock;
+
+nix_err nix_gc_incref(nix_c_context * context, const void * p)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        std::scoped_lock lock(nix_refcount_lock);
+        auto f = nix_refcounts.find(p);
+        if (f != nix_refcounts.end()) {
+            f->second++;
+        } else {
+            nix_refcounts[p] = 1;
+        }
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_gc_decref(nix_c_context * context, const void * p)
+{
+
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        std::scoped_lock lock(nix_refcount_lock);
+        auto f = nix_refcounts.find(p);
+        if (f != nix_refcounts.end()) {
+            if (--f->second == 0)
+                nix_refcounts.erase(f);
+        } else
+            throw std::runtime_error("nix_gc_decref: object was not referenced");
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_gc_now()
+{
+    GC_gcollect();
+}
+
+#else
+nix_err nix_gc_incref(nix_c_context * context, const void *)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    return NIX_OK;
+}
+nix_err nix_gc_decref(nix_c_context * context, const void *)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    return NIX_OK;
+}
+void nix_gc_now() {}
+#endif
+
+void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd))
+{
+#ifdef HAVE_BOEHMGC
+    GC_REGISTER_FINALIZER(obj, finalizer, cd, 0, 0);
+#endif
+}

src/libexpr-c/nix_api_expr.h

@@ -0,0 +1,213 @@
+#ifndef NIX_API_EXPR_H
+#define NIX_API_EXPR_H
+/** @defgroup libexpr libexpr
+ * @brief Bindings to the Nix language evaluator
+ *
+ * Example (without error handling):
+ * @code{.c}
+ * int main() {
+ *    nix_libexpr_init(NULL);
+ *
+ *    Store* store = nix_store_open(NULL, "dummy", NULL);
+ *    EvalState* state = nix_state_create(NULL, NULL, store); // empty nix path
+ *    Value *value = nix_alloc_value(NULL, state);
+ *
+ *    nix_expr_eval_from_string(NULL, state, "builtins.nixVersion", ".", value);
+ *    nix_value_force(NULL, state, value);
+ *    printf("nix version: %s\n", nix_get_string(NULL, value));
+ *
+ *    nix_gc_decref(NULL, value);
+ *    nix_state_free(state);
+ *    nix_store_free(store);
+ *    return 0;
+ *    }
+ * @endcode
+ * @{
+ */
+/** @file
+ * @brief Main entry for the libexpr C bindings
+ */
+
+#include "nix_api_store.h"
+#include "nix_api_util.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+// Type definitions
+/**
+ * @brief Represents a state of the Nix language evaluator.
+ *
+ * Multiple states can be created for multi-threaded
+ * operation.
+ * @struct EvalState
+ * @see nix_state_create
+ */
+typedef struct EvalState EvalState; // nix::EvalState
+/**
+ * @brief Represents a value in the Nix language.
+ *
+ * Owned by the garbage collector.
+ * @struct Value
+ * @see value_manip
+ */
+typedef void Value; // nix::Value
+
+// Function prototypes
+/**
+ * @brief Initialize the Nix language evaluator.
+ *
+ * This function must be called at least once,
+ * at some point before constructing a EvalState for the first time.
+ * This function can be called multiple times, and is idempotent.
+ *
+ * @param[out] context Optional, stores error information
+ * @return NIX_OK if the initialization was successful, an error code otherwise.
+ */
+nix_err nix_libexpr_init(nix_c_context * context);
+
+/**
+ * @brief Parses and evaluates a Nix expression from a string.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in] expr The Nix expression to parse.
+ * @param[in] path The file path to associate with the expression.
+ * This is required for expressions that contain relative paths (such as `./.`) that are resolved relative to the given
+ * directory.
+ * @param[out] value The result of the evaluation. You must allocate this
+ * yourself.
+ * @return NIX_OK if the evaluation was successful, an error code otherwise.
+ */
+nix_err nix_expr_eval_from_string(
+    nix_c_context * context, EvalState * state, const char * expr, const char * path, Value * value);
+
+/**
+ * @brief Calls a Nix function with an argument.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in] fn The Nix function to call.
+ * @param[in] arg The argument to pass to the function.
+ * @param[out] value The result of the function call.
+ * @return NIX_OK if the function call was successful, an error code otherwise.
+ */
+nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, Value * arg, Value * value);
+
+/**
+ * @brief Forces the evaluation of a Nix value.
+ *
+ * The Nix interpreter is lazy, and not-yet-evaluated Values can be
+ * of type NIX_TYPE_THUNK instead of their actual value.
+ *
+ * This function converts these Values into their final type.
+ *
+ * @note You don't need this function for basic API usage, since all functions
+ * that return a value call it for you. The only place you will see a
+ * NIX_TYPE_THUNK is in the arguments that are passed to a PrimOp function
+ * you supplied to nix_alloc_primop.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in,out] value The Nix value to force.
+ * @post value is not of type NIX_TYPE_THUNK
+ * @return NIX_OK if the force operation was successful, an error code
+ * otherwise.
+ */
+nix_err nix_value_force(nix_c_context * context, EvalState * state, Value * value);
+
+/**
+ * @brief Forces the deep evaluation of a Nix value.
+ *
+ * Recursively calls nix_value_force
+ *
+ * @see nix_value_force
+ * @warning Calling this function on a recursive data structure will cause a
+ * stack overflow.
+ * @param[out] context Optional, stores error information
+ * @param[in] state The state of the evaluation.
+ * @param[in,out] value The Nix value to force.
+ * @return NIX_OK if the deep force operation was successful, an error code
+ * otherwise.
+ */
+nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, Value * value);
+
+/**
+ * @brief Create a new Nix language evaluator state.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] searchPath Array of strings corresponding to entries in NIX_PATH.
+ * @param[in] store The Nix store to use.
+ * @return A new Nix state or NULL on failure.
+ */
+EvalState * nix_state_create(nix_c_context * context, const char ** searchPath, Store * store);
+
+/**
+ * @brief Frees a Nix state.
+ *
+ * Does not fail.
+ *
+ * @param[in] state The state to free.
+ */
+void nix_state_free(EvalState * state);
+
+/** @addtogroup GC
+ * @brief Reference counting and garbage collector operations
+ *
+ * The Nix language evaluator uses a garbage collector. To ease C interop, we implement
+ * a reference counting scheme, where objects will be deallocated
+ * when there are no references from the Nix side, and the reference count kept
+ * by the C API reaches `0`.
+ *
+ * Functions returning a garbage-collected object will automatically increase
+ * the refcount for you. You should make sure to call `nix_gc_decref` when
+ * you're done with a value returned by the evaluator.
+ * @{
+ */
+/**
+ * @brief Increment the garbage collector reference counter for the given object.
+ *
+ * The Nix language evaluator C API keeps track of alive objects by reference counting.
+ * When you're done with a refcounted pointer, call nix_gc_decref().
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] object The object to keep alive
+ */
+nix_err nix_gc_incref(nix_c_context * context, const void * object);
+/**
+ * @brief Decrement the garbage collector reference counter for the given object
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] object The object to stop referencing
+ */
+nix_err nix_gc_decref(nix_c_context * context, const void * object);
+
+/**
+ * @brief Trigger the garbage collector manually
+ *
+ * You should not need to do this, but it can be useful for debugging.
+ */
+void nix_gc_now();
+
+/**
+ * @brief Register a callback that gets called when the object is garbage
+ * collected.
+ * @note Objects can only have a single finalizer. This function overwrites existing values
+ * silently.
+ * @param[in] obj the object to watch
+ * @param[in] cd the data to pass to the finalizer
+ * @param[in] finalizer the callback function, called with obj and cd
+ */
+void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * obj, void * cd));
+
+/** @} */
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+
+/** @} */
+
+#endif // NIX_API_EXPR_H

src/libexpr-c/nix_api_expr_internal.h

@@ -0,0 +1,44 @@
+#ifndef NIX_API_EXPR_INTERNAL_H
+#define NIX_API_EXPR_INTERNAL_H
+
+#include "eval.hh"
+#include "attr-set.hh"
+#include "nix_api_value.h"
+
+struct EvalState
+{
+    nix::EvalState state;
+};
+
+struct BindingsBuilder
+{
+    nix::BindingsBuilder builder;
+};
+
+struct ListBuilder
+{
+    nix::ListBuilder builder;
+};
+
+struct nix_string_return
+{
+    std::string str;
+};
+
+struct nix_printer
+{
+    std::ostream & s;
+};
+
+struct nix_string_context
+{
+    nix::NixStringContext & ctx;
+};
+
+struct nix_realised_string
+{
+    std::string str;
+    std::vector<StorePath> storePaths;
+};
+
+#endif // NIX_API_EXPR_INTERNAL_H

src/libexpr-c/nix_api_external.cc

@@ -0,0 +1,198 @@
+#include "attr-set.hh"
+#include "config.hh"
+#include "eval.hh"
+#include "globals.hh"
+#include "value.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_external.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+#include "nix_api_value.h"
+#include "value/context.hh"
+
+#include <nlohmann/json.hpp>
+
+#ifdef HAVE_BOEHMGC
+# include "gc/gc.h"
+# define GC_INCLUDE_NEW 1
+# include "gc_cpp.h"
+#endif
+
+void nix_set_string_return(nix_string_return * str, const char * c)
+{
+    str->str = c;
+}
+
+nix_err nix_external_print(nix_c_context * context, nix_printer * printer, const char * c)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        printer->s << c;
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_external_add_string_context(nix_c_context * context, nix_string_context * ctx, const char * c)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto r = nix::NixStringContextElem::parse(c);
+        ctx->ctx.insert(r);
+    }
+    NIXC_CATCH_ERRS
+}
+
+class NixCExternalValue : public nix::ExternalValueBase
+{
+    NixCExternalValueDesc & desc;
+    void * v;
+
+public:
+    NixCExternalValue(NixCExternalValueDesc & desc, void * v)
+        : desc(desc)
+        , v(v){};
+    void * get_ptr()
+    {
+        return v;
+    }
+    /**
+     * Print out the value
+     */
+    virtual std::ostream & print(std::ostream & str) const override
+    {
+        nix_printer p{str};
+        desc.print(v, &p);
+        return str;
+    }
+
+    /**
+     * Return a simple string describing the type
+     */
+    virtual std::string showType() const override
+    {
+        nix_string_return res;
+        desc.showType(v, &res);
+        return std::move(res.str);
+    }
+
+    /**
+     * Return a string to be used in builtins.typeOf
+     */
+    virtual std::string typeOf() const override
+    {
+        nix_string_return res;
+        desc.typeOf(v, &res);
+        return std::move(res.str);
+    }
+
+    /**
+     * Coerce the value to a string.
+     */
+    virtual std::string coerceToString(
+        nix::EvalState & state,
+        const nix::PosIdx & pos,
+        nix::NixStringContext & context,
+        bool copyMore,
+        bool copyToStore) const override
+    {
+        if (!desc.coerceToString) {
+            return nix::ExternalValueBase::coerceToString(state, pos, context, copyMore, copyToStore);
+        }
+        nix_string_context ctx{context};
+        nix_string_return res{""};
+        // todo: pos, errors
+        desc.coerceToString(v, &ctx, copyMore, copyToStore, &res);
+        if (res.str.empty()) {
+            return nix::ExternalValueBase::coerceToString(state, pos, context, copyMore, copyToStore);
+        }
+        return std::move(res.str);
+    }
+
+    /**
+     * Compare to another value of the same type.
+     */
+    virtual bool operator==(const ExternalValueBase & b) const override
+    {
+        if (!desc.equal) {
+            return false;
+        }
+        auto r = dynamic_cast<const NixCExternalValue *>(&b);
+        if (!r)
+            return false;
+        return desc.equal(v, r->v);
+    }
+
+    /**
+     * Print the value as JSON.
+     */
+    virtual nlohmann::json printValueAsJSON(
+        nix::EvalState & state, bool strict, nix::NixStringContext & context, bool copyToStore = true) const override
+    {
+        if (!desc.printValueAsJSON) {
+            return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore);
+        }
+        nix_string_context ctx{context};
+        nix_string_return res{""};
+        desc.printValueAsJSON(v, (EvalState *) &state, strict, &ctx, copyToStore, &res);
+        if (res.str.empty()) {
+            return nix::ExternalValueBase::printValueAsJSON(state, strict, context, copyToStore);
+        }
+        return nlohmann::json::parse(res.str);
+    }
+
+    /**
+     * Print the value as XML.
+     */
+    virtual void printValueAsXML(
+        nix::EvalState & state,
+        bool strict,
+        bool location,
+        nix::XMLWriter & doc,
+        nix::NixStringContext & context,
+        nix::PathSet & drvsSeen,
+        const nix::PosIdx pos) const override
+    {
+        if (!desc.printValueAsXML) {
+            return nix::ExternalValueBase::printValueAsXML(state, strict, location, doc, context, drvsSeen, pos);
+        }
+        nix_string_context ctx{context};
+        desc.printValueAsXML(
+            v, (EvalState *) &state, strict, location, &doc, &ctx, &drvsSeen,
+            *reinterpret_cast<const uint32_t *>(&pos));
+    }
+
+    virtual ~NixCExternalValue() override{};
+};
+
+ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto ret = new
+#ifdef HAVE_BOEHMGC
+            (GC)
+#endif
+                NixCExternalValue(*desc, v);
+        nix_gc_incref(nullptr, ret);
+        return (ExternalValue *) ret;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto r = dynamic_cast<NixCExternalValue *>((nix::ExternalValueBase *) b);
+        if (r)
+            return r->get_ptr();
+        return nullptr;
+    }
+    NIXC_CATCH_ERRS_NULL
+}

src/libexpr-c/nix_api_external.h

@@ -0,0 +1,196 @@
+#ifndef NIX_API_EXTERNAL_H
+#define NIX_API_EXTERNAL_H
+/** @ingroup libexpr
+ * @addtogroup Externals
+ * @brief Deal with external values
+ * @{
+ */
+/** @file
+ * @brief libexpr C bindings dealing with external values
+ */
+
+#include "nix_api_expr.h"
+#include "nix_api_util.h"
+#include "nix_api_value.h"
+#include "stdbool.h"
+#include "stddef.h"
+#include "stdint.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+/**
+ * @brief Represents a string owned by the Nix language evaluator.
+ * @see nix_set_owned_string
+ */
+typedef struct nix_string_return nix_string_return;
+/**
+ * @brief Wraps a stream that can output multiple string pieces.
+ */
+typedef struct nix_printer nix_printer;
+/**
+ * @brief A list of string context items
+ */
+typedef struct nix_string_context nix_string_context;
+
+/**
+ * @brief Sets the contents of a nix_string_return
+ *
+ * Copies the passed string.
+ * @param[out] str the nix_string_return to write to
+ * @param[in]  c   The string to copy
+ */
+void nix_set_string_return(nix_string_return * str, const char * c);
+
+/**
+ * Print to the nix_printer
+ *
+ * @param[out] context Optional, stores error information
+ * @param printer The nix_printer to print to
+ * @param[in] str The string to print
+ * @returns NIX_OK if everything worked
+ */
+nix_err nix_external_print(nix_c_context * context, nix_printer * printer, const char * str);
+
+/**
+ * Add string context to the nix_string_context object
+ * @param[out] context Optional, stores error information
+ * @param[out] string_context The nix_string_context to add to
+ * @param[in] c The context string to add
+ * @returns NIX_OK if everything worked
+ */
+nix_err nix_external_add_string_context(nix_c_context * context, nix_string_context * string_context, const char * c);
+
+/**
+ * @brief Definition for a class of external values
+ *
+ * Create and implement one of these, then pass it to nix_create_external_value
+ * Make sure to keep it alive while the external value lives.
+ *
+ * Optional functions can be set to NULL
+ *
+ * @see nix_create_external_value
+ */
+typedef struct NixCExternalValueDesc
+{
+    /**
+     * @brief Called when printing the external value
+     *
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] printer The printer to print to, pass to nix_external_print
+     */
+    void (*print)(void * self, nix_printer * printer);
+    /**
+     * @brief Called on :t
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] res the return value
+     */
+    void (*showType)(void * self, nix_string_return * res);
+    /**
+     * @brief Called on `builtins.typeOf`
+     * @param self the void* passed to nix_create_external_value
+     * @param[out] res the return value
+     */
+    void (*typeOf)(void * self, nix_string_return * res);
+    /**
+     * @brief Called on "${str}" and builtins.toString.
+     *
+     * The latter with coerceMore=true
+     * Optional, the default is to throw an error.
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[out] c writable string context for the resulting string
+     * @param[in] coerceMore boolean, try to coerce to strings in more cases
+     * instead of throwing an error
+     * @param[in] copyToStore boolean, whether to copy referenced paths to store
+     * or keep them as-is
+     * @param[out] res the return value. Not touching this, or setting it to the
+     * empty string, will make the conversion throw an error.
+     */
+    void (*coerceToString)(
+        void * self, nix_string_context * c, int coerceMore, int copyToStore, nix_string_return * res);
+    /**
+     * @brief Try to compare two external values
+     *
+     * Optional, the default is always false.
+     * If the other object was not a Nix C external value, this comparison will
+     * also return false
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] other the void* passed to the other object's
+     * nix_create_external_value
+     * @returns true if the objects are deemed to be equal
+     */
+    int (*equal)(void * self, void * other);
+    /**
+     * @brief Convert the external value to json
+     *
+     * Optional, the default is to throw an error
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] state The evaluator state
+     * @param[in] strict boolean Whether to force the value before printing
+     * @param[out] c writable string context for the resulting string
+     * @param[in] copyToStore whether to copy referenced paths to store or keep
+     * them as-is
+     * @param[out] res the return value. Gets parsed as JSON. Not touching this,
+     * or setting it to the empty string, will make the conversion throw an error.
+     */
+    void (*printValueAsJSON)(
+        void * self, EvalState *, bool strict, nix_string_context * c, bool copyToStore, nix_string_return * res);
+    /**
+     * @brief Convert the external value to XML
+     *
+     * Optional, the default is to throw an error
+     * @todo The mechanisms for this call are incomplete. There are no C
+     *       bindings to work with XML, pathsets and positions.
+     * @param[in] self the void* passed to nix_create_external_value
+     * @param[in] state The evaluator state
+     * @param[in] strict boolean Whether to force the value before printing
+     * @param[in] location boolean Whether to include position information in the
+     * xml
+     * @param[out] doc XML document to output to
+     * @param[out] c writable string context for the resulting string
+     * @param[in,out] drvsSeen a path set to avoid duplicating derivations
+     * @param[in] pos The position of the call.
+     */
+    void (*printValueAsXML)(
+        void * self,
+        EvalState *,
+        int strict,
+        int location,
+        void * doc,
+        nix_string_context * c,
+        void * drvsSeen,
+        int pos);
+} NixCExternalValueDesc;
+
+/**
+ * @brief Create an external value, that can be given to nix_init_external
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] desc a NixCExternalValueDesc, you should keep this alive as long
+ * as the ExternalValue lives
+ * @param[in] v the value to store
+ * @returns external value, owned by the garbage collector
+ * @see nix_init_external
+ */
+ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v);
+
+/**
+ * @brief Extract the pointer from a nix c external value.
+ * @param[out] context Optional, stores error information
+ * @param[in] b The external value
+ * @returns The pointer, or null if the external value was not from nix c.
+ * @see nix_get_external
+ */
+void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b);
+
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+/** @} */
+
+#endif // NIX_API_EXTERNAL_H

src/libexpr-c/nix_api_value.cc

@@ -0,0 +1,582 @@
+#include "attr-set.hh"
+#include "config.hh"
+#include "eval.hh"
+#include "globals.hh"
+#include "path.hh"
+#include "primops.hh"
+#include "value.hh"
+
+#include "nix_api_expr.h"
+#include "nix_api_expr_internal.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+#include "nix_api_store_internal.h"
+#include "nix_api_value.h"
+#include "value/context.hh"
+
+#ifdef HAVE_BOEHMGC
+#  include "gc/gc.h"
+#  define GC_INCLUDE_NEW 1
+#  include "gc_cpp.h"
+#endif
+
+// Helper function to throw an exception if value is null
+static const nix::Value & check_value_not_null(const Value * value)
+{
+    if (!value) {
+        throw std::runtime_error("Value is null");
+    }
+    return *((const nix::Value *) value);
+}
+
+static nix::Value & check_value_not_null(Value * value)
+{
+    if (!value) {
+        throw std::runtime_error("Value is null");
+    }
+    return *((nix::Value *) value);
+}
+
+/**
+ * Helper function to convert calls from nix into C API.
+ *
+ * Deals with errors and converts arguments from C++ into C types.
+ */
+static void nix_c_primop_wrapper(
+    PrimOpFun f, void * userdata, nix::EvalState & state, const nix::PosIdx pos, nix::Value ** args, nix::Value & v)
+{
+    nix_c_context ctx;
+    f(userdata, &ctx, (EvalState *) &state, (Value **) args, (Value *) &v);
+    /* TODO: In the future, this should throw different errors depending on the error code */
+    if (ctx.last_err_code != NIX_OK)
+        state.error<nix::EvalError>("Error from builtin function: %s", *ctx.last_err).atPos(pos).debugThrow();
+}
+
+PrimOp * nix_alloc_primop(
+    nix_c_context * context,
+    PrimOpFun fun,
+    int arity,
+    const char * name,
+    const char ** args,
+    const char * doc,
+    void * user_data)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        using namespace std::placeholders;
+        auto p = new
+#ifdef HAVE_BOEHMGC
+            (GC)
+#endif
+                nix::PrimOp{
+                    .name = name,
+                    .args = {},
+                    .arity = (size_t) arity,
+                    .doc = doc,
+                    .fun = std::bind(nix_c_primop_wrapper, fun, user_data, _1, _2, _3, _4)};
+        if (args)
+            for (size_t i = 0; args[i]; i++)
+                p->args.emplace_back(*args);
+        nix_gc_incref(nullptr, p);
+        return (PrimOp *) p;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::RegisterPrimOp r(std::move(*((nix::PrimOp *) primOp)));
+    }
+    NIXC_CATCH_ERRS
+}
+
+Value * nix_alloc_value(nix_c_context * context, EvalState * state)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        Value * res = state->state.allocValue();
+        nix_gc_incref(nullptr, res);
+        return res;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+ValueType nix_get_type(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        using namespace nix;
+        switch (v.type()) {
+        case nThunk:
+            return NIX_TYPE_THUNK;
+        case nInt:
+            return NIX_TYPE_INT;
+        case nFloat:
+            return NIX_TYPE_FLOAT;
+        case nBool:
+            return NIX_TYPE_BOOL;
+        case nString:
+            return NIX_TYPE_STRING;
+        case nPath:
+            return NIX_TYPE_PATH;
+        case nNull:
+            return NIX_TYPE_NULL;
+        case nAttrs:
+            return NIX_TYPE_ATTRS;
+        case nList:
+            return NIX_TYPE_LIST;
+        case nFunction:
+            return NIX_TYPE_FUNCTION;
+        case nExternal:
+            return NIX_TYPE_EXTERNAL;
+        }
+        return NIX_TYPE_NULL;
+    }
+    NIXC_CATCH_ERRS_RES(NIX_TYPE_NULL);
+}
+
+const char * nix_get_typename(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        auto s = nix::showType(v);
+        return strdup(s.c_str());
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+bool nix_get_bool(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nBool);
+        return v.boolean();
+    }
+    NIXC_CATCH_ERRS_RES(false);
+}
+
+nix_err nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nString);
+        call_nix_get_string_callback(v.c_str(), callback, user_data);
+    }
+    NIXC_CATCH_ERRS
+}
+
+const char * nix_get_path_string(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nPath);
+        // NOTE (from @yorickvP)
+        // v._path.path should work but may not be how Eelco intended it.
+        // Long-term this function should be rewritten to copy some data into a
+        // user-allocated string.
+        // We could use v.path().to_string().c_str(), but I'm concerned this
+        // crashes. Looks like .path() allocates a CanonPath with a copy of the
+        // string, then it gets the underlying data from that.
+        return v.payload.path.path;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+unsigned int nix_get_list_size(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nList);
+        return v.listSize();
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nAttrs);
+        return v.attrs()->size();
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+double nix_get_float(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nFloat);
+        return v.fpoint();
+    }
+    NIXC_CATCH_ERRS_RES(0.0);
+}
+
+int64_t nix_get_int(nix_c_context * context, const Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nInt);
+        return v.integer();
+    }
+    NIXC_CATCH_ERRS_RES(0);
+}
+
+ExternalValue * nix_get_external(nix_c_context * context, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nExternal);
+        return (ExternalValue *) v.external();
+    }
+    NIXC_CATCH_ERRS_NULL;
+}
+
+Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nList);
+        auto * p = v.listElems()[ix];
+        nix_gc_incref(nullptr, p);
+        if (p != nullptr)
+            state->state.forceValue(*p, nix::noPos);
+        return (Value *) p;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nAttrs);
+        nix::Symbol s = state->state.symbols.create(name);
+        auto attr = v.attrs()->get(s);
+        if (attr) {
+            nix_gc_incref(nullptr, attr->value);
+            state->state.forceValue(*attr->value, nix::noPos);
+            return attr->value;
+        }
+        nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute");
+        return nullptr;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        assert(v.type() == nix::nAttrs);
+        nix::Symbol s = state->state.symbols.create(name);
+        auto attr = v.attrs()->get(s);
+        if (attr)
+            return true;
+        return false;
+    }
+    NIXC_CATCH_ERRS_RES(false);
+}
+
+Value *
+nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        const nix::Attr & a = (*v.attrs())[i];
+        *name = ((const std::string &) (state->state.symbols[a.name])).c_str();
+        nix_gc_incref(nullptr, a.value);
+        state->state.forceValue(*a.value, nix::noPos);
+        return a.value;
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        const nix::Attr & a = (*v.attrs())[i];
+        return ((const std::string &) (state->state.symbols[a.name])).c_str();
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_init_bool(nix_c_context * context, Value * value, bool b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkBool(b);
+    }
+    NIXC_CATCH_ERRS
+}
+
+// todo string context
+nix_err nix_init_string(nix_c_context * context, Value * value, const char * str)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkString(std::string_view(str));
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkPath(s->state.rootPath(nix::CanonPath(str)));
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_float(nix_c_context * context, Value * value, double d)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkFloat(d);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkInt(i);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_null(nix_c_context * context, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkNull();
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        auto r = (nix::ExternalValueBase *) val;
+        v.mkExternal(r);
+    }
+    NIXC_CATCH_ERRS
+}
+
+ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto builder = state->state.buildList(capacity);
+        return new
+#if HAVE_BOEHMGC
+            (NoGC)
+#endif
+                ListBuilder{std::move(builder)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & e = check_value_not_null(value);
+        list_builder->builder[index] = &e;
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_list_builder_free(ListBuilder * list_builder)
+{
+#if HAVE_BOEHMGC
+    GC_FREE(list_builder);
+#else
+    delete list_builder;
+#endif
+}
+
+nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkList(list_builder->builder);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * p)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkPrimOp((nix::PrimOp *) p);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_copy_value(nix_c_context * context, Value * value, Value * source)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        auto & s = check_value_not_null(source);
+        v = s;
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        v.mkAttrs(b->builder);
+    }
+    NIXC_CATCH_ERRS
+}
+
+BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto bb = state->state.buildBindings(capacity);
+        return new
+#if HAVE_BOEHMGC
+            (NoGC)
+#endif
+                BindingsBuilder{std::move(bb)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * bb, const char * name, Value * value)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        nix::Symbol s = bb->builder.state.symbols.create(name);
+        bb->builder.insert(s, &v);
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_bindings_builder_free(BindingsBuilder * bb)
+{
+#if HAVE_BOEHMGC
+    GC_FREE((nix::BindingsBuilder *) bb);
+#else
+    delete (nix::BindingsBuilder *) bb;
+#endif
+}
+
+nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto & v = check_value_not_null(value);
+        nix::NixStringContext stringContext;
+        auto rawStr = state->state.coerceToString(nix::noPos, v, stringContext, "while realising a string").toOwned();
+        nix::StorePathSet storePaths;
+        auto rewrites = state->state.realiseContext(stringContext, &storePaths);
+
+        auto s = nix::rewriteStrings(rawStr, rewrites);
+
+        // Convert to the C API StorePath type and convert to vector for index-based access
+        std::vector<StorePath> vec;
+        for (auto & sp : storePaths) {
+            vec.push_back(StorePath{sp});
+        }
+
+        return new nix_realised_string{.str = s, .storePaths = vec};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_realised_string_free(nix_realised_string * s)
+{
+    delete s;
+}
+
+size_t nix_realised_string_get_buffer_size(nix_realised_string * s)
+{
+    return s->str.size();
+}
+
+const char * nix_realised_string_get_buffer_start(nix_realised_string * s)
+{
+    return s->str.data();
+}
+
+size_t nix_realised_string_get_store_path_count(nix_realised_string * s)
+{
+    return s->storePaths.size();
+}
+
+const StorePath * nix_realised_string_get_store_path(nix_realised_string * s, size_t i)
+{
+    return &s->storePaths[i];
+}

src/libexpr-c/nix_api_value.h

@@ -0,0 +1,494 @@
+#ifndef NIX_API_VALUE_H
+#define NIX_API_VALUE_H
+
+/** @addtogroup libexpr
+ * @{
+ */
+/** @file
+ * @brief libexpr C bindings dealing with values
+ */
+
+#include "nix_api_util.h"
+#include "nix_api_store.h"
+#include "stdbool.h"
+#include "stddef.h"
+#include "stdint.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+// Type definitions
+typedef enum {
+    NIX_TYPE_THUNK,
+    NIX_TYPE_INT,
+    NIX_TYPE_FLOAT,
+    NIX_TYPE_BOOL,
+    NIX_TYPE_STRING,
+    NIX_TYPE_PATH,
+    NIX_TYPE_NULL,
+    NIX_TYPE_ATTRS,
+    NIX_TYPE_LIST,
+    NIX_TYPE_FUNCTION,
+    NIX_TYPE_EXTERNAL
+} ValueType;
+
+// forward declarations
+typedef void Value;
+typedef struct EvalState EvalState;
+// type defs
+/** @brief Stores an under-construction set of bindings
+ * @ingroup value_manip
+ *
+ * Do not reuse.
+ * @see nix_make_bindings_builder, nix_bindings_builder_free, nix_make_attrs
+ * @see nix_bindings_builder_insert
+ */
+typedef struct BindingsBuilder BindingsBuilder;
+
+/** @brief Stores an under-construction list
+ * @ingroup value_manip
+ *
+ * Do not reuse.
+ * @see nix_make_list_builder, nix_list_builder_free, nix_make_list
+ * @see nix_list_builder_insert
+ */
+typedef struct ListBuilder ListBuilder;
+
+/** @brief PrimOp function
+ * @ingroup primops
+ *
+ * Owned by the GC
+ * @see nix_alloc_primop, nix_init_primop
+ */
+typedef struct PrimOp PrimOp;
+/** @brief External Value
+ * @ingroup Externals
+ *
+ * Owned by the GC
+ */
+typedef struct ExternalValue ExternalValue;
+
+/** @brief String without placeholders, and realised store paths
+ */
+typedef struct nix_realised_string nix_realised_string;
+
+/** @defgroup primops
+ * @brief Create your own primops
+ * @{
+ */
+/** @brief Function pointer for primops
+ * When you want to return an error, call nix_set_err_msg(context, NIX_ERR_UNKNOWN, "your error message here").
+ *
+ * @param[in] user_data Arbitrary data that was initially supplied to nix_alloc_primop
+ * @param[out] context Stores error information.
+ * @param[in] state Evaluator state
+ * @param[in] args list of arguments. Note that these can be thunks and should be forced using nix_value_force before
+ * use.
+ * @param[out] ret return value
+ * @see nix_alloc_primop, nix_init_primop
+ */
+typedef void (*PrimOpFun)(void * user_data, nix_c_context * context, EvalState * state, Value ** args, Value * ret);
+
+/** @brief Allocate a PrimOp
+ *
+ * Owned by the garbage collector.
+ * Use nix_gc_decref() when you're done with the returned PrimOp.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] fun callback
+ * @param[in] arity expected number of function arguments
+ * @param[in] name function name
+ * @param[in] args array of argument names, NULL-terminated
+ * @param[in] doc optional, documentation for this primop
+ * @param[in] user_data optional, arbitrary data, passed to the callback when it's called
+ * @return primop, or null in case of errors
+ * @see nix_init_primop
+ */
+PrimOp * nix_alloc_primop(
+    nix_c_context * context,
+    PrimOpFun fun,
+    int arity,
+    const char * name,
+    const char ** args,
+    const char * doc,
+    void * user_data);
+
+/** @brief add a primop to the `builtins` attribute set
+ *
+ * Only applies to States created after this call.
+ *
+ * Moves your PrimOp content into the global evaluator
+ * registry, meaning your input PrimOp pointer is no longer usable.
+ * You are free to remove your references to it,
+ * after which it will be garbage collected.
+ *
+ * @param[out] context Optional, stores error information
+ * @return primop, or null in case of errors
+ *
+ */
+nix_err nix_register_primop(nix_c_context * context, PrimOp * primOp);
+/** @} */
+
+// Function prototypes
+
+/** @brief Allocate a Nix value
+ *
+ * Owned by the GC. Use nix_gc_decref() when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] state nix evaluator state
+ * @return value, or null in case of errors
+ *
+ */
+Value * nix_alloc_value(nix_c_context * context, EvalState * state);
+
+/** @addtogroup value_manip Manipulating values
+ * @brief Functions to inspect and change Nix language values, represented by Value.
+ * @{
+ */
+/** @name Getters
+ */
+/**@{*/
+/** @brief Get value type
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return type of nix value
+ */
+ValueType nix_get_type(nix_c_context * context, const Value * value);
+
+/** @brief Get type name of value as defined in the evaluator
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return type name, owned string
+ * @todo way to free the result
+ */
+const char * nix_get_typename(nix_c_context * context, const Value * value);
+
+/** @brief Get boolean value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return true or false, error info via context
+ */
+bool nix_get_bool(nix_c_context * context, const Value * value);
+
+/** @brief Get the raw string
+ *
+ * This may contain placeholders.
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] callback Called with the string value.
+ * @param[in] user_data optional, arbitrary data, passed to the callback when it's called.
+ * @return string
+ * @return error code, NIX_OK on success.
+ */
+nix_err
+nix_get_string(nix_c_context * context, const Value * value, nix_get_string_callback callback, void * user_data);
+
+/** @brief Get path as string
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return string
+ * @return NULL in case of error.
+ */
+const char * nix_get_path_string(nix_c_context * context, const Value * value);
+
+/** @brief Get the length of a list
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return length of list, error info via context
+ */
+unsigned int nix_get_list_size(nix_c_context * context, const Value * value);
+
+/** @brief Get the element count of an attrset
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return attrset element count, error info via context
+ */
+unsigned int nix_get_attrs_size(nix_c_context * context, const Value * value);
+
+/** @brief Get float value in 64 bits
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return float contents, error info via context
+ */
+double nix_get_float(nix_c_context * context, const Value * value);
+
+/** @brief Get int value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return int contents, error info via context
+ */
+int64_t nix_get_int(nix_c_context * context, const Value * value);
+
+/** @brief Get external reference
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @return reference to external, NULL in case of error
+ */
+ExternalValue * nix_get_external(nix_c_context * context, Value *);
+
+/** @brief Get the ix'th element of a list
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] ix list element to get
+ * @return value, NULL in case of errors
+ */
+Value * nix_get_list_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int ix);
+
+/** @brief Get an attr by name
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] name attribute name
+ * @return value, NULL in case of errors
+ */
+Value * nix_get_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name);
+
+/** @brief Check if an attribute name exists on a value
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] name attribute name
+ * @return value, error info via context
+ */
+bool nix_has_attr_byname(nix_c_context * context, const Value * value, EvalState * state, const char * name);
+
+/** @brief Get an attribute by index in the sorted bindings
+ *
+ * Also gives you the name.
+ *
+ * Owned by the GC. Use nix_gc_decref when you're done with the pointer
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] i attribute index
+ * @param[out] name will store a pointer to the attribute name
+ * @return value, NULL in case of errors
+ */
+Value *
+nix_get_attr_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i, const char ** name);
+
+/** @brief Get an attribute name by index in the sorted bindings
+ *
+ * Useful when you want the name but want to avoid evaluation.
+ *
+ * Owned by the nix EvalState
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value to inspect
+ * @param[in] state nix evaluator state
+ * @param[in] i attribute index
+ * @return name, NULL in case of errors
+ */
+const char * nix_get_attr_name_byidx(nix_c_context * context, const Value * value, EvalState * state, unsigned int i);
+
+/**@}*/
+/** @name Initializers
+ *
+ * Values are typically "returned" by initializing already allocated memory that serves as the return value.
+ * For this reason, the construction of values is not tied their allocation.
+ * Nix is a language with immutable values. Respect this property by only initializing Values once; and only initialize
+ * Values that are meant to be initialized by you. Failing to adhere to these rules may lead to undefined behavior.
+ */
+/**@{*/
+/** @brief Set boolean value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] b the boolean value
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_bool(nix_c_context * context, Value * value, bool b);
+
+/** @brief Set a string
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] str the string, copied
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_string(nix_c_context * context, Value * value, const char * str);
+
+/** @brief Set a path
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] str the path string, copied
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_path_string(nix_c_context * context, EvalState * s, Value * value, const char * str);
+
+/** @brief Set a float
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] d the float, 64-bits
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_float(nix_c_context * context, Value * value, double d);
+
+/** @brief Set an int
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] i the int
+ * @return error code, NIX_OK on success.
+ */
+
+nix_err nix_init_int(nix_c_context * context, Value * value, int64_t i);
+/** @brief Set null
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @return error code, NIX_OK on success.
+ */
+
+nix_err nix_init_null(nix_c_context * context, Value * value);
+/** @brief Set an external value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] val the external value to set. Will be GC-referenced by the value.
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_external(nix_c_context * context, Value * value, ExternalValue * val);
+
+/** @brief Create a list from a list builder
+ * @param[out] context Optional, stores error information
+ * @param[in] list_builder list builder to use. Make sure to unref this afterwards.
+ * @param[out] value Nix value to modify
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_make_list(nix_c_context * context, ListBuilder * list_builder, Value * value);
+
+/** @brief Create a list builder
+ * @param[out] context Optional, stores error information
+ * @param[in] state nix evaluator state
+ * @param[in] capacity how many bindings you'll add. Don't exceed.
+ * @return owned reference to a list builder. Make sure to unref when you're done.
+ */
+ListBuilder * nix_make_list_builder(nix_c_context * context, EvalState * state, size_t capacity);
+
+/** @brief Insert bindings into a builder
+ * @param[out] context Optional, stores error information
+ * @param[in] list_builder ListBuilder to insert into
+ * @param[in] index index to manipulate
+ * @param[in] value value to insert
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_list_builder_insert(nix_c_context * context, ListBuilder * list_builder, unsigned int index, Value * value);
+
+/** @brief Free a list builder
+ *
+ * Does not fail.
+ * @param[in] builder the builder to free
+ */
+void nix_list_builder_free(ListBuilder * list_builder);
+
+/** @brief Create an attribute set from a bindings builder
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] b bindings builder to use. Make sure to unref this afterwards.
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_make_attrs(nix_c_context * context, Value * value, BindingsBuilder * b);
+
+/** @brief Set primop
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] op primop, will be gc-referenced by the value
+ * @see nix_alloc_primop
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_init_primop(nix_c_context * context, Value * value, PrimOp * op);
+/** @brief Copy from another value
+ * @param[out] context Optional, stores error information
+ * @param[out] value Nix value to modify
+ * @param[in] source value to copy from
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_copy_value(nix_c_context * context, Value * value, Value * source);
+/**@}*/
+
+/** @brief Create a bindings builder
+* @param[out] context Optional, stores error information
+* @param[in] state nix evaluator state
+* @param[in] capacity how many bindings you'll add. Don't exceed.
+* @return owned reference to a bindings builder. Make sure to unref when you're
+done.
+*/
+BindingsBuilder * nix_make_bindings_builder(nix_c_context * context, EvalState * state, size_t capacity);
+
+/** @brief Insert bindings into a builder
+ * @param[out] context Optional, stores error information
+ * @param[in] builder BindingsBuilder to insert into
+ * @param[in] name attribute name, copied into the symbol store
+ * @param[in] value value to give the binding
+ * @return error code, NIX_OK on success.
+ */
+nix_err
+nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * builder, const char * name, Value * value);
+
+/** @brief Free a bindings builder
+ *
+ * Does not fail.
+ * @param[in] builder the builder to free
+ */
+void nix_bindings_builder_free(BindingsBuilder * builder);
+/**@}*/
+
+/** @brief Realise a string context.
+ *
+ * This will
+ *  - realise the store paths referenced by the string's context, and
+ *  - perform the replacement of placeholders.
+ *  - create temporary garbage collection roots for the store paths, for
+ *    the lifetime of the current process.
+ *  - log to stderr
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] value Nix value, which must be a string
+ * @param[in] state Nix evaluator state
+ * @param[in] isIFD If true, disallow derivation outputs if setting `allow-import-from-derivation` is false.
+                    You should set this to true when this call is part of a primop.
+                    You should set this to false when building for your application's purpose.
+ * @return NULL if failed, are a new nix_realised_string, which must be freed with nix_realised_string_free
+ */
+nix_realised_string * nix_string_realise(nix_c_context * context, EvalState * state, Value * value, bool isIFD);
+
+/** @brief Start of the string
+ * @param[in] realised_string
+ * @return pointer to the start of the string. It may not be null-terminated.
+ */
+const char * nix_realised_string_get_buffer_start(nix_realised_string * realised_string);
+
+/** @brief Length of the string
+ * @param[in] realised_string
+ * @return length of the string in bytes
+ */
+size_t nix_realised_string_get_buffer_size(nix_realised_string * realised_string);
+
+/** @brief Number of realised store paths
+ * @param[in] realised_string
+ * @return number of realised store paths that were referenced by the string via its context
+ */
+size_t nix_realised_string_get_store_path_count(nix_realised_string * realised_string);
+
+/** @brief Get a store path. The store paths are stored in an arbitrary order.
+ * @param[in] realised_string
+ * @param[in] index index of the store path, must be less than the count
+ * @return store path
+ */
+const StorePath * nix_realised_string_get_store_path(nix_realised_string * realised_string, size_t index);
+
+/** @brief Free a realised string
+ * @param[in] realised_string
+ */
+void nix_realised_string_free(nix_realised_string * realised_string);
+
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+
+/** @} */
+#endif // NIX_API_VALUE_H

src/libexpr/eval.cc

@@ -892,7 +892,6 @@ void Value::mkStringMove(const char * s, const NixStringContext & context)
     mkString(s, encodeContext(context));
 }
 
-
 void Value::mkPath(const SourcePath & path)
 {
     mkPath(&*path.accessor, makeImmutableString(path.path.abs()));
@@ -1660,7 +1659,8 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                 try {
                     fn->fun(*this, vCur.determinePos(noPos), args, vCur);
                 } catch (Error & e) {
-                    addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
+                    if (fn->addTrace)
+                        addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
                     throw;
                 }
 
@@ -1708,7 +1708,8 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
                     //    so the debugger allows to inspect the wrong parameters passed to the builtin.
                     fn->fun(*this, vCur.determinePos(noPos), vArgs, vCur);
                 } catch (Error & e) {
-                    addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
+                    if (fn->addTrace)
+                        addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name);
                     throw;
                 }
 

src/libexpr/eval.hh

@@ -17,6 +17,7 @@
 #include <optional>
 #include <unordered_map>
 #include <mutex>
+#include <functional>
 
 namespace nix {
 
@@ -69,10 +70,17 @@ struct PrimOp
      */
     const char * doc = nullptr;
 
+    /**
+     * Add a trace item, `while calling the '<name>' builtin`
+     *
+     * This is used to remove the redundant item for `builtins.addErrorContext`.
+     */
+    bool addTrace = true;
+
     /**
      * Implementation of the primop.
      */
-    PrimOpFun fun;
+    std::function<std::remove_pointer<PrimOpFun>::type> fun;
 
     /**
      * Optional experimental for this to be gated on.
@@ -722,10 +730,12 @@ public:
     bool fullGC();
 
     /**
-     * Realise the given context, and return a mapping from the placeholders
+     * Realise the given context
-     * used to construct the associated value to their final store path
+     * @param[in] context the context to realise
+     * @param[out] maybePaths if not nullptr, all built or referenced store paths will be added to this set
+     * @return a mapping from the placeholders used to construct the associated value to their final store path.
      */
-    [[nodiscard]] StringMap realiseContext(const NixStringContext & context);
+    [[nodiscard]] StringMap realiseContext(const NixStringContext & context, StorePathSet * maybePaths = nullptr, bool isIFD = true);
 
     /* Call the binary path filter predicate used builtins.path etc. */
     bool callPathFilter(

src/libexpr/flake/flake.cc

@@ -10,6 +10,7 @@
 #include "finally.hh"
 #include "fetch-settings.hh"
 #include "value-to-json.hh"
+#include "local-fs-store.hh"
 
 namespace nix {
 
@@ -755,7 +756,17 @@ void callFlake(EvalState & state,
 
         auto lockedNode = node.dynamic_pointer_cast<const LockedNode>();
 
-        auto [storePath, subdir] = state.store->toStorePath(sourcePath.path.abs());
+        // FIXME: This is a hack to support chroot stores. Remove this
+        // once we can pass a sourcePath rather than a storePath to
+        // call-flake.nix.
+        auto path = sourcePath.path.abs();
+        if (auto store = state.store.dynamic_pointer_cast<LocalFSStore>()) {
+            auto realStoreDir = store->getRealStoreDir();
+            if (isInDir(path, realStoreDir))
+                path = store->storeDir + path.substr(realStoreDir.size());
+        }
+
+        auto [storePath, subdir] = state.store->toStorePath(path);
 
         emitTreeAttrs(
             state,
@@ -857,10 +868,13 @@ static RegisterPrimOp r3({
       Parse a flake reference, and return its exploded form.
 
       For example:
+
       ```nix
       builtins.parseFlakeRef "github:NixOS/nixpkgs/23.05?dir=lib"
       ```
+
       evaluates to:
+
       ```nix
       { dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github"; }
       ```
@@ -909,12 +923,15 @@ static RegisterPrimOp r4({
       Convert a flake reference from attribute set format to URL format.
 
       For example:
+
       ```nix
       builtins.flakeRefToString {
         dir = "lib"; owner = "NixOS"; ref = "23.05"; repo = "nixpkgs"; type = "github";
       }
       ```
+
       evaluates to
+
       ```nix
       "github:NixOS/nixpkgs/23.05?dir=lib"
       ```

src/libexpr/local.mk

@@ -11,8 +11,11 @@ libexpr_SOURCES := \
   $(wildcard $(d)/flake/*.cc) \
   $(d)/lexer-tab.cc \
   $(d)/parser-tab.cc
+# Not just for this library itself, but also for downstream libraries using this library
 
-libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
+INCLUDE_libexpr := -I $(d)
+
+libexpr_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers) $(INCLUDE_libmain) $(INCLUDE_libexpr)
 
 libexpr_LIBS = libutil libstore libfetchers
 

src/libexpr/primops.cc

@@ -39,7 +39,7 @@ namespace nix {
  * Miscellaneous
  *************************************************************/
 
-StringMap EvalState::realiseContext(const NixStringContext & context)
+StringMap EvalState::realiseContext(const NixStringContext & context, StorePathSet * maybePathsOut, bool isIFD)
 {
     std::vector<DerivedPath::Built> drvs;
     StringMap res;
@@ -59,21 +59,23 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
             },
             [&](const NixStringContextElem::Opaque & o) {
                 auto ctxS = store->printStorePath(o.path);
-                res.insert_or_assign(ctxS, ctxS);
                 ensureValid(o.path);
+                if (maybePathsOut)
+                    maybePathsOut->emplace(o.path);
             },
             [&](const NixStringContextElem::DrvDeep & d) {
                 /* Treat same as Opaque */
                 auto ctxS = store->printStorePath(d.drvPath);
-                res.insert_or_assign(ctxS, ctxS);
                 ensureValid(d.drvPath);
+                if (maybePathsOut)
+                    maybePathsOut->emplace(d.drvPath);
             },
         }, c.raw);
     }
 
     if (drvs.empty()) return {};
 
-    if (!evalSettings.enableImportFromDerivation)
+    if (isIFD && !evalSettings.enableImportFromDerivation)
         error<EvalError>(
             "cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled",
             drvs.begin()->to_string(*store)
@@ -90,6 +92,8 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
         auto outputs = resolveDerivedPath(*buildStore, drv, &*store);
         for (auto & [outputName, outputPath] : outputs) {
             outputsToCopyAndAllow.insert(outputPath);
+            if (maybePathsOut)
+                maybePathsOut->emplace(outputPath);
 
             /* Get all the output paths corresponding to the placeholders we had */
             if (experimentalFeatureSettings.isEnabled(Xp::CaDerivations)) {
@@ -106,10 +110,13 @@ StringMap EvalState::realiseContext(const NixStringContext & context)
     }
 
     if (store != buildStore) copyClosure(*buildStore, *store, outputsToCopyAndAllow);
-    for (auto & outputPath : outputsToCopyAndAllow) {
+
-        /* Add the output of this derivations to the allowed
+    if (isIFD) {
-           paths. */
+        for (auto & outputPath : outputsToCopyAndAllow) {
-        allowPath(outputPath);
+            /* Add the output of this derivations to the allowed
+            paths. */
+            allowPath(outputPath);
+        }
     }
 
     return res;
@@ -826,7 +833,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * *
         auto message = state.coerceToString(pos, *args[0], context,
                 "while evaluating the error message passed to builtins.addErrorContext",
                 false, false).toOwned();
-        e.addTrace(nullptr, HintFmt(message));
+        e.addTrace(nullptr, HintFmt(message), TracePrint::Always);
         throw;
     }
 }
@@ -834,6 +841,8 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * *
 static RegisterPrimOp primop_addErrorContext(PrimOp {
     .name = "__addErrorContext",
     .arity = 2,
+    // The normal trace item is redundant
+    .addTrace = false,
     .fun = prim_addErrorContext,
 });
 
@@ -1132,7 +1141,7 @@ static void derivationStrictInternal(
     bool contentAddressed = false;
     bool isImpure = false;
     std::optional<std::string> outputHash;
-    std::string outputHashAlgo;
+    std::optional<HashAlgorithm> outputHashAlgo;
     std::optional<ContentAddressMethod> ingestionMethod;
 
     StringSet outputs;
@@ -1144,18 +1153,20 @@ static void derivationStrictInternal(
         vomit("processing attribute '%1%'", key);
 
         auto handleHashMode = [&](const std::string_view s) {
-            if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive;
+            if (s == "recursive") {
-            else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
+                // back compat, new name is "nar"
-            else if (s == "git") {
+                ingestionMethod = FileIngestionMethod::Recursive;
-                experimentalFeatureSettings.require(Xp::GitHashing);
+            } else try {
-                ingestionMethod = FileIngestionMethod::Git;
+                ingestionMethod = ContentAddressMethod::parse(s);
-            } else if (s == "text") {
+            } catch (UsageError &) {
-                experimentalFeatureSettings.require(Xp::DynamicDerivations);
-                ingestionMethod = TextIngestionMethod {};
-            } else
                 state.error<EvalError>(
                     "invalid value '%s' for 'outputHashMode' attribute", s
                 ).atPos(v).debugThrow();
+            }
+            if (ingestionMethod == TextIngestionMethod {})
+                experimentalFeatureSettings.require(Xp::DynamicDerivations);
+            if (ingestionMethod == FileIngestionMethod::Git)
+                experimentalFeatureSettings.require(Xp::GitHashing);
         };
 
         auto handleOutputs = [&](const Strings & ss) {
@@ -1231,7 +1242,7 @@ static void derivationStrictInternal(
                     else if (i->name == state.sOutputHash)
                         outputHash = state.forceStringNoCtx(*i->value, pos, context_below);
                     else if (i->name == state.sOutputHashAlgo)
-                        outputHashAlgo = state.forceStringNoCtx(*i->value, pos, context_below);
+                        outputHashAlgo = parseHashAlgoOpt(state.forceStringNoCtx(*i->value, pos, context_below));
                     else if (i->name == state.sOutputHashMode)
                         handleHashMode(state.forceStringNoCtx(*i->value, pos, context_below));
                     else if (i->name == state.sOutputs) {
@@ -1249,7 +1260,7 @@ static void derivationStrictInternal(
                     if (i->name == state.sBuilder) drv.builder = std::move(s);
                     else if (i->name == state.sSystem) drv.platform = std::move(s);
                     else if (i->name == state.sOutputHash) outputHash = std::move(s);
-                    else if (i->name == state.sOutputHashAlgo) outputHashAlgo = std::move(s);
+                    else if (i->name == state.sOutputHashAlgo) outputHashAlgo = parseHashAlgoOpt(s);
                     else if (i->name == state.sOutputHashMode) handleHashMode(s);
                     else if (i->name == state.sOutputs)
                         handleOutputs(tokenizeString<Strings>(s));
@@ -1332,7 +1343,7 @@ static void derivationStrictInternal(
                 "multiple outputs are not supported in fixed-output derivations"
             ).atPos(v).debugThrow();
 
-        auto h = newHashAllowEmpty(*outputHash, parseHashAlgoOpt(outputHashAlgo));
+        auto h = newHashAllowEmpty(*outputHash, outputHashAlgo);
 
         auto method = ingestionMethod.value_or(FileIngestionMethod::Flat);
 
@@ -1352,7 +1363,7 @@ static void derivationStrictInternal(
             state.error<EvalError>("derivation cannot be both content-addressed and impure")
                 .atPos(v).debugThrow();
 
-        auto ha = parseHashAlgoOpt(outputHashAlgo).value_or(HashAlgorithm::SHA256);
+        auto ha = outputHashAlgo.value_or(HashAlgorithm::SHA256);
         auto method = ingestionMethod.value_or(FileIngestionMethod::Recursive);
 
         for (auto & i : outputs) {
@@ -1575,23 +1586,50 @@ static RegisterPrimOp primop_pathExists({
     .fun = prim_pathExists,
 });
 
+// Ideally, all trailing slashes should have been removed, but it's been like this for
+// almost a decade as of writing. Changing it will affect reproducibility.
+static std::string_view legacyBaseNameOf(std::string_view path)
+{
+    if (path.empty())
+        return "";
+
+    auto last = path.size() - 1;
+    if (path[last] == '/' && last > 0)
+        last -= 1;
+
+    auto pos = path.rfind('/', last);
+    if (pos == path.npos)
+        pos = 0;
+    else
+        pos += 1;
+
+    return path.substr(pos, last - pos + 1);
+}
+
 /* Return the base name of the given string, i.e., everything
    following the last slash. */
 static void prim_baseNameOf(EvalState & state, const PosIdx pos, Value * * args, Value & v)
 {
     NixStringContext context;
-    v.mkString(baseNameOf(*state.coerceToString(pos, *args[0], context,
+    v.mkString(legacyBaseNameOf(*state.coerceToString(pos, *args[0], context,
             "while evaluating the first argument passed to builtins.baseNameOf",
             false, false)), context);
 }
 
 static RegisterPrimOp primop_baseNameOf({
     .name = "baseNameOf",
-    .args = {"s"},
+    .args = {"x"},
     .doc = R"(
-      Return the *base name* of the string *s*, that is, everything
+      Return the *base name* of either a [path value](@docroot@/language/values.md#type-path) *x* or a string *x*, depending on which type is passed, and according to the following rules.
-      following the final slash in the string. This is similar to the GNU
+
-      `basename` command.
+      For a path value, the *base name* is considered to be the part of the path after the last directory separator, including any file extensions.
+      This is the simple case, as path values don't have trailing slashes.
+
+      When the argument is a string, a more involved logic applies. If the string ends with a `/`, only this one final slash is removed.
+
+      After this, the *base name* is returned as previously described, assuming `/` as the directory separator. (Note that evaluation must be platform independent.)
+
+      This is somewhat similar to the [GNU `basename`](https://www.gnu.org/software/coreutils/manual/html_node/basename-invocation.html) command, but GNU `basename` will strip any number of trailing slashes.
     )",
     .fun = prim_baseNameOf,
 });
@@ -1893,11 +1931,13 @@ static RegisterPrimOp primop_outputOf({
       *`derivation reference`* must be a string that may contain a regular store path to a derivation, or may be a placeholder reference. If the derivation is produced by a derivation, you must explicitly select `drv.outPath`.
       This primop can be chained arbitrarily deeply.
       For instance,
+
       ```nix
       builtins.outputOf
         (builtins.outputOf myDrv "out")
         "out"
       ```
+
       will return a placeholder for the output of the output of `myDrv`.
 
       This primop corresponds to the `^` sigil for derivable paths, e.g. as part of installable syntax on the command line.
@@ -3379,10 +3419,11 @@ static void prim_sort(EvalState & state, const PosIdx pos, Value * * args, Value
     auto comparator = [&](Value * a, Value * b) {
         /* Optimization: if the comparator is lessThan, bypass
            callFunction. */
-        /* TODO: (layus) this is absurd. An optimisation like this
+        if (args[0]->isPrimOp()) {
-           should be outside the lambda creation */
+            auto ptr = args[0]->primOp()->fun.target<decltype(&prim_lessThan)>();
-        if (args[0]->isPrimOp() && args[0]->primOp()->fun == prim_lessThan)
+            if (ptr && *ptr == prim_lessThan)
-            return CompareValues(state, noPos, "while evaluating the ordering function passed to builtins.sort")(a, b);
+                return CompareValues(state, noPos, "while evaluating the ordering function passed to builtins.sort")(a, b);
+        }
 
         Value * vs[] = {a, b};
         Value vBool;
@@ -3838,7 +3879,7 @@ static RegisterPrimOp primop_stringLength({
     .name = "__stringLength",
     .args = {"e"},
     .doc = R"(
-      Return the length of the string *e*. If *e* is not a string,
+      Return the number of bytes of the string *e*. If *e* is not a string,
       evaluation is aborted.
     )",
     .fun = prim_stringLength,

src/libexpr/primops/fetchTree.cc

@@ -473,7 +473,7 @@ static void fetch(EvalState & state, const PosIdx pos, Value * * args, Value & v
     auto storePath =
         unpack
         ? fetchToStore(*state.store, fetchers::downloadTarball(*url).accessor, FetchMode::Copy, name)
-        : fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
+        : fetchers::downloadFile(state.store, *url, name).storePath;
 
     if (expectedHash) {
         auto hash = unpack
@@ -650,12 +650,14 @@ static RegisterPrimOp primop_fetchGit({
 
         The public keys against which `rev` is verified if `verifyCommit` is enabled.
         Must be given as a list of attribute sets with the following form:
+
         ```nix
         {
           key = "<public key>";
           type = "<key type>"; # optional, default: "ssh-ed25519"
         }
         ```
+
         Requires the [`verified-fetches` experimental feature](@docroot@/contributing/experimental-features.md#xp-feature-verified-fetches).
 
 

src/libexpr/search-path.cc

@@ -44,7 +44,7 @@ SearchPath::Elem SearchPath::Elem::parse(std::string_view rawElem)
 }
 
 
-SearchPath parseSearchPath(const Strings & rawElems)
+SearchPath SearchPath::parse(const Strings & rawElems)
 {
     SearchPath res;
     for (auto & rawElem : rawElems)

src/libexpr/value.hh

@@ -321,6 +321,7 @@ public:
     }
 
     void mkPath(const SourcePath & path);
+    void mkPath(std::string_view path);
 
     inline void mkPath(InputAccessor * accessor, const char * path)
     {

src/libfetchers/fetch-settings.hh

@@ -78,7 +78,6 @@ struct FetchSettings : public Config
         )",
         {}, true, Xp::Flakes};
 
-
     Setting<bool> useRegistries{this, true, "use-registries",
         "Whether to use flake registries to resolve flake references.",
         {}, true, Xp::Flakes};
@@ -94,6 +93,22 @@ struct FetchSettings : public Config
           empty, the summary is generated based on the action performed.
         )",
         {}, true, Xp::Flakes};
+
+    Setting<bool> trustTarballsFromGitForges{
+        this, true, "trust-tarballs-from-git-forges",
+        R"(
+          If enabled (the default), Nix will consider tarballs from
+          GitHub and similar Git forges to be locked if a Git revision
+          is specified,
+          e.g. `github:NixOS/patchelf/7c2f768bf9601268a4e71c2ebe91e2011918a70f`.
+          This requires Nix to trust that the provider will return the
+          correct contents for the specified Git revision.
+
+          If disabled, such tarballs are only considered locked if a
+          `narHash` attribute is specified,
+          e.g. `github:NixOS/patchelf/7c2f768bf9601268a4e71c2ebe91e2011918a70f?narHash=sha256-PPXqKY2hJng4DBVE0I4xshv/vGLUskL7jl53roB8UdU%3D`.
+        )"};
+
 };
 
 // FIXME: don't use a global variable.

src/libfetchers/fetchers.cc

@@ -3,6 +3,7 @@
 #include "input-accessor.hh"
 #include "source-path.hh"
 #include "fetch-to-store.hh"
+#include "json-utils.hh"
 
 #include <nlohmann/json.hpp>
 
@@ -412,3 +413,20 @@ std::string publicKeys_to_string(const std::vector<PublicKey>& publicKeys)
 }
 
 }
+
+namespace nlohmann {
+
+using namespace nix;
+
+fetchers::PublicKey adl_serializer<fetchers::PublicKey>::from_json(const json & json) {
+    auto type = optionalValueAt(json, "type").value_or("ssh-ed25519");
+    auto key = valueAt(json, "key");
+    return fetchers::PublicKey { getString(type), getString(key) };
+}
+
+void adl_serializer<fetchers::PublicKey>::to_json(json & json, fetchers::PublicKey p) {
+    json["type"] = p.type;
+    json["key"] = p.key;
+}
+
+}

src/libfetchers/fetchers.hh

@@ -4,6 +4,7 @@
 #include "types.hh"
 #include "hash.hh"
 #include "canon-path.hh"
+#include "json-impls.hh"
 #include "attrs.hh"
 #include "url.hh"
 
@@ -230,8 +231,9 @@ struct PublicKey
     std::string type = "ssh-ed25519";
     std::string key;
 };
-NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE_WITH_DEFAULT(PublicKey, type, key)
 
 std::string publicKeys_to_string(const std::vector<PublicKey>&);
 
 }
+
+JSON_IMPL(fetchers::PublicKey)

src/libfetchers/filtering-input-accessor.cc

@@ -38,7 +38,7 @@ std::string FilteringInputAccessor::readLink(const CanonPath & path)
 
 std::string FilteringInputAccessor::showPath(const CanonPath & path)
 {
-    return next->showPath(prefix / path);
+    return displayPrefix + next->showPath(prefix / path) + displaySuffix;
 }
 
 void FilteringInputAccessor::checkAccess(const CanonPath & path)

src/libfetchers/filtering-input-accessor.hh

@@ -27,7 +27,9 @@ struct FilteringInputAccessor : InputAccessor
         : next(src.accessor)
         , prefix(src.path)
         , makeNotAllowedError(std::move(makeNotAllowedError))
-    { }
+    {
+        displayPrefix.clear();
+    }
 
     std::string readFile(const CanonPath & path) override;
 

src/libfetchers/fs-input-accessor.cc

@@ -24,7 +24,10 @@ ref<InputAccessor> makeStorePathAccessor(
     const StorePath & storePath)
 {
     // FIXME: should use `store->getFSAccessor()`
-    return makeFSInputAccessor(std::filesystem::path { store->toRealPath(storePath) });
+    auto root = std::filesystem::path { store->toRealPath(storePath) };
+    auto accessor = makeFSInputAccessor(root);
+    accessor->setPathDisplay(root);
+    return accessor;
 }
 
 SourcePath getUnfilteredRootPath(CanonPath path)

src/libfetchers/git-utils.cc

@@ -198,6 +198,12 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         return git_repository_is_shallow(*this);
     }
 
+    void setRemote(const std::string & name, const std::string & url) override
+    {
+        if (git_remote_set_url(*this, name.c_str(), url.c_str()))
+            throw Error("setting remote '%s' URL to '%s': %s", name, url, git_error_last()->message);
+    }
+
     Hash resolveRef(std::string ref) override
     {
         Object object;
@@ -302,9 +308,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
 
     std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev, bool exportIgnore) override;
 
-    std::string resolveSubmoduleUrl(
+    std::string resolveSubmoduleUrl(const std::string & url) override
-        const std::string & url,
-        const std::string & base) override
     {
         git_buf buf = GIT_BUF_INIT;
         if (git_submodule_resolve_url(&buf, *this, url.c_str()))
@@ -312,10 +316,6 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
         Finally cleanup = [&]() { git_buf_dispose(&buf); };
 
         std::string res(buf.ptr);
-
-        if (!hasPrefix(res, "/") && res.find("://") == res.npos)
-            res = parseURL(base + "/" + res).canonicalise().to_string();
-
         return res;
     }
 
@@ -348,7 +348,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
     {
         auto act = (Activity *) payload;
         act->result(resFetchStatus, trim(std::string_view(str, len)));
-        return _isInterrupted ? -1 : 0;
+        return getInterrupted() ? -1 : 0;
     }
 
     static int transferProgressCallback(const git_indexer_progress * stats, void * payload)
@@ -361,7 +361,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this<GitRepoImpl>
                 stats->indexed_deltas,
                 stats->total_deltas,
                 stats->received_bytes / (1024.0 * 1024.0)));
-        return _isInterrupted ? -1 : 0;
+        return getInterrupted() ? -1 : 0;
     }
 
     void fetch(

src/libfetchers/git-utils.hh

@@ -32,6 +32,8 @@ struct GitRepo
     /* Return the commit hash to which a ref points. */
     virtual Hash resolveRef(std::string ref) = 0;
 
+    virtual void setRemote(const std::string & name, const std::string & url) = 0;
+
     /**
      * Info about a submodule.
      */
@@ -69,9 +71,7 @@ struct GitRepo
      */
     virtual std::vector<std::tuple<Submodule, Hash>> getSubmodules(const Hash & rev, bool exportIgnore) = 0;
 
-    virtual std::string resolveSubmoduleUrl(
+    virtual std::string resolveSubmoduleUrl(const std::string & url) = 0;
-        const std::string & url,
-        const std::string & base) = 0;
 
     virtual bool hasObject(const Hash & oid) = 0;
 

src/libfetchers/github.cc

@@ -294,7 +294,9 @@ struct GitArchiveInputScheme : InputScheme
            Git revision alone, we also require a NAR hash for
            locking. FIXME: in the future, we may want to require a Git
            tree hash instead of a NAR hash. */
-        return input.getRev().has_value() && input.getNarHash().has_value();
+        return input.getRev().has_value()
+            && (fetchSettings.trustTarballsFromGitForges ||
+                input.getNarHash().has_value());
     }
 
     std::optional<ExperimentalFeature> experimentalFeature() const override
@@ -355,7 +357,7 @@ struct GitHubInputScheme : GitArchiveInputScheme
         auto json = nlohmann::json::parse(
             readFile(
                 store->toRealPath(
-                    downloadFile(store, url, "source", false, headers).storePath)));
+                    downloadFile(store, url, "source", headers).storePath)));
 
         return RefInfo {
             .rev = Hash::parseAny(std::string { json["sha"] }, HashAlgorithm::SHA1),
@@ -429,7 +431,7 @@ struct GitLabInputScheme : GitArchiveInputScheme
         auto json = nlohmann::json::parse(
             readFile(
                 store->toRealPath(
-                    downloadFile(store, url, "source", false, headers).storePath)));
+                    downloadFile(store, url, "source", headers).storePath)));
 
         return RefInfo {
             .rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)
@@ -493,7 +495,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         std::string refUri;
         if (ref == "HEAD") {
             auto file = store->toRealPath(
-                downloadFile(store, fmt("%s/HEAD", base_url), "source", false, headers).storePath);
+                downloadFile(store, fmt("%s/HEAD", base_url), "source", headers).storePath);
             std::ifstream is(file);
             std::string line;
             getline(is, line);
@@ -509,7 +511,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
         std::regex refRegex(refUri);
 
         auto file = store->toRealPath(
-            downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath);
+            downloadFile(store, fmt("%s/info/refs", base_url), "source", headers).storePath);
         std::ifstream is(file);
 
         std::string line;

src/libfetchers/local.mk

@@ -5,8 +5,18 @@ libfetchers_NAME = libnixfetchers
 libfetchers_DIR := $(d)
 
 libfetchers_SOURCES := $(wildcard $(d)/*.cc)
+ifdef HOST_UNIX
+  libfetchers_SOURCES += $(wildcard $(d)/unix/*.cc)
+endif
 
-libfetchers_CXXFLAGS += -I src/libutil -I src/libstore
+# Not just for this library itself, but also for downstream libraries using this library
+
+INCLUDE_libfetchers := -I $(d)
+ifdef HOST_UNIX
+  INCLUDE_libfetchers += -I $(d)/unix
+endif
+
+libfetchers_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libfetchers)
 
 libfetchers_LDFLAGS += $(THREAD_LDFLAGS) $(LIBGIT2_LIBS) -larchive
 

src/libfetchers/mounted-input-accessor.cc

@@ -9,6 +9,8 @@ struct MountedInputAccessor : InputAccessor
     MountedInputAccessor(std::map<CanonPath, ref<InputAccessor>> _mounts)
         : mounts(std::move(_mounts))
     {
+        displayPrefix.clear();
+
         // Currently we require a root filesystem. This could be relaxed.
         assert(mounts.contains(CanonPath::root));
 
@@ -48,7 +50,7 @@ struct MountedInputAccessor : InputAccessor
     std::string showPath(const CanonPath & path) override
     {
         auto [accessor, subpath] = resolve(path);
-        return accessor->showPath(subpath);
+        return displayPrefix + accessor->showPath(subpath) + displaySuffix;
     }
 
     std::pair<ref<InputAccessor>, CanonPath> resolve(CanonPath path)

src/libfetchers/registry.cc

@@ -158,7 +158,7 @@ static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
         }
 
         if (!hasPrefix(path, "/")) {
-            auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;
+            auto storePath = downloadFile(store, path, "flake-registry.json").storePath;
             if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
                 store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json");
             path = store->toRealPath(storePath);

src/libfetchers/tarball.cc

@@ -19,7 +19,6 @@ DownloadFileResult downloadFile(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
-    bool locked,
     const Headers & headers)
 {
     // FIXME: check store
@@ -101,7 +100,7 @@ DownloadFileResult downloadFile(
             inAttrs,
             infoAttrs,
             *storePath,
-            locked);
+            false);
     }
 
     return {
@@ -306,7 +305,7 @@ struct FileInputScheme : CurlInputScheme
            the Nix store directly, since there is little deduplication
            benefit in using the Git cache for single big files like
            tarballs. */
-        auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName(), false);
+        auto file = downloadFile(store, getStrAttr(input.attrs, "url"), input.getName());
 
         auto narHash = store->queryPathInfo(file.storePath)->narHash;
         input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true));

src/libfetchers/tarball.hh

@@ -25,7 +25,6 @@ DownloadFileResult downloadFile(
     ref<Store> store,
     const std::string & url,
     const std::string & name,
-    bool locked,
     const Headers & headers = {});
 
 struct DownloadTarballResult

src/libfetchers/unix/git.cc

@@ -147,9 +147,12 @@ std::vector<PublicKey> getPublicKeys(const Attrs & attrs)
 {
     std::vector<PublicKey> publicKeys;
     if (attrs.contains("publicKeys")) {
-        nlohmann::json publicKeysJson = nlohmann::json::parse(getStrAttr(attrs, "publicKeys"));
+        auto pubKeysJson = nlohmann::json::parse(getStrAttr(attrs, "publicKeys"));
-        ensureType(publicKeysJson, nlohmann::json::value_t::array);
+        auto & pubKeys = getArray(pubKeysJson);
-        publicKeys = publicKeysJson.get<std::vector<PublicKey>>();
+
+        for (auto & key : pubKeys) {
+            publicKeys.push_back(key);
+        }
     }
     if (attrs.contains("publicKey"))
         publicKeys.push_back(PublicKey{maybeGetStrAttr(attrs, "keytype").value_or("ssh-ed25519"),getStrAttr(attrs, "publicKey")});
@@ -523,6 +526,9 @@ struct GitInputScheme : InputScheme
 
             auto repo = GitRepo::openRepo(cacheDir, true, true);
 
+            // We need to set the origin so resolving submodule URLs works
+            repo->setRemote("origin", repoInfo.url);
+
             Path localRefFile =
                 ref.compare(0, 5, "refs/") == 0
                 ? cacheDir + "/" + ref
@@ -626,7 +632,7 @@ struct GitInputScheme : InputScheme
             std::map<CanonPath, nix::ref<InputAccessor>> mounts;
 
             for (auto & [submodule, submoduleRev] : repo->getSubmodules(rev, exportIgnore)) {
-                auto resolved = repo->resolveSubmoduleUrl(submodule.url, repoInfo.url);
+                auto resolved = repo->resolveSubmoduleUrl(submodule.url);
                 debug("Git submodule %s: %s %s %s -> %s",
                     submodule.path, submodule.url, submodule.branch, submoduleRev.gitRev(), resolved);
                 fetchers::Attrs attrs;
@@ -639,6 +645,7 @@ struct GitInputScheme : InputScheme
                 auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
                     submoduleInput.getAccessor(store);
+                submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»");
                 mounts.insert_or_assign(submodule.path, submoduleAccessor);
             }
 
@@ -675,6 +682,8 @@ struct GitInputScheme : InputScheme
                 exportIgnore,
                 makeNotAllowedError(repoInfo.url));
 
+        accessor->setPathDisplay(repoInfo.url);
+
         /* If the repo has submodules, return a mounted input accessor
            consisting of the accessor for the top-level repo and the
            accessors for the submodule workdirs. */
@@ -691,6 +700,7 @@ struct GitInputScheme : InputScheme
                 auto submoduleInput = fetchers::Input::fromAttrs(std::move(attrs));
                 auto [submoduleAccessor, submoduleInput2] =
                     submoduleInput.getAccessor(store);
+                submoduleAccessor->setPathDisplay("«" + submoduleInput.to_string() + "»");
 
                 /* If the submodule is dirty, mark this repo dirty as
                    well. */

src/libfetchers/unix/mercurial.cc

@@ -352,7 +352,11 @@ struct MercurialInputScheme : InputScheme
 
         auto storePath = fetchToStore(store, input);
 
-        return {makeStorePathAccessor(store, storePath), input};
+        auto accessor = makeStorePathAccessor(store, storePath);
+
+        accessor->setPathDisplay("«" + input.to_string() + "»");
+
+        return {accessor, input};
     }
 
     bool isLocked(const Input & input) const override

src/libmain/local.mk

@@ -5,8 +5,13 @@ libmain_NAME = libnixmain
 libmain_DIR := $(d)
 
 libmain_SOURCES := $(wildcard $(d)/*.cc)
+ifdef HOST_UNIX
+  libmain_SOURCES += $(wildcard $(d)/unix/*.cc)
+endif
 
-libmain_CXXFLAGS += -I src/libutil -I src/libstore
+INCLUDE_libmain := -I $(d)
+
+libmain_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libstore) $(INCLUDE_libmain)
 
 libmain_LDFLAGS += $(OPENSSL_LIBS)
 

src/libmain/progress-bar.cc

@@ -123,14 +123,18 @@ public:
     }
 
     void pause() override {
-        state_.lock()->paused = true;
+        auto state (state_.lock());
-        writeToStderr("\r\e[K");
+        state->paused = true;
+        if (state->active)
+            writeToStderr("\r\e[K");
     }
 
     void resume() override {
-        state_.lock()->paused = false;
+        auto state (state_.lock());
-        writeToStderr("\r\e[K");
+        state->paused = false;
-        state_.lock()->haveUpdate = true;
+        if (state->active)
+            writeToStderr("\r\e[K");
+        state->haveUpdate = true;
         updateCV.notify_one();
     }
 
@@ -162,9 +166,7 @@ public:
             writeToStderr("\r\e[K" + filterANSIEscapes(s, !isTTY) + ANSI_NORMAL "\n");
             draw(state);
         } else {
-            auto s2 = s + ANSI_NORMAL "\n";
+            writeToStderr(filterANSIEscapes(s, !isTTY) + "\n");
-            if (!isTTY) s2 = filterANSIEscapes(s2, true);
-            writeToStderr(s2);
         }
     }
 
@@ -519,7 +521,7 @@ public:
     std::optional<char> ask(std::string_view msg) override
     {
         auto state(state_.lock());
-        if (!state->active || !isatty(STDIN_FILENO)) return {};
+        if (!state->active) return {};
         std::cerr << fmt("\r\e[K%s ", msg);
         auto s = trim(readLine(STDIN_FILENO));
         if (s.size() != 1) return {};
@@ -535,7 +537,7 @@ public:
 
 Logger * makeProgressBar()
 {
-    return new ProgressBar(shouldANSI());
+    return new ProgressBar(isTTY());
 }
 
 void startProgressBar()

src/libmain/shared.cc

@@ -121,7 +121,7 @@ void initNix()
 
     initLibStore();
 
-    startSignalHandlerThread();
+    unix::startSignalHandlerThread();
 
     /* Reset SIGCHLD to its default. */
     struct sigaction act;
@@ -308,7 +308,7 @@ void printVersion(const std::string & programName)
 void showManPage(const std::string & name)
 {
     restoreProcessContext();
-    setenv("MANPATH", settings.nixManDir.c_str(), 1);
+    setEnv("MANPATH", settings.nixManDir.c_str());
     execlp("man", "man", name.c_str(), nullptr);
     throw SysError("command 'man %1%' failed", name.c_str());
 }
@@ -369,7 +369,7 @@ RunPager::RunPager()
         if (dup2(toPager.readSide.get(), STDIN_FILENO) == -1)
             throw SysError("dupping stdin");
         if (!getenv("LESS"))
-            setenv("LESS", "FRSXMK", 1);
+            setEnv("LESS", "FRSXMK");
         restoreProcessContext();
         if (pager)
             execl("/bin/sh", "sh", "-c", pager, nullptr);

src/libstore-c/local.mk

@@ -0,0 +1,21 @@
+libraries += libstorec
+
+libstorec_NAME = libnixstorec
+
+libstorec_DIR := $(d)
+
+libstorec_SOURCES := $(wildcard $(d)/*.cc)
+
+libstorec_LIBS = libutil libstore libutilc
+
+libstorec_LDFLAGS += $(THREAD_LDFLAGS)
+
+# Not just for this library itself, but also for downstream libraries using this library
+
+INCLUDE_libstorec := -I $(d)
+libstorec_CXXFLAGS += $(INCLUDE_libutil) $(INCLUDE_libutilc) \
+		    $(INCLUDE_libstore) $(INCLUDE_libstorec)
+
+$(eval $(call install-file-in, $(d)/nix-store-c.pc, $(libdir)/pkgconfig, 0644))
+
+libstorec_FORCE_INSTALL := 1

src/libstore-c/nix-store-c.pc.in

@@ -0,0 +1,9 @@
+prefix=@prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Nix
+Description: Nix Store - C API
+Version: @PACKAGE_VERSION@
+Libs: -L${libdir} -lnixstorec -lnixutilc
+Cflags: -I${includedir}/nix

src/libstore-c/nix_api_store.cc

@@ -0,0 +1,146 @@
+#include "nix_api_store.h"
+#include "nix_api_store_internal.h"
+#include "nix_api_util.h"
+#include "nix_api_util_internal.h"
+
+#include "path.hh"
+#include "store-api.hh"
+#include "build-result.hh"
+
+#include "globals.hh"
+
+nix_err nix_libstore_init(nix_c_context * context)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::initLibStore();
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err nix_init_plugins(nix_c_context * context)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::initPlugins();
+    }
+    NIXC_CATCH_ERRS
+}
+
+Store * nix_store_open(nix_c_context * context, const char * uri, const char *** params)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        std::string uri_str = uri ? uri : "";
+
+        if (uri_str.empty())
+            return new Store{nix::openStore()};
+
+        if (!params)
+            return new Store{nix::openStore(uri_str)};
+
+        nix::Store::Params params_map;
+        for (size_t i = 0; params[i] != nullptr; i++) {
+            params_map[params[i][0]] = params[i][1];
+        }
+        return new Store{nix::openStore(uri_str, params_map)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+void nix_store_free(Store * store)
+{
+    delete store;
+}
+
+nix_err nix_store_get_uri(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto res = store->ptr->getUri();
+        return call_nix_get_string_callback(res, callback, user_data);
+    }
+    NIXC_CATCH_ERRS
+}
+
+nix_err
+nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        auto res = store->ptr->getVersion();
+        return call_nix_get_string_callback(res.value_or(""), callback, user_data);
+    }
+    NIXC_CATCH_ERRS
+}
+
+bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * path)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        return store->ptr->isValidPath(path->path);
+    }
+    NIXC_CATCH_ERRS_RES(false);
+}
+
+StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path)
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+        nix::StorePath s = store->ptr->parseStorePath(path);
+        return new StorePath{std::move(s)};
+    }
+    NIXC_CATCH_ERRS_NULL
+}
+
+nix_err nix_store_realise(
+    nix_c_context * context,
+    Store * store,
+    StorePath * path,
+    void * userdata,
+    void (*callback)(void * userdata, const char *, const char *))
+{
+    if (context)
+        context->last_err_code = NIX_OK;
+    try {
+
+        const std::vector<nix::DerivedPath> paths{nix::DerivedPath::Built{
+            .drvPath = nix::makeConstantStorePathRef(path->path), .outputs = nix::OutputsSpec::All{}}};
+
+        const auto nixStore = store->ptr;
+        auto results = nixStore->buildPathsWithResults(paths, nix::bmNormal, nixStore);
+
+        if (callback) {
+            for (const auto & result : results) {
+                for (const auto & [outputName, realisation] : result.builtOutputs) {
+                    auto op = store->ptr->printStorePath(realisation.outPath);
+                    callback(userdata, outputName.c_str(), op.c_str());
+                }
+            }
+        }
+    }
+    NIXC_CATCH_ERRS
+}
+
+void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data)
+{
+    std::string_view name = store_path->path.name();
+    callback(name.data(), name.size(), user_data);
+}
+
+void nix_store_path_free(StorePath * sp)
+{
+    delete sp;
+}
+
+StorePath * nix_store_path_clone(const StorePath * p)
+{
+    return new StorePath{p->path};
+}

src/libstore-c/nix_api_store.h

@@ -0,0 +1,169 @@
+#ifndef NIX_API_STORE_H
+#define NIX_API_STORE_H
+/**
+ * @defgroup libstore libstore
+ * @brief C bindings for nix libstore
+ *
+ * libstore is used for talking to a Nix store
+ * @{
+ */
+/** @file
+ * @brief Main entry for the libstore C bindings
+ */
+
+#include "nix_api_util.h"
+#include <stdbool.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+// cffi start
+
+/** @brief Reference to a Nix store */
+typedef struct Store Store;
+/** @brief Nix store path */
+typedef struct StorePath StorePath;
+
+/**
+ * @brief Initializes the Nix store library
+ *
+ * This function should be called before creating a Store
+ * This function can be called multiple times.
+ *
+ * @param[out] context Optional, stores error information
+ * @return NIX_OK if the initialization was successful, an error code otherwise.
+ */
+nix_err nix_libstore_init(nix_c_context * context);
+
+/**
+ * @brief Loads the plugins specified in Nix's plugin-files setting.
+ *
+ * Call this once, after calling your desired init functions and setting
+ * relevant settings.
+ *
+ * @param[out] context Optional, stores error information
+ * @return NIX_OK if the initialization was successful, an error code otherwise.
+ */
+nix_err nix_init_plugins(nix_c_context * context);
+
+/**
+ * @brief Open a nix store
+ * Store instances may share state and resources behind the scenes.
+ * @param[out] context Optional, stores error information
+ * @param[in] uri URI of the nix store, copied
+ * @param[in] params optional, array of key-value pairs, {{"endpoint",
+ * "https://s3.local"}}
+ * @return a Store pointer, NULL in case of errors
+ * @see nix_store_free
+ */
+Store * nix_store_open(nix_c_context *, const char * uri, const char *** params);
+
+/**
+ * @brief Deallocate a nix store and free any resources if not also held by other Store instances.
+ *
+ * Does not fail.
+ *
+ * @param[in] store the store to free
+ */
+void nix_store_free(Store * store);
+
+/**
+ * @brief get the URI of a nix store
+ * @param[out] context Optional, stores error information
+ * @param[in] store nix store reference
+ * @param[in] callback Called with the URI.
+ * @param[in] user_data optional, arbitrary data, passed to the callback when it's called.
+ * @see nix_get_string_callback
+ * @return error code, NIX_OK on success.
+ */
+nix_err nix_store_get_uri(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data);
+
+// returns: owned StorePath*
+/**
+ * @brief Parse a Nix store path into a StorePath
+ *
+ * @note Don't forget to free this path using nix_store_path_free()!
+ * @param[out] context Optional, stores error information
+ * @param[in] store nix store reference
+ * @param[in] path Path string to parse, copied
+ * @return owned store path, NULL on error
+ */
+StorePath * nix_store_parse_path(nix_c_context * context, Store * store, const char * path);
+
+/**
+ * @brief Get the path name (e.g. "name" in /nix/store/...-name)
+ *
+ * @param[in] store_path the path to get the name from
+ * @param[in] callback called with the name
+ * @param[in] user_data arbitrary data, passed to the callback when it's called.
+ */
+void nix_store_path_name(const StorePath * store_path, nix_get_string_callback callback, void * user_data);
+
+/**
+ * @brief Copy a StorePath
+ *
+ * @param[in] p the path to copy
+ * @return a new StorePath
+ */
+StorePath * nix_store_path_clone(const StorePath * p);
+
+/** @brief Deallocate a StorePath
+ *
+ * Does not fail.
+ * @param[in] p the path to free
+ */
+void nix_store_path_free(StorePath * p);
+
+/**
+ * @brief Check if a StorePath is valid (i.e. that corresponding store object and its closure of references exists in
+ * the store)
+ * @param[out] context Optional, stores error information
+ * @param[in] store Nix Store reference
+ * @param[in] path Path to check
+ * @return true or false, error info in context
+ */
+bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * path);
+// nix_err nix_store_ensure(Store*, const char*);
+// nix_err nix_store_build_paths(Store*);
+/**
+ * @brief Realise a Nix store path
+ *
+ * Blocking, calls callback once for each realised output.
+ *
+ * @note When working with expressions, consider using e.g. nix_string_realise to get the output. `.drvPath` may not be
+ * accurate or available in the future. See https://github.com/NixOS/nix/issues/6507
+ *
+ * @param[out] context Optional, stores error information
+ * @param[in] store Nix Store reference
+ * @param[in] path Path to build
+ * @param[in] userdata data to pass to every callback invocation
+ * @param[in] callback called for every realised output
+ */
+nix_err nix_store_realise(
+    nix_c_context * context,
+    Store * store,
+    StorePath * path,
+    void * userdata,
+    void (*callback)(void * userdata, const char * outname, const char * out));
+
+/**
+ * @brief get the version of a nix store.
+ * If the store doesn't have a version (like the dummy store), returns an empty string.
+ * @param[out] context Optional, stores error information
+ * @param[in] store nix store reference
+ * @param[in] callback Called with the version.
+ * @param[in] user_data optional, arbitrary data, passed to the callback when it's called.
+ * @see nix_get_string_callback
+ * @return error code, NIX_OK on success.
+ */
+nix_err
+nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data);
+
+// cffi end
+#ifdef __cplusplus
+}
+#endif
+/**
+ * @}
+ */
+#endif // NIX_API_STORE_H

src/libstore-c/nix_api_store_internal.h

@@ -0,0 +1,15 @@
+#ifndef NIX_API_STORE_INTERNAL_H
+#define NIX_API_STORE_INTERNAL_H
+#include "store-api.hh"
+
+struct Store
+{
+    nix::ref<nix::Store> ptr;
+};
+
+struct StorePath
+{
+    nix::StorePath path;
+};
+
+#endif

src/libstore/binary-cache-store.cc

@@ -124,14 +124,6 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
         diskCache->upsertNarInfo(getUri(), std::string(narInfo->path.hashPart()), std::shared_ptr<NarInfo>(narInfo));
 }
 
-AutoCloseFD openFile(const Path & path)
-{
-    auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
-    if (!fd)
-        throw SysError("opening file '%1%'", path);
-    return fd;
-}
-
 ref<const ValidPathInfo> BinaryCacheStore::addToStoreCommon(
     Source & narSource, RepairFlag repair, CheckSigsFlag checkSigs,
     std::function<ValidPathInfo(HashResult)> mkInfo)

src/libstore/build/derivation-goal.cc

@@ -1138,7 +1138,7 @@ void DerivationGoal::resolvedFinished()
 
 HookReply DerivationGoal::tryBuildHook()
 {
-    if (!worker.tryBuildHook || !useDerivation) return rpDecline;
+    if (settings.buildHook.get().empty() || !worker.tryBuildHook || !useDerivation) return rpDecline;
 
     if (!worker.hook)
         worker.hook = std::make_unique<HookInstance>();

src/libstore/build/local-derivation-goal.cc

@@ -14,10 +14,8 @@
 #include "topo-sort.hh"
 #include "callback.hh"
 #include "json-utils.hh"
-#include "cgroup.hh"
 #include "personality.hh"
 #include "current-process.hh"
-#include "namespaces.hh"
 #include "child.hh"
 #include "unix-domain-socket.hh"
 #include "posix-fs-canonicalise.hh"
@@ -40,18 +38,20 @@
 
 /* Includes required for chroot support. */
 #if __linux__
-#include <sys/ioctl.h>
+# include <sys/ioctl.h>
-#include <net/if.h>
+# include <net/if.h>
-#include <netinet/ip.h>
+# include <netinet/ip.h>
-#include <sys/mman.h>
+# include <sys/mman.h>
-#include <sched.h>
+# include <sched.h>
-#include <sys/param.h>
+# include <sys/param.h>
-#include <sys/mount.h>
+# include <sys/mount.h>
-#include <sys/syscall.h>
+# include <sys/syscall.h>
-#if HAVE_SECCOMP
+# include "namespaces.hh"
-#include <seccomp.h>
+# if HAVE_SECCOMP
-#endif
+#   include <seccomp.h>
-#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
+# endif
+# define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
+# include "cgroup.hh"
 #endif
 
 #if __APPLE__
@@ -395,21 +395,33 @@ void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
 #if __linux__
 static void doBind(const Path & source, const Path & target, bool optional = false) {
     debug("bind mounting '%1%' to '%2%'", source, target);
-    struct stat st;
+
-    if (stat(source.c_str(), &st) == -1) {
+    auto bindMount = [&]() {
-        if (optional && errno == ENOENT)
+        if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
+            throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
+    };
+
+    auto maybeSt = maybeLstat(source);
+    if (!maybeSt) {
+        if (optional)
             return;
         else
             throw SysError("getting attributes of path '%1%'", source);
     }
-    if (S_ISDIR(st.st_mode))
+    auto st = *maybeSt;
+
+    if (S_ISDIR(st.st_mode)) {
         createDirs(target);
-    else {
+        bindMount();
+    } else if (S_ISLNK(st.st_mode)) {
+        // Symlinks can (apparently) not be bind-mounted, so just copy it
+        createDirs(dirOf(target));
+        copyFile(source, target, /* andDelete */ false);
+    } else {
         createDirs(dirOf(target));
         writeFile(target, "");
+        bindMount();
     }
-    if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
-        throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
 };
 #endif
 
@@ -488,7 +500,7 @@ void LocalDerivationGoal::startBuilder()
 
     /* Create a temporary directory where the build will take
        place. */
-    tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
+    tmpDir = createTempDir(settings.buildDir.get().value_or(""), "nix-build-" + std::string(drvPath.name()), false, false, 0700);
 
     chownToBuilder(tmpDir);
 
@@ -1811,11 +1823,18 @@ void LocalDerivationGoal::runChild()
                     if (pathExists(path))
                         ss.push_back(path);
 
-                if (settings.caFile != "")
+                if (settings.caFile != "" && pathExists(settings.caFile)) {
-                    pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", settings.caFile, true);
+                    Path caFile = settings.caFile;
+                    pathsInChroot.try_emplace("/etc/ssl/certs/ca-certificates.crt", canonPath(caFile, true), true);
+                }
             }
 
-            for (auto & i : ss) pathsInChroot.emplace(i, i);
+            for (auto & i : ss) {
+                // For backwards-compatibiliy, resolve all the symlinks in the
+                // chroot paths
+                auto canonicalPath = canonPath(i, true);
+                pathsInChroot.emplace(i, canonicalPath);
+            }
 
             /* Bind-mount all the directories from the "host"
                filesystem that we want in the chroot
@@ -2053,13 +2072,13 @@ void LocalDerivationGoal::runChild()
                             i.first, i.second.source);
 
                     std::string path = i.first;
-                    struct stat st;
+                    auto optSt = maybeLstat(path.c_str());
-                    if (lstat(path.c_str(), &st)) {
+                    if (!optSt) {
-                        if (i.second.optional && errno == ENOENT)
+                        if (i.second.optional)
                             continue;
-                        throw SysError("getting attributes of path '%s", path);
+                        throw SysError("getting attributes of required path '%s", path);
                     }
-                    if (S_ISDIR(st.st_mode))
+                    if (S_ISDIR(optSt->st_mode))
                         sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
                     else
                         sandboxProfile += fmt("\t(literal \"%s\")\n", path);
@@ -2089,7 +2108,7 @@ void LocalDerivationGoal::runChild()
             bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
 
             /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
-               to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
+               to find temporary directories, so we want to open up a broader place for them to put their files, if needed. */
             Path globalTmpDir = canonPath(defaultTempDir(), true);
 
             /* They don't like trailing slashes on subpath directives */
@@ -2271,14 +2290,12 @@ SingleDrvOutputs LocalDerivationGoal::registerOutputs()
             continue;
         }
 
-        struct stat st;
+        auto optSt = maybeLstat(actualPath.c_str());
-        if (lstat(actualPath.c_str(), &st) == -1) {
+        if (!optSt)
-            if (errno == ENOENT)
+            throw BuildError(
-                throw BuildError(
+                "builder for '%s' failed to produce output path for output '%s' at '%s'",
-                    "builder for '%s' failed to produce output path for output '%s' at '%s'",
+                worker.store.printStorePath(drvPath), outputName, actualPath);
-                    worker.store.printStorePath(drvPath), outputName, actualPath);
+        struct stat & st = *optSt;
-            throw SysError("getting attributes of path '%s'", actualPath);
-        }
 
 #ifndef __CYGWIN__
         /* Check that the output is not group or world writable, as
@@ -2950,16 +2967,25 @@ bool LocalDerivationGoal::isReadDesc(int fd)
 
 StorePath LocalDerivationGoal::makeFallbackPath(OutputNameView outputName)
 {
+    // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path
+    // See doc/manual/src/protocols/store-path.md for details
+    // TODO: We may want to separate the responsibilities of constructing the path fingerprint and of actually doing the hashing
+    auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName);
     return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
+        pathType,
+        // pass an all-zeroes hash
         Hash(HashAlgorithm::SHA256), outputPathName(drv->name, outputName));
 }
 
 
 StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path)
 {
+    // This is a bogus path type, constructed this way to ensure that it doesn't collide with any other store path
+    // See doc/manual/src/protocols/store-path.md for details
+    auto pathType = "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string());
     return worker.store.makeStorePath(
-        "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
+        pathType,
+        // pass an all-zeroes hash
         Hash(HashAlgorithm::SHA256), path.name());
 }
 

src/libstore/builtins/buildenv.cc

@@ -64,9 +64,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
             continue;
 
         else if (S_ISDIR(srcSt.st_mode)) {
-            struct stat dstSt;
+            auto dstStOpt = maybeLstat(dstFile.c_str());
-            auto res = lstat(dstFile.c_str(), &dstSt);
+            if (dstStOpt) {
-            if (res == 0) {
+                auto & dstSt = *dstStOpt;
                 if (S_ISDIR(dstSt.st_mode)) {
                     createLinks(state, srcFile, dstFile, priority);
                     continue;
@@ -82,14 +82,13 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                     createLinks(state, srcFile, dstFile, priority);
                     continue;
                 }
-            } else if (errno != ENOENT)
+            }
-                throw SysError("getting status of '%1%'", dstFile);
         }
 
         else {
-            struct stat dstSt;
+            auto dstStOpt = maybeLstat(dstFile.c_str());
-            auto res = lstat(dstFile.c_str(), &dstSt);
+            if (dstStOpt) {
-            if (res == 0) {
+                auto & dstSt = *dstStOpt;
                 if (S_ISLNK(dstSt.st_mode)) {
                     auto prevPriority = state.priorities[dstFile];
                     if (prevPriority == priority)
@@ -104,8 +103,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
                         throw SysError("unlinking '%1%'", dstFile);
                 } else if (S_ISDIR(dstSt.st_mode))
                     throw Error("collision between non-directory '%1%' and directory '%2%'", srcFile, dstFile);
-            } else if (errno != ENOENT)
+            }
-                throw SysError("getting status of '%1%'", dstFile);
         }
 
         createSymlink(srcFile, dstFile);

src/libstore/daemon.cc

@@ -1,5 +1,6 @@
 #include "daemon.hh"
 #include "monitor-fd.hh"
+#include "signals.hh"
 #include "worker-protocol.hh"
 #include "worker-protocol-impl.hh"
 #include "build-result.hh"
@@ -1038,7 +1039,7 @@ void processConnection(
     unsigned int opCount = 0;
 
     Finally finally([&]() {
-        _isInterrupted = false;
+        setInterrupted(false);
         printMsgUsing(prevLogger, lvlDebug, "%d operations", opCount);
     });
 

src/libstore/derivations.cc

@@ -1239,16 +1239,14 @@ DerivationOutput DerivationOutput::fromJSON(
     const ExperimentalFeatureSettings & xpSettings)
 {
     std::set<std::string_view> keys;
-    ensureType(_json, nlohmann::detail::value_t::object);
+    auto & json = getObject(_json);
-    auto json = (std::map<std::string, nlohmann::json>) _json;
 
     for (const auto & [key, _] : json)
         keys.insert(key);
 
     auto methodAlgo = [&]() -> std::pair<ContentAddressMethod, HashAlgorithm> {
-        std::string hashAlgoStr = json["hashAlgo"];
+        auto & str = getString(valueAt(json, "hashAlgo"));
-        // remaining to parse, will be mutated by parsers
+        std::string_view s = str;
-        std::string_view s = hashAlgoStr;
         ContentAddressMethod method = ContentAddressMethod::parsePrefix(s);
         if (method == TextIngestionMethod {})
             xpSettings.require(Xp::DynamicDerivations);
@@ -1258,7 +1256,7 @@ DerivationOutput DerivationOutput::fromJSON(
 
     if (keys == (std::set<std::string_view> { "path" })) {
         return DerivationOutput::InputAddressed {
-            .path = store.parseStorePath((std::string) json["path"]),
+            .path = store.parseStorePath(getString(valueAt(json, "path"))),
         };
     }
 
@@ -1267,10 +1265,10 @@ DerivationOutput DerivationOutput::fromJSON(
         auto dof = DerivationOutput::CAFixed {
             .ca = ContentAddress {
                 .method = std::move(method),
-                .hash = Hash::parseNonSRIUnprefixed((std::string) json["hash"], hashAlgo),
+                .hash = Hash::parseNonSRIUnprefixed(getString(valueAt(json, "hash")), hashAlgo),
             },
         };
-        if (dof.path(store, drvName, outputName) != store.parseStorePath((std::string) json["path"]))
+        if (dof.path(store, drvName, outputName) != store.parseStorePath(getString(valueAt(json, "path"))))
             throw Error("Path doesn't match derivation output");
         return dof;
     }
@@ -1357,20 +1355,19 @@ nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const
 
 Derivation Derivation::fromJSON(
     const StoreDirConfig & store,
-    const nlohmann::json & json,
+    const nlohmann::json & _json,
     const ExperimentalFeatureSettings & xpSettings)
 {
     using nlohmann::detail::value_t;
 
     Derivation res;
 
-    ensureType(json, value_t::object);
+    auto & json = getObject(_json);
 
-    res.name = ensureType(valueAt(json, "name"), value_t::string);
+    res.name = getString(valueAt(json, "name"));
 
     try {
-        auto & outputsObj = ensureType(valueAt(json, "outputs"), value_t::object);
+        for (auto & [outputName, output] : getObject(valueAt(json, "outputs"))) {
-        for (auto & [outputName, output] : outputsObj.items()) {
             res.outputs.insert_or_assign(
                 outputName,
                 DerivationOutput::fromJSON(store, res.name, outputName, output));
@@ -1381,8 +1378,7 @@ Derivation Derivation::fromJSON(
     }
 
     try {
-        auto & inputsList = ensureType(valueAt(json, "inputSrcs"), value_t::array);
+        for (auto & input : getArray(valueAt(json, "inputSrcs")))
-        for (auto & input : inputsList)
             res.inputSrcs.insert(store.parseStorePath(static_cast<const std::string &>(input)));
     } catch (Error & e) {
         e.addTrace({}, "while reading key 'inputSrcs'");
@@ -1391,18 +1387,17 @@ Derivation Derivation::fromJSON(
 
     try {
         std::function<DerivedPathMap<StringSet>::ChildNode(const nlohmann::json &)> doInput;
-        doInput = [&](const auto & json) {
+        doInput = [&](const auto & _json) {
+            auto & json = getObject(_json);
             DerivedPathMap<StringSet>::ChildNode node;
-            node.value = static_cast<const StringSet &>(
+            node.value = getStringSet(valueAt(json, "outputs"));
-                ensureType(valueAt(json, "outputs"), value_t::array));
+            for (auto & [outputId, childNode] : getObject(valueAt(json, "dynamicOutputs"))) {
-            for (auto & [outputId, childNode] : ensureType(valueAt(json, "dynamicOutputs"), value_t::object).items()) {
                 xpSettings.require(Xp::DynamicDerivations);
                 node.childMap[outputId] = doInput(childNode);
             }
             return node;
         };
-        auto & inputDrvsObj = ensureType(valueAt(json, "inputDrvs"), value_t::object);
+        for (auto & [inputDrvPath, inputOutputs] : getObject(valueAt(json, "inputDrvs")))
-        for (auto & [inputDrvPath, inputOutputs] : inputDrvsObj.items())
             res.inputDrvs.map[store.parseStorePath(inputDrvPath)] =
                 doInput(inputOutputs);
     } catch (Error & e) {
@@ -1410,10 +1405,10 @@ Derivation Derivation::fromJSON(
         throw;
     }
 
-    res.platform = ensureType(valueAt(json, "system"), value_t::string);
+    res.platform = getString(valueAt(json, "system"));
-    res.builder = ensureType(valueAt(json, "builder"), value_t::string);
+    res.builder = getString(valueAt(json, "builder"));
-    res.args = ensureType(valueAt(json, "args"), value_t::array);
+    res.args = getStringList(valueAt(json, "args"));
-    res.env = ensureType(valueAt(json, "env"), value_t::object);
+    res.env = getStringMap(valueAt(json, "env"));
 
     return res;
 }

src/libstore/filetransfer.cc

@@ -1,5 +1,4 @@
 #include "filetransfer.hh"
-#include "namespaces.hh"
 #include "globals.hh"
 #include "store-api.hh"
 #include "s3.hh"
@@ -12,6 +11,10 @@
 #include <aws/core/client/ClientConfiguration.h>
 #endif
 
+#if __linux__
+# include "namespaces.hh"
+#endif
+
 #include <unistd.h>
 #include <fcntl.h>
 
@@ -255,11 +258,11 @@ struct curlFileTransfer : public FileTransfer
         int progressCallback(double dltotal, double dlnow)
         {
             try {
-              act.progress(dlnow, dltotal);
+                act.progress(dlnow, dltotal);
             } catch (nix::Interrupted &) {
-              assert(_isInterrupted);
+                assert(getInterrupted());
             }
-            return _isInterrupted;
+            return getInterrupted();
         }
 
         static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow)
@@ -463,7 +466,7 @@ struct curlFileTransfer : public FileTransfer
                 if (errorSink)
                     response = std::move(errorSink->s);
                 auto exc =
-                    code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
+                    code == CURLE_ABORTED_BY_CALLBACK && getInterrupted()
                     ? FileTransferError(Interrupted, std::move(response), "%s of '%s' was interrupted", request.verb(), request.uri)
                     : httpStatus != 0
                     ? FileTransferError(err,
@@ -568,7 +571,9 @@ struct curlFileTransfer : public FileTransfer
             stopWorkerThread();
         });
 
+        #if __linux__
         unshareFilesystem();
+        #endif
 
         std::map<CURL *, std::shared_ptr<TransferItem>> items;
 

src/libstore/gc.cc

@@ -665,7 +665,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
         results.paths.insert(path);
 
         uint64_t bytesFreed;
-        deletePath(realPath, bytesFreed);
+        deleteStorePath(realPath, bytesFreed);
+
         results.bytesFreed += bytesFreed;
 
         if (results.bytesFreed > options.maxFreed) {
@@ -752,7 +753,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
                 auto i = referrersCache.find(*path);
                 if (i == referrersCache.end()) {
                     StorePathSet referrers;
-                    queryReferrers(*path, referrers);
+                    queryGCReferrers(*path, referrers);
                     referrersCache.emplace(*path, std::move(referrers));
                     i = referrersCache.find(*path);
                 }
@@ -879,7 +880,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
             if (unlink(path.c_str()) == -1)
                 throw SysError("deleting '%1%'", path);
 
-            /* Do not accound for deleted file here. Rely on deletePath()
+            /* Do not account for deleted file here. Rely on deletePath()
                accounting.  */
         }
 

src/libstore/globals.cc

@@ -2,7 +2,6 @@
 #include "current-process.hh"
 #include "archive.hh"
 #include "args.hh"
-#include "users.hh"
 #include "abstract-setting-to-json.hh"
 #include "compute-levels.hh"
 
@@ -57,7 +56,7 @@ Settings::Settings()
     , nixManDir(canonPath(NIX_MAN_DIR))
     , nixDaemonSocketFile(canonPath(getEnvNonEmpty("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH)))
 {
-    buildUsersGroup = getuid() == 0 ? "nixbld" : "";
+    buildUsersGroup = isRootUser() ? "nixbld" : "";
     allowSymlinkedStore = getEnv("NIX_IGNORE_SYMLINK_STORE") == "1";
 
     auto sslOverride = getEnv("NIX_SSL_CERT_FILE").value_or(getEnv("SSL_CERT_FILE").value_or(""));
@@ -346,6 +345,12 @@ void initPlugins()
                 dlopen(file.c_str(), RTLD_LAZY | RTLD_LOCAL);
             if (!handle)
                 throw Error("could not dynamically open plugin file '%s': %s", file, dlerror());
+
+            /* Older plugins use a statically initialized object to run their code.
+               Newer plugins can also export nix_plugin_entry() */
+            void (*nix_plugin_entry)() = (void (*)())dlsym(handle, "nix_plugin_entry");
+            if (nix_plugin_entry)
+                nix_plugin_entry();
         }
     }
 
@@ -404,6 +409,7 @@ void assertLibStoreInitialized() {
 }
 
 void initLibStore() {
+    if (initLibStoreDone) return;
 
     initLibUtil();
 

src/libstore/globals.hh

@@ -5,6 +5,7 @@
 #include "config.hh"
 #include "environment-variables.hh"
 #include "experimental-features.hh"
+#include "users.hh"
 
 #include <map>
 #include <limits>
@@ -665,7 +666,7 @@ public:
     Setting<bool> sandboxFallback{this, true, "sandbox-fallback",
         "Whether to disable sandboxing when the kernel doesn't allow it."};
 
-    Setting<bool> requireDropSupplementaryGroups{this, getuid() == 0, "require-drop-supplementary-groups",
+    Setting<bool> requireDropSupplementaryGroups{this, isRootUser(), "require-drop-supplementary-groups",
         R"(
           Following the principle of least privilege,
           Nix will attempt to drop supplementary groups when building with sandboxing.
@@ -687,16 +688,36 @@ public:
     Setting<std::string> sandboxShmSize{
         this, "50%", "sandbox-dev-shm-size",
         R"(
-          This option determines the maximum size of the `tmpfs` filesystem
+            *Linux only*
-          mounted on `/dev/shm` in Linux sandboxes. For the format, see the
+
-          description of the `size` option of `tmpfs` in mount(8). The default
+            This option determines the maximum size of the `tmpfs` filesystem
-          is `50%`.
+            mounted on `/dev/shm` in Linux sandboxes. For the format, see the
+            description of the `size` option of `tmpfs` in mount(8). The default
+            is `50%`.
         )"};
 
     Setting<Path> sandboxBuildDir{this, "/build", "sandbox-build-dir",
-        "The build directory inside the sandbox."};
+        R"(
+            *Linux only*
+
+            The build directory inside the sandbox.
+
+            This directory is backed by [`build-dir`](#conf-build-dir) on the host.
+        )"};
 #endif
 
+    Setting<std::optional<Path>> buildDir{this, std::nullopt, "build-dir",
+        R"(
+            The directory on the host, in which derivations' temporary build directories are created.
+
+            If not set, Nix will use the system temporary directory indicated by the `TMPDIR` environment variable.
+            Note that builds are often performed by the Nix daemon, so its `TMPDIR` is used, and not that of the Nix command line interface.
+
+            This is also the location where [`--keep-failed`](@docroot@/command-ref/opt-common.md#opt-keep-failed) leaves its files.
+
+            If Nix runs without sandbox, or if the platform does not support sandboxing with bind mounts (e.g. macOS), then the [`builder`](@docroot@/language/derivations.md#attr-builder)'s environment will contain this directory, instead of the virtual location [`sandbox-build-dir`](#conf-sandbox-build-dir).
+        )"};
+
     Setting<PathSet> allowedImpureHostPrefixes{this, {}, "allowed-impure-host-deps",
         "Which prefixes to allow derivations to ask for access to (primarily for Darwin)."};
 
@@ -1136,9 +1157,10 @@ public:
         this, {}, "plugin-files",
         R"(
           A list of plugin files to be loaded by Nix. Each of these files will
-          be dlopened by Nix, allowing them to affect execution through static
+          be dlopened by Nix. If they contain the symbol `nix_plugin_entry()`,
-          initialization. In particular, these plugins may construct static
+          this symbol will be called. Alternatively, they can affect execution
-          instances of RegisterPrimOp to add new primops or constants to the
+          through static initialization. In particular, these plugins may construct
+          static instances of RegisterPrimOp to add new primops or constants to the
           expression language, RegisterStoreImplementation to add new store
           implementations, RegisterCommand to add new subcommands to the `nix`
           command, and RegisterSetting to add new nix config settings. See the

src/libstore/local-fs-store.cc

@@ -33,6 +33,10 @@ struct LocalStoreAccessor : PosixSourceAccessor
 
     std::optional<Stat> maybeLstat(const CanonPath & path) override
     {
+        /* Handle the case where `path` is (a parent of) the store. */
+        if (isDirOrInDir(store->storeDir, path.abs()))
+            return Stat{ .type = tDirectory };
+
         return PosixSourceAccessor::maybeLstat(toRealPath(path));
     }
 

src/libstore/local-overlay-store.cc

@@ -0,0 +1,292 @@
+#include "local-overlay-store.hh"
+#include "callback.hh"
+#include "realisation.hh"
+#include "processes.hh"
+#include "url.hh"
+#include <regex>
+
+namespace nix {
+
+std::string LocalOverlayStoreConfig::doc()
+{
+    return
+        #include "local-overlay-store.md"
+        ;
+}
+
+Path LocalOverlayStoreConfig::toUpperPath(const StorePath & path) {
+    return upperLayer + "/" + path.to_string();
+}
+
+LocalOverlayStore::LocalOverlayStore(const Params & params)
+    : StoreConfig(params)
+    , LocalFSStoreConfig(params)
+    , LocalStoreConfig(params)
+    , LocalOverlayStoreConfig(params)
+    , Store(params)
+    , LocalFSStore(params)
+    , LocalStore(params)
+    , lowerStore(openStore(percentDecode(lowerStoreUri.get())).dynamic_pointer_cast<LocalFSStore>())
+{
+    if (checkMount.get()) {
+        std::smatch match;
+        std::string mountInfo;
+        auto mounts = readFile("/proc/self/mounts");
+        auto regex = std::regex(R"((^|\n)overlay )" + realStoreDir.get() + R"( .*(\n|$))");
+
+        // Mount points can be stacked, so there might be multiple matching entries.
+        // Loop until the last match, which will be the current state of the mount point.
+        while (std::regex_search(mounts, match, regex)) {
+            mountInfo = match.str();
+            mounts = match.suffix();
+        }
+
+        auto checkOption = [&](std::string option, std::string value) {
+            return std::regex_search(mountInfo, std::regex("\\b" + option + "=" + value + "( |,)"));
+        };
+
+        auto expectedLowerDir = lowerStore->realStoreDir.get();
+        if (!checkOption("lowerdir", expectedLowerDir) || !checkOption("upperdir", upperLayer)) {
+            debug("expected lowerdir: %s", expectedLowerDir);
+            debug("expected upperdir: %s", upperLayer);
+            debug("actual mount: %s", mountInfo);
+            throw Error("overlay filesystem '%s' mounted incorrectly",
+                realStoreDir.get());
+        }
+    }
+}
+
+
+void LocalOverlayStore::registerDrvOutput(const Realisation & info)
+{
+    // First do queryRealisation on lower layer to populate DB
+    auto res = lowerStore->queryRealisation(info.id);
+    if (res)
+        LocalStore::registerDrvOutput(*res);
+
+    LocalStore::registerDrvOutput(info);
+}
+
+
+void LocalOverlayStore::queryPathInfoUncached(const StorePath & path,
+    Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
+{
+    auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
+
+    LocalStore::queryPathInfoUncached(path,
+        {[this, path, callbackPtr](std::future<std::shared_ptr<const ValidPathInfo>> fut) {
+            try {
+                auto info = fut.get();
+                if (info)
+                    return (*callbackPtr)(std::move(info));
+            } catch (...) {
+                return callbackPtr->rethrow();
+            }
+            // If we don't have it, check lower store
+            lowerStore->queryPathInfo(path,
+                {[path, callbackPtr](std::future<ref<const ValidPathInfo>> fut) {
+                    try {
+                        (*callbackPtr)(fut.get().get_ptr());
+                    } catch (...) {
+                        return callbackPtr->rethrow();
+                    }
+                }});
+        }});
+}
+
+
+void LocalOverlayStore::queryRealisationUncached(const DrvOutput & drvOutput,
+    Callback<std::shared_ptr<const Realisation>> callback) noexcept
+{
+    auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
+
+    LocalStore::queryRealisationUncached(drvOutput,
+        {[this, drvOutput, callbackPtr](std::future<std::shared_ptr<const Realisation>> fut) {
+            try {
+                auto info = fut.get();
+                if (info)
+                    return (*callbackPtr)(std::move(info));
+            } catch (...) {
+                return callbackPtr->rethrow();
+            }
+            // If we don't have it, check lower store
+            lowerStore->queryRealisation(drvOutput,
+                {[callbackPtr](std::future<std::shared_ptr<const Realisation>> fut) {
+                    try {
+                        (*callbackPtr)(fut.get());
+                    } catch (...) {
+                        return callbackPtr->rethrow();
+                    }
+                }});
+        }});
+}
+
+
+bool LocalOverlayStore::isValidPathUncached(const StorePath & path)
+{
+    auto res = LocalStore::isValidPathUncached(path);
+    if (res) return res;
+    res = lowerStore->isValidPath(path);
+    if (res) {
+        // Get path info from lower store so upper DB genuinely has it.
+        auto p = lowerStore->queryPathInfo(path);
+        // recur on references, syncing entire closure.
+        for (auto & r : p->references)
+            if (r != path)
+                isValidPath(r);
+        LocalStore::registerValidPath(*p);
+    }
+    return res;
+}
+
+
+void LocalOverlayStore::queryReferrers(const StorePath & path, StorePathSet & referrers)
+{
+    LocalStore::queryReferrers(path, referrers);
+    lowerStore->queryReferrers(path, referrers);
+}
+
+
+void LocalOverlayStore::queryGCReferrers(const StorePath & path, StorePathSet & referrers)
+{
+    LocalStore::queryReferrers(path, referrers);
+}
+
+
+StorePathSet LocalOverlayStore::queryValidDerivers(const StorePath & path)
+{
+    auto res = LocalStore::queryValidDerivers(path);
+    for (auto p : lowerStore->queryValidDerivers(path))
+        res.insert(p);
+    return res;
+}
+
+
+std::optional<StorePath> LocalOverlayStore::queryPathFromHashPart(const std::string & hashPart)
+{
+    auto res = LocalStore::queryPathFromHashPart(hashPart);
+    if (res)
+        return res;
+    else
+        return lowerStore->queryPathFromHashPart(hashPart);
+}
+
+
+void LocalOverlayStore::registerValidPaths(const ValidPathInfos & infos)
+{
+    // First, get any from lower store so we merge
+    {
+        StorePathSet notInUpper;
+        for (auto & [p, _] : infos)
+            if (!LocalStore::isValidPathUncached(p)) // avoid divergence
+                notInUpper.insert(p);
+        auto pathsInLower = lowerStore->queryValidPaths(notInUpper);
+        ValidPathInfos inLower;
+        for (auto & p : pathsInLower)
+            inLower.insert_or_assign(p, *lowerStore->queryPathInfo(p));
+        LocalStore::registerValidPaths(inLower);
+    }
+    // Then do original request
+    LocalStore::registerValidPaths(infos);
+}
+
+
+void LocalOverlayStore::collectGarbage(const GCOptions & options, GCResults & results)
+{
+    LocalStore::collectGarbage(options, results);
+
+    remountIfNecessary();
+}
+
+
+void LocalOverlayStore::deleteStorePath(const Path & path, uint64_t & bytesFreed)
+{
+    auto mergedDir = realStoreDir.get() + "/";
+    if (path.substr(0, mergedDir.length()) != mergedDir) {
+        warn("local-overlay: unexpected gc path '%s' ", path);
+        return;
+    }
+
+    StorePath storePath = {path.substr(mergedDir.length())};
+    auto upperPath = toUpperPath(storePath);
+
+    if (pathExists(upperPath)) {
+        debug("upper exists: %s", path);
+        if (lowerStore->isValidPath(storePath)) {
+            debug("lower exists: %s", storePath.to_string());
+            // Path also exists in lower store.
+            // We must delete via upper layer to avoid creating a whiteout.
+            deletePath(upperPath, bytesFreed);
+            _remountRequired = true;
+        } else {
+            // Path does not exist in lower store.
+            // So we can delete via overlayfs and not need to remount.
+            LocalStore::deleteStorePath(path, bytesFreed);
+        }
+    }
+}
+
+
+void LocalOverlayStore::optimiseStore()
+{
+    Activity act(*logger, actOptimiseStore);
+
+    // Note for LocalOverlayStore, queryAllValidPaths only returns paths in upper layer
+    auto paths = queryAllValidPaths();
+
+    act.progress(0, paths.size());
+
+    uint64_t done = 0;
+
+    for (auto & path : paths) {
+        if (lowerStore->isValidPath(path)) {
+            uint64_t bytesFreed = 0;
+            // Deduplicate store path
+            deleteStorePath(Store::toRealPath(path), bytesFreed);
+        }
+        done++;
+        act.progress(done, paths.size());
+    }
+
+    remountIfNecessary();
+}
+
+
+LocalStore::VerificationResult LocalOverlayStore::verifyAllValidPaths(RepairFlag repair)
+{
+    StorePathSet done;
+
+    auto existsInStoreDir = [&](const StorePath & storePath) {
+        return pathExists(realStoreDir.get() + "/" + storePath.to_string());
+    };
+
+    bool errors = false;
+    StorePathSet validPaths;
+
+    for (auto & i : queryAllValidPaths())
+        verifyPath(i, existsInStoreDir, done, validPaths, repair, errors);
+
+    return {
+        .errors = errors,
+        .validPaths = validPaths,
+    };
+}
+
+
+void LocalOverlayStore::remountIfNecessary()
+{
+    if (!_remountRequired) return;
+
+    if (remountHook.get().empty()) {
+        warn("'%s' needs remounting, set remount-hook to do this automatically", realStoreDir.get());
+    } else {
+        runProgram(remountHook, false, {realStoreDir});
+    }
+
+    _remountRequired = false;
+}
+
+
+static RegisterStoreImplementation<LocalOverlayStore, LocalOverlayStoreConfig> regLocalOverlayStore;
+
+}

src/libstore/local-overlay-store.hh

@@ -0,0 +1,215 @@
+#include "local-store.hh"
+
+namespace nix {
+
+/**
+ * Configuration for `LocalOverlayStore`.
+ */
+struct LocalOverlayStoreConfig : virtual LocalStoreConfig
+{
+    LocalOverlayStoreConfig(const StringMap & params)
+        : StoreConfig(params)
+        , LocalFSStoreConfig(params)
+        , LocalStoreConfig(params)
+    { }
+
+    const Setting<std::string> lowerStoreUri{(StoreConfig*) this, "", "lower-store",
+        R"(
+          [Store URL](@docroot@/command-ref/new-cli/nix3-help-stores.md#store-url-format)
+          for the lower store. The default is `auto` (i.e. use the Nix daemon or `/nix/store` directly).
+
+          Must be a store with a store dir on the file system.
+          Must be used as OverlayFS lower layer for this store's store dir.
+        )"};
+
+    const PathSetting upperLayer{(StoreConfig*) this, "", "upper-layer",
+        R"(
+          Directory containing the OverlayFS upper layer for this store's store dir.
+        )"};
+
+    Setting<bool> checkMount{(StoreConfig*) this, true, "check-mount",
+        R"(
+          Check that the overlay filesystem is correctly mounted.
+
+          Nix does not manage the overlayfs mount point itself, but the correct
+          functioning of the overlay store does depend on this mount point being set up
+          correctly. Rather than just assume this is the case, check that the lowerdir
+          and upperdir options are what we expect them to be. This check is on by
+          default, but can be disabled if needed.
+        )"};
+
+    const PathSetting remountHook{(StoreConfig*) this, "", "remount-hook",
+        R"(
+          Script or other executable to run when overlay filesystem needs remounting.
+
+          This is occasionally necessary when deleting a store path that exists in both upper and lower layers.
+          In such a situation, bypassing OverlayFS and deleting the path in the upper layer directly
+          is the only way to perform the deletion without creating a "whiteout".
+          However this causes the OverlayFS kernel data structures to get out-of-sync,
+          and can lead to 'stale file handle' errors; remounting solves the problem.
+
+          The store directory is passed as an argument to the invoked executable.
+        )"};
+
+    const std::string name() override { return "Experimental Local Overlay Store"; }
+
+    std::optional<ExperimentalFeature> experimentalFeature() const override
+    {
+        return ExperimentalFeature::LocalOverlayStore;
+    }
+
+    std::string doc() override;
+
+protected:
+    /**
+     * @return The host OS path corresponding to the store path for the
+     * upper layer.
+     *
+     * @note The there is no guarantee a store object is actually stored
+     * at that file path. It might be stored in the lower layer instead,
+     * or it might not be part of this store at all.
+     */
+    Path toUpperPath(const StorePath & path);
+};
+
+/**
+ * Variation of local store using OverlayFS for the store directory.
+ *
+ * Documentation on overridden methods states how they differ from their
+ * `LocalStore` counterparts.
+ */
+class LocalOverlayStore : public virtual LocalOverlayStoreConfig, public virtual LocalStore
+{
+    /**
+     * The store beneath us.
+     *
+     * Our store dir should be an overlay fs where the lower layer
+     * is that store's store dir, and the upper layer is some
+     * scratch storage just for us.
+     */
+    ref<LocalFSStore> lowerStore;
+
+public:
+    LocalOverlayStore(const Params & params);
+
+    LocalOverlayStore(std::string scheme, std::string path, const Params & params)
+        : LocalOverlayStore(params)
+    {
+        if (!path.empty())
+            throw UsageError("local-overlay:// store url doesn't support path part, only scheme and query params");
+    }
+
+    static std::set<std::string> uriSchemes()
+    {
+        return { "local-overlay" };
+    }
+
+    std::string getUri() override
+    {
+        return "local-overlay://";
+    }
+
+private:
+    /**
+     * First copy up any lower store realisation with the same key, so we
+     * merge rather than mask it.
+     */
+    void registerDrvOutput(const Realisation & info) override;
+
+    /**
+     * Check lower store if upper DB does not have.
+     */
+    void queryPathInfoUncached(const StorePath & path,
+        Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override;
+
+    /**
+     * Check lower store if upper DB does not have.
+     *
+     * In addition, copy up metadata for lower store objects (and their
+     * closure). (I.e. Optimistically cache in the upper DB.)
+     */
+    bool isValidPathUncached(const StorePath & path) override;
+
+    /**
+     * Check the lower store and upper DB.
+     */
+    void queryReferrers(const StorePath & path, StorePathSet & referrers) override;
+
+    /**
+     * Check the lower store and upper DB.
+     */
+    StorePathSet queryValidDerivers(const StorePath & path) override;
+
+    /**
+     * Check lower store if upper DB does not have.
+     */
+    std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
+
+    /**
+     * First copy up any lower store realisation with the same key, so we
+     * merge rather than mask it.
+     */
+    void registerValidPaths(const ValidPathInfos & infos) override;
+
+    /**
+     * Check lower store if upper DB does not have.
+     */
+    void queryRealisationUncached(const DrvOutput&,
+        Callback<std::shared_ptr<const Realisation>> callback) noexcept override;
+
+    /**
+     * Call `remountIfNecessary` after collecting garbage normally.
+     */
+    void collectGarbage(const GCOptions & options, GCResults & results) override;
+
+    /**
+     * Check which layers the store object exists in to try to avoid
+     * needing to remount.
+     */
+    void deleteStorePath(const Path & path, uint64_t & bytesFreed) override;
+
+    /**
+     * Deduplicate by removing store objects from the upper layer that
+     * are now in the lower layer.
+     *
+     * Operations on a layered store will not cause duplications, but addition of
+     * new store objects to the lower layer can instill induce them
+     * (there is no way to prevent that). This cleans up those
+     * duplications.
+     *
+     * @note We do not yet optomise the upper layer in the normal way
+     * (hardlink) yet. We would like to, but it requires more
+     * refactoring of existing code to support this sustainably.
+     */
+    void optimiseStore() override;
+
+    /**
+     * Check all paths registered in the upper DB.
+     *
+     * Note that this includes store objects that reside in either overlayfs layer;
+     * just enumerating the contents of the upper layer would skip them.
+     *
+     * We don't verify the contents of both layers on the assumption that the lower layer is far bigger,
+     * and also the observation that anything not in the upper db the overlayfs doesn't yet care about.
+     */
+    VerificationResult verifyAllValidPaths(RepairFlag repair) override;
+
+    /**
+     * Deletion only effects the upper layer, so we ignore lower-layer referrers.
+     */
+    void queryGCReferrers(const StorePath & path, StorePathSet & referrers) override;
+
+    /**
+     * Call the `remountHook` if we have done something such that the
+     * OverlayFS needed to be remounted. See that hook's user-facing
+     * documentation for further details.
+     */
+    void remountIfNecessary();
+
+    /**
+     * State for `remountIfNecessary`
+     */
+    std::atomic_bool _remountRequired = false;
+};
+
+}