mirror of
https://github.com/NixOS/nix.git
synced 2024-10-30 13:50:51 +00:00
Merge remote-tracking branch 'origin/master' into fix-and-ci-static-builds
This commit is contained in:
commit
d7ffe327ae
@ -1,6 +1,7 @@
|
||||
((c++-mode . (
|
||||
(c-file-style . "k&r")
|
||||
(c-basic-offset . 4)
|
||||
(c-block-comment-prefix . " ")
|
||||
(indent-tabs-mode . nil)
|
||||
(tab-width . 4)
|
||||
(show-trailing-whitespace . t)
|
||||
|
5
.github/workflows/test.yml
vendored
5
.github/workflows/test.yml
vendored
@ -10,5 +10,8 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: cachix/install-nix-action@v10
|
||||
- run: nix-build release.nix --arg nix '{ outPath = ./.; revCount = 123; shortRev = "abcdefgh"; }' --arg systems '[ builtins.currentSystem ]' -A installerScript -A perlBindings
|
||||
#- run: nix flake check
|
||||
- run: nix-build -A checks.$(if [[ `uname` = Linux ]]; then echo x86_64-linux; else echo x86_64-darwin; fi)
|
||||
|
1
Makefile
1
Makefile
@ -11,6 +11,7 @@ makefiles = \
|
||||
src/resolve-system-dependencies/local.mk \
|
||||
scripts/local.mk \
|
||||
corepkgs/local.mk \
|
||||
misc/bash/local.mk \
|
||||
misc/systemd/local.mk \
|
||||
misc/launchd/local.mk \
|
||||
misc/upstart/local.mk \
|
||||
|
@ -19,6 +19,7 @@ LIBLZMA_LIBS = @LIBLZMA_LIBS@
|
||||
OPENSSL_LIBS = @OPENSSL_LIBS@
|
||||
PACKAGE_NAME = @PACKAGE_NAME@
|
||||
PACKAGE_VERSION = @PACKAGE_VERSION@
|
||||
SHELL = @bash@
|
||||
SODIUM_LIBS = @SODIUM_LIBS@
|
||||
SQLITE3_LIBS = @SQLITE3_LIBS@
|
||||
bash = @bash@
|
||||
|
25
README.md
25
README.md
@ -12,7 +12,7 @@ for more details.
|
||||
On Linux and macOS the easiest way to Install Nix is to run the following shell command
|
||||
(as a user other than root):
|
||||
|
||||
```
|
||||
```console
|
||||
$ curl -L https://nixos.org/nix/install | sh
|
||||
```
|
||||
|
||||
@ -20,27 +20,8 @@ Information on additional installation methods is available on the [Nix download
|
||||
|
||||
## Building And Developing
|
||||
|
||||
### Building Nix
|
||||
|
||||
You can build Nix using one of the targets provided by [release.nix](./release.nix):
|
||||
|
||||
```
|
||||
$ nix-build ./release.nix -A build.aarch64-linux
|
||||
$ nix-build ./release.nix -A build.x86_64-darwin
|
||||
$ nix-build ./release.nix -A build.i686-linux
|
||||
$ nix-build ./release.nix -A build.x86_64-linux
|
||||
```
|
||||
|
||||
### Development Environment
|
||||
|
||||
You can use the provided `shell.nix` to get a working development environment:
|
||||
|
||||
```
|
||||
$ nix-shell
|
||||
$ ./bootstrap.sh
|
||||
$ ./configure
|
||||
$ make
|
||||
```
|
||||
See our [Hacking guide](https://hydra.nixos.org/job/nix/master/build.x86_64-linux/latest/download-by-type/doc/manual#chap-hacking) in our manual for instruction on how to
|
||||
build nix from source with nix-build or how to get a development environment.
|
||||
|
||||
## Additional Resources
|
||||
|
||||
|
@ -123,6 +123,7 @@ AC_PATH_PROG(flex, flex, false)
|
||||
AC_PATH_PROG(bison, bison, false)
|
||||
AC_PATH_PROG(dot, dot)
|
||||
AC_PATH_PROG(lsof, lsof, lsof)
|
||||
NEED_PROG(jq, jq)
|
||||
|
||||
|
||||
AC_SUBST(coreutils, [$(dirname $(type -p cat))])
|
||||
|
3
default.nix
Normal file
3
default.nix
Normal file
@ -0,0 +1,3 @@
|
||||
(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
|
||||
src = ./.;
|
||||
}).defaultNix
|
@ -1,119 +0,0 @@
|
||||
<section xmlns="http://docbook.org/ns/docbook"
|
||||
xmlns:xlink="http://www.w3.org/1999/xlink"
|
||||
xmlns:xi="http://www.w3.org/2001/XInclude"
|
||||
version="5.0"
|
||||
xml:id='sec-builder-syntax'>
|
||||
|
||||
<title>Builder Syntax</title>
|
||||
|
||||
<example xml:id='ex-hello-builder'><title>Build script for GNU Hello
|
||||
(<filename>builder.sh</filename>)</title>
|
||||
<programlisting>
|
||||
source $stdenv/setup <co xml:id='ex-hello-builder-co-1' />
|
||||
|
||||
PATH=$perl/bin:$PATH <co xml:id='ex-hello-builder-co-2' />
|
||||
|
||||
tar xvfz $src <co xml:id='ex-hello-builder-co-3' />
|
||||
cd hello-*
|
||||
./configure --prefix=$out <co xml:id='ex-hello-builder-co-4' />
|
||||
make <co xml:id='ex-hello-builder-co-5' />
|
||||
make install</programlisting>
|
||||
</example>
|
||||
|
||||
<para><xref linkend='ex-hello-builder' /> shows the builder referenced
|
||||
from Hello's Nix expression (stored in
|
||||
<filename>pkgs/applications/misc/hello/ex-1/builder.sh</filename>).
|
||||
The builder can actually be made a lot shorter by using the
|
||||
<emphasis>generic builder</emphasis> functions provided by
|
||||
<varname>stdenv</varname>, but here we write out the build steps to
|
||||
elucidate what a builder does. It performs the following
|
||||
steps:</para>
|
||||
|
||||
<calloutlist>
|
||||
|
||||
<callout arearefs='ex-hello-builder-co-1'>
|
||||
|
||||
<para>When Nix runs a builder, it initially completely clears the
|
||||
environment (except for the attributes declared in the
|
||||
derivation). For instance, the <envar>PATH</envar> variable is
|
||||
empty<footnote><para>Actually, it's initialised to
|
||||
<filename>/path-not-set</filename> to prevent Bash from setting it
|
||||
to a default value.</para></footnote>. This is done to prevent
|
||||
undeclared inputs from being used in the build process. If for
|
||||
example the <envar>PATH</envar> contained
|
||||
<filename>/usr/bin</filename>, then you might accidentally use
|
||||
<filename>/usr/bin/gcc</filename>.</para>
|
||||
|
||||
<para>So the first step is to set up the environment. This is
|
||||
done by calling the <filename>setup</filename> script of the
|
||||
standard environment. The environment variable
|
||||
<envar>stdenv</envar> points to the location of the standard
|
||||
environment being used. (It wasn't specified explicitly as an
|
||||
attribute in <xref linkend='ex-hello-nix' />, but
|
||||
<varname>mkDerivation</varname> adds it automatically.)</para>
|
||||
|
||||
</callout>
|
||||
|
||||
<callout arearefs='ex-hello-builder-co-2'>
|
||||
|
||||
<para>Since Hello needs Perl, we have to make sure that Perl is in
|
||||
the <envar>PATH</envar>. The <envar>perl</envar> environment
|
||||
variable points to the location of the Perl package (since it
|
||||
was passed in as an attribute to the derivation), so
|
||||
<filename><replaceable>$perl</replaceable>/bin</filename> is the
|
||||
directory containing the Perl interpreter.</para>
|
||||
|
||||
</callout>
|
||||
|
||||
<callout arearefs='ex-hello-builder-co-3'>
|
||||
|
||||
<para>Now we have to unpack the sources. The
|
||||
<varname>src</varname> attribute was bound to the result of
|
||||
fetching the Hello source tarball from the network, so the
|
||||
<envar>src</envar> environment variable points to the location in
|
||||
the Nix store to which the tarball was downloaded. After
|
||||
unpacking, we <command>cd</command> to the resulting source
|
||||
directory.</para>
|
||||
|
||||
<para>The whole build is performed in a temporary directory
|
||||
created in <varname>/tmp</varname>, by the way. This directory is
|
||||
removed after the builder finishes, so there is no need to clean
|
||||
up the sources afterwards. Also, the temporary directory is
|
||||
always newly created, so you don't have to worry about files from
|
||||
previous builds interfering with the current build.</para>
|
||||
|
||||
</callout>
|
||||
|
||||
<callout arearefs='ex-hello-builder-co-4'>
|
||||
|
||||
<para>GNU Hello is a typical Autoconf-based package, so we first
|
||||
have to run its <filename>configure</filename> script. In Nix
|
||||
every package is stored in a separate location in the Nix store,
|
||||
for instance
|
||||
<filename>/nix/store/9a54ba97fb71b65fda531012d0443ce2-hello-2.1.1</filename>.
|
||||
Nix computes this path by cryptographically hashing all attributes
|
||||
of the derivation. The path is passed to the builder through the
|
||||
<envar>out</envar> environment variable. So here we give
|
||||
<filename>configure</filename> the parameter
|
||||
<literal>--prefix=$out</literal> to cause Hello to be installed in
|
||||
the expected location.</para>
|
||||
|
||||
</callout>
|
||||
|
||||
<callout arearefs='ex-hello-builder-co-5'>
|
||||
|
||||
<para>Finally we build Hello (<literal>make</literal>) and install
|
||||
it into the location specified by <envar>out</envar>
|
||||
(<literal>make install</literal>).</para>
|
||||
|
||||
</callout>
|
||||
|
||||
</calloutlist>
|
||||
|
||||
<para>If you are wondering about the absence of error checking on the
|
||||
result of various commands called in the builder: this is because the
|
||||
shell script is evaluated with Bash's <option>-e</option> option,
|
||||
which causes the script to be aborted if any command fails without an
|
||||
error check.</para>
|
||||
|
||||
</section>
|
@ -7,15 +7,34 @@
|
||||
<para>This section provides some notes on how to hack on Nix. To get
|
||||
the latest version of Nix from GitHub:
|
||||
<screen>
|
||||
$ git clone git://github.com/NixOS/nix.git
|
||||
$ git clone https://github.com/NixOS/nix.git
|
||||
$ cd nix
|
||||
</screen>
|
||||
</para>
|
||||
|
||||
<para>To build it and its dependencies:
|
||||
<para>To build Nix for the current operating system/architecture use
|
||||
|
||||
<screen>
|
||||
$ nix-build release.nix -A build.x86_64-linux
|
||||
$ nix-build
|
||||
</screen>
|
||||
|
||||
or if you have a flakes-enabled nix:
|
||||
|
||||
<screen>
|
||||
$ nix build
|
||||
</screen>
|
||||
|
||||
This will build <literal>defaultPackage</literal> attribute defined in the <literal>flake.nix</literal> file.
|
||||
|
||||
To build for other platforms add one of the following suffixes to it: aarch64-linux,
|
||||
i686-linux, x86_64-darwin, x86_64-linux.
|
||||
|
||||
i.e.
|
||||
|
||||
<screen>
|
||||
nix-build -A defaultPackage.x86_64-linux
|
||||
</screen>
|
||||
|
||||
</para>
|
||||
|
||||
<para>To build all dependencies and start a shell in which all
|
||||
@ -27,13 +46,27 @@ $ nix-shell
|
||||
To build Nix itself in this shell:
|
||||
<screen>
|
||||
[nix-shell]$ ./bootstrap.sh
|
||||
[nix-shell]$ configurePhase
|
||||
[nix-shell]$ make
|
||||
[nix-shell]$ ./configure $configureFlags
|
||||
[nix-shell]$ make -j $NIX_BUILD_CORES
|
||||
</screen>
|
||||
To install it in <literal>$(pwd)/inst</literal> and test it:
|
||||
<screen>
|
||||
[nix-shell]$ make install
|
||||
[nix-shell]$ make installcheck
|
||||
[nix-shell]$ ./inst/bin/nix --version
|
||||
nix (Nix) 2.4
|
||||
</screen>
|
||||
|
||||
If you have a flakes-enabled nix you can replace:
|
||||
|
||||
<screen>
|
||||
$ nix-shell
|
||||
</screen>
|
||||
|
||||
by:
|
||||
|
||||
<screen>
|
||||
$ nix develop
|
||||
</screen>
|
||||
|
||||
</para>
|
||||
|
26
flake.lock
Normal file
26
flake.lock
Normal file
@ -0,0 +1,26 @@
|
||||
{
|
||||
"nodes": {
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1591633336,
|
||||
"narHash": "sha256-oVXv4xAnDJB03LvZGbC72vSVlIbbJr8tpjEW5o/Fdek=",
|
||||
"owner": "NixOS",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "70717a337f7ae4e486ba71a500367cad697e5f09",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"id": "nixpkgs",
|
||||
"ref": "nixos-20.03-small",
|
||||
"type": "indirect"
|
||||
}
|
||||
},
|
||||
"root": {
|
||||
"inputs": {
|
||||
"nixpkgs": "nixpkgs"
|
||||
}
|
||||
}
|
||||
},
|
||||
"root": "root",
|
||||
"version": 6
|
||||
}
|
497
flake.nix
Normal file
497
flake.nix
Normal file
@ -0,0 +1,497 @@
|
||||
{
|
||||
description = "The purely functional package manager";
|
||||
|
||||
inputs.nixpkgs.url = "nixpkgs/nixos-20.03-small";
|
||||
|
||||
outputs = { self, nixpkgs }:
|
||||
|
||||
let
|
||||
|
||||
version = builtins.readFile ./.version + versionSuffix;
|
||||
versionSuffix =
|
||||
if officialRelease
|
||||
then ""
|
||||
else "pre${builtins.substring 0 8 (self.lastModifiedDate or self.lastModified)}_${self.shortRev or "dirty"}";
|
||||
|
||||
officialRelease = false;
|
||||
|
||||
systems = [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ];
|
||||
|
||||
forAllSystems = f: nixpkgs.lib.genAttrs systems (system: f system);
|
||||
|
||||
# Memoize nixpkgs for different platforms for efficiency.
|
||||
nixpkgsFor = forAllSystems (system:
|
||||
import nixpkgs {
|
||||
inherit system;
|
||||
overlays = [ self.overlay ];
|
||||
}
|
||||
);
|
||||
|
||||
commonDeps = pkgs: with pkgs; rec {
|
||||
# Use "busybox-sandbox-shell" if present,
|
||||
# if not (legacy) fallback and hope it's sufficient.
|
||||
sh = pkgs.busybox-sandbox-shell or (busybox.override {
|
||||
useMusl = true;
|
||||
enableStatic = true;
|
||||
enableMinimal = true;
|
||||
extraConfig = ''
|
||||
CONFIG_FEATURE_FANCY_ECHO y
|
||||
CONFIG_FEATURE_SH_MATH y
|
||||
CONFIG_FEATURE_SH_MATH_64 y
|
||||
|
||||
CONFIG_ASH y
|
||||
CONFIG_ASH_OPTIMIZE_FOR_SIZE y
|
||||
|
||||
CONFIG_ASH_ALIAS y
|
||||
CONFIG_ASH_BASH_COMPAT y
|
||||
CONFIG_ASH_CMDCMD y
|
||||
CONFIG_ASH_ECHO y
|
||||
CONFIG_ASH_GETOPTS y
|
||||
CONFIG_ASH_INTERNAL_GLOB y
|
||||
CONFIG_ASH_JOB_CONTROL y
|
||||
CONFIG_ASH_PRINTF y
|
||||
CONFIG_ASH_TEST y
|
||||
'';
|
||||
});
|
||||
|
||||
configureFlags =
|
||||
lib.optionals stdenv.isLinux [
|
||||
"--with-sandbox-shell=${sh}/bin/busybox"
|
||||
];
|
||||
|
||||
|
||||
nativeBuildDeps =
|
||||
[
|
||||
buildPackages.bison
|
||||
buildPackages.flex
|
||||
buildPackages.libxml2
|
||||
buildPackages.libxslt
|
||||
buildPackages.docbook5
|
||||
buildPackages.docbook_xsl_ns
|
||||
buildPackages.autoreconfHook
|
||||
buildPackages.pkgconfig
|
||||
|
||||
# Tests
|
||||
buildPackages.git
|
||||
buildPackages.mercurial
|
||||
];
|
||||
|
||||
buildDeps =
|
||||
[ bison
|
||||
flex
|
||||
libxml2
|
||||
libxslt
|
||||
docbook5
|
||||
docbook_xsl_ns
|
||||
autoconf-archive
|
||||
autoreconfHook
|
||||
|
||||
curl
|
||||
bzip2 xz brotli zlib editline
|
||||
openssl pkgconfig sqlite
|
||||
libarchive
|
||||
boost
|
||||
(if lib.versionAtLeast lib.version "20.03pre"
|
||||
then nlohmann_json
|
||||
else nlohmann_json.override { multipleHeaders = true; })
|
||||
nlohmann_json
|
||||
gmock
|
||||
]
|
||||
++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
|
||||
++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
|
||||
|
||||
awsDeps = lib.optional (stdenv.isLinux || stdenv.isDarwin)
|
||||
(aws-sdk-cpp.override {
|
||||
apis = ["s3" "transfer"];
|
||||
customMemoryManagement = false;
|
||||
});
|
||||
|
||||
propagatedDeps =
|
||||
[ (boehmgc.override { enableLargeConfig = true; })
|
||||
];
|
||||
|
||||
perlDeps =
|
||||
[ perl
|
||||
perlPackages.DBDSQLite
|
||||
];
|
||||
};
|
||||
|
||||
in {
|
||||
|
||||
# A Nixpkgs overlay that overrides the 'nix' and
|
||||
# 'nix.perl-bindings' packages.
|
||||
overlay = final: prev: {
|
||||
|
||||
nix = with final; with commonDeps pkgs; (stdenv.mkDerivation {
|
||||
name = "nix-${version}";
|
||||
|
||||
src = self;
|
||||
|
||||
VERSION_SUFFIX = versionSuffix;
|
||||
|
||||
outputs = [ "out" "dev" "doc" ];
|
||||
|
||||
nativeBuildInputs = nativeBuildDeps;
|
||||
buildInputs = buildDeps ++ awsDeps;
|
||||
|
||||
propagatedBuildInputs = propagatedDeps;
|
||||
|
||||
preConfigure =
|
||||
''
|
||||
# Copy libboost_context so we don't get all of Boost in our closure.
|
||||
# https://github.com/NixOS/nixpkgs/issues/45462
|
||||
mkdir -p $out/lib
|
||||
cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
|
||||
rm -f $out/lib/*.a
|
||||
${lib.optionalString stdenv.isLinux ''
|
||||
chmod u+w $out/lib/*.so.*
|
||||
patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
|
||||
''}
|
||||
'';
|
||||
|
||||
configureFlags = configureFlags ++
|
||||
[ "--sysconfdir=/etc" ];
|
||||
|
||||
enableParallelBuilding = true;
|
||||
|
||||
makeFlags = "profiledir=$(out)/etc/profile.d";
|
||||
|
||||
doCheck = true;
|
||||
|
||||
installFlags = "sysconfdir=$(out)/etc";
|
||||
|
||||
postInstall = ''
|
||||
mkdir -p $doc/nix-support
|
||||
echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
|
||||
'';
|
||||
|
||||
doInstallCheck = true;
|
||||
installCheckFlags = "sysconfdir=$(out)/etc";
|
||||
|
||||
separateDebugInfo = true;
|
||||
}) // {
|
||||
|
||||
perl-bindings = with final; stdenv.mkDerivation {
|
||||
name = "nix-perl-${version}";
|
||||
|
||||
src = self;
|
||||
|
||||
buildInputs =
|
||||
[ autoconf-archive
|
||||
autoreconfHook
|
||||
nix
|
||||
curl
|
||||
bzip2
|
||||
xz
|
||||
pkgconfig
|
||||
pkgs.perl
|
||||
boost
|
||||
]
|
||||
++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
|
||||
|
||||
configureFlags = ''
|
||||
--with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
|
||||
--with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}
|
||||
'';
|
||||
|
||||
enableParallelBuilding = true;
|
||||
|
||||
postUnpack = "sourceRoot=$sourceRoot/perl";
|
||||
};
|
||||
|
||||
};
|
||||
|
||||
};
|
||||
|
||||
hydraJobs = {
|
||||
|
||||
# Binary package for various platforms.
|
||||
build = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix);
|
||||
|
||||
build-static = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix-static);
|
||||
|
||||
# Perl bindings for various platforms.
|
||||
perlBindings = nixpkgs.lib.genAttrs systems (system: self.packages.${system}.nix.perl-bindings);
|
||||
|
||||
# Binary tarball for various platforms, containing a Nix store
|
||||
# with the closure of 'nix' package, and the second half of
|
||||
# the installation script.
|
||||
binaryTarball = nixpkgs.lib.genAttrs systems (system:
|
||||
|
||||
with nixpkgsFor.${system};
|
||||
|
||||
let
|
||||
installerClosureInfo = closureInfo { rootPaths = [ nix cacert ]; };
|
||||
in
|
||||
|
||||
runCommand "nix-binary-tarball-${version}"
|
||||
{ #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
|
||||
meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
|
||||
}
|
||||
''
|
||||
cp ${installerClosureInfo}/registration $TMPDIR/reginfo
|
||||
substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
|
||||
--subst-var-by nix ${nix} \
|
||||
--subst-var-by cacert ${cacert}
|
||||
|
||||
substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
|
||||
--subst-var-by nix ${nix} \
|
||||
--subst-var-by cacert ${cacert}
|
||||
substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
|
||||
--subst-var-by nix ${nix} \
|
||||
--subst-var-by cacert ${cacert}
|
||||
substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
|
||||
--subst-var-by nix ${nix} \
|
||||
--subst-var-by cacert ${cacert}
|
||||
|
||||
if type -p shellcheck; then
|
||||
# SC1090: Don't worry about not being able to find
|
||||
# $nix/etc/profile.d/nix.sh
|
||||
shellcheck --exclude SC1090 $TMPDIR/install
|
||||
shellcheck $TMPDIR/install-darwin-multi-user.sh
|
||||
shellcheck $TMPDIR/install-systemd-multi-user.sh
|
||||
|
||||
# SC1091: Don't panic about not being able to source
|
||||
# /etc/profile
|
||||
# SC2002: Ignore "useless cat" "error", when loading
|
||||
# .reginfo, as the cat is a much cleaner
|
||||
# implementation, even though it is "useless"
|
||||
# SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
|
||||
# root's home directory
|
||||
shellcheck --external-sources \
|
||||
--exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
|
||||
fi
|
||||
|
||||
chmod +x $TMPDIR/install
|
||||
chmod +x $TMPDIR/install-darwin-multi-user.sh
|
||||
chmod +x $TMPDIR/install-systemd-multi-user.sh
|
||||
chmod +x $TMPDIR/install-multi-user
|
||||
dir=nix-${version}-${system}
|
||||
fn=$out/$dir.tar.xz
|
||||
mkdir -p $out/nix-support
|
||||
echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
|
||||
tar cvfJ $fn \
|
||||
--owner=0 --group=0 --mode=u+rw,uga+r \
|
||||
--absolute-names \
|
||||
--hard-dereference \
|
||||
--transform "s,$TMPDIR/install,$dir/install," \
|
||||
--transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
|
||||
--transform "s,$NIX_STORE,$dir/store,S" \
|
||||
$TMPDIR/install $TMPDIR/install-darwin-multi-user.sh \
|
||||
$TMPDIR/install-systemd-multi-user.sh \
|
||||
$TMPDIR/install-multi-user $TMPDIR/reginfo \
|
||||
$(cat ${installerClosureInfo}/store-paths)
|
||||
'');
|
||||
|
||||
# The first half of the installation script. This is uploaded
|
||||
# to https://nixos.org/nix/install. It downloads the binary
|
||||
# tarball for the user's system and calls the second half of the
|
||||
# installation script.
|
||||
installerScript =
|
||||
with nixpkgsFor.x86_64-linux;
|
||||
runCommand "installer-script"
|
||||
{ buildInputs = [ nix ];
|
||||
}
|
||||
''
|
||||
mkdir -p $out/nix-support
|
||||
|
||||
substitute ${./scripts/install.in} $out/install \
|
||||
${pkgs.lib.concatMapStrings
|
||||
(system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${self.hydraJobs.binaryTarball.${system}}/*.tar.xz) ")
|
||||
[ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
|
||||
} \
|
||||
--replace '@nixVersion@' ${version}
|
||||
|
||||
echo "file installer $out/install" >> $out/nix-support/hydra-build-products
|
||||
'';
|
||||
|
||||
# Line coverage analysis.
|
||||
coverage =
|
||||
with nixpkgsFor.x86_64-linux;
|
||||
with commonDeps pkgs;
|
||||
|
||||
releaseTools.coverageAnalysis {
|
||||
name = "nix-coverage-${version}";
|
||||
|
||||
src = self;
|
||||
|
||||
enableParallelBuilding = true;
|
||||
|
||||
nativeBuildInputs = nativeBuildDeps;
|
||||
buildInputs = buildDeps ++ propagatedDeps ++ awsDeps;
|
||||
|
||||
dontInstall = false;
|
||||
|
||||
doInstallCheck = true;
|
||||
|
||||
lcovFilter = [ "*/boost/*" "*-tab.*" ];
|
||||
|
||||
# We call `dot', and even though we just use it to
|
||||
# syntax-check generated dot files, it still requires some
|
||||
# fonts. So provide those.
|
||||
FONTCONFIG_FILE = texFunctions.fontsConf;
|
||||
|
||||
# To test building without precompiled headers.
|
||||
makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ];
|
||||
};
|
||||
|
||||
# System tests.
|
||||
tests.remoteBuilds = import ./tests/remote-builds.nix {
|
||||
system = "x86_64-linux";
|
||||
inherit nixpkgs;
|
||||
inherit (self) overlay;
|
||||
};
|
||||
|
||||
tests.nix-copy-closure = import ./tests/nix-copy-closure.nix {
|
||||
system = "x86_64-linux";
|
||||
inherit nixpkgs;
|
||||
inherit (self) overlay;
|
||||
};
|
||||
|
||||
tests.githubFlakes = (import ./tests/github-flakes.nix rec {
|
||||
system = "x86_64-linux";
|
||||
inherit nixpkgs;
|
||||
inherit (self) overlay;
|
||||
});
|
||||
|
||||
tests.setuid = nixpkgs.lib.genAttrs
|
||||
["i686-linux" "x86_64-linux"]
|
||||
(system:
|
||||
import ./tests/setuid.nix rec {
|
||||
inherit nixpkgs system;
|
||||
inherit (self) overlay;
|
||||
});
|
||||
|
||||
# Test whether the binary tarball works in an Ubuntu system.
|
||||
tests.binaryTarball =
|
||||
with nixpkgsFor.x86_64-linux;
|
||||
vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test"
|
||||
{ diskImage = vmTools.diskImages.ubuntu1204x86_64;
|
||||
}
|
||||
''
|
||||
set -x
|
||||
useradd -m alice
|
||||
su - alice -c 'tar xf ${self.hydraJobs.binaryTarball.x86_64-linux}/*.tar.*'
|
||||
mkdir /dest-nix
|
||||
mount -o bind /dest-nix /nix # Provide a writable /nix.
|
||||
chown alice /nix
|
||||
su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install'
|
||||
su - alice -c 'nix-store --verify'
|
||||
su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}'
|
||||
|
||||
# Check whether 'nix upgrade-nix' works.
|
||||
cat > /tmp/paths.nix <<EOF
|
||||
{
|
||||
x86_64-linux = "${self.hydraJobs.build.x86_64-linux}";
|
||||
}
|
||||
EOF
|
||||
su - alice -c 'nix --experimental-features nix-command upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
|
||||
(! [ -L /home/alice/.profile-1-link ])
|
||||
su - alice -c 'PAGER= nix-store -qR ${self.hydraJobs.build.x86_64-linux}'
|
||||
|
||||
mkdir -p $out/nix-support
|
||||
touch $out/nix-support/hydra-build-products
|
||||
umount /nix
|
||||
'');
|
||||
|
||||
/*
|
||||
# Check whether we can still evaluate all of Nixpkgs.
|
||||
tests.evalNixpkgs =
|
||||
import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
|
||||
# FIXME: fix pkgs/top-level/make-tarball.nix in NixOS to not require a revCount.
|
||||
inherit nixpkgs;
|
||||
pkgs = nixpkgsFor.x86_64-linux;
|
||||
officialRelease = false;
|
||||
};
|
||||
|
||||
# Check whether we can still evaluate NixOS.
|
||||
tests.evalNixOS =
|
||||
with nixpkgsFor.x86_64-linux;
|
||||
runCommand "eval-nixos" { buildInputs = [ nix ]; }
|
||||
''
|
||||
export NIX_STATE_DIR=$TMPDIR
|
||||
|
||||
nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \
|
||||
--arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }'
|
||||
|
||||
touch $out
|
||||
'';
|
||||
*/
|
||||
|
||||
};
|
||||
|
||||
checks = forAllSystems (system: {
|
||||
binaryTarball = self.hydraJobs.binaryTarball.${system};
|
||||
build-static = self.hydraJobs.build-static.${system};
|
||||
perlBindings = self.hydraJobs.perlBindings.${system};
|
||||
});
|
||||
|
||||
packages = forAllSystems (system: {
|
||||
inherit (nixpkgsFor.${system}) nix;
|
||||
|
||||
nix-static = let
|
||||
nixpkgs = nixpkgsFor.${system}.pkgsStatic;
|
||||
in with commonDeps nixpkgs; nixpkgs.stdenv.mkDerivation {
|
||||
name = "nix-${version}";
|
||||
|
||||
src = self;
|
||||
|
||||
VERSION_SUFFIX = versionSuffix;
|
||||
|
||||
outputs = [ "out" "dev" "doc" ];
|
||||
|
||||
nativeBuildInputs = nativeBuildDeps;
|
||||
buildInputs = buildDeps ++ propagatedDeps;
|
||||
|
||||
configureFlags = [ "--sysconfdir=/etc" ];
|
||||
|
||||
enableParallelBuilding = true;
|
||||
|
||||
makeFlags = "profiledir=$(out)/etc/profile.d";
|
||||
|
||||
doCheck = true;
|
||||
|
||||
installFlags = "sysconfdir=$(out)/etc";
|
||||
|
||||
postInstall = ''
|
||||
mkdir -p $doc/nix-support
|
||||
echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
|
||||
'';
|
||||
|
||||
doInstallCheck = true;
|
||||
installCheckFlags = "sysconfdir=$(out)/etc";
|
||||
|
||||
stripAllList = ["bin"];
|
||||
};
|
||||
});
|
||||
|
||||
defaultPackage = forAllSystems (system: self.packages.${system}.nix);
|
||||
|
||||
devShell = forAllSystems (system:
|
||||
with nixpkgsFor.${system};
|
||||
with commonDeps pkgs;
|
||||
|
||||
stdenv.mkDerivation {
|
||||
name = "nix";
|
||||
|
||||
nativeBuildInputs = nativeBuildDeps;
|
||||
buildInputs = buildDeps ++ propagatedDeps ++ awsDeps ++ perlDeps;
|
||||
|
||||
inherit configureFlags;
|
||||
|
||||
enableParallelBuilding = true;
|
||||
|
||||
installFlags = "sysconfdir=$(out)/etc";
|
||||
|
||||
shellHook =
|
||||
''
|
||||
export prefix=$(pwd)/inst
|
||||
configureFlags+=" --prefix=$prefix"
|
||||
PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH
|
||||
PATH=$prefix/bin:$PATH
|
||||
unset PYTHONPATH
|
||||
'';
|
||||
});
|
||||
|
||||
};
|
||||
}
|
2
local.mk
2
local.mk
@ -8,7 +8,7 @@ clean-files += Makefile.config
|
||||
|
||||
GLOBAL_CXXFLAGS += -Wno-deprecated-declarations
|
||||
|
||||
$(foreach i, config.h $(call rwildcard, src/lib*, *.hh), \
|
||||
$(foreach i, config.h $(wildcard src/lib*/*.hh), \
|
||||
$(eval $(call install-file-in, $(i), $(includedir)/nix, 0644)))
|
||||
|
||||
$(GCH) $(PCH): src/libutil/util.hh config.h
|
||||
|
@ -170,15 +170,5 @@ $channelsBucket->add_key(
|
||||
chdir("/home/eelco/Dev/nix-pristine") or die;
|
||||
system("git remote update origin") == 0 or die;
|
||||
system("git tag --force --sign $version $nixRev -m 'Tagging release $version'") == 0 or die;
|
||||
|
||||
# Update the website.
|
||||
my $siteDir = "/home/eelco/Dev/nixos-homepage-pristine";
|
||||
|
||||
system("cd $siteDir && git pull") == 0 or die;
|
||||
|
||||
write_file("$siteDir/nix-release.tt",
|
||||
"[%-\n" .
|
||||
"latestNixVersion = \"$version\"\n" .
|
||||
"-%]\n");
|
||||
|
||||
system("cd $siteDir && git commit -a -m 'Nix $version released'") == 0 or die;
|
||||
system("git push --tags") == 0 or die;
|
||||
system("git push --force-with-lease origin $nixRev:refs/heads/latest-release") == 0 or die;
|
||||
|
19
misc/bash/completion.sh
Normal file
19
misc/bash/completion.sh
Normal file
@ -0,0 +1,19 @@
|
||||
function _complete_nix {
|
||||
local -a words
|
||||
local cword cur
|
||||
_get_comp_words_by_ref -n ':=&' words cword cur
|
||||
local have_type
|
||||
while IFS= read -r line; do
|
||||
if [[ -z $have_type ]]; then
|
||||
have_type=1
|
||||
if [[ $line = filenames ]]; then
|
||||
compopt -o filenames
|
||||
fi
|
||||
else
|
||||
COMPREPLY+=("$line")
|
||||
fi
|
||||
done < <(NIX_GET_COMPLETIONS=$cword "${words[@]}")
|
||||
__ltrim_colon_completions "$cur"
|
||||
}
|
||||
|
||||
complete -F _complete_nix nix
|
1
misc/bash/local.mk
Normal file
1
misc/bash/local.mk
Normal file
@ -0,0 +1 @@
|
||||
$(eval $(call install-file-as, $(d)/completion.sh, $(datarootdir)/bash-completion/completions/nix, 0644))
|
@ -21,13 +21,13 @@ clean-files += $(GCH) $(PCH)
|
||||
|
||||
ifeq ($(PRECOMPILE_HEADERS), 1)
|
||||
|
||||
ifeq ($(CXX), g++)
|
||||
ifeq ($(findstring g++,$(CXX)), g++)
|
||||
|
||||
GLOBAL_CXXFLAGS_PCH += -include $(buildprefix)precompiled-headers.h -Winvalid-pch
|
||||
|
||||
GLOBAL_ORDER_AFTER += $(GCH)
|
||||
|
||||
else ifeq ($(CXX), clang++)
|
||||
else ifeq ($(findstring clang++,$(CXX)), clang++)
|
||||
|
||||
GLOBAL_CXXFLAGS_PCH += -include-pch $(PCH) -Winvalid-pch
|
||||
|
||||
|
28
mk/run_test.sh
Executable file
28
mk/run_test.sh
Executable file
@ -0,0 +1,28 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -u
|
||||
|
||||
red=""
|
||||
green=""
|
||||
yellow=""
|
||||
normal=""
|
||||
|
||||
post_run_msg="ran test $1..."
|
||||
if [ -t 1 ]; then
|
||||
red="[31;1m"
|
||||
green="[32;1m"
|
||||
yellow="[33;1m"
|
||||
normal="[m"
|
||||
fi
|
||||
(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} init.sh 2>/dev/null > /dev/null)
|
||||
log="$(cd $(dirname $1) && env ${TESTS_ENVIRONMENT} $(basename $1) 2>&1)"
|
||||
status=$?
|
||||
if [ $status -eq 0 ]; then
|
||||
echo "$post_run_msg [${green}PASS$normal]"
|
||||
elif [ $status -eq 99 ]; then
|
||||
echo "$post_run_msg [${yellow}SKIP$normal]"
|
||||
else
|
||||
echo "$post_run_msg [${red}FAIL$normal]"
|
||||
echo "$log" | sed 's/^/ /'
|
||||
exit "$status"
|
||||
fi
|
44
mk/tests.mk
44
mk/tests.mk
@ -1,45 +1,15 @@
|
||||
# Run program $1 as part of ‘make installcheck’.
|
||||
|
||||
test-deps =
|
||||
|
||||
define run-install-test
|
||||
|
||||
installcheck: $1
|
||||
installcheck: $1.test
|
||||
|
||||
_installcheck-list += $1
|
||||
.PHONY: $1.test
|
||||
$1.test: $1 $(test-deps)
|
||||
@env TEST_NAME=$(notdir $(basename $1)) TESTS_ENVIRONMENT="$(tests-environment)" mk/run_test.sh $1
|
||||
|
||||
endef
|
||||
|
||||
# Color code from https://unix.stackexchange.com/a/10065
|
||||
installcheck:
|
||||
@total=0; failed=0; \
|
||||
red=""; \
|
||||
green=""; \
|
||||
yellow=""; \
|
||||
normal=""; \
|
||||
if [ -t 1 ]; then \
|
||||
red="[31;1m"; \
|
||||
green="[32;1m"; \
|
||||
yellow="[33;1m"; \
|
||||
normal="[m"; \
|
||||
fi; \
|
||||
for i in $(_installcheck-list); do \
|
||||
total=$$((total + 1)); \
|
||||
printf "running test $$i..."; \
|
||||
log="$$(cd $$(dirname $$i) && $(tests-environment) $$(basename $$i) 2>&1)"; \
|
||||
status=$$?; \
|
||||
if [ $$status -eq 0 ]; then \
|
||||
echo " [$${green}PASS$$normal]"; \
|
||||
elif [ $$status -eq 99 ]; then \
|
||||
echo " [$${yellow}SKIP$$normal]"; \
|
||||
else \
|
||||
echo " [$${red}FAIL$$normal]"; \
|
||||
echo "$$log" | sed 's/^/ /'; \
|
||||
failed=$$((failed + 1)); \
|
||||
fi; \
|
||||
done; \
|
||||
if [ "$$failed" != 0 ]; then \
|
||||
echo "$${red}$$failed out of $$total tests failed $$normal"; \
|
||||
exit 1; \
|
||||
else \
|
||||
echo "$${green}All tests succeeded$$normal"; \
|
||||
fi
|
||||
|
||||
.PHONY: check installcheck
|
||||
|
@ -80,7 +80,7 @@ SV * queryReferences(char * path)
|
||||
SV * queryPathHash(char * path)
|
||||
PPCODE:
|
||||
try {
|
||||
auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash.to_string(Base32, true);
|
||||
auto s = store()->queryPathInfo(store()->parseStorePath(path))->narHash->to_string(Base32, true);
|
||||
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
|
||||
} catch (Error & e) {
|
||||
croak("%s", e.what());
|
||||
@ -106,7 +106,7 @@ SV * queryPathInfo(char * path, int base32)
|
||||
XPUSHs(&PL_sv_undef);
|
||||
else
|
||||
XPUSHs(sv_2mortal(newSVpv(store()->printStorePath(*info->deriver).c_str(), 0)));
|
||||
auto s = info->narHash.to_string(base32 ? Base32 : Base16, true);
|
||||
auto s = info->narHash->to_string(base32 ? Base32 : Base16, true);
|
||||
XPUSHs(sv_2mortal(newSVpv(s.c_str(), 0)));
|
||||
mXPUSHi(info->registrationTime);
|
||||
mXPUSHi(info->narSize);
|
||||
@ -182,7 +182,7 @@ void importPaths(int fd, int dontCheckSigs)
|
||||
PPCODE:
|
||||
try {
|
||||
FdSource source(fd);
|
||||
store()->importPaths(source, nullptr, dontCheckSigs ? NoCheckSigs : CheckSigs);
|
||||
store()->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs);
|
||||
} catch (Error & e) {
|
||||
croak("%s", e.what());
|
||||
}
|
||||
@ -304,7 +304,10 @@ SV * derivationFromPath(char * drvPath)
|
||||
|
||||
HV * outputs = newHV();
|
||||
for (auto & i : drv.outputs)
|
||||
hv_store(outputs, i.first.c_str(), i.first.size(), newSVpv(store()->printStorePath(i.second.path).c_str(), 0), 0);
|
||||
hv_store(
|
||||
outputs, i.first.c_str(), i.first.size(),
|
||||
newSVpv(store()->printStorePath(i.second.path(*store(), drv.name)).c_str(), 0),
|
||||
0);
|
||||
hv_stores(hash, "outputs", newRV((SV *) outputs));
|
||||
|
||||
AV * inputDrvs = newAV();
|
||||
|
@ -1,89 +0,0 @@
|
||||
{ pkgs, enableStatic }:
|
||||
|
||||
with pkgs;
|
||||
|
||||
rec {
|
||||
# Use "busybox-sandbox-shell" if present,
|
||||
# if not (legacy) fallback and hope it's sufficient.
|
||||
sh = pkgs.busybox-sandbox-shell or (busybox.override {
|
||||
useMusl = true;
|
||||
enableStatic = true;
|
||||
enableMinimal = true;
|
||||
extraConfig = ''
|
||||
CONFIG_FEATURE_FANCY_ECHO y
|
||||
CONFIG_FEATURE_SH_MATH y
|
||||
CONFIG_FEATURE_SH_MATH_64 y
|
||||
|
||||
CONFIG_ASH y
|
||||
CONFIG_ASH_OPTIMIZE_FOR_SIZE y
|
||||
|
||||
CONFIG_ASH_ALIAS y
|
||||
CONFIG_ASH_BASH_COMPAT y
|
||||
CONFIG_ASH_CMDCMD y
|
||||
CONFIG_ASH_ECHO y
|
||||
CONFIG_ASH_GETOPTS y
|
||||
CONFIG_ASH_INTERNAL_GLOB y
|
||||
CONFIG_ASH_JOB_CONTROL y
|
||||
CONFIG_ASH_PRINTF y
|
||||
CONFIG_ASH_TEST y
|
||||
'';
|
||||
});
|
||||
|
||||
configureFlags =
|
||||
lib.optionals (!enableStatic && stdenv.isLinux) [
|
||||
"--with-sandbox-shell=${sh}/bin/busybox"
|
||||
];
|
||||
|
||||
nativeBuildDeps =
|
||||
[
|
||||
buildPackages.bison
|
||||
buildPackages.flex
|
||||
buildPackages.libxml2
|
||||
buildPackages.libxslt
|
||||
buildPackages.docbook5
|
||||
buildPackages.docbook_xsl_ns
|
||||
buildPackages.autoreconfHook
|
||||
buildPackages.pkgconfig
|
||||
|
||||
# Tests
|
||||
buildPackages.git
|
||||
buildPackages.mercurial
|
||||
buildPackages.ipfs
|
||||
];
|
||||
|
||||
buildDeps =
|
||||
[ autoreconfHook
|
||||
autoconf-archive
|
||||
|
||||
curl
|
||||
bzip2 xz brotli zlib editline
|
||||
openssl sqlite
|
||||
libarchive
|
||||
boost
|
||||
nlohmann_json
|
||||
gmock
|
||||
]
|
||||
++ lib.optionals stdenv.isLinux [libseccomp utillinuxMinimal]
|
||||
++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium
|
||||
++ lib.optional (!enableStatic && (stdenv.isLinux || stdenv.isDarwin))
|
||||
((aws-sdk-cpp.override {
|
||||
apis = ["s3" "transfer"];
|
||||
customMemoryManagement = false;
|
||||
}).overrideDerivation (args: {
|
||||
/*
|
||||
patches = args.patches or [] ++ [ (fetchpatch {
|
||||
url = https://github.com/edolstra/aws-sdk-cpp/commit/3e07e1f1aae41b4c8b340735ff9e8c735f0c063f.patch;
|
||||
sha256 = "1pij0v449p166f9l29x7ppzk8j7g9k9mp15ilh5qxp29c7fnvxy2";
|
||||
}) ];
|
||||
*/
|
||||
}));
|
||||
|
||||
propagatedDeps =
|
||||
[ (boehmgc.override { enableLargeConfig = true; })
|
||||
];
|
||||
|
||||
perlDeps =
|
||||
[ perl
|
||||
perlPackages.DBDSQLite
|
||||
];
|
||||
}
|
313
release.nix
313
release.nix
@ -1,313 +0,0 @@
|
||||
{ nix ? builtins.fetchGit ./.
|
||||
, nixpkgs ? builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz
|
||||
, officialRelease ? false
|
||||
, systems ? [ "x86_64-linux" "i686-linux" "x86_64-darwin" "aarch64-linux" ]
|
||||
}:
|
||||
|
||||
let
|
||||
|
||||
pkgs = import nixpkgs { system = builtins.currentSystem or "x86_64-linux"; };
|
||||
|
||||
version =
|
||||
builtins.readFile ./.version
|
||||
+ (if officialRelease then "" else "pre${toString nix.revCount}_${nix.shortRev}");
|
||||
|
||||
buildFun = pkgs: enableStatic:
|
||||
with pkgs; with import ./release-common.nix { inherit pkgs enableStatic; };
|
||||
stdenv.mkDerivation {
|
||||
name = "nix-${version}";
|
||||
|
||||
src = nix;
|
||||
|
||||
outputs = [ "out" "dev" "doc" ];
|
||||
|
||||
buildInputs = buildDeps;
|
||||
|
||||
nativeBuildInputs = nativeBuildDeps;
|
||||
|
||||
propagatedBuildInputs = propagatedDeps;
|
||||
|
||||
preConfigure =
|
||||
lib.optionalString (!enableStatic) ''
|
||||
# Copy libboost_context so we don't get all of Boost in our closure.
|
||||
# https://github.com/NixOS/nixpkgs/issues/45462
|
||||
mkdir -p $out/lib
|
||||
cp -pd ${boost}/lib/{libboost_context*,libboost_thread*,libboost_system*} $out/lib
|
||||
rm -f $out/lib/*.a
|
||||
${lib.optionalString stdenv.isLinux ''
|
||||
chmod u+w $out/lib/*.so.*
|
||||
patchelf --set-rpath $out/lib:${stdenv.cc.cc.lib}/lib $out/lib/libboost_thread.so.*
|
||||
''}
|
||||
|
||||
(cd perl; autoreconf --install --force --verbose)
|
||||
'';
|
||||
|
||||
configureFlags = configureFlags ++
|
||||
[ "--sysconfdir=/etc" ];
|
||||
|
||||
dontUpdateAutotoolsGnuConfigScripts = true;
|
||||
|
||||
enableParallelBuilding = true;
|
||||
|
||||
makeFlags = [ "profiledir=$(out)/etc/profile.d" "PRECOMPILE_HEADERS=0" ];
|
||||
|
||||
installFlags = "sysconfdir=$(out)/etc";
|
||||
|
||||
postInstall = ''
|
||||
mkdir -p $doc/nix-support
|
||||
echo "doc manual $doc/share/doc/nix/manual" >> $doc/nix-support/hydra-build-products
|
||||
'';
|
||||
|
||||
doCheck = true;
|
||||
|
||||
doInstallCheck = true;
|
||||
installCheckFlags = "sysconfdir=$(out)/etc";
|
||||
|
||||
separateDebugInfo = !enableStatic;
|
||||
|
||||
stripAllList = ["bin"];
|
||||
};
|
||||
|
||||
|
||||
jobs = rec {
|
||||
|
||||
|
||||
build-static = pkgs.lib.genAttrs systems (system:
|
||||
buildFun (import nixpkgs { inherit system; }).pkgsStatic true);
|
||||
|
||||
|
||||
build = pkgs.lib.genAttrs systems (system:
|
||||
buildFun (import nixpkgs { inherit system; }) false);
|
||||
|
||||
|
||||
perlBindings = pkgs.lib.genAttrs systems (system:
|
||||
|
||||
let pkgs = import nixpkgs { inherit system; }; in with pkgs;
|
||||
|
||||
releaseTools.nixBuild {
|
||||
name = "nix-perl-${version}";
|
||||
|
||||
src = nix;
|
||||
|
||||
buildInputs =
|
||||
[ autoconf-archive
|
||||
autoreconfHook
|
||||
jobs.build.${system}
|
||||
curl
|
||||
bzip2
|
||||
xz
|
||||
pkgconfig
|
||||
pkgs.perl
|
||||
boost
|
||||
]
|
||||
++ lib.optional (stdenv.isLinux || stdenv.isDarwin) libsodium;
|
||||
|
||||
configureFlags = ''
|
||||
--with-dbi=${perlPackages.DBI}/${pkgs.perl.libPrefix}
|
||||
--with-dbd-sqlite=${perlPackages.DBDSQLite}/${pkgs.perl.libPrefix}
|
||||
'';
|
||||
|
||||
enableParallelBuilding = true;
|
||||
|
||||
postUnpack = "sourceRoot=$sourceRoot/perl";
|
||||
});
|
||||
|
||||
|
||||
binaryTarball = pkgs.lib.genAttrs systems (system:
|
||||
|
||||
with import nixpkgs { inherit system; };
|
||||
|
||||
let
|
||||
toplevel = builtins.getAttr system jobs.build;
|
||||
installerClosureInfo = closureInfo { rootPaths = [ toplevel cacert ]; };
|
||||
in
|
||||
|
||||
runCommand "nix-binary-tarball-${version}"
|
||||
{ #nativeBuildInputs = lib.optional (system != "aarch64-linux") shellcheck;
|
||||
meta.description = "Distribution-independent Nix bootstrap binaries for ${system}";
|
||||
}
|
||||
''
|
||||
cp ${installerClosureInfo}/registration $TMPDIR/reginfo
|
||||
cp ${./scripts/create-darwin-volume.sh} $TMPDIR/create-darwin-volume.sh
|
||||
substitute ${./scripts/install-nix-from-closure.sh} $TMPDIR/install \
|
||||
--subst-var-by nix ${toplevel} \
|
||||
--subst-var-by cacert ${cacert}
|
||||
substitute ${./scripts/install-darwin-multi-user.sh} $TMPDIR/install-darwin-multi-user.sh \
|
||||
--subst-var-by nix ${toplevel} \
|
||||
--subst-var-by cacert ${cacert}
|
||||
substitute ${./scripts/install-systemd-multi-user.sh} $TMPDIR/install-systemd-multi-user.sh \
|
||||
--subst-var-by nix ${toplevel} \
|
||||
--subst-var-by cacert ${cacert}
|
||||
substitute ${./scripts/install-multi-user.sh} $TMPDIR/install-multi-user \
|
||||
--subst-var-by nix ${toplevel} \
|
||||
--subst-var-by cacert ${cacert}
|
||||
|
||||
if type -p shellcheck; then
|
||||
# SC1090: Don't worry about not being able to find
|
||||
# $nix/etc/profile.d/nix.sh
|
||||
shellcheck --exclude SC1090 $TMPDIR/install
|
||||
shellcheck $TMPDIR/create-darwin-volume.sh
|
||||
shellcheck $TMPDIR/install-darwin-multi-user.sh
|
||||
shellcheck $TMPDIR/install-systemd-multi-user.sh
|
||||
|
||||
# SC1091: Don't panic about not being able to source
|
||||
# /etc/profile
|
||||
# SC2002: Ignore "useless cat" "error", when loading
|
||||
# .reginfo, as the cat is a much cleaner
|
||||
# implementation, even though it is "useless"
|
||||
# SC2116: Allow ROOT_HOME=$(echo ~root) for resolving
|
||||
# root's home directory
|
||||
shellcheck --external-sources \
|
||||
--exclude SC1091,SC2002,SC2116 $TMPDIR/install-multi-user
|
||||
fi
|
||||
|
||||
chmod +x $TMPDIR/install
|
||||
chmod +x $TMPDIR/create-darwin-volume.sh
|
||||
chmod +x $TMPDIR/install-darwin-multi-user.sh
|
||||
chmod +x $TMPDIR/install-systemd-multi-user.sh
|
||||
chmod +x $TMPDIR/install-multi-user
|
||||
dir=nix-${version}-${system}
|
||||
fn=$out/$dir.tar.xz
|
||||
mkdir -p $out/nix-support
|
||||
echo "file binary-dist $fn" >> $out/nix-support/hydra-build-products
|
||||
tar cvfJ $fn \
|
||||
--owner=0 --group=0 --mode=u+rw,uga+r \
|
||||
--absolute-names \
|
||||
--hard-dereference \
|
||||
--transform "s,$TMPDIR/install,$dir/install," \
|
||||
--transform "s,$TMPDIR/create-darwin-volume.sh,$dir/create-darwin-volume.sh," \
|
||||
--transform "s,$TMPDIR/reginfo,$dir/.reginfo," \
|
||||
--transform "s,$NIX_STORE,$dir/store,S" \
|
||||
$TMPDIR/install \
|
||||
$TMPDIR/create-darwin-volume.sh \
|
||||
$TMPDIR/install-darwin-multi-user.sh \
|
||||
$TMPDIR/install-systemd-multi-user.sh \
|
||||
$TMPDIR/install-multi-user \
|
||||
$TMPDIR/reginfo \
|
||||
$(cat ${installerClosureInfo}/store-paths)
|
||||
'');
|
||||
|
||||
|
||||
coverage =
|
||||
with pkgs;
|
||||
|
||||
with import ./release-common.nix { inherit pkgs; enableStatic = false; };
|
||||
|
||||
releaseTools.coverageAnalysis {
|
||||
name = "nix-coverage-${version}";
|
||||
|
||||
src = nix;
|
||||
|
||||
enableParallelBuilding = true;
|
||||
|
||||
nativeBuildInputs = nativeBuildDeps;
|
||||
buildInputs = buildDeps ++ propagatedDeps;
|
||||
|
||||
dontInstall = false;
|
||||
|
||||
doInstallCheck = true;
|
||||
|
||||
lcovFilter = [ "*/boost/*" "*-tab.*" ];
|
||||
|
||||
# We call `dot', and even though we just use it to
|
||||
# syntax-check generated dot files, it still requires some
|
||||
# fonts. So provide those.
|
||||
FONTCONFIG_FILE = texFunctions.fontsConf;
|
||||
|
||||
# To test building without precompiled headers.
|
||||
makeFlagsArray = [ "PRECOMPILE_HEADERS=0" ];
|
||||
};
|
||||
|
||||
|
||||
# System tests.
|
||||
tests.remoteBuilds = (import ./tests/remote-builds.nix rec {
|
||||
inherit nixpkgs;
|
||||
nix = build.x86_64-linux; system = "x86_64-linux";
|
||||
});
|
||||
|
||||
tests.nix-copy-closure = (import ./tests/nix-copy-closure.nix rec {
|
||||
inherit nixpkgs;
|
||||
nix = build.x86_64-linux; system = "x86_64-linux";
|
||||
});
|
||||
|
||||
tests.setuid = pkgs.lib.genAttrs
|
||||
["i686-linux" "x86_64-linux"]
|
||||
(system:
|
||||
import ./tests/setuid.nix rec {
|
||||
inherit nixpkgs;
|
||||
nix = build.${system}; inherit system;
|
||||
});
|
||||
|
||||
tests.binaryTarball =
|
||||
with import nixpkgs { system = "x86_64-linux"; };
|
||||
vmTools.runInLinuxImage (runCommand "nix-binary-tarball-test"
|
||||
{ diskImage = vmTools.diskImages.ubuntu1204x86_64;
|
||||
}
|
||||
''
|
||||
set -x
|
||||
useradd -m alice
|
||||
su - alice -c 'tar xf ${binaryTarball.x86_64-linux}/*.tar.*'
|
||||
mkdir /dest-nix
|
||||
mount -o bind /dest-nix /nix # Provide a writable /nix.
|
||||
chown alice /nix
|
||||
su - alice -c '_NIX_INSTALLER_TEST=1 ./nix-*/install'
|
||||
su - alice -c 'nix-store --verify'
|
||||
su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
|
||||
|
||||
# Check whether 'nix upgrade-nix' works.
|
||||
cat > /tmp/paths.nix <<EOF
|
||||
{
|
||||
x86_64-linux = "${build.x86_64-linux}";
|
||||
}
|
||||
EOF
|
||||
su - alice -c 'nix --experimental-features nix-command upgrade-nix -vvv --nix-store-paths-url file:///tmp/paths.nix'
|
||||
(! [ -L /home/alice/.profile-1-link ])
|
||||
su - alice -c 'PAGER= nix-store -qR ${build.x86_64-linux}'
|
||||
|
||||
mkdir -p $out/nix-support
|
||||
touch $out/nix-support/hydra-build-products
|
||||
umount /nix
|
||||
''); # */
|
||||
|
||||
/*
|
||||
tests.evalNixpkgs =
|
||||
import (nixpkgs + "/pkgs/top-level/make-tarball.nix") {
|
||||
inherit nixpkgs;
|
||||
inherit pkgs;
|
||||
nix = build.x86_64-linux;
|
||||
officialRelease = false;
|
||||
};
|
||||
|
||||
tests.evalNixOS =
|
||||
pkgs.runCommand "eval-nixos" { buildInputs = [ build.x86_64-linux ]; }
|
||||
''
|
||||
export NIX_STATE_DIR=$TMPDIR
|
||||
|
||||
nix-instantiate ${nixpkgs}/nixos/release-combined.nix -A tested --dry-run \
|
||||
--arg nixpkgs '{ outPath = ${nixpkgs}; revCount = 123; shortRev = "abcdefgh"; }'
|
||||
|
||||
touch $out
|
||||
'';
|
||||
*/
|
||||
|
||||
|
||||
installerScript =
|
||||
pkgs.runCommand "installer-script"
|
||||
{ buildInputs = [ build.${builtins.currentSystem or "x86_64-linux"} ]; }
|
||||
''
|
||||
mkdir -p $out/nix-support
|
||||
|
||||
substitute ${./scripts/install.in} $out/install \
|
||||
${pkgs.lib.concatMapStrings
|
||||
(system: "--replace '@binaryTarball_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${binaryTarball.${system}}/*.tar.xz) ")
|
||||
systems
|
||||
} \
|
||||
--replace '@nixVersion@' ${version}
|
||||
|
||||
echo "file installer $out/install" >> $out/nix-support/hydra-build-products
|
||||
'';
|
||||
|
||||
};
|
||||
|
||||
|
||||
in jobs
|
@ -207,7 +207,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
|
||||
if [ -w "$fn" ]; then
|
||||
if ! grep -q "$p" "$fn"; then
|
||||
echo "modifying $fn..." >&2
|
||||
echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
|
||||
echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
|
||||
fi
|
||||
added=1
|
||||
break
|
||||
@ -218,7 +218,7 @@ if [ -z "$NIX_INSTALLER_NO_MODIFY_PROFILE" ]; then
|
||||
if [ -w "$fn" ]; then
|
||||
if ! grep -q "$p" "$fn"; then
|
||||
echo "modifying $fn..." >&2
|
||||
echo "if [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
|
||||
echo -e "\nif [ -e $p ]; then . $p; fi # added by Nix installer" >> "$fn"
|
||||
fi
|
||||
added=1
|
||||
break
|
||||
|
30
shell.nix
30
shell.nix
@ -1,27 +1,3 @@
|
||||
{ useClang ? false, enableStatic ? false }:
|
||||
|
||||
with import (builtins.fetchTarball https://github.com/NixOS/nixpkgs/archive/nixos-20.03-small.tar.gz) {};
|
||||
|
||||
with import ./release-common.nix { inherit pkgs enableStatic; };
|
||||
|
||||
(if useClang then clangStdenv else stdenv).mkDerivation {
|
||||
name = "nix";
|
||||
|
||||
nativeBuildInputs = nativeBuildDeps;
|
||||
|
||||
buildInputs = buildDeps ++ propagatedDeps ++ perlDeps;
|
||||
|
||||
inherit configureFlags;
|
||||
|
||||
enableParallelBuilding = true;
|
||||
|
||||
installFlags = "sysconfdir=$(out)/etc";
|
||||
|
||||
shellHook =
|
||||
''
|
||||
export prefix=$(pwd)/inst
|
||||
configureFlags+=" --prefix=$prefix"
|
||||
PKG_CONFIG_PATH=$prefix/lib/pkgconfig:$PKG_CONFIG_PATH
|
||||
PATH=$prefix/bin:$PATH
|
||||
'';
|
||||
}
|
||||
(import (fetchTarball https://github.com/edolstra/flake-compat/archive/master.tar.gz) {
|
||||
src = ./.;
|
||||
}).shellNix
|
||||
|
@ -33,7 +33,7 @@ std::string escapeUri(std::string uri)
|
||||
|
||||
static string currentLoad;
|
||||
|
||||
static AutoCloseFD openSlotLock(const Machine & m, unsigned long long slot)
|
||||
static AutoCloseFD openSlotLock(const Machine & m, uint64_t slot)
|
||||
{
|
||||
return openLockFile(fmt("%s/%s-%d", currentLoad, escapeUri(m.storeUri), slot), true);
|
||||
}
|
||||
@ -119,7 +119,7 @@ static int _main(int argc, char * * argv)
|
||||
bool rightType = false;
|
||||
|
||||
Machine * bestMachine = nullptr;
|
||||
unsigned long long bestLoad = 0;
|
||||
uint64_t bestLoad = 0;
|
||||
for (auto & m : machines) {
|
||||
debug("considering building on remote machine '%s'", m.storeUri);
|
||||
|
||||
@ -130,8 +130,8 @@ static int _main(int argc, char * * argv)
|
||||
m.mandatoryMet(requiredFeatures)) {
|
||||
rightType = true;
|
||||
AutoCloseFD free;
|
||||
unsigned long long load = 0;
|
||||
for (unsigned long long slot = 0; slot < m.maxJobs; ++slot) {
|
||||
uint64_t load = 0;
|
||||
for (uint64_t slot = 0; slot < m.maxJobs; ++slot) {
|
||||
auto slotLock = openSlotLock(m, slot);
|
||||
if (lockFile(slotLock.get(), ltWrite, false)) {
|
||||
if (!free) {
|
||||
|
@ -130,7 +130,7 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what)
|
||||
|
||||
Symbol file = state.symbols.create(filename);
|
||||
|
||||
return { file, lineno, 0 };
|
||||
return { foFile, file, lineno, 0 };
|
||||
}
|
||||
|
||||
|
||||
|
@ -78,7 +78,7 @@ public:
|
||||
if (!a)
|
||||
throw Error({
|
||||
.hint = hintfmt("attribute '%s' missing", name),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
return *a;
|
||||
|
@ -4,6 +4,8 @@
|
||||
#include "util.hh"
|
||||
#include "eval.hh"
|
||||
#include "fetchers.hh"
|
||||
#include "registry.hh"
|
||||
#include "flake/flakeref.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
namespace nix {
|
||||
@ -31,6 +33,27 @@ MixEvalArgs::MixEvalArgs()
|
||||
.labels = {"path"},
|
||||
.handler = {[&](std::string s) { searchPath.push_back(s); }}
|
||||
});
|
||||
|
||||
addFlag({
|
||||
.longName = "impure",
|
||||
.description = "allow access to mutable paths and repositories",
|
||||
.handler = {[&]() {
|
||||
evalSettings.pureEval = false;
|
||||
}},
|
||||
});
|
||||
|
||||
addFlag({
|
||||
.longName = "override-flake",
|
||||
.description = "override a flake registry value",
|
||||
.labels = {"original-ref", "resolved-ref"},
|
||||
.handler = {[&](std::string _from, std::string _to) {
|
||||
auto from = parseFlakeRef(_from, absPath("."));
|
||||
auto to = parseFlakeRef(_to, absPath("."));
|
||||
fetchers::Attrs extraAttrs;
|
||||
if (to.subdir != "") extraAttrs["dir"] = to.subdir;
|
||||
fetchers::overrideRegistry(from.input, to.input, extraAttrs);
|
||||
}}
|
||||
});
|
||||
}
|
||||
|
||||
Bindings * MixEvalArgs::getAutoArgs(EvalState & state)
|
||||
@ -53,7 +76,7 @@ Path lookupFileArg(EvalState & state, string s)
|
||||
if (isUri(s)) {
|
||||
return state.store->toRealPath(
|
||||
fetchers::downloadTarball(
|
||||
state.store, resolveUri(s), "source", false).storePath);
|
||||
state.store, resolveUri(s), "source", false).first.storePath);
|
||||
} else if (s.size() > 2 && s.at(0) == '<' && s.at(s.size() - 1) == '>') {
|
||||
Path p = s.substr(1, s.size() - 2);
|
||||
return state.findFile(p);
|
||||
|
616
src/libexpr/eval-cache.cc
Normal file
616
src/libexpr/eval-cache.cc
Normal file
@ -0,0 +1,616 @@
|
||||
#include "eval-cache.hh"
|
||||
#include "sqlite.hh"
|
||||
#include "eval.hh"
|
||||
#include "eval-inline.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
namespace nix::eval_cache {
|
||||
|
||||
static const char * schema = R"sql(
|
||||
create table if not exists Attributes (
|
||||
parent integer not null,
|
||||
name text,
|
||||
type integer not null,
|
||||
value text,
|
||||
context text,
|
||||
primary key (parent, name)
|
||||
);
|
||||
)sql";
|
||||
|
||||
struct AttrDb
|
||||
{
|
||||
std::atomic_bool failed{false};
|
||||
|
||||
struct State
|
||||
{
|
||||
SQLite db;
|
||||
SQLiteStmt insertAttribute;
|
||||
SQLiteStmt insertAttributeWithContext;
|
||||
SQLiteStmt queryAttribute;
|
||||
SQLiteStmt queryAttributes;
|
||||
std::unique_ptr<SQLiteTxn> txn;
|
||||
};
|
||||
|
||||
std::unique_ptr<Sync<State>> _state;
|
||||
|
||||
AttrDb(const Hash & fingerprint)
|
||||
: _state(std::make_unique<Sync<State>>())
|
||||
{
|
||||
auto state(_state->lock());
|
||||
|
||||
Path cacheDir = getCacheDir() + "/nix/eval-cache-v2";
|
||||
createDirs(cacheDir);
|
||||
|
||||
Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite";
|
||||
|
||||
state->db = SQLite(dbPath);
|
||||
state->db.isCache();
|
||||
state->db.exec(schema);
|
||||
|
||||
state->insertAttribute.create(state->db,
|
||||
"insert or replace into Attributes(parent, name, type, value) values (?, ?, ?, ?)");
|
||||
|
||||
state->insertAttributeWithContext.create(state->db,
|
||||
"insert or replace into Attributes(parent, name, type, value, context) values (?, ?, ?, ?, ?)");
|
||||
|
||||
state->queryAttribute.create(state->db,
|
||||
"select rowid, type, value, context from Attributes where parent = ? and name = ?");
|
||||
|
||||
state->queryAttributes.create(state->db,
|
||||
"select name from Attributes where parent = ?");
|
||||
|
||||
state->txn = std::make_unique<SQLiteTxn>(state->db);
|
||||
}
|
||||
|
||||
~AttrDb()
|
||||
{
|
||||
try {
|
||||
auto state(_state->lock());
|
||||
if (!failed)
|
||||
state->txn->commit();
|
||||
state->txn.reset();
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
}
|
||||
|
||||
template<typename F>
|
||||
AttrId doSQLite(F && fun)
|
||||
{
|
||||
if (failed) return 0;
|
||||
try {
|
||||
return fun();
|
||||
} catch (SQLiteError &) {
|
||||
ignoreException();
|
||||
failed = true;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
AttrId setAttrs(
|
||||
AttrKey key,
|
||||
const std::vector<Symbol> & attrs)
|
||||
{
|
||||
return doSQLite([&]()
|
||||
{
|
||||
auto state(_state->lock());
|
||||
|
||||
state->insertAttribute.use()
|
||||
(key.first)
|
||||
(key.second)
|
||||
(AttrType::FullAttrs)
|
||||
(0, false).exec();
|
||||
|
||||
AttrId rowId = state->db.getLastInsertedRowId();
|
||||
assert(rowId);
|
||||
|
||||
for (auto & attr : attrs)
|
||||
state->insertAttribute.use()
|
||||
(rowId)
|
||||
(attr)
|
||||
(AttrType::Placeholder)
|
||||
(0, false).exec();
|
||||
|
||||
return rowId;
|
||||
});
|
||||
}
|
||||
|
||||
AttrId setString(
|
||||
AttrKey key,
|
||||
std::string_view s,
|
||||
const char * * context = nullptr)
|
||||
{
|
||||
return doSQLite([&]()
|
||||
{
|
||||
auto state(_state->lock());
|
||||
|
||||
if (context) {
|
||||
std::string ctx;
|
||||
for (const char * * p = context; *p; ++p) {
|
||||
if (p != context) ctx.push_back(' ');
|
||||
ctx.append(*p);
|
||||
}
|
||||
state->insertAttributeWithContext.use()
|
||||
(key.first)
|
||||
(key.second)
|
||||
(AttrType::String)
|
||||
(s)
|
||||
(ctx).exec();
|
||||
} else {
|
||||
state->insertAttribute.use()
|
||||
(key.first)
|
||||
(key.second)
|
||||
(AttrType::String)
|
||||
(s).exec();
|
||||
}
|
||||
|
||||
return state->db.getLastInsertedRowId();
|
||||
});
|
||||
}
|
||||
|
||||
AttrId setBool(
|
||||
AttrKey key,
|
||||
bool b)
|
||||
{
|
||||
return doSQLite([&]()
|
||||
{
|
||||
auto state(_state->lock());
|
||||
|
||||
state->insertAttribute.use()
|
||||
(key.first)
|
||||
(key.second)
|
||||
(AttrType::Bool)
|
||||
(b ? 1 : 0).exec();
|
||||
|
||||
return state->db.getLastInsertedRowId();
|
||||
});
|
||||
}
|
||||
|
||||
AttrId setPlaceholder(AttrKey key)
|
||||
{
|
||||
return doSQLite([&]()
|
||||
{
|
||||
auto state(_state->lock());
|
||||
|
||||
state->insertAttribute.use()
|
||||
(key.first)
|
||||
(key.second)
|
||||
(AttrType::Placeholder)
|
||||
(0, false).exec();
|
||||
|
||||
return state->db.getLastInsertedRowId();
|
||||
});
|
||||
}
|
||||
|
||||
AttrId setMissing(AttrKey key)
|
||||
{
|
||||
return doSQLite([&]()
|
||||
{
|
||||
auto state(_state->lock());
|
||||
|
||||
state->insertAttribute.use()
|
||||
(key.first)
|
||||
(key.second)
|
||||
(AttrType::Missing)
|
||||
(0, false).exec();
|
||||
|
||||
return state->db.getLastInsertedRowId();
|
||||
});
|
||||
}
|
||||
|
||||
AttrId setMisc(AttrKey key)
|
||||
{
|
||||
return doSQLite([&]()
|
||||
{
|
||||
auto state(_state->lock());
|
||||
|
||||
state->insertAttribute.use()
|
||||
(key.first)
|
||||
(key.second)
|
||||
(AttrType::Misc)
|
||||
(0, false).exec();
|
||||
|
||||
return state->db.getLastInsertedRowId();
|
||||
});
|
||||
}
|
||||
|
||||
AttrId setFailed(AttrKey key)
|
||||
{
|
||||
return doSQLite([&]()
|
||||
{
|
||||
auto state(_state->lock());
|
||||
|
||||
state->insertAttribute.use()
|
||||
(key.first)
|
||||
(key.second)
|
||||
(AttrType::Failed)
|
||||
(0, false).exec();
|
||||
|
||||
return state->db.getLastInsertedRowId();
|
||||
});
|
||||
}
|
||||
|
||||
std::optional<std::pair<AttrId, AttrValue>> getAttr(
|
||||
AttrKey key,
|
||||
SymbolTable & symbols)
|
||||
{
|
||||
auto state(_state->lock());
|
||||
|
||||
auto queryAttribute(state->queryAttribute.use()(key.first)(key.second));
|
||||
if (!queryAttribute.next()) return {};
|
||||
|
||||
auto rowId = (AttrType) queryAttribute.getInt(0);
|
||||
auto type = (AttrType) queryAttribute.getInt(1);
|
||||
|
||||
switch (type) {
|
||||
case AttrType::Placeholder:
|
||||
return {{rowId, placeholder_t()}};
|
||||
case AttrType::FullAttrs: {
|
||||
// FIXME: expensive, should separate this out.
|
||||
std::vector<Symbol> attrs;
|
||||
auto queryAttributes(state->queryAttributes.use()(rowId));
|
||||
while (queryAttributes.next())
|
||||
attrs.push_back(symbols.create(queryAttributes.getStr(0)));
|
||||
return {{rowId, attrs}};
|
||||
}
|
||||
case AttrType::String: {
|
||||
std::vector<std::pair<Path, std::string>> context;
|
||||
if (!queryAttribute.isNull(3))
|
||||
for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
|
||||
context.push_back(decodeContext(s));
|
||||
return {{rowId, string_t{queryAttribute.getStr(2), context}}};
|
||||
}
|
||||
case AttrType::Bool:
|
||||
return {{rowId, queryAttribute.getInt(2) != 0}};
|
||||
case AttrType::Missing:
|
||||
return {{rowId, missing_t()}};
|
||||
case AttrType::Misc:
|
||||
return {{rowId, misc_t()}};
|
||||
case AttrType::Failed:
|
||||
return {{rowId, failed_t()}};
|
||||
default:
|
||||
throw Error("unexpected type in evaluation cache");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
static std::shared_ptr<AttrDb> makeAttrDb(const Hash & fingerprint)
|
||||
{
|
||||
try {
|
||||
return std::make_shared<AttrDb>(fingerprint);
|
||||
} catch (SQLiteError &) {
|
||||
ignoreException();
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
EvalCache::EvalCache(
|
||||
std::optional<std::reference_wrapper<const Hash>> useCache,
|
||||
EvalState & state,
|
||||
RootLoader rootLoader)
|
||||
: db(useCache ? makeAttrDb(*useCache) : nullptr)
|
||||
, state(state)
|
||||
, rootLoader(rootLoader)
|
||||
{
|
||||
}
|
||||
|
||||
Value * EvalCache::getRootValue()
|
||||
{
|
||||
if (!value) {
|
||||
debug("getting root value");
|
||||
value = allocRootValue(rootLoader());
|
||||
}
|
||||
return *value;
|
||||
}
|
||||
|
||||
std::shared_ptr<AttrCursor> EvalCache::getRoot()
|
||||
{
|
||||
return std::make_shared<AttrCursor>(ref(shared_from_this()), std::nullopt);
|
||||
}
|
||||
|
||||
AttrCursor::AttrCursor(
|
||||
ref<EvalCache> root,
|
||||
Parent parent,
|
||||
Value * value,
|
||||
std::optional<std::pair<AttrId, AttrValue>> && cachedValue)
|
||||
: root(root), parent(parent), cachedValue(std::move(cachedValue))
|
||||
{
|
||||
if (value)
|
||||
_value = allocRootValue(value);
|
||||
}
|
||||
|
||||
AttrKey AttrCursor::getKey()
|
||||
{
|
||||
if (!parent)
|
||||
return {0, root->state.sEpsilon};
|
||||
if (!parent->first->cachedValue) {
|
||||
parent->first->cachedValue = root->db->getAttr(
|
||||
parent->first->getKey(), root->state.symbols);
|
||||
assert(parent->first->cachedValue);
|
||||
}
|
||||
return {parent->first->cachedValue->first, parent->second};
|
||||
}
|
||||
|
||||
Value & AttrCursor::getValue()
|
||||
{
|
||||
if (!_value) {
|
||||
if (parent) {
|
||||
auto & vParent = parent->first->getValue();
|
||||
root->state.forceAttrs(vParent);
|
||||
auto attr = vParent.attrs->get(parent->second);
|
||||
if (!attr)
|
||||
throw Error("attribute '%s' is unexpectedly missing", getAttrPathStr());
|
||||
_value = allocRootValue(attr->value);
|
||||
} else
|
||||
_value = allocRootValue(root->getRootValue());
|
||||
}
|
||||
return **_value;
|
||||
}
|
||||
|
||||
std::vector<Symbol> AttrCursor::getAttrPath() const
|
||||
{
|
||||
if (parent) {
|
||||
auto attrPath = parent->first->getAttrPath();
|
||||
attrPath.push_back(parent->second);
|
||||
return attrPath;
|
||||
} else
|
||||
return {};
|
||||
}
|
||||
|
||||
std::vector<Symbol> AttrCursor::getAttrPath(Symbol name) const
|
||||
{
|
||||
auto attrPath = getAttrPath();
|
||||
attrPath.push_back(name);
|
||||
return attrPath;
|
||||
}
|
||||
|
||||
std::string AttrCursor::getAttrPathStr() const
|
||||
{
|
||||
return concatStringsSep(".", getAttrPath());
|
||||
}
|
||||
|
||||
std::string AttrCursor::getAttrPathStr(Symbol name) const
|
||||
{
|
||||
return concatStringsSep(".", getAttrPath(name));
|
||||
}
|
||||
|
||||
Value & AttrCursor::forceValue()
|
||||
{
|
||||
debug("evaluating uncached attribute %s", getAttrPathStr());
|
||||
|
||||
auto & v = getValue();
|
||||
|
||||
try {
|
||||
root->state.forceValue(v);
|
||||
} catch (EvalError &) {
|
||||
debug("setting '%s' to failed", getAttrPathStr());
|
||||
if (root->db)
|
||||
cachedValue = {root->db->setFailed(getKey()), failed_t()};
|
||||
throw;
|
||||
}
|
||||
|
||||
if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
|
||||
if (v.type == tString)
|
||||
cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), v.string.s};
|
||||
else if (v.type == tPath)
|
||||
cachedValue = {root->db->setString(getKey(), v.path), v.path};
|
||||
else if (v.type == tBool)
|
||||
cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
|
||||
else if (v.type == tAttrs)
|
||||
; // FIXME: do something?
|
||||
else
|
||||
cachedValue = {root->db->setMisc(getKey()), misc_t()};
|
||||
}
|
||||
|
||||
return v;
|
||||
}
|
||||
|
||||
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name)
|
||||
{
|
||||
if (root->db) {
|
||||
if (!cachedValue)
|
||||
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
|
||||
|
||||
if (cachedValue) {
|
||||
if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
|
||||
for (auto & attr : *attrs)
|
||||
if (attr == name)
|
||||
return std::make_shared<AttrCursor>(root, std::make_pair(shared_from_this(), name));
|
||||
return nullptr;
|
||||
} else if (std::get_if<placeholder_t>(&cachedValue->second)) {
|
||||
auto attr = root->db->getAttr({cachedValue->first, name}, root->state.symbols);
|
||||
if (attr) {
|
||||
if (std::get_if<missing_t>(&attr->second))
|
||||
return nullptr;
|
||||
else if (std::get_if<failed_t>(&attr->second))
|
||||
throw EvalError("cached failure of attribute '%s'", getAttrPathStr(name));
|
||||
else
|
||||
return std::make_shared<AttrCursor>(root,
|
||||
std::make_pair(shared_from_this(), name), nullptr, std::move(attr));
|
||||
}
|
||||
// Incomplete attrset, so need to fall thru and
|
||||
// evaluate to see whether 'name' exists
|
||||
} else
|
||||
return nullptr;
|
||||
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
|
||||
}
|
||||
}
|
||||
|
||||
auto & v = forceValue();
|
||||
|
||||
if (v.type != tAttrs)
|
||||
return nullptr;
|
||||
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
|
||||
|
||||
auto attr = v.attrs->get(name);
|
||||
|
||||
if (!attr) {
|
||||
if (root->db) {
|
||||
if (!cachedValue)
|
||||
cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
|
||||
root->db->setMissing({cachedValue->first, name});
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::optional<std::pair<AttrId, AttrValue>> cachedValue2;
|
||||
if (root->db) {
|
||||
if (!cachedValue)
|
||||
cachedValue = {root->db->setPlaceholder(getKey()), placeholder_t()};
|
||||
cachedValue2 = {root->db->setPlaceholder({cachedValue->first, name}), placeholder_t()};
|
||||
}
|
||||
|
||||
return std::make_shared<AttrCursor>(
|
||||
root, std::make_pair(shared_from_this(), name), attr->value, std::move(cachedValue2));
|
||||
}
|
||||
|
||||
std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(std::string_view name)
|
||||
{
|
||||
return maybeGetAttr(root->state.symbols.create(name));
|
||||
}
|
||||
|
||||
std::shared_ptr<AttrCursor> AttrCursor::getAttr(Symbol name)
|
||||
{
|
||||
auto p = maybeGetAttr(name);
|
||||
if (!p)
|
||||
throw Error("attribute '%s' does not exist", getAttrPathStr(name));
|
||||
return p;
|
||||
}
|
||||
|
||||
std::shared_ptr<AttrCursor> AttrCursor::getAttr(std::string_view name)
|
||||
{
|
||||
return getAttr(root->state.symbols.create(name));
|
||||
}
|
||||
|
||||
std::shared_ptr<AttrCursor> AttrCursor::findAlongAttrPath(const std::vector<Symbol> & attrPath)
|
||||
{
|
||||
auto res = shared_from_this();
|
||||
for (auto & attr : attrPath) {
|
||||
res = res->maybeGetAttr(attr);
|
||||
if (!res) return {};
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
std::string AttrCursor::getString()
|
||||
{
|
||||
if (root->db) {
|
||||
if (!cachedValue)
|
||||
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
|
||||
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
|
||||
if (auto s = std::get_if<string_t>(&cachedValue->second)) {
|
||||
debug("using cached string attribute '%s'", getAttrPathStr());
|
||||
return s->first;
|
||||
} else
|
||||
throw TypeError("'%s' is not a string", getAttrPathStr());
|
||||
}
|
||||
}
|
||||
|
||||
auto & v = forceValue();
|
||||
|
||||
if (v.type != tString && v.type != tPath)
|
||||
throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
|
||||
|
||||
return v.type == tString ? v.string.s : v.path;
|
||||
}
|
||||
|
||||
string_t AttrCursor::getStringWithContext()
|
||||
{
|
||||
if (root->db) {
|
||||
if (!cachedValue)
|
||||
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
|
||||
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
|
||||
if (auto s = std::get_if<string_t>(&cachedValue->second)) {
|
||||
debug("using cached string attribute '%s'", getAttrPathStr());
|
||||
return *s;
|
||||
} else
|
||||
throw TypeError("'%s' is not a string", getAttrPathStr());
|
||||
}
|
||||
}
|
||||
|
||||
auto & v = forceValue();
|
||||
|
||||
if (v.type == tString)
|
||||
return {v.string.s, v.getContext()};
|
||||
else if (v.type == tPath)
|
||||
return {v.path, {}};
|
||||
else
|
||||
throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
|
||||
}
|
||||
|
||||
bool AttrCursor::getBool()
|
||||
{
|
||||
if (root->db) {
|
||||
if (!cachedValue)
|
||||
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
|
||||
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
|
||||
if (auto b = std::get_if<bool>(&cachedValue->second)) {
|
||||
debug("using cached Boolean attribute '%s'", getAttrPathStr());
|
||||
return *b;
|
||||
} else
|
||||
throw TypeError("'%s' is not a Boolean", getAttrPathStr());
|
||||
}
|
||||
}
|
||||
|
||||
auto & v = forceValue();
|
||||
|
||||
if (v.type != tBool)
|
||||
throw TypeError("'%s' is not a Boolean", getAttrPathStr());
|
||||
|
||||
return v.boolean;
|
||||
}
|
||||
|
||||
std::vector<Symbol> AttrCursor::getAttrs()
|
||||
{
|
||||
if (root->db) {
|
||||
if (!cachedValue)
|
||||
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
|
||||
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
|
||||
if (auto attrs = std::get_if<std::vector<Symbol>>(&cachedValue->second)) {
|
||||
debug("using cached attrset attribute '%s'", getAttrPathStr());
|
||||
return *attrs;
|
||||
} else
|
||||
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
|
||||
}
|
||||
}
|
||||
|
||||
auto & v = forceValue();
|
||||
|
||||
if (v.type != tAttrs)
|
||||
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
|
||||
|
||||
std::vector<Symbol> attrs;
|
||||
for (auto & attr : *getValue().attrs)
|
||||
attrs.push_back(attr.name);
|
||||
std::sort(attrs.begin(), attrs.end(), [](const Symbol & a, const Symbol & b) {
|
||||
return (const string &) a < (const string &) b;
|
||||
});
|
||||
|
||||
if (root->db)
|
||||
cachedValue = {root->db->setAttrs(getKey(), attrs), attrs};
|
||||
|
||||
return attrs;
|
||||
}
|
||||
|
||||
bool AttrCursor::isDerivation()
|
||||
{
|
||||
auto aType = maybeGetAttr("type");
|
||||
return aType && aType->getString() == "derivation";
|
||||
}
|
||||
|
||||
StorePath AttrCursor::forceDerivation()
|
||||
{
|
||||
auto aDrvPath = getAttr(root->state.sDrvPath);
|
||||
auto drvPath = root->state.store->parseStorePath(aDrvPath->getString());
|
||||
if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) {
|
||||
/* The eval cache contains 'drvPath', but the actual path has
|
||||
been garbage-collected. So force it to be regenerated. */
|
||||
aDrvPath->forceValue();
|
||||
if (!root->state.store->isValidPath(drvPath))
|
||||
throw Error("don't know how to recreate store derivation '%s'!",
|
||||
root->state.store->printStorePath(drvPath));
|
||||
}
|
||||
return drvPath;
|
||||
}
|
||||
|
||||
}
|
121
src/libexpr/eval-cache.hh
Normal file
121
src/libexpr/eval-cache.hh
Normal file
@ -0,0 +1,121 @@
|
||||
#pragma once
|
||||
|
||||
#include "sync.hh"
|
||||
#include "hash.hh"
|
||||
#include "eval.hh"
|
||||
|
||||
#include <functional>
|
||||
#include <variant>
|
||||
|
||||
namespace nix::eval_cache {
|
||||
|
||||
class AttrDb;
|
||||
class AttrCursor;
|
||||
|
||||
class EvalCache : public std::enable_shared_from_this<EvalCache>
|
||||
{
|
||||
friend class AttrCursor;
|
||||
|
||||
std::shared_ptr<AttrDb> db;
|
||||
EvalState & state;
|
||||
typedef std::function<Value *()> RootLoader;
|
||||
RootLoader rootLoader;
|
||||
RootValue value;
|
||||
|
||||
Value * getRootValue();
|
||||
|
||||
public:
|
||||
|
||||
EvalCache(
|
||||
std::optional<std::reference_wrapper<const Hash>> useCache,
|
||||
EvalState & state,
|
||||
RootLoader rootLoader);
|
||||
|
||||
std::shared_ptr<AttrCursor> getRoot();
|
||||
};
|
||||
|
||||
enum AttrType {
|
||||
Placeholder = 0,
|
||||
FullAttrs = 1,
|
||||
String = 2,
|
||||
Missing = 3,
|
||||
Misc = 4,
|
||||
Failed = 5,
|
||||
Bool = 6,
|
||||
};
|
||||
|
||||
struct placeholder_t {};
|
||||
struct missing_t {};
|
||||
struct misc_t {};
|
||||
struct failed_t {};
|
||||
typedef uint64_t AttrId;
|
||||
typedef std::pair<AttrId, Symbol> AttrKey;
|
||||
typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
|
||||
|
||||
typedef std::variant<
|
||||
std::vector<Symbol>,
|
||||
string_t,
|
||||
placeholder_t,
|
||||
missing_t,
|
||||
misc_t,
|
||||
failed_t,
|
||||
bool
|
||||
> AttrValue;
|
||||
|
||||
class AttrCursor : public std::enable_shared_from_this<AttrCursor>
|
||||
{
|
||||
friend class EvalCache;
|
||||
|
||||
ref<EvalCache> root;
|
||||
typedef std::optional<std::pair<std::shared_ptr<AttrCursor>, Symbol>> Parent;
|
||||
Parent parent;
|
||||
RootValue _value;
|
||||
std::optional<std::pair<AttrId, AttrValue>> cachedValue;
|
||||
|
||||
AttrKey getKey();
|
||||
|
||||
Value & getValue();
|
||||
|
||||
public:
|
||||
|
||||
AttrCursor(
|
||||
ref<EvalCache> root,
|
||||
Parent parent,
|
||||
Value * value = nullptr,
|
||||
std::optional<std::pair<AttrId, AttrValue>> && cachedValue = {});
|
||||
|
||||
std::vector<Symbol> getAttrPath() const;
|
||||
|
||||
std::vector<Symbol> getAttrPath(Symbol name) const;
|
||||
|
||||
std::string getAttrPathStr() const;
|
||||
|
||||
std::string getAttrPathStr(Symbol name) const;
|
||||
|
||||
std::shared_ptr<AttrCursor> maybeGetAttr(Symbol name);
|
||||
|
||||
std::shared_ptr<AttrCursor> maybeGetAttr(std::string_view name);
|
||||
|
||||
std::shared_ptr<AttrCursor> getAttr(Symbol name);
|
||||
|
||||
std::shared_ptr<AttrCursor> getAttr(std::string_view name);
|
||||
|
||||
std::shared_ptr<AttrCursor> findAlongAttrPath(const std::vector<Symbol> & attrPath);
|
||||
|
||||
std::string getString();
|
||||
|
||||
string_t getStringWithContext();
|
||||
|
||||
bool getBool();
|
||||
|
||||
std::vector<Symbol> getAttrs();
|
||||
|
||||
bool isDerivation();
|
||||
|
||||
Value & forceValue();
|
||||
|
||||
/* Force creation of the .drv file in the Nix store. */
|
||||
StorePath forceDerivation();
|
||||
};
|
||||
|
||||
}
|
@ -11,7 +11,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
|
||||
{
|
||||
throw EvalError({
|
||||
.hint = hintfmt(s),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
@ -25,7 +25,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
|
||||
{
|
||||
throw TypeError({
|
||||
.hint = hintfmt(s, showType(v)),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -199,6 +199,18 @@ string showType(const Value & v)
|
||||
}
|
||||
|
||||
|
||||
bool Value::isTrivial() const
|
||||
{
|
||||
return
|
||||
type != tApp
|
||||
&& type != tPrimOpApp
|
||||
&& (type != tThunk
|
||||
|| (dynamic_cast<ExprAttrs *>(thunk.expr)
|
||||
&& ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
|
||||
|| dynamic_cast<ExprLambda *>(thunk.expr));
|
||||
}
|
||||
|
||||
|
||||
#if HAVE_BOEHMGC
|
||||
/* Called when the Boehm GC runs out of memory. */
|
||||
static void * oomHandler(size_t requested)
|
||||
@ -337,6 +349,9 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
|
||||
, sOutputHashAlgo(symbols.create("outputHashAlgo"))
|
||||
, sOutputHashMode(symbols.create("outputHashMode"))
|
||||
, sRecurseForDerivations(symbols.create("recurseForDerivations"))
|
||||
, sDescription(symbols.create("description"))
|
||||
, sSelf(symbols.create("self"))
|
||||
, sEpsilon(symbols.create(""))
|
||||
, repair(NoRepair)
|
||||
, store(store)
|
||||
, baseEnv(allocEnv(128))
|
||||
@ -366,7 +381,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
|
||||
|
||||
if (store->isInStore(r.second)) {
|
||||
StorePathSet closure;
|
||||
store->computeFSClosure(store->parseStorePath(store->toStorePath(r.second)), closure);
|
||||
store->computeFSClosure(store->toStorePath(r.second).first, closure);
|
||||
for (auto & path : closure)
|
||||
allowedPaths->insert(store->printStorePath(path));
|
||||
} else
|
||||
@ -529,7 +544,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const
|
||||
{
|
||||
throw EvalError({
|
||||
.hint = hintfmt(s, s2),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
@ -542,7 +557,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const
|
||||
{
|
||||
throw EvalError({
|
||||
.hint = hintfmt(s, s2, s3),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
@ -551,7 +566,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const
|
||||
// p1 is where the error occurred; p2 is a position mentioned in the message.
|
||||
throw EvalError({
|
||||
.hint = hintfmt(s, sym, p2),
|
||||
.nixCode = NixCode { .errPos = p1 }
|
||||
.errPos = p1
|
||||
});
|
||||
}
|
||||
|
||||
@ -559,7 +574,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
|
||||
{
|
||||
throw TypeError({
|
||||
.hint = hintfmt(s),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
@ -572,7 +587,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
|
||||
{
|
||||
throw TypeError({
|
||||
.hint = hintfmt(s, fun.showNamePos(), s2),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
@ -580,7 +595,7 @@ LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s,
|
||||
{
|
||||
throw AssertionError({
|
||||
.hint = hintfmt(s, s1),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
@ -588,23 +603,18 @@ LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char *
|
||||
{
|
||||
throw UndefinedVarError({
|
||||
.hint = hintfmt(s, s1),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2))
|
||||
LocalNoInline(void addErrorTrace(Error & e, const char * s, const string & s2))
|
||||
{
|
||||
e.addPrefix(format(s) % s2);
|
||||
e.addTrace(std::nullopt, s, s2);
|
||||
}
|
||||
|
||||
LocalNoInline(void addErrorPrefix(Error & e, const char * s, const ExprLambda & fun, const Pos & pos))
|
||||
LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, const string & s2))
|
||||
{
|
||||
e.addPrefix(format(s) % fun.showNamePos() % pos);
|
||||
}
|
||||
|
||||
LocalNoInline(void addErrorPrefix(Error & e, const char * s, const string & s2, const Pos & pos))
|
||||
{
|
||||
e.addPrefix(format(s) % s2 % pos);
|
||||
e.addTrace(pos, s, s2);
|
||||
}
|
||||
|
||||
|
||||
@ -787,7 +797,7 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env)
|
||||
}
|
||||
|
||||
|
||||
void EvalState::evalFile(const Path & path_, Value & v)
|
||||
void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial)
|
||||
{
|
||||
auto path = checkSourcePath(path_);
|
||||
|
||||
@ -816,9 +826,14 @@ void EvalState::evalFile(const Path & path_, Value & v)
|
||||
fileParseCache[path2] = e;
|
||||
|
||||
try {
|
||||
// Enforce that 'flake.nix' is a direct attrset, not a
|
||||
// computation.
|
||||
if (mustBeTrivial &&
|
||||
!(dynamic_cast<ExprAttrs *>(e)))
|
||||
throw Error("file '%s' must be an attribute set", path);
|
||||
eval(e, v);
|
||||
} catch (Error & e) {
|
||||
addErrorPrefix(e, "while evaluating the file '%1%':\n", path2);
|
||||
addErrorTrace(e, "while evaluating the file '%1%':", path2);
|
||||
throw;
|
||||
}
|
||||
|
||||
@ -1068,8 +1083,8 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
|
||||
|
||||
} catch (Error & e) {
|
||||
if (pos2 && pos2->file != state.sDerivationNix)
|
||||
addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n",
|
||||
showAttrPath(state, env, attrPath), *pos2);
|
||||
addErrorTrace(e, *pos2, "while evaluating the attribute '%1%'",
|
||||
showAttrPath(state, env, attrPath));
|
||||
throw;
|
||||
}
|
||||
|
||||
@ -1237,11 +1252,15 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
|
||||
|
||||
/* Evaluate the body. This is conditional on showTrace, because
|
||||
catching exceptions makes this function not tail-recursive. */
|
||||
if (settings.showTrace)
|
||||
if (loggerSettings.showTrace.get())
|
||||
try {
|
||||
lambda.body->eval(*this, env2, v);
|
||||
} catch (Error & e) {
|
||||
addErrorPrefix(e, "while evaluating %1%, called from %2%:\n", lambda, pos);
|
||||
addErrorTrace(e, lambda.pos, "while evaluating %s",
|
||||
(lambda.name.set()
|
||||
? "'" + (string) lambda.name + "'"
|
||||
: "anonymous lambdaction"));
|
||||
addErrorTrace(e, pos, "from call site%s", "");
|
||||
throw;
|
||||
}
|
||||
else
|
||||
@ -1516,7 +1535,7 @@ void EvalState::forceValueDeep(Value & v)
|
||||
try {
|
||||
recurse(*i.value);
|
||||
} catch (Error & e) {
|
||||
addErrorPrefix(e, "while evaluating the attribute '%1%' at %2%:\n", i.name, *i.pos);
|
||||
addErrorTrace(e, *i.pos, "while evaluating the attribute '%1%'", i.name);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@ -1587,6 +1606,18 @@ string EvalState::forceString(Value & v, const Pos & pos)
|
||||
}
|
||||
|
||||
|
||||
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
|
||||
name>. */
|
||||
std::pair<string, string> decodeContext(std::string_view s)
|
||||
{
|
||||
if (s.at(0) == '!') {
|
||||
size_t index = s.find("!", 1);
|
||||
return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
|
||||
} else
|
||||
return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
|
||||
}
|
||||
|
||||
|
||||
void copyContext(const Value & v, PathSet & context)
|
||||
{
|
||||
if (v.string.context)
|
||||
@ -1595,6 +1626,17 @@ void copyContext(const Value & v, PathSet & context)
|
||||
}
|
||||
|
||||
|
||||
std::vector<std::pair<Path, std::string>> Value::getContext()
|
||||
{
|
||||
std::vector<std::pair<Path, std::string>> res;
|
||||
assert(type == tString);
|
||||
if (string.context)
|
||||
for (const char * * p = string.context; *p; ++p)
|
||||
res.push_back(decodeContext(*p));
|
||||
return res;
|
||||
}
|
||||
|
||||
|
||||
string EvalState::forceString(Value & v, PathSet & context, const Pos & pos)
|
||||
{
|
||||
string s = forceString(v, pos);
|
||||
@ -1936,7 +1978,7 @@ string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, boo
|
||||
{
|
||||
throw TypeError({
|
||||
.hint = hintfmt("cannot coerce %1% to a string", showType()),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
|
@ -4,13 +4,13 @@
|
||||
#include "value.hh"
|
||||
#include "nixexpr.hh"
|
||||
#include "symbol-table.hh"
|
||||
#include "hash.hh"
|
||||
#include "config.hh"
|
||||
|
||||
#include <regex>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <unordered_map>
|
||||
#include <mutex>
|
||||
|
||||
|
||||
namespace nix {
|
||||
@ -75,7 +75,8 @@ public:
|
||||
sFile, sLine, sColumn, sFunctor, sToString,
|
||||
sRight, sWrong, sStructuredAttrs, sBuilder, sArgs,
|
||||
sOutputHash, sOutputHashAlgo, sOutputHashMode,
|
||||
sRecurseForDerivations;
|
||||
sRecurseForDerivations,
|
||||
sDescription, sSelf, sEpsilon;
|
||||
Symbol sDerivationNix;
|
||||
|
||||
/* If set, force copying files to the Nix store even if they
|
||||
@ -90,6 +91,7 @@ public:
|
||||
|
||||
const ref<Store> store;
|
||||
|
||||
|
||||
private:
|
||||
SrcToStore srcToStore;
|
||||
|
||||
@ -152,8 +154,9 @@ public:
|
||||
Expr * parseStdin();
|
||||
|
||||
/* Evaluate an expression read from the given file to normal
|
||||
form. */
|
||||
void evalFile(const Path & path, Value & v);
|
||||
form. Optionally enforce that the top-level expression is
|
||||
trivial (i.e. doesn't require arbitrary computation). */
|
||||
void evalFile(const Path & path, Value & v, bool mustBeTrivial = false);
|
||||
|
||||
void resetFileCache();
|
||||
|
||||
@ -250,7 +253,7 @@ private:
|
||||
friend struct ExprAttrs;
|
||||
friend struct ExprLet;
|
||||
|
||||
Expr * parse(const char * text, const Path & path,
|
||||
Expr * parse(const char * text, FileOrigin origin, const Path & path,
|
||||
const Path & basePath, StaticEnv & staticEnv);
|
||||
|
||||
public:
|
||||
@ -330,7 +333,7 @@ string showType(const Value & v);
|
||||
|
||||
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
|
||||
name>. */
|
||||
std::pair<string, string> decodeContext(const string & s);
|
||||
std::pair<string, string> decodeContext(std::string_view s);
|
||||
|
||||
/* If `path' refers to a directory, then append "/default.nix". */
|
||||
Path resolveExprPath(Path path);
|
||||
|
56
src/libexpr/flake/call-flake.nix
Normal file
56
src/libexpr/flake/call-flake.nix
Normal file
@ -0,0 +1,56 @@
|
||||
lockFileStr: rootSrc: rootSubdir:
|
||||
|
||||
let
|
||||
|
||||
lockFile = builtins.fromJSON lockFileStr;
|
||||
|
||||
allNodes =
|
||||
builtins.mapAttrs
|
||||
(key: node:
|
||||
let
|
||||
|
||||
sourceInfo =
|
||||
if key == lockFile.root
|
||||
then rootSrc
|
||||
else fetchTree (node.info or {} // removeAttrs node.locked ["dir"]);
|
||||
|
||||
subdir = if key == lockFile.root then rootSubdir else node.locked.dir or "";
|
||||
|
||||
flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix");
|
||||
|
||||
inputs = builtins.mapAttrs
|
||||
(inputName: inputSpec: allNodes.${resolveInput inputSpec})
|
||||
(node.inputs or {});
|
||||
|
||||
# Resolve a input spec into a node name. An input spec is
|
||||
# either a node name, or a 'follows' path from the root
|
||||
# node.
|
||||
resolveInput = inputSpec:
|
||||
if builtins.isList inputSpec
|
||||
then getInputByPath lockFile.root inputSpec
|
||||
else inputSpec;
|
||||
|
||||
# Follow an input path (e.g. ["dwarffs" "nixpkgs"]) from the
|
||||
# root node, returning the final node.
|
||||
getInputByPath = nodeName: path:
|
||||
if path == []
|
||||
then nodeName
|
||||
else
|
||||
getInputByPath
|
||||
# Since this could be a 'follows' input, call resolveInput.
|
||||
(resolveInput lockFile.nodes.${nodeName}.inputs.${builtins.head path})
|
||||
(builtins.tail path);
|
||||
|
||||
outputs = flake.outputs (inputs // { self = result; });
|
||||
|
||||
result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; };
|
||||
in
|
||||
if node.flake or true then
|
||||
assert builtins.isFunction flake.outputs;
|
||||
result
|
||||
else
|
||||
sourceInfo
|
||||
)
|
||||
lockFile.nodes;
|
||||
|
||||
in allNodes.${lockFile.root}
|
609
src/libexpr/flake/flake.cc
Normal file
609
src/libexpr/flake/flake.cc
Normal file
@ -0,0 +1,609 @@
|
||||
#include "flake.hh"
|
||||
#include "lockfile.hh"
|
||||
#include "primops.hh"
|
||||
#include "eval-inline.hh"
|
||||
#include "store-api.hh"
|
||||
#include "fetchers.hh"
|
||||
#include "finally.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
using namespace flake;
|
||||
|
||||
namespace flake {
|
||||
|
||||
typedef std::pair<Tree, FlakeRef> FetchedFlake;
|
||||
typedef std::vector<std::pair<FlakeRef, FetchedFlake>> FlakeCache;
|
||||
|
||||
static std::optional<FetchedFlake> lookupInFlakeCache(
|
||||
const FlakeCache & flakeCache,
|
||||
const FlakeRef & flakeRef)
|
||||
{
|
||||
// FIXME: inefficient.
|
||||
for (auto & i : flakeCache) {
|
||||
if (flakeRef == i.first) {
|
||||
debug("mapping '%s' to previously seen input '%s' -> '%s",
|
||||
flakeRef, i.first, i.second.second);
|
||||
return i.second;
|
||||
}
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
|
||||
EvalState & state,
|
||||
const FlakeRef & originalRef,
|
||||
bool allowLookup,
|
||||
FlakeCache & flakeCache)
|
||||
{
|
||||
auto fetched = lookupInFlakeCache(flakeCache, originalRef);
|
||||
FlakeRef resolvedRef = originalRef;
|
||||
|
||||
if (!fetched) {
|
||||
if (originalRef.input.isDirect()) {
|
||||
fetched.emplace(originalRef.fetchTree(state.store));
|
||||
} else {
|
||||
if (allowLookup) {
|
||||
resolvedRef = originalRef.resolve(state.store);
|
||||
auto fetchedResolved = lookupInFlakeCache(flakeCache, originalRef);
|
||||
if (!fetchedResolved) fetchedResolved.emplace(resolvedRef.fetchTree(state.store));
|
||||
flakeCache.push_back({resolvedRef, fetchedResolved.value()});
|
||||
fetched.emplace(fetchedResolved.value());
|
||||
}
|
||||
else {
|
||||
throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", originalRef);
|
||||
}
|
||||
}
|
||||
flakeCache.push_back({originalRef, fetched.value()});
|
||||
}
|
||||
|
||||
auto [tree, lockedRef] = fetched.value();
|
||||
|
||||
debug("got tree '%s' from '%s'",
|
||||
state.store->printStorePath(tree.storePath), lockedRef);
|
||||
|
||||
if (state.allowedPaths)
|
||||
state.allowedPaths->insert(tree.actualPath);
|
||||
|
||||
assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store));
|
||||
|
||||
return {std::move(tree), resolvedRef, lockedRef};
|
||||
}
|
||||
|
||||
static void expectType(EvalState & state, ValueType type,
|
||||
Value & value, const Pos & pos)
|
||||
{
|
||||
if (value.type == tThunk && value.isTrivial())
|
||||
state.forceValue(value, pos);
|
||||
if (value.type != type)
|
||||
throw Error("expected %s but got %s at %s",
|
||||
showType(type), showType(value.type), pos);
|
||||
}
|
||||
|
||||
static std::map<FlakeId, FlakeInput> parseFlakeInputs(
|
||||
EvalState & state, Value * value, const Pos & pos);
|
||||
|
||||
static FlakeInput parseFlakeInput(EvalState & state,
|
||||
const std::string & inputName, Value * value, const Pos & pos)
|
||||
{
|
||||
expectType(state, tAttrs, *value, pos);
|
||||
|
||||
FlakeInput input;
|
||||
|
||||
auto sInputs = state.symbols.create("inputs");
|
||||
auto sUrl = state.symbols.create("url");
|
||||
auto sFlake = state.symbols.create("flake");
|
||||
auto sFollows = state.symbols.create("follows");
|
||||
|
||||
fetchers::Attrs attrs;
|
||||
std::optional<std::string> url;
|
||||
|
||||
for (nix::Attr attr : *(value->attrs)) {
|
||||
try {
|
||||
if (attr.name == sUrl) {
|
||||
expectType(state, tString, *attr.value, *attr.pos);
|
||||
url = attr.value->string.s;
|
||||
attrs.emplace("url", *url);
|
||||
} else if (attr.name == sFlake) {
|
||||
expectType(state, tBool, *attr.value, *attr.pos);
|
||||
input.isFlake = attr.value->boolean;
|
||||
} else if (attr.name == sInputs) {
|
||||
input.overrides = parseFlakeInputs(state, attr.value, *attr.pos);
|
||||
} else if (attr.name == sFollows) {
|
||||
expectType(state, tString, *attr.value, *attr.pos);
|
||||
input.follows = parseInputPath(attr.value->string.s);
|
||||
} else {
|
||||
state.forceValue(*attr.value);
|
||||
if (attr.value->type == tString)
|
||||
attrs.emplace(attr.name, attr.value->string.s);
|
||||
else
|
||||
throw TypeError("flake input attribute '%s' is %s while a string is expected",
|
||||
attr.name, showType(*attr.value));
|
||||
}
|
||||
} catch (Error & e) {
|
||||
e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name));
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
if (attrs.count("type"))
|
||||
try {
|
||||
input.ref = FlakeRef::fromAttrs(attrs);
|
||||
} catch (Error & e) {
|
||||
e.addTrace(pos, hintfmt("in flake input"));
|
||||
throw;
|
||||
}
|
||||
else {
|
||||
attrs.erase("url");
|
||||
if (!attrs.empty())
|
||||
throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos);
|
||||
if (url)
|
||||
input.ref = parseFlakeRef(*url, {}, true);
|
||||
}
|
||||
|
||||
if (!input.follows && !input.ref)
|
||||
input.ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}});
|
||||
|
||||
return input;
|
||||
}
|
||||
|
||||
static std::map<FlakeId, FlakeInput> parseFlakeInputs(
|
||||
EvalState & state, Value * value, const Pos & pos)
|
||||
{
|
||||
std::map<FlakeId, FlakeInput> inputs;
|
||||
|
||||
expectType(state, tAttrs, *value, pos);
|
||||
|
||||
for (nix::Attr & inputAttr : *(*value).attrs) {
|
||||
inputs.emplace(inputAttr.name,
|
||||
parseFlakeInput(state,
|
||||
inputAttr.name,
|
||||
inputAttr.value,
|
||||
*inputAttr.pos));
|
||||
}
|
||||
|
||||
return inputs;
|
||||
}
|
||||
|
||||
static Flake getFlake(
|
||||
EvalState & state,
|
||||
const FlakeRef & originalRef,
|
||||
bool allowLookup,
|
||||
FlakeCache & flakeCache)
|
||||
{
|
||||
auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
|
||||
state, originalRef, allowLookup, flakeCache);
|
||||
|
||||
// Guard against symlink attacks.
|
||||
auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix");
|
||||
if (!isInDir(flakeFile, sourceInfo.actualPath))
|
||||
throw Error("'flake.nix' file of flake '%s' escapes from '%s'",
|
||||
lockedRef, state.store->printStorePath(sourceInfo.storePath));
|
||||
|
||||
Flake flake {
|
||||
.originalRef = originalRef,
|
||||
.resolvedRef = resolvedRef,
|
||||
.lockedRef = lockedRef,
|
||||
.sourceInfo = std::make_shared<fetchers::Tree>(std::move(sourceInfo))
|
||||
};
|
||||
|
||||
if (!pathExists(flakeFile))
|
||||
throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir);
|
||||
|
||||
Value vInfo;
|
||||
state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack
|
||||
|
||||
expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0));
|
||||
|
||||
auto sEdition = state.symbols.create("edition"); // FIXME: remove soon
|
||||
|
||||
if (vInfo.attrs->get(sEdition))
|
||||
warn("flake '%s' has deprecated attribute 'edition'", lockedRef);
|
||||
|
||||
if (auto description = vInfo.attrs->get(state.sDescription)) {
|
||||
expectType(state, tString, *description->value, *description->pos);
|
||||
flake.description = description->value->string.s;
|
||||
}
|
||||
|
||||
auto sInputs = state.symbols.create("inputs");
|
||||
|
||||
if (auto inputs = vInfo.attrs->get(sInputs))
|
||||
flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos);
|
||||
|
||||
auto sOutputs = state.symbols.create("outputs");
|
||||
|
||||
if (auto outputs = vInfo.attrs->get(sOutputs)) {
|
||||
expectType(state, tLambda, *outputs->value, *outputs->pos);
|
||||
flake.vOutputs = allocRootValue(outputs->value);
|
||||
|
||||
if ((*flake.vOutputs)->lambda.fun->matchAttrs) {
|
||||
for (auto & formal : (*flake.vOutputs)->lambda.fun->formals->formals) {
|
||||
if (formal.name != state.sSelf)
|
||||
flake.inputs.emplace(formal.name, FlakeInput {
|
||||
.ref = parseFlakeRef(formal.name)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
} else
|
||||
throw Error("flake '%s' lacks attribute 'outputs'", lockedRef);
|
||||
|
||||
for (auto & attr : *vInfo.attrs) {
|
||||
if (attr.name != sEdition &&
|
||||
attr.name != state.sDescription &&
|
||||
attr.name != sInputs &&
|
||||
attr.name != sOutputs)
|
||||
throw Error("flake '%s' has an unsupported attribute '%s', at %s",
|
||||
lockedRef, attr.name, *attr.pos);
|
||||
}
|
||||
|
||||
return flake;
|
||||
}
|
||||
|
||||
Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup)
|
||||
{
|
||||
FlakeCache flakeCache;
|
||||
return getFlake(state, originalRef, allowLookup, flakeCache);
|
||||
}
|
||||
|
||||
/* Compute an in-memory lock file for the specified top-level flake,
|
||||
and optionally write it to file, it the flake is writable. */
|
||||
LockedFlake lockFlake(
|
||||
EvalState & state,
|
||||
const FlakeRef & topRef,
|
||||
const LockFlags & lockFlags)
|
||||
{
|
||||
settings.requireExperimentalFeature("flakes");
|
||||
|
||||
FlakeCache flakeCache;
|
||||
|
||||
auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache);
|
||||
|
||||
// FIXME: symlink attack
|
||||
auto oldLockFile = LockFile::read(
|
||||
flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock");
|
||||
|
||||
debug("old lock file: %s", oldLockFile);
|
||||
|
||||
// FIXME: check whether all overrides are used.
|
||||
std::map<InputPath, FlakeInput> overrides;
|
||||
std::set<InputPath> overridesUsed, updatesUsed;
|
||||
|
||||
for (auto & i : lockFlags.inputOverrides)
|
||||
overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
|
||||
|
||||
LockFile newLockFile;
|
||||
|
||||
std::vector<FlakeRef> parents;
|
||||
|
||||
std::function<void(
|
||||
const FlakeInputs & flakeInputs,
|
||||
std::shared_ptr<Node> node,
|
||||
const InputPath & inputPathPrefix,
|
||||
std::shared_ptr<const Node> oldNode)>
|
||||
computeLocks;
|
||||
|
||||
computeLocks = [&](
|
||||
const FlakeInputs & flakeInputs,
|
||||
std::shared_ptr<Node> node,
|
||||
const InputPath & inputPathPrefix,
|
||||
std::shared_ptr<const Node> oldNode)
|
||||
{
|
||||
debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
|
||||
|
||||
/* Get the overrides (i.e. attributes of the form
|
||||
'inputs.nixops.inputs.nixpkgs.url = ...'). */
|
||||
// FIXME: check this
|
||||
for (auto & [id, input] : flake.inputs) {
|
||||
for (auto & [idOverride, inputOverride] : input.overrides) {
|
||||
auto inputPath(inputPathPrefix);
|
||||
inputPath.push_back(id);
|
||||
inputPath.push_back(idOverride);
|
||||
overrides.insert_or_assign(inputPath, inputOverride);
|
||||
}
|
||||
}
|
||||
|
||||
/* Go over the flake inputs, resolve/fetch them if
|
||||
necessary (i.e. if they're new or the flakeref changed
|
||||
from what's in the lock file). */
|
||||
for (auto & [id, input2] : flakeInputs) {
|
||||
auto inputPath(inputPathPrefix);
|
||||
inputPath.push_back(id);
|
||||
auto inputPathS = printInputPath(inputPath);
|
||||
debug("computing input '%s'", inputPathS);
|
||||
|
||||
/* Do we have an override for this input from one of the
|
||||
ancestors? */
|
||||
auto i = overrides.find(inputPath);
|
||||
bool hasOverride = i != overrides.end();
|
||||
if (hasOverride) overridesUsed.insert(inputPath);
|
||||
auto & input = hasOverride ? i->second : input2;
|
||||
|
||||
/* Resolve 'follows' later (since it may refer to an input
|
||||
path we haven't processed yet. */
|
||||
if (input.follows) {
|
||||
InputPath target;
|
||||
if (hasOverride || input.absolute)
|
||||
/* 'follows' from an override is relative to the
|
||||
root of the graph. */
|
||||
target = *input.follows;
|
||||
else {
|
||||
/* Otherwise, it's relative to the current flake. */
|
||||
target = inputPathPrefix;
|
||||
for (auto & i : *input.follows) target.push_back(i);
|
||||
}
|
||||
debug("input '%s' follows '%s'", inputPathS, printInputPath(target));
|
||||
node->inputs.insert_or_assign(id, target);
|
||||
continue;
|
||||
}
|
||||
|
||||
assert(input.ref);
|
||||
|
||||
/* Do we have an entry in the existing lock file? And we
|
||||
don't have a --update-input flag for this input? */
|
||||
std::shared_ptr<LockedNode> oldLock;
|
||||
|
||||
updatesUsed.insert(inputPath);
|
||||
|
||||
if (oldNode && !lockFlags.inputUpdates.count(inputPath))
|
||||
if (auto oldLock2 = get(oldNode->inputs, id))
|
||||
if (auto oldLock3 = std::get_if<0>(&*oldLock2))
|
||||
oldLock = *oldLock3;
|
||||
|
||||
if (oldLock
|
||||
&& oldLock->originalRef == *input.ref
|
||||
&& !hasOverride)
|
||||
{
|
||||
debug("keeping existing input '%s'", inputPathS);
|
||||
|
||||
/* Copy the input from the old lock since its flakeref
|
||||
didn't change and there is no override from a
|
||||
higher level flake. */
|
||||
auto childNode = std::make_shared<LockedNode>(
|
||||
oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake);
|
||||
|
||||
node->inputs.insert_or_assign(id, childNode);
|
||||
|
||||
/* If we have an --update-input flag for an input
|
||||
of this input, then we must fetch the flake to
|
||||
to update it. */
|
||||
auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
|
||||
|
||||
auto hasChildUpdate =
|
||||
lb != lockFlags.inputUpdates.end()
|
||||
&& lb->size() > inputPath.size()
|
||||
&& std::equal(inputPath.begin(), inputPath.end(), lb->begin());
|
||||
|
||||
if (hasChildUpdate) {
|
||||
auto inputFlake = getFlake(
|
||||
state, oldLock->lockedRef, false, flakeCache);
|
||||
computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
|
||||
} else {
|
||||
/* No need to fetch this flake, we can be
|
||||
lazy. However there may be new overrides on the
|
||||
inputs of this flake, so we need to check
|
||||
those. */
|
||||
FlakeInputs fakeInputs;
|
||||
|
||||
for (auto & i : oldLock->inputs) {
|
||||
if (auto lockedNode = std::get_if<0>(&i.second)) {
|
||||
fakeInputs.emplace(i.first, FlakeInput {
|
||||
.ref = (*lockedNode)->originalRef,
|
||||
.isFlake = (*lockedNode)->isFlake,
|
||||
});
|
||||
} else if (auto follows = std::get_if<1>(&i.second)) {
|
||||
fakeInputs.emplace(i.first, FlakeInput {
|
||||
.follows = *follows,
|
||||
.absolute = true
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
computeLocks(fakeInputs, childNode, inputPath, oldLock);
|
||||
}
|
||||
|
||||
} else {
|
||||
/* We need to create a new lock file entry. So fetch
|
||||
this input. */
|
||||
debug("creating new input '%s'", inputPathS);
|
||||
|
||||
if (!lockFlags.allowMutable && !input.ref->input.isImmutable())
|
||||
throw Error("cannot update flake input '%s' in pure mode", inputPathS);
|
||||
|
||||
if (input.isFlake) {
|
||||
auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache);
|
||||
|
||||
/* Note: in case of an --override-input, we use
|
||||
the *original* ref (input2.ref) for the
|
||||
"original" field, rather than the
|
||||
override. This ensures that the override isn't
|
||||
nuked the next time we update the lock
|
||||
file. That is, overrides are sticky unless you
|
||||
use --no-write-lock-file. */
|
||||
auto childNode = std::make_shared<LockedNode>(
|
||||
inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref);
|
||||
|
||||
node->inputs.insert_or_assign(id, childNode);
|
||||
|
||||
/* Guard against circular flake imports. */
|
||||
for (auto & parent : parents)
|
||||
if (parent == *input.ref)
|
||||
throw Error("found circular import of flake '%s'", parent);
|
||||
parents.push_back(*input.ref);
|
||||
Finally cleanup([&]() { parents.pop_back(); });
|
||||
|
||||
/* Recursively process the inputs of this
|
||||
flake. Also, unless we already have this flake
|
||||
in the top-level lock file, use this flake's
|
||||
own lock file. */
|
||||
computeLocks(
|
||||
inputFlake.inputs, childNode, inputPath,
|
||||
oldLock
|
||||
? std::dynamic_pointer_cast<const Node>(oldLock)
|
||||
: LockFile::read(
|
||||
inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root);
|
||||
}
|
||||
|
||||
else {
|
||||
auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
|
||||
state, *input.ref, lockFlags.useRegistries, flakeCache);
|
||||
node->inputs.insert_or_assign(id,
|
||||
std::make_shared<LockedNode>(lockedRef, *input.ref, false));
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
computeLocks(
|
||||
flake.inputs, newLockFile.root, {},
|
||||
lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
|
||||
|
||||
for (auto & i : lockFlags.inputOverrides)
|
||||
if (!overridesUsed.count(i.first))
|
||||
warn("the flag '--override-input %s %s' does not match any input",
|
||||
printInputPath(i.first), i.second);
|
||||
|
||||
for (auto & i : lockFlags.inputUpdates)
|
||||
if (!updatesUsed.count(i))
|
||||
warn("the flag '--update-input %s' does not match any input", printInputPath(i));
|
||||
|
||||
/* Check 'follows' inputs. */
|
||||
newLockFile.check();
|
||||
|
||||
debug("new lock file: %s", newLockFile);
|
||||
|
||||
/* Check whether we need to / can write the new lock file. */
|
||||
if (!(newLockFile == oldLockFile)) {
|
||||
|
||||
auto diff = LockFile::diff(oldLockFile, newLockFile);
|
||||
|
||||
if (lockFlags.writeLockFile) {
|
||||
if (auto sourcePath = topRef.input.getSourcePath()) {
|
||||
if (!newLockFile.isImmutable()) {
|
||||
if (settings.warnDirty)
|
||||
warn("will not write lock file of flake '%s' because it has a mutable input", topRef);
|
||||
} else {
|
||||
if (!lockFlags.updateLockFile)
|
||||
throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
|
||||
|
||||
auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
|
||||
|
||||
auto path = *sourcePath + "/" + relPath;
|
||||
|
||||
bool lockFileExists = pathExists(path);
|
||||
|
||||
if (lockFileExists) {
|
||||
auto s = chomp(diff);
|
||||
if (s.empty())
|
||||
warn("updating lock file '%s'", path);
|
||||
else
|
||||
warn("updating lock file '%s':\n%s", path, s);
|
||||
} else
|
||||
warn("creating lock file '%s'", path);
|
||||
|
||||
newLockFile.write(path);
|
||||
|
||||
topRef.input.markChangedFile(
|
||||
(topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
|
||||
lockFlags.commitLockFile
|
||||
? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s",
|
||||
relPath, lockFileExists ? "Update" : "Add", diff))
|
||||
: std::nullopt);
|
||||
|
||||
/* Rewriting the lockfile changed the top-level
|
||||
repo, so we should re-read it. FIXME: we could
|
||||
also just clear the 'rev' field... */
|
||||
auto prevLockedRef = flake.lockedRef;
|
||||
FlakeCache dummyCache;
|
||||
flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache);
|
||||
|
||||
if (lockFlags.commitLockFile &&
|
||||
flake.lockedRef.input.getRev() &&
|
||||
prevLockedRef.input.getRev() != flake.lockedRef.input.getRev())
|
||||
warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev());
|
||||
|
||||
/* Make sure that we picked up the change,
|
||||
i.e. the tree should usually be dirty
|
||||
now. Corner case: we could have reverted from a
|
||||
dirty to a clean tree! */
|
||||
if (flake.lockedRef.input == prevLockedRef.input
|
||||
&& !flake.lockedRef.input.isImmutable())
|
||||
throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
|
||||
}
|
||||
} else
|
||||
throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
|
||||
} else
|
||||
warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff));
|
||||
}
|
||||
|
||||
return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
|
||||
}
|
||||
|
||||
void callFlake(EvalState & state,
|
||||
const LockedFlake & lockedFlake,
|
||||
Value & vRes)
|
||||
{
|
||||
auto vLocks = state.allocValue();
|
||||
auto vRootSrc = state.allocValue();
|
||||
auto vRootSubdir = state.allocValue();
|
||||
auto vTmp1 = state.allocValue();
|
||||
auto vTmp2 = state.allocValue();
|
||||
|
||||
mkString(*vLocks, lockedFlake.lockFile.to_string());
|
||||
|
||||
emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc);
|
||||
|
||||
mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir);
|
||||
|
||||
static RootValue vCallFlake = nullptr;
|
||||
|
||||
if (!vCallFlake) {
|
||||
vCallFlake = allocRootValue(state.allocValue());
|
||||
state.eval(state.parseExprFromString(
|
||||
#include "call-flake.nix.gen.hh"
|
||||
, "/"), **vCallFlake);
|
||||
}
|
||||
|
||||
state.callFunction(**vCallFlake, *vLocks, *vTmp1, noPos);
|
||||
state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos);
|
||||
state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos);
|
||||
}
|
||||
|
||||
static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
||||
{
|
||||
auto flakeRefS = state.forceStringNoCtx(*args[0], pos);
|
||||
auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
|
||||
if (evalSettings.pureEval && !flakeRef.input.isImmutable())
|
||||
throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos);
|
||||
|
||||
callFlake(state,
|
||||
lockFlake(state, flakeRef,
|
||||
LockFlags {
|
||||
.updateLockFile = false,
|
||||
.useRegistries = !evalSettings.pureEval,
|
||||
.allowMutable = !evalSettings.pureEval,
|
||||
}),
|
||||
v);
|
||||
}
|
||||
|
||||
static RegisterPrimOp r2("__getFlake", 1, prim_getFlake, "flakes");
|
||||
|
||||
}
|
||||
|
||||
Fingerprint LockedFlake::getFingerprint() const
|
||||
{
|
||||
// FIXME: as an optimization, if the flake contains a lock file
|
||||
// and we haven't changed it, then it's sufficient to use
|
||||
// flake.sourceInfo.storePath for the fingerprint.
|
||||
return hashString(htSHA256,
|
||||
fmt("%s;%d;%d;%s",
|
||||
flake.sourceInfo->storePath.to_string(),
|
||||
flake.lockedRef.input.getRevCount().value_or(0),
|
||||
flake.lockedRef.input.getLastModified().value_or(0),
|
||||
lockFile));
|
||||
}
|
||||
|
||||
Flake::~Flake() { }
|
||||
|
||||
}
|
111
src/libexpr/flake/flake.hh
Normal file
111
src/libexpr/flake/flake.hh
Normal file
@ -0,0 +1,111 @@
|
||||
#pragma once
|
||||
|
||||
#include "types.hh"
|
||||
#include "flakeref.hh"
|
||||
#include "lockfile.hh"
|
||||
#include "value.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
class EvalState;
|
||||
|
||||
namespace fetchers { struct Tree; }
|
||||
|
||||
namespace flake {
|
||||
|
||||
struct FlakeInput;
|
||||
|
||||
typedef std::map<FlakeId, FlakeInput> FlakeInputs;
|
||||
|
||||
struct FlakeInput
|
||||
{
|
||||
std::optional<FlakeRef> ref;
|
||||
bool isFlake = true;
|
||||
std::optional<InputPath> follows;
|
||||
bool absolute = false; // whether 'follows' is relative to the flake root
|
||||
FlakeInputs overrides;
|
||||
};
|
||||
|
||||
struct Flake
|
||||
{
|
||||
FlakeRef originalRef;
|
||||
FlakeRef resolvedRef;
|
||||
FlakeRef lockedRef;
|
||||
std::optional<std::string> description;
|
||||
std::shared_ptr<const fetchers::Tree> sourceInfo;
|
||||
FlakeInputs inputs;
|
||||
RootValue vOutputs;
|
||||
~Flake();
|
||||
};
|
||||
|
||||
Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup);
|
||||
|
||||
/* Fingerprint of a locked flake; used as a cache key. */
|
||||
typedef Hash Fingerprint;
|
||||
|
||||
struct LockedFlake
|
||||
{
|
||||
Flake flake;
|
||||
LockFile lockFile;
|
||||
|
||||
Fingerprint getFingerprint() const;
|
||||
};
|
||||
|
||||
struct LockFlags
|
||||
{
|
||||
/* Whether to ignore the existing lock file, creating a new one
|
||||
from scratch. */
|
||||
bool recreateLockFile = false;
|
||||
|
||||
/* Whether to update the lock file at all. If set to false, if any
|
||||
change to the lock file is needed (e.g. when an input has been
|
||||
added to flake.nix), you get a fatal error. */
|
||||
bool updateLockFile = true;
|
||||
|
||||
/* Whether to write the lock file to disk. If set to true, if the
|
||||
any changes to the lock file are needed and the flake is not
|
||||
writable (i.e. is not a local Git working tree or similar), you
|
||||
get a fatal error. If set to false, Nix will use the modified
|
||||
lock file in memory only, without writing it to disk. */
|
||||
bool writeLockFile = true;
|
||||
|
||||
/* Whether to use the registries to lookup indirect flake
|
||||
references like 'nixpkgs'. */
|
||||
bool useRegistries = true;
|
||||
|
||||
/* Whether mutable flake references (i.e. those without a Git
|
||||
revision or similar) without a corresponding lock are
|
||||
allowed. Mutable flake references with a lock are always
|
||||
allowed. */
|
||||
bool allowMutable = true;
|
||||
|
||||
/* Whether to commit changes to flake.lock. */
|
||||
bool commitLockFile = false;
|
||||
|
||||
/* Flake inputs to be overriden. */
|
||||
std::map<InputPath, FlakeRef> inputOverrides;
|
||||
|
||||
/* Flake inputs to be updated. This means that any existing lock
|
||||
for those inputs will be ignored. */
|
||||
std::set<InputPath> inputUpdates;
|
||||
};
|
||||
|
||||
LockedFlake lockFlake(
|
||||
EvalState & state,
|
||||
const FlakeRef & flakeRef,
|
||||
const LockFlags & lockFlags);
|
||||
|
||||
void callFlake(
|
||||
EvalState & state,
|
||||
const LockedFlake & lockedFlake,
|
||||
Value & v);
|
||||
|
||||
}
|
||||
|
||||
void emitTreeAttrs(
|
||||
EvalState & state,
|
||||
const fetchers::Tree & tree,
|
||||
const fetchers::Input & input,
|
||||
Value & v, bool emptyRevFallback = false);
|
||||
|
||||
}
|
204
src/libexpr/flake/flakeref.cc
Normal file
204
src/libexpr/flake/flakeref.cc
Normal file
@ -0,0 +1,204 @@
|
||||
#include "flakeref.hh"
|
||||
#include "store-api.hh"
|
||||
#include "url.hh"
|
||||
#include "fetchers.hh"
|
||||
#include "registry.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
#if 0
|
||||
// 'dir' path elements cannot start with a '.'. We also reject
|
||||
// potentially dangerous characters like ';'.
|
||||
const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)";
|
||||
const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*";
|
||||
#endif
|
||||
|
||||
std::string FlakeRef::to_string() const
|
||||
{
|
||||
auto url = input.toURL();
|
||||
if (subdir != "")
|
||||
url.query.insert_or_assign("dir", subdir);
|
||||
return url.to_string();
|
||||
}
|
||||
|
||||
fetchers::Attrs FlakeRef::toAttrs() const
|
||||
{
|
||||
auto attrs = input.toAttrs();
|
||||
if (subdir != "")
|
||||
attrs.emplace("dir", subdir);
|
||||
return attrs;
|
||||
}
|
||||
|
||||
std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef)
|
||||
{
|
||||
str << flakeRef.to_string();
|
||||
return str;
|
||||
}
|
||||
|
||||
bool FlakeRef::operator ==(const FlakeRef & other) const
|
||||
{
|
||||
return input == other.input && subdir == other.subdir;
|
||||
}
|
||||
|
||||
FlakeRef FlakeRef::resolve(ref<Store> store) const
|
||||
{
|
||||
auto [input2, extraAttrs] = lookupInRegistries(store, input);
|
||||
return FlakeRef(std::move(input2), fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir));
|
||||
}
|
||||
|
||||
FlakeRef parseFlakeRef(
|
||||
const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
|
||||
{
|
||||
auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir, allowMissing);
|
||||
if (fragment != "")
|
||||
throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url);
|
||||
return flakeRef;
|
||||
}
|
||||
|
||||
std::optional<FlakeRef> maybeParseFlakeRef(
|
||||
const std::string & url, const std::optional<Path> & baseDir)
|
||||
{
|
||||
try {
|
||||
return parseFlakeRef(url, baseDir);
|
||||
} catch (Error &) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
|
||||
const std::string & url, const std::optional<Path> & baseDir, bool allowMissing)
|
||||
{
|
||||
using namespace fetchers;
|
||||
|
||||
static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+";
|
||||
|
||||
static std::regex pathUrlRegex(
|
||||
"(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)"
|
||||
+ "(?:\\?(" + queryRegex + "))?"
|
||||
+ "(?:#(" + queryRegex + "))?",
|
||||
std::regex::ECMAScript);
|
||||
|
||||
static std::regex flakeRegex(
|
||||
"((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)"
|
||||
+ "(?:#(" + queryRegex + "))?",
|
||||
std::regex::ECMAScript);
|
||||
|
||||
std::smatch match;
|
||||
|
||||
/* Check if 'url' is a flake ID. This is an abbreviated syntax for
|
||||
'flake:<flake-id>?ref=<ref>&rev=<rev>'. */
|
||||
|
||||
if (std::regex_match(url, match, flakeRegex)) {
|
||||
auto parsedURL = ParsedURL{
|
||||
.url = url,
|
||||
.base = "flake:" + std::string(match[1]),
|
||||
.scheme = "flake",
|
||||
.authority = "",
|
||||
.path = match[1],
|
||||
};
|
||||
|
||||
return std::make_pair(
|
||||
FlakeRef(Input::fromURL(parsedURL), ""),
|
||||
percentDecode(std::string(match[6])));
|
||||
}
|
||||
|
||||
else if (std::regex_match(url, match, pathUrlRegex)) {
|
||||
std::string path = match[1];
|
||||
std::string fragment = percentDecode(std::string(match[3]));
|
||||
|
||||
if (baseDir) {
|
||||
/* Check if 'url' is a path (either absolute or relative
|
||||
to 'baseDir'). If so, search upward to the root of the
|
||||
repo (i.e. the directory containing .git). */
|
||||
|
||||
path = absPath(path, baseDir, true);
|
||||
|
||||
if (!S_ISDIR(lstat(path).st_mode))
|
||||
throw BadURL("path '%s' is not a flake (because it's not a directory)", path);
|
||||
|
||||
if (!allowMissing && !pathExists(path + "/flake.nix"))
|
||||
throw BadURL("path '%s' is not a flake (because it doesn't contain a 'flake.nix' file)", path);
|
||||
|
||||
auto flakeRoot = path;
|
||||
std::string subdir;
|
||||
|
||||
while (flakeRoot != "/") {
|
||||
if (pathExists(flakeRoot + "/.git")) {
|
||||
auto base = std::string("git+file://") + flakeRoot;
|
||||
|
||||
auto parsedURL = ParsedURL{
|
||||
.url = base, // FIXME
|
||||
.base = base,
|
||||
.scheme = "git+file",
|
||||
.authority = "",
|
||||
.path = flakeRoot,
|
||||
.query = decodeQuery(match[2]),
|
||||
};
|
||||
|
||||
if (subdir != "") {
|
||||
if (parsedURL.query.count("dir"))
|
||||
throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url);
|
||||
parsedURL.query.insert_or_assign("dir", subdir);
|
||||
}
|
||||
|
||||
if (pathExists(flakeRoot + "/.git/shallow"))
|
||||
parsedURL.query.insert_or_assign("shallow", "1");
|
||||
|
||||
return std::make_pair(
|
||||
FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
|
||||
fragment);
|
||||
}
|
||||
|
||||
subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir);
|
||||
flakeRoot = dirOf(flakeRoot);
|
||||
}
|
||||
|
||||
} else {
|
||||
if (!hasPrefix(path, "/"))
|
||||
throw BadURL("flake reference '%s' is not an absolute path", url);
|
||||
path = canonPath(path);
|
||||
}
|
||||
|
||||
fetchers::Attrs attrs;
|
||||
attrs.insert_or_assign("type", "path");
|
||||
attrs.insert_or_assign("path", path);
|
||||
|
||||
return std::make_pair(FlakeRef(Input::fromAttrs(std::move(attrs)), ""), fragment);
|
||||
}
|
||||
|
||||
else {
|
||||
auto parsedURL = parseURL(url);
|
||||
std::string fragment;
|
||||
std::swap(fragment, parsedURL.fragment);
|
||||
return std::make_pair(
|
||||
FlakeRef(Input::fromURL(parsedURL), get(parsedURL.query, "dir").value_or("")),
|
||||
fragment);
|
||||
}
|
||||
}
|
||||
|
||||
std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
|
||||
const std::string & url, const std::optional<Path> & baseDir)
|
||||
{
|
||||
try {
|
||||
return parseFlakeRefWithFragment(url, baseDir);
|
||||
} catch (Error & e) {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs)
|
||||
{
|
||||
auto attrs2(attrs);
|
||||
attrs2.erase("dir");
|
||||
return FlakeRef(
|
||||
fetchers::Input::fromAttrs(std::move(attrs2)),
|
||||
fetchers::maybeGetStrAttr(attrs, "dir").value_or(""));
|
||||
}
|
||||
|
||||
std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const
|
||||
{
|
||||
auto [tree, lockedInput] = input.fetch(store);
|
||||
return {std::move(tree), FlakeRef(std::move(lockedInput), subdir)};
|
||||
}
|
||||
|
||||
}
|
53
src/libexpr/flake/flakeref.hh
Normal file
53
src/libexpr/flake/flakeref.hh
Normal file
@ -0,0 +1,53 @@
|
||||
#pragma once
|
||||
|
||||
#include "types.hh"
|
||||
#include "hash.hh"
|
||||
#include "fetchers.hh"
|
||||
|
||||
#include <variant>
|
||||
|
||||
namespace nix {
|
||||
|
||||
class Store;
|
||||
|
||||
typedef std::string FlakeId;
|
||||
|
||||
struct FlakeRef
|
||||
{
|
||||
fetchers::Input input;
|
||||
|
||||
Path subdir;
|
||||
|
||||
bool operator==(const FlakeRef & other) const;
|
||||
|
||||
FlakeRef(fetchers::Input && input, const Path & subdir)
|
||||
: input(std::move(input)), subdir(subdir)
|
||||
{ }
|
||||
|
||||
// FIXME: change to operator <<.
|
||||
std::string to_string() const;
|
||||
|
||||
fetchers::Attrs toAttrs() const;
|
||||
|
||||
FlakeRef resolve(ref<Store> store) const;
|
||||
|
||||
static FlakeRef fromAttrs(const fetchers::Attrs & attrs);
|
||||
|
||||
std::pair<fetchers::Tree, FlakeRef> fetchTree(ref<Store> store) const;
|
||||
};
|
||||
|
||||
std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef);
|
||||
|
||||
FlakeRef parseFlakeRef(
|
||||
const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
|
||||
|
||||
std::optional<FlakeRef> maybeParseFlake(
|
||||
const std::string & url, const std::optional<Path> & baseDir = {});
|
||||
|
||||
std::pair<FlakeRef, std::string> parseFlakeRefWithFragment(
|
||||
const std::string & url, const std::optional<Path> & baseDir = {}, bool allowMissing = false);
|
||||
|
||||
std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment(
|
||||
const std::string & url, const std::optional<Path> & baseDir = {});
|
||||
|
||||
}
|
338
src/libexpr/flake/lockfile.cc
Normal file
338
src/libexpr/flake/lockfile.cc
Normal file
@ -0,0 +1,338 @@
|
||||
#include "lockfile.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace nix::flake {
|
||||
|
||||
FlakeRef getFlakeRef(
|
||||
const nlohmann::json & json,
|
||||
const char * attr,
|
||||
const char * info)
|
||||
{
|
||||
auto i = json.find(attr);
|
||||
if (i != json.end()) {
|
||||
auto attrs = jsonToAttrs(*i);
|
||||
// FIXME: remove when we drop support for version 5.
|
||||
if (info) {
|
||||
auto j = json.find(info);
|
||||
if (j != json.end()) {
|
||||
for (auto k : jsonToAttrs(*j))
|
||||
attrs.insert_or_assign(k.first, k.second);
|
||||
}
|
||||
}
|
||||
return FlakeRef::fromAttrs(attrs);
|
||||
}
|
||||
|
||||
throw Error("attribute '%s' missing in lock file", attr);
|
||||
}
|
||||
|
||||
LockedNode::LockedNode(const nlohmann::json & json)
|
||||
: lockedRef(getFlakeRef(json, "locked", "info"))
|
||||
, originalRef(getFlakeRef(json, "original", nullptr))
|
||||
, isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
|
||||
{
|
||||
if (!lockedRef.input.isImmutable())
|
||||
throw Error("lockfile contains mutable lock '%s'", attrsToJson(lockedRef.input.toAttrs()));
|
||||
}
|
||||
|
||||
StorePath LockedNode::computeStorePath(Store & store) const
|
||||
{
|
||||
return lockedRef.input.computeStorePath(store);
|
||||
}
|
||||
|
||||
std::shared_ptr<Node> LockFile::findInput(const InputPath & path)
|
||||
{
|
||||
auto pos = root;
|
||||
|
||||
if (!pos) return {};
|
||||
|
||||
for (auto & elem : path) {
|
||||
if (auto i = get(pos->inputs, elem)) {
|
||||
if (auto node = std::get_if<0>(&*i))
|
||||
pos = *node;
|
||||
else if (auto follows = std::get_if<1>(&*i)) {
|
||||
pos = findInput(*follows);
|
||||
if (!pos) return {};
|
||||
}
|
||||
} else
|
||||
return {};
|
||||
}
|
||||
|
||||
return pos;
|
||||
}
|
||||
|
||||
LockFile::LockFile(const nlohmann::json & json, const Path & path)
|
||||
{
|
||||
auto version = json.value("version", 0);
|
||||
if (version < 5 || version > 7)
|
||||
throw Error("lock file '%s' has unsupported version %d", path, version);
|
||||
|
||||
std::unordered_map<std::string, std::shared_ptr<Node>> nodeMap;
|
||||
|
||||
std::function<void(Node & node, const nlohmann::json & jsonNode)> getInputs;
|
||||
|
||||
getInputs = [&](Node & node, const nlohmann::json & jsonNode)
|
||||
{
|
||||
if (jsonNode.find("inputs") == jsonNode.end()) return;
|
||||
for (auto & i : jsonNode["inputs"].items()) {
|
||||
if (i.value().is_array()) {
|
||||
InputPath path;
|
||||
for (auto & j : i.value())
|
||||
path.push_back(j);
|
||||
node.inputs.insert_or_assign(i.key(), path);
|
||||
} else {
|
||||
std::string inputKey = i.value();
|
||||
auto k = nodeMap.find(inputKey);
|
||||
if (k == nodeMap.end()) {
|
||||
auto jsonNode2 = json["nodes"][inputKey];
|
||||
auto input = std::make_shared<LockedNode>(jsonNode2);
|
||||
k = nodeMap.insert_or_assign(inputKey, input).first;
|
||||
getInputs(*input, jsonNode2);
|
||||
}
|
||||
if (auto child = std::dynamic_pointer_cast<LockedNode>(k->second))
|
||||
node.inputs.insert_or_assign(i.key(), child);
|
||||
else
|
||||
// FIXME: replace by follows node
|
||||
throw Error("lock file contains cycle to root node");
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
std::string rootKey = json["root"];
|
||||
nodeMap.insert_or_assign(rootKey, root);
|
||||
getInputs(*root, json["nodes"][rootKey]);
|
||||
|
||||
// FIXME: check that there are no cycles in version >= 7. Cycles
|
||||
// between inputs are only possible using 'follows' indirections.
|
||||
// Once we drop support for version <= 6, we can simplify the code
|
||||
// a bit since we don't need to worry about cycles.
|
||||
}
|
||||
|
||||
nlohmann::json LockFile::toJson() const
|
||||
{
|
||||
nlohmann::json nodes;
|
||||
std::unordered_map<std::shared_ptr<const Node>, std::string> nodeKeys;
|
||||
std::unordered_set<std::string> keys;
|
||||
|
||||
std::function<std::string(const std::string & key, std::shared_ptr<const Node> node)> dumpNode;
|
||||
|
||||
dumpNode = [&](std::string key, std::shared_ptr<const Node> node) -> std::string
|
||||
{
|
||||
auto k = nodeKeys.find(node);
|
||||
if (k != nodeKeys.end())
|
||||
return k->second;
|
||||
|
||||
if (!keys.insert(key).second) {
|
||||
for (int n = 2; ; ++n) {
|
||||
auto k = fmt("%s_%d", key, n);
|
||||
if (keys.insert(k).second) {
|
||||
key = k;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
nodeKeys.insert_or_assign(node, key);
|
||||
|
||||
auto n = nlohmann::json::object();
|
||||
|
||||
if (!node->inputs.empty()) {
|
||||
auto inputs = nlohmann::json::object();
|
||||
for (auto & i : node->inputs) {
|
||||
if (auto child = std::get_if<0>(&i.second)) {
|
||||
inputs[i.first] = dumpNode(i.first, *child);
|
||||
} else if (auto follows = std::get_if<1>(&i.second)) {
|
||||
auto arr = nlohmann::json::array();
|
||||
for (auto & x : *follows)
|
||||
arr.push_back(x);
|
||||
inputs[i.first] = std::move(arr);
|
||||
}
|
||||
}
|
||||
n["inputs"] = std::move(inputs);
|
||||
}
|
||||
|
||||
if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) {
|
||||
n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs());
|
||||
n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs());
|
||||
if (!lockedNode->isFlake) n["flake"] = false;
|
||||
}
|
||||
|
||||
nodes[key] = std::move(n);
|
||||
|
||||
return key;
|
||||
};
|
||||
|
||||
nlohmann::json json;
|
||||
json["version"] = 7;
|
||||
json["root"] = dumpNode("root", root);
|
||||
json["nodes"] = std::move(nodes);
|
||||
|
||||
return json;
|
||||
}
|
||||
|
||||
std::string LockFile::to_string() const
|
||||
{
|
||||
return toJson().dump(2);
|
||||
}
|
||||
|
||||
LockFile LockFile::read(const Path & path)
|
||||
{
|
||||
if (!pathExists(path)) return LockFile();
|
||||
return LockFile(nlohmann::json::parse(readFile(path)), path);
|
||||
}
|
||||
|
||||
std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
|
||||
{
|
||||
stream << lockFile.toJson().dump(2);
|
||||
return stream;
|
||||
}
|
||||
|
||||
void LockFile::write(const Path & path) const
|
||||
{
|
||||
createDirs(dirOf(path));
|
||||
writeFile(path, fmt("%s\n", *this));
|
||||
}
|
||||
|
||||
bool LockFile::isImmutable() const
|
||||
{
|
||||
std::unordered_set<std::shared_ptr<const Node>> nodes;
|
||||
|
||||
std::function<void(std::shared_ptr<const Node> node)> visit;
|
||||
|
||||
visit = [&](std::shared_ptr<const Node> node)
|
||||
{
|
||||
if (!nodes.insert(node).second) return;
|
||||
for (auto & i : node->inputs)
|
||||
if (auto child = std::get_if<0>(&i.second))
|
||||
visit(*child);
|
||||
};
|
||||
|
||||
visit(root);
|
||||
|
||||
for (auto & i : nodes) {
|
||||
if (i == root) continue;
|
||||
auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(i);
|
||||
if (lockedNode && !lockedNode->lockedRef.input.isImmutable()) return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool LockFile::operator ==(const LockFile & other) const
|
||||
{
|
||||
// FIXME: slow
|
||||
return toJson() == other.toJson();
|
||||
}
|
||||
|
||||
InputPath parseInputPath(std::string_view s)
|
||||
{
|
||||
InputPath path;
|
||||
|
||||
for (auto & elem : tokenizeString<std::vector<std::string>>(s, "/")) {
|
||||
if (!std::regex_match(elem, flakeIdRegex))
|
||||
throw UsageError("invalid flake input path element '%s'", elem);
|
||||
path.push_back(elem);
|
||||
}
|
||||
|
||||
return path;
|
||||
}
|
||||
|
||||
std::map<InputPath, Node::Edge> LockFile::getAllInputs() const
|
||||
{
|
||||
std::unordered_set<std::shared_ptr<Node>> done;
|
||||
std::map<InputPath, Node::Edge> res;
|
||||
|
||||
std::function<void(const InputPath & prefix, std::shared_ptr<Node> node)> recurse;
|
||||
|
||||
recurse = [&](const InputPath & prefix, std::shared_ptr<Node> node)
|
||||
{
|
||||
if (!done.insert(node).second) return;
|
||||
|
||||
for (auto &[id, input] : node->inputs) {
|
||||
auto inputPath(prefix);
|
||||
inputPath.push_back(id);
|
||||
res.emplace(inputPath, input);
|
||||
if (auto child = std::get_if<0>(&input))
|
||||
recurse(inputPath, *child);
|
||||
}
|
||||
};
|
||||
|
||||
recurse({}, root);
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
std::ostream & operator <<(std::ostream & stream, const Node::Edge & edge)
|
||||
{
|
||||
if (auto node = std::get_if<0>(&edge))
|
||||
stream << "'" << (*node)->lockedRef << "'";
|
||||
else if (auto follows = std::get_if<1>(&edge))
|
||||
stream << fmt("follows '%s'", printInputPath(*follows));
|
||||
return stream;
|
||||
}
|
||||
|
||||
static bool equals(const Node::Edge & e1, const Node::Edge & e2)
|
||||
{
|
||||
if (auto n1 = std::get_if<0>(&e1))
|
||||
if (auto n2 = std::get_if<0>(&e2))
|
||||
return (*n1)->lockedRef == (*n2)->lockedRef;
|
||||
if (auto f1 = std::get_if<1>(&e1))
|
||||
if (auto f2 = std::get_if<1>(&e2))
|
||||
return *f1 == *f2;
|
||||
return false;
|
||||
}
|
||||
|
||||
std::string LockFile::diff(const LockFile & oldLocks, const LockFile & newLocks)
|
||||
{
|
||||
auto oldFlat = oldLocks.getAllInputs();
|
||||
auto newFlat = newLocks.getAllInputs();
|
||||
|
||||
auto i = oldFlat.begin();
|
||||
auto j = newFlat.begin();
|
||||
std::string res;
|
||||
|
||||
while (i != oldFlat.end() || j != newFlat.end()) {
|
||||
if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) {
|
||||
res += fmt("* Added '%s': %s\n", printInputPath(j->first), j->second);
|
||||
++j;
|
||||
} else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) {
|
||||
res += fmt("* Removed '%s'\n", printInputPath(i->first));
|
||||
++i;
|
||||
} else {
|
||||
if (!equals(i->second, j->second)) {
|
||||
res += fmt("* Updated '%s': %s -> %s\n",
|
||||
printInputPath(i->first),
|
||||
i->second,
|
||||
j->second);
|
||||
}
|
||||
++i;
|
||||
++j;
|
||||
}
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
void LockFile::check()
|
||||
{
|
||||
auto inputs = getAllInputs();
|
||||
|
||||
for (auto & [inputPath, input] : inputs) {
|
||||
if (auto follows = std::get_if<1>(&input)) {
|
||||
if (!follows->empty() && !get(inputs, *follows))
|
||||
throw Error("input '%s' follows a non-existent input '%s'",
|
||||
printInputPath(inputPath),
|
||||
printInputPath(*follows));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void check();
|
||||
|
||||
std::string printInputPath(const InputPath & path)
|
||||
{
|
||||
return concatStringsSep("/", path);
|
||||
}
|
||||
|
||||
}
|
85
src/libexpr/flake/lockfile.hh
Normal file
85
src/libexpr/flake/lockfile.hh
Normal file
@ -0,0 +1,85 @@
|
||||
#pragma once
|
||||
|
||||
#include "flakeref.hh"
|
||||
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
||||
namespace nix {
|
||||
class Store;
|
||||
struct StorePath;
|
||||
}
|
||||
|
||||
namespace nix::flake {
|
||||
|
||||
using namespace fetchers;
|
||||
|
||||
typedef std::vector<FlakeId> InputPath;
|
||||
|
||||
struct LockedNode;
|
||||
|
||||
/* A node in the lock file. It has outgoing edges to other nodes (its
|
||||
inputs). Only the root node has this type; all other nodes have
|
||||
type LockedNode. */
|
||||
struct Node : std::enable_shared_from_this<Node>
|
||||
{
|
||||
typedef std::variant<std::shared_ptr<LockedNode>, InputPath> Edge;
|
||||
|
||||
std::map<FlakeId, Edge> inputs;
|
||||
|
||||
virtual ~Node() { }
|
||||
};
|
||||
|
||||
/* A non-root node in the lock file. */
|
||||
struct LockedNode : Node
|
||||
{
|
||||
FlakeRef lockedRef, originalRef;
|
||||
bool isFlake = true;
|
||||
|
||||
LockedNode(
|
||||
const FlakeRef & lockedRef,
|
||||
const FlakeRef & originalRef,
|
||||
bool isFlake = true)
|
||||
: lockedRef(lockedRef), originalRef(originalRef), isFlake(isFlake)
|
||||
{ }
|
||||
|
||||
LockedNode(const nlohmann::json & json);
|
||||
|
||||
StorePath computeStorePath(Store & store) const;
|
||||
};
|
||||
|
||||
struct LockFile
|
||||
{
|
||||
std::shared_ptr<Node> root = std::make_shared<Node>();
|
||||
|
||||
LockFile() {};
|
||||
LockFile(const nlohmann::json & json, const Path & path);
|
||||
|
||||
nlohmann::json toJson() const;
|
||||
|
||||
std::string to_string() const;
|
||||
|
||||
static LockFile read(const Path & path);
|
||||
|
||||
void write(const Path & path) const;
|
||||
|
||||
bool isImmutable() const;
|
||||
|
||||
bool operator ==(const LockFile & other) const;
|
||||
|
||||
std::shared_ptr<Node> findInput(const InputPath & path);
|
||||
|
||||
std::map<InputPath, Node::Edge> getAllInputs() const;
|
||||
|
||||
static std::string diff(const LockFile & oldLocks, const LockFile & newLocks);
|
||||
|
||||
/* Check that every 'follows' input target exists. */
|
||||
void check();
|
||||
};
|
||||
|
||||
std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile);
|
||||
|
||||
InputPath parseInputPath(std::string_view s);
|
||||
|
||||
std::string printInputPath(const InputPath & path);
|
||||
|
||||
}
|
@ -39,7 +39,7 @@ DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPat
|
||||
if (i == drv.outputs.end())
|
||||
throw Error("derivation '%s' does not have output '%s'", store->printStorePath(drvPath), outputName);
|
||||
|
||||
outPath = store->printStorePath(i->second.path);
|
||||
outPath = store->printStorePath(i->second.path(*store, drv.name));
|
||||
}
|
||||
|
||||
|
||||
|
@ -4,7 +4,12 @@ libexpr_NAME = libnixexpr
|
||||
|
||||
libexpr_DIR := $(d)
|
||||
|
||||
libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc
|
||||
libexpr_SOURCES := \
|
||||
$(wildcard $(d)/*.cc) \
|
||||
$(wildcard $(d)/primops/*.cc) \
|
||||
$(wildcard $(d)/flake/*.cc) \
|
||||
$(d)/lexer-tab.cc \
|
||||
$(d)/parser-tab.cc
|
||||
|
||||
libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr
|
||||
|
||||
@ -34,4 +39,9 @@ dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer
|
||||
|
||||
$(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
|
||||
|
||||
$(foreach i, $(wildcard src/libexpr/flake/*.hh), \
|
||||
$(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
|
||||
|
||||
$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh
|
||||
|
||||
$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh
|
||||
|
@ -197,7 +197,22 @@ std::ostream & operator << (std::ostream & str, const Pos & pos)
|
||||
if (!pos)
|
||||
str << "undefined position";
|
||||
else
|
||||
str << (format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%") % (string) pos.file % pos.line % pos.column).str();
|
||||
{
|
||||
auto f = format(ANSI_BOLD "%1%" ANSI_NORMAL ":%2%:%3%");
|
||||
switch (pos.origin) {
|
||||
case foFile:
|
||||
f % (string) pos.file;
|
||||
break;
|
||||
case foStdin:
|
||||
case foString:
|
||||
f % "(string)";
|
||||
break;
|
||||
default:
|
||||
throw Error("unhandled Pos origin!");
|
||||
}
|
||||
str << (f % pos.line % pos.column).str();
|
||||
}
|
||||
|
||||
return str;
|
||||
}
|
||||
|
||||
@ -270,7 +285,7 @@ void ExprVar::bindVars(const StaticEnv & env)
|
||||
if (withLevel == -1)
|
||||
throw UndefinedVarError({
|
||||
.hint = hintfmt("undefined variable '%1%'", name),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
fromWith = true;
|
||||
this->level = withLevel;
|
||||
|
@ -24,11 +24,12 @@ MakeError(RestrictedPathError, Error);
|
||||
|
||||
struct Pos
|
||||
{
|
||||
FileOrigin origin;
|
||||
Symbol file;
|
||||
unsigned int line, column;
|
||||
Pos() : line(0), column(0) { };
|
||||
Pos(const Symbol & file, unsigned int line, unsigned int column)
|
||||
: file(file), line(line), column(column) { };
|
||||
Pos() : origin(foString), line(0), column(0) { };
|
||||
Pos(FileOrigin origin, const Symbol & file, unsigned int line, unsigned int column)
|
||||
: origin(origin), file(file), line(line), column(column) { };
|
||||
operator bool() const
|
||||
{
|
||||
return line != 0;
|
||||
@ -238,7 +239,7 @@ struct ExprLambda : Expr
|
||||
if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
|
||||
throw ParseError({
|
||||
.hint = hintfmt("duplicate formal function argument '%1%'", arg),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
};
|
||||
void setName(Symbol & name);
|
||||
|
@ -30,7 +30,8 @@ namespace nix {
|
||||
SymbolTable & symbols;
|
||||
Expr * result;
|
||||
Path basePath;
|
||||
Symbol path;
|
||||
Symbol file;
|
||||
FileOrigin origin;
|
||||
ErrorInfo error;
|
||||
Symbol sLetBody;
|
||||
ParseData(EvalState & state)
|
||||
@ -67,16 +68,15 @@ static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prev
|
||||
throw ParseError({
|
||||
.hint = hintfmt("attribute '%1%' already defined at %2%",
|
||||
showAttrPath(attrPath), prevPos),
|
||||
.nixCode = NixCode { .errPos = pos },
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
|
||||
{
|
||||
throw ParseError({
|
||||
.hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
|
||||
.nixCode = NixCode { .errPos = pos },
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
@ -148,7 +148,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
|
||||
throw ParseError({
|
||||
.hint = hintfmt("duplicate formal function argument '%1%'",
|
||||
formal.name),
|
||||
.nixCode = NixCode { .errPos = pos },
|
||||
.errPos = pos
|
||||
});
|
||||
formals->formals.push_front(formal);
|
||||
}
|
||||
@ -246,7 +246,7 @@ static Expr * stripIndentation(const Pos & pos, SymbolTable & symbols, vector<Ex
|
||||
|
||||
static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
|
||||
{
|
||||
return Pos(data->path, loc.first_line, loc.first_column);
|
||||
return Pos(data->origin, data->file, loc.first_line, loc.first_column);
|
||||
}
|
||||
|
||||
#define CUR_POS makeCurPos(*yylocp, data)
|
||||
@ -259,7 +259,7 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err
|
||||
{
|
||||
data->error = {
|
||||
.hint = hintfmt(error),
|
||||
.nixCode = NixCode { .errPos = makeCurPos(*loc, data) }
|
||||
.errPos = makeCurPos(*loc, data)
|
||||
};
|
||||
}
|
||||
|
||||
@ -339,7 +339,7 @@ expr_function
|
||||
{ if (!$2->dynamicAttrs.empty())
|
||||
throw ParseError({
|
||||
.hint = hintfmt("dynamic attributes not allowed in let"),
|
||||
.nixCode = NixCode { .errPos = CUR_POS },
|
||||
.errPos = CUR_POS
|
||||
});
|
||||
$$ = new ExprLet($2, $4);
|
||||
}
|
||||
@ -419,7 +419,7 @@ expr_simple
|
||||
if (noURLLiterals)
|
||||
throw ParseError({
|
||||
.hint = hintfmt("URL literals are disabled"),
|
||||
.nixCode = NixCode { .errPos = CUR_POS }
|
||||
.errPos = CUR_POS
|
||||
});
|
||||
$$ = new ExprString(data->symbols.create($1));
|
||||
}
|
||||
@ -492,7 +492,7 @@ attrs
|
||||
} else
|
||||
throw ParseError({
|
||||
.hint = hintfmt("dynamic attributes not allowed in inherit"),
|
||||
.nixCode = NixCode { .errPos = makeCurPos(@2, data) },
|
||||
.errPos = makeCurPos(@2, data)
|
||||
});
|
||||
}
|
||||
| { $$ = new AttrPath; }
|
||||
@ -569,13 +569,24 @@ formal
|
||||
namespace nix {
|
||||
|
||||
|
||||
Expr * EvalState::parse(const char * text,
|
||||
Expr * EvalState::parse(const char * text, FileOrigin origin,
|
||||
const Path & path, const Path & basePath, StaticEnv & staticEnv)
|
||||
{
|
||||
yyscan_t scanner;
|
||||
ParseData data(*this);
|
||||
data.origin = origin;
|
||||
switch (origin) {
|
||||
case foFile:
|
||||
data.file = data.symbols.create(path);
|
||||
break;
|
||||
case foStdin:
|
||||
case foString:
|
||||
data.file = data.symbols.create(text);
|
||||
break;
|
||||
default:
|
||||
assert(false);
|
||||
}
|
||||
data.basePath = basePath;
|
||||
data.path = data.symbols.create(path);
|
||||
|
||||
yylex_init(&scanner);
|
||||
yy_scan_string(text, scanner);
|
||||
@ -625,13 +636,13 @@ Expr * EvalState::parseExprFromFile(const Path & path)
|
||||
|
||||
Expr * EvalState::parseExprFromFile(const Path & path, StaticEnv & staticEnv)
|
||||
{
|
||||
return parse(readFile(path).c_str(), path, dirOf(path), staticEnv);
|
||||
return parse(readFile(path).c_str(), foFile, path, dirOf(path), staticEnv);
|
||||
}
|
||||
|
||||
|
||||
Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath, StaticEnv & staticEnv)
|
||||
{
|
||||
return parse(s.data(), "(string)", basePath, staticEnv);
|
||||
return parse(s.data(), foString, "", basePath, staticEnv);
|
||||
}
|
||||
|
||||
|
||||
@ -644,7 +655,7 @@ Expr * EvalState::parseExprFromString(std::string_view s, const Path & basePath)
|
||||
Expr * EvalState::parseStdin()
|
||||
{
|
||||
//Activity act(*logger, lvlTalkative, format("parsing standard input"));
|
||||
return parseExprFromString(drainFD(0), absPath("."));
|
||||
return parse(drainFD(0).data(), foStdin, "", absPath("."), staticBaseEnv);
|
||||
}
|
||||
|
||||
|
||||
@ -693,7 +704,7 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
|
||||
? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
|
||||
: "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
|
||||
path),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
@ -708,7 +719,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
|
||||
if (isUri(elem.second)) {
|
||||
try {
|
||||
res = { true, store->toRealPath(fetchers::downloadTarball(
|
||||
store, resolveUri(elem.second), "source", false).storePath) };
|
||||
store, resolveUri(elem.second), "source", false).first.storePath) };
|
||||
} catch (FileTransferError & e) {
|
||||
logWarning({
|
||||
.name = "Entry download",
|
||||
|
@ -30,18 +30,6 @@ namespace nix {
|
||||
*************************************************************/
|
||||
|
||||
|
||||
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
|
||||
name>. */
|
||||
std::pair<string, string> decodeContext(const string & s)
|
||||
{
|
||||
if (s.at(0) == '!') {
|
||||
size_t index = s.find("!", 1);
|
||||
return std::pair<string, string>(string(s, index + 1), string(s, 1, index - 1));
|
||||
} else
|
||||
return std::pair<string, string>(s.at(0) == '/' ? s : string(s, 1), "");
|
||||
}
|
||||
|
||||
|
||||
InvalidPathError::InvalidPathError(const Path & path) :
|
||||
EvalError("path '%s' is not valid", path), path(path) {}
|
||||
|
||||
@ -64,7 +52,7 @@ void EvalState::realiseContext(const PathSet & context)
|
||||
DerivationOutputs::iterator i = drv.outputs.find(outputName);
|
||||
if (i == drv.outputs.end())
|
||||
throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName);
|
||||
allowedPaths->insert(store->printStorePath(i->second.path));
|
||||
allowedPaths->insert(store->printStorePath(i->second.path(*store, drv.name)));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -77,7 +65,7 @@ void EvalState::realiseContext(const PathSet & context)
|
||||
|
||||
/* For performance, prefetch all substitute info. */
|
||||
StorePathSet willBuild, willSubstitute, unknown;
|
||||
unsigned long long downloadSize, narSize;
|
||||
uint64_t downloadSize, narSize;
|
||||
store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
|
||||
store->buildPaths(drvs);
|
||||
@ -96,15 +84,24 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
|
||||
} catch (InvalidPathError & e) {
|
||||
throw EvalError({
|
||||
.hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
Path realPath = state.checkSourcePath(state.toRealPath(path, context));
|
||||
|
||||
// FIXME
|
||||
if (state.store->isStorePath(path) && state.store->isValidPath(state.store->parseStorePath(path)) && isDerivation(path)) {
|
||||
Derivation drv = readDerivation(*state.store, realPath);
|
||||
auto isValidDerivationInStore = [&]() -> std::optional<StorePath> {
|
||||
if (!state.store->isStorePath(path))
|
||||
return std::nullopt;
|
||||
auto storePath = state.store->parseStorePath(path);
|
||||
if (!(state.store->isValidPath(storePath) && isDerivation(path)))
|
||||
return std::nullopt;
|
||||
return storePath;
|
||||
};
|
||||
if (auto optStorePath = isValidDerivationInStore()) {
|
||||
auto storePath = *optStorePath;
|
||||
Derivation drv = readDerivation(*state.store, realPath, Derivation::nameFromPath(storePath));
|
||||
Value & w = *state.allocValue();
|
||||
state.mkAttrs(w, 3 + drv.outputs.size());
|
||||
Value * v2 = state.allocAttr(w, state.sDrvPath);
|
||||
@ -118,7 +115,7 @@ static void prim_scopedImport(EvalState & state, const Pos & pos, Value * * args
|
||||
|
||||
for (const auto & o : drv.outputs) {
|
||||
v2 = state.allocAttr(w, state.symbols.create(o.first));
|
||||
mkString(*v2, state.store->printStorePath(o.second.path), {"!" + o.first + "!" + path});
|
||||
mkString(*v2, state.store->printStorePath(o.second.path(*state.store, drv.name)), {"!" + o.first + "!" + path});
|
||||
outputsVal->listElems()[outputs_index] = state.allocValue();
|
||||
mkString(*(outputsVal->listElems()[outputs_index++]), o.first);
|
||||
}
|
||||
@ -177,7 +174,7 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
|
||||
.hint = hintfmt(
|
||||
"cannot import '%1%', since path '%2%' is not valid",
|
||||
path, e.path),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
@ -215,7 +212,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
||||
if (count == 0) {
|
||||
throw EvalError({
|
||||
.hint = hintfmt("at least one argument to 'exec' required"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
PathSet context;
|
||||
@ -230,7 +227,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
||||
throw EvalError({
|
||||
.hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid",
|
||||
program, e.path),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
@ -239,13 +236,13 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
||||
try {
|
||||
parsed = state.parseExprFromString(output, pos.file);
|
||||
} catch (Error & e) {
|
||||
e.addPrefix(fmt("While parsing the output from '%1%', at %2%\n", program, pos));
|
||||
e.addTrace(pos, "While parsing the output from '%1%'", program);
|
||||
throw;
|
||||
}
|
||||
try {
|
||||
state.eval(parsed, v);
|
||||
} catch (Error & e) {
|
||||
e.addPrefix(fmt("While evaluating the output from '%1%', at %2%\n", program, pos));
|
||||
e.addTrace(pos, "While evaluating the output from '%1%'", program);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@ -385,7 +382,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
|
||||
if (startSet == args[0]->attrs->end())
|
||||
throw EvalError({
|
||||
.hint = hintfmt("attribute 'startSet' required"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
state.forceList(*startSet->value, pos);
|
||||
|
||||
@ -399,7 +396,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
|
||||
if (op == args[0]->attrs->end())
|
||||
throw EvalError({
|
||||
.hint = hintfmt("attribute 'operator' required"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
state.forceValue(*op->value, pos);
|
||||
|
||||
@ -421,7 +418,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
|
||||
if (key == e->attrs->end())
|
||||
throw EvalError({
|
||||
.hint = hintfmt("attribute 'key' required"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
state.forceValue(*key->value, pos);
|
||||
|
||||
@ -471,7 +468,7 @@ static void prim_addErrorContext(EvalState & state, const Pos & pos, Value * * a
|
||||
v = *args[1];
|
||||
} catch (Error & e) {
|
||||
PathSet context;
|
||||
e.addPrefix(format("%1%\n") % state.coerceToString(pos, *args[0], context));
|
||||
e.addTrace(std::nullopt, state.coerceToString(pos, *args[0], context));
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@ -556,14 +553,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
|
||||
if (attr == args[0]->attrs->end())
|
||||
throw EvalError({
|
||||
.hint = hintfmt("required attribute 'name' missing"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
string drvName;
|
||||
Pos & posDrvName(*attr->pos);
|
||||
try {
|
||||
drvName = state.forceStringNoCtx(*attr->value, pos);
|
||||
} catch (Error & e) {
|
||||
e.addPrefix(fmt("while evaluating the derivation attribute 'name' at %1%:\n", posDrvName));
|
||||
e.addTrace(posDrvName, "while evaluating the derivation attribute 'name'");
|
||||
throw;
|
||||
}
|
||||
|
||||
@ -582,6 +579,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
|
||||
|
||||
/* Build the derivation expression by processing the attributes. */
|
||||
Derivation drv;
|
||||
drv.name = drvName;
|
||||
|
||||
PathSet context;
|
||||
|
||||
@ -603,7 +601,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
|
||||
else
|
||||
throw EvalError({
|
||||
.hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
|
||||
.nixCode = NixCode { .errPos = posDrvName }
|
||||
.errPos = posDrvName
|
||||
});
|
||||
};
|
||||
|
||||
@ -613,7 +611,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
|
||||
if (outputs.find(j) != outputs.end())
|
||||
throw EvalError({
|
||||
.hint = hintfmt("duplicate derivation output '%1%'", j),
|
||||
.nixCode = NixCode { .errPos = posDrvName }
|
||||
.errPos = posDrvName
|
||||
});
|
||||
/* !!! Check whether j is a valid attribute
|
||||
name. */
|
||||
@ -623,14 +621,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
|
||||
if (j == "drv")
|
||||
throw EvalError({
|
||||
.hint = hintfmt("invalid derivation output name 'drv'" ),
|
||||
.nixCode = NixCode { .errPos = posDrvName }
|
||||
.errPos = posDrvName
|
||||
});
|
||||
outputs.insert(j);
|
||||
}
|
||||
if (outputs.empty())
|
||||
throw EvalError({
|
||||
.hint = hintfmt("derivation cannot have an empty set of outputs"),
|
||||
.nixCode = NixCode { .errPos = posDrvName }
|
||||
.errPos = posDrvName
|
||||
});
|
||||
};
|
||||
|
||||
@ -696,8 +694,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
|
||||
}
|
||||
|
||||
} catch (Error & e) {
|
||||
e.addPrefix(format("while evaluating the attribute '%1%' of the derivation '%2%' at %3%:\n")
|
||||
% key % drvName % posDrvName);
|
||||
e.addTrace(posDrvName,
|
||||
"while evaluating the attribute '%1%' of the derivation '%2%'",
|
||||
key, drvName);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
@ -745,20 +744,20 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
|
||||
if (drv.builder == "")
|
||||
throw EvalError({
|
||||
.hint = hintfmt("required attribute 'builder' missing"),
|
||||
.nixCode = NixCode { .errPos = posDrvName }
|
||||
.errPos = posDrvName
|
||||
});
|
||||
|
||||
if (drv.platform == "")
|
||||
throw EvalError({
|
||||
.hint = hintfmt("required attribute 'system' missing"),
|
||||
.nixCode = NixCode { .errPos = posDrvName }
|
||||
.errPos = posDrvName
|
||||
});
|
||||
|
||||
/* Check whether the derivation name is valid. */
|
||||
if (isDerivation(drvName))
|
||||
throw EvalError({
|
||||
.hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension),
|
||||
.nixCode = NixCode { .errPos = posDrvName }
|
||||
.errPos = posDrvName
|
||||
});
|
||||
|
||||
if (outputHash) {
|
||||
@ -766,7 +765,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
|
||||
if (outputs.size() != 1 || *(outputs.begin()) != "out")
|
||||
throw Error({
|
||||
.hint = hintfmt("multiple outputs are not supported in fixed-output derivations"),
|
||||
.nixCode = NixCode { .errPos = posDrvName }
|
||||
.errPos = posDrvName
|
||||
});
|
||||
|
||||
std::optional<HashType> ht = parseHashTypeOpt(outputHashAlgo);
|
||||
@ -775,11 +774,12 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
|
||||
auto outPath = state.store->makeFixedOutputPath(ingestionMethod, h, drvName);
|
||||
if (!jsonObject) drv.env["out"] = state.store->printStorePath(outPath);
|
||||
drv.outputs.insert_or_assign("out", DerivationOutput {
|
||||
.path = std::move(outPath),
|
||||
.output = DerivationOutputFixed {
|
||||
.hash = FixedOutputHash {
|
||||
.method = ingestionMethod,
|
||||
.hash = std::move(h),
|
||||
},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@ -794,8 +794,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
|
||||
if (!jsonObject) drv.env[i] = "";
|
||||
drv.outputs.insert_or_assign(i,
|
||||
DerivationOutput {
|
||||
.output = DerivationOutputInputAddressed {
|
||||
.path = StorePath::dummy,
|
||||
.hash = std::optional<FixedOutputHash> {},
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@ -806,8 +807,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
|
||||
if (!jsonObject) drv.env[i] = state.store->printStorePath(outPath);
|
||||
drv.outputs.insert_or_assign(i,
|
||||
DerivationOutput {
|
||||
.output = DerivationOutputInputAddressed {
|
||||
.path = std::move(outPath),
|
||||
.hash = std::optional<FixedOutputHash>(),
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -828,7 +830,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
|
||||
mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
|
||||
for (auto & i : drv.outputs) {
|
||||
mkString(*state.allocAttr(v, state.symbols.create(i.first)),
|
||||
state.store->printStorePath(i.second.path), {"!" + i.first + "!" + drvPathS});
|
||||
state.store->printStorePath(i.second.path(*state.store, drv.name)), {"!" + i.first + "!" + drvPathS});
|
||||
}
|
||||
v.attrs->sort();
|
||||
}
|
||||
@ -880,12 +882,12 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
|
||||
if (!state.store->isInStore(path))
|
||||
throw EvalError({
|
||||
.hint = hintfmt("path '%1%' is not in the Nix store", path),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
Path path2 = state.store->toStorePath(path);
|
||||
auto path2 = state.store->toStorePath(path).first;
|
||||
if (!settings.readOnlyMode)
|
||||
state.store->ensurePath(state.store->parseStorePath(path2));
|
||||
context.insert(path2);
|
||||
state.store->ensurePath(path2);
|
||||
context.insert(state.store->printStorePath(path2));
|
||||
mkString(v, path, context);
|
||||
}
|
||||
|
||||
@ -901,7 +903,7 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
|
||||
.hint = hintfmt(
|
||||
"cannot check the existence of '%1%', since path '%2%' is not valid",
|
||||
path, e.path),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
@ -947,7 +949,7 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
|
||||
} catch (InvalidPathError & e) {
|
||||
throw EvalError({
|
||||
.hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
string s = readFile(state.checkSourcePath(state.toRealPath(path, context)));
|
||||
@ -978,7 +980,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
|
||||
if (i == v2.attrs->end())
|
||||
throw EvalError({
|
||||
.hint = hintfmt("attribute 'path' missing"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
PathSet context;
|
||||
@ -989,7 +991,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
|
||||
} catch (InvalidPathError & e) {
|
||||
throw EvalError({
|
||||
.hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
@ -1009,7 +1011,7 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
|
||||
if (!ht)
|
||||
throw Error({
|
||||
.hint = hintfmt("unknown hash type '%1%'", type),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
PathSet context; // discarded
|
||||
@ -1028,7 +1030,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
|
||||
} catch (InvalidPathError & e) {
|
||||
throw EvalError({
|
||||
.hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
|
||||
@ -1104,7 +1106,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
|
||||
"in 'toFile': the file named '%1%' must not contain a reference "
|
||||
"to a derivation but contains (%2%)",
|
||||
name, path),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
refs.insert(state.store->parseStorePath(path));
|
||||
}
|
||||
@ -1122,7 +1124,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
|
||||
|
||||
|
||||
static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
|
||||
Value * filterFun, FileIngestionMethod method, const Hash & expectedHash, Value & v)
|
||||
Value * filterFun, FileIngestionMethod method, const std::optional<Hash> expectedHash, Value & v)
|
||||
{
|
||||
const auto path = evalSettings.pureEval && expectedHash ?
|
||||
path_ :
|
||||
@ -1153,7 +1155,7 @@ static void addPath(EvalState & state, const Pos & pos, const string & name, con
|
||||
|
||||
std::optional<StorePath> expectedStorePath;
|
||||
if (expectedHash)
|
||||
expectedStorePath = state.store->makeFixedOutputPath(method, expectedHash, name);
|
||||
expectedStorePath = state.store->makeFixedOutputPath(method, *expectedHash, name);
|
||||
Path dstPath;
|
||||
if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
|
||||
dstPath = state.store->printStorePath(settings.readOnlyMode
|
||||
@ -1175,7 +1177,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
|
||||
if (!context.empty())
|
||||
throw EvalError({
|
||||
.hint = hintfmt("string '%1%' cannot refer to other paths", path),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
state.forceValue(*args[0], pos);
|
||||
@ -1184,10 +1186,10 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
|
||||
.hint = hintfmt(
|
||||
"first argument in call to 'filterSource' is not a function but %1%",
|
||||
showType(*args[0])),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, Hash(), v);
|
||||
addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v);
|
||||
}
|
||||
|
||||
static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
||||
@ -1197,7 +1199,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
|
||||
string name;
|
||||
Value * filterFun = nullptr;
|
||||
auto method = FileIngestionMethod::Recursive;
|
||||
Hash expectedHash;
|
||||
Hash expectedHash(htSHA256);
|
||||
|
||||
for (auto & attr : *args[0]->attrs) {
|
||||
const string & n(attr.name);
|
||||
@ -1207,7 +1209,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
|
||||
if (!context.empty())
|
||||
throw EvalError({
|
||||
.hint = hintfmt("string '%1%' cannot refer to other paths", path),
|
||||
.nixCode = NixCode { .errPos = *attr.pos }
|
||||
.errPos = *attr.pos
|
||||
});
|
||||
} else if (attr.name == state.sName)
|
||||
name = state.forceStringNoCtx(*attr.value, *attr.pos);
|
||||
@ -1221,13 +1223,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
|
||||
else
|
||||
throw EvalError({
|
||||
.hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name),
|
||||
.nixCode = NixCode { .errPos = *attr.pos }
|
||||
.errPos = *attr.pos
|
||||
});
|
||||
}
|
||||
if (path.empty())
|
||||
throw EvalError({
|
||||
.hint = hintfmt("'path' required"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
if (name.empty())
|
||||
name = baseNameOf(path);
|
||||
@ -1288,7 +1290,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
||||
if (i == args[1]->attrs->end())
|
||||
throw EvalError({
|
||||
.hint = hintfmt("attribute '%1%' missing", attr),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
// !!! add to stack trace?
|
||||
if (state.countCalls && i->pos) state.attrSelects[*i->pos]++;
|
||||
@ -1371,7 +1373,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
|
||||
if (j == v2.attrs->end())
|
||||
throw TypeError({
|
||||
.hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
string name = state.forceStringNoCtx(*j->value, pos);
|
||||
|
||||
@ -1381,7 +1383,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
|
||||
if (j2 == v2.attrs->end())
|
||||
throw TypeError({
|
||||
.hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
v.attrs->push_back(Attr(sym, j2->value, j2->pos));
|
||||
}
|
||||
@ -1457,7 +1459,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
|
||||
if (args[0]->type != tLambda)
|
||||
throw TypeError({
|
||||
.hint = hintfmt("'functionArgs' requires a function"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
if (!args[0]->lambda.fun->matchAttrs) {
|
||||
@ -1513,7 +1515,7 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu
|
||||
if (n < 0 || (unsigned int) n >= list.listSize())
|
||||
throw Error({
|
||||
.hint = hintfmt("list index %1% is out of bounds", n),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
state.forceValue(*list.listElems()[n], pos);
|
||||
v = *list.listElems()[n];
|
||||
@ -1543,7 +1545,7 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
|
||||
if (args[0]->listSize() == 0)
|
||||
throw Error({
|
||||
.hint = hintfmt("'tail' called on an empty list"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
state.mkList(v, args[0]->listSize() - 1);
|
||||
@ -1688,7 +1690,7 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val
|
||||
if (len < 0)
|
||||
throw EvalError({
|
||||
.hint = hintfmt("cannot create list of size %1%", len),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
state.mkList(v, len);
|
||||
@ -1850,7 +1852,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
|
||||
if (f2 == 0)
|
||||
throw EvalError({
|
||||
.hint = hintfmt("division by zero"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
if (args[0]->type == tFloat || args[1]->type == tFloat) {
|
||||
@ -1862,7 +1864,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
|
||||
if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
|
||||
throw EvalError({
|
||||
.hint = hintfmt("overflow in integer division"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
mkInt(v, i1 / i2);
|
||||
@ -1923,7 +1925,7 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V
|
||||
if (start < 0)
|
||||
throw EvalError({
|
||||
.hint = hintfmt("negative start position in 'substring'"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
mkString(v, (unsigned int) start >= s.size() ? "" : string(s, start, len), context);
|
||||
@ -1946,7 +1948,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
|
||||
if (!ht)
|
||||
throw Error({
|
||||
.hint = hintfmt("unknown hash type '%1%'", type),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
PathSet context; // discarded
|
||||
@ -1992,12 +1994,12 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
||||
// limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
|
||||
throw EvalError({
|
||||
.hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
} else {
|
||||
throw EvalError({
|
||||
.hint = hintfmt("invalid regular expression '%s'", re),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -2065,12 +2067,12 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value
|
||||
// limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
|
||||
throw EvalError({
|
||||
.hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
} else {
|
||||
throw EvalError({
|
||||
.hint = hintfmt("invalid regular expression '%s'", re),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
}
|
||||
@ -2104,7 +2106,7 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
|
||||
if (args[0]->listSize() != args[1]->listSize())
|
||||
throw EvalError({
|
||||
.hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
vector<string> from;
|
||||
|
@ -148,7 +148,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
|
||||
if (!state.store->isStorePath(i.name))
|
||||
throw EvalError({
|
||||
.hint = hintfmt("Context key '%s' is not a store path", i.name),
|
||||
.nixCode = NixCode { .errPos = *i.pos }
|
||||
.errPos = *i.pos
|
||||
});
|
||||
if (!settings.readOnlyMode)
|
||||
state.store->ensurePath(state.store->parseStorePath(i.name));
|
||||
@ -165,7 +165,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
|
||||
if (!isDerivation(i.name)) {
|
||||
throw EvalError({
|
||||
.hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
|
||||
.nixCode = NixCode { .errPos = *i.pos }
|
||||
.errPos = *i.pos
|
||||
});
|
||||
}
|
||||
context.insert("=" + string(i.name));
|
||||
@ -178,7 +178,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
|
||||
if (iter->value->listSize() && !isDerivation(i.name)) {
|
||||
throw EvalError({
|
||||
.hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
|
||||
.nixCode = NixCode { .errPos = *i.pos }
|
||||
.errPos = *i.pos
|
||||
});
|
||||
}
|
||||
for (unsigned int n = 0; n < iter->value->listSize(); ++n) {
|
||||
|
@ -1,91 +0,0 @@
|
||||
#include "primops.hh"
|
||||
#include "eval-inline.hh"
|
||||
#include "store-api.hh"
|
||||
#include "hash.hh"
|
||||
#include "fetchers.hh"
|
||||
#include "url.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
||||
{
|
||||
std::string url;
|
||||
std::optional<std::string> ref;
|
||||
std::optional<Hash> rev;
|
||||
std::string name = "source";
|
||||
bool fetchSubmodules = false;
|
||||
PathSet context;
|
||||
|
||||
state.forceValue(*args[0]);
|
||||
|
||||
if (args[0]->type == tAttrs) {
|
||||
|
||||
state.forceAttrs(*args[0], pos);
|
||||
|
||||
for (auto & attr : *args[0]->attrs) {
|
||||
string n(attr.name);
|
||||
if (n == "url")
|
||||
url = state.coerceToString(*attr.pos, *attr.value, context, false, false);
|
||||
else if (n == "ref")
|
||||
ref = state.forceStringNoCtx(*attr.value, *attr.pos);
|
||||
else if (n == "rev")
|
||||
rev = Hash(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA1);
|
||||
else if (n == "name")
|
||||
name = state.forceStringNoCtx(*attr.value, *attr.pos);
|
||||
else if (n == "submodules")
|
||||
fetchSubmodules = state.forceBool(*attr.value, *attr.pos);
|
||||
else
|
||||
throw EvalError({
|
||||
.hint = hintfmt("unsupported argument '%s' to 'fetchGit'", attr.name),
|
||||
.nixCode = NixCode { .errPos = *attr.pos }
|
||||
});
|
||||
}
|
||||
|
||||
if (url.empty())
|
||||
throw EvalError({
|
||||
.hint = hintfmt("'url' argument required"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
});
|
||||
|
||||
} else
|
||||
url = state.coerceToString(pos, *args[0], context, false, false);
|
||||
|
||||
// FIXME: git externals probably can be used to bypass the URI
|
||||
// whitelist. Ah well.
|
||||
state.checkURI(url);
|
||||
|
||||
if (evalSettings.pureEval && !rev)
|
||||
throw Error("in pure evaluation mode, 'fetchGit' requires a Git revision");
|
||||
|
||||
fetchers::Attrs attrs;
|
||||
attrs.insert_or_assign("type", "git");
|
||||
attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
|
||||
if (ref) attrs.insert_or_assign("ref", *ref);
|
||||
if (rev) attrs.insert_or_assign("rev", rev->gitRev());
|
||||
if (fetchSubmodules) attrs.insert_or_assign("submodules", true);
|
||||
auto input = fetchers::inputFromAttrs(attrs);
|
||||
|
||||
// FIXME: use name?
|
||||
auto [tree, input2] = input->fetchTree(state.store);
|
||||
|
||||
state.mkAttrs(v, 8);
|
||||
auto storePath = state.store->printStorePath(tree.storePath);
|
||||
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
|
||||
// Backward compatibility: set 'rev' to
|
||||
// 0000000000000000000000000000000000000000 for a dirty tree.
|
||||
auto rev2 = input2->getRev().value_or(Hash(htSHA1));
|
||||
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
|
||||
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev2.gitShortRev());
|
||||
// Backward compatibility: set 'revCount' to 0 for a dirty tree.
|
||||
mkInt(*state.allocAttr(v, state.symbols.create("revCount")),
|
||||
tree.info.revCount.value_or(0));
|
||||
mkBool(*state.allocAttr(v, state.symbols.create("submodules")), fetchSubmodules);
|
||||
v.attrs->sort();
|
||||
|
||||
if (state.allowedPaths)
|
||||
state.allowedPaths->insert(tree.actualPath);
|
||||
}
|
||||
|
||||
static RegisterPrimOp r("fetchGit", 1, prim_fetchGit);
|
||||
|
||||
}
|
@ -40,14 +40,14 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
|
||||
else
|
||||
throw EvalError({
|
||||
.hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
|
||||
.nixCode = NixCode { .errPos = *attr.pos }
|
||||
.errPos = *attr.pos
|
||||
});
|
||||
}
|
||||
|
||||
if (url.empty())
|
||||
throw EvalError({
|
||||
.hint = hintfmt("'url' argument required"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
} else
|
||||
@ -65,23 +65,23 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
|
||||
attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url);
|
||||
if (ref) attrs.insert_or_assign("ref", *ref);
|
||||
if (rev) attrs.insert_or_assign("rev", rev->gitRev());
|
||||
auto input = fetchers::inputFromAttrs(attrs);
|
||||
auto input = fetchers::Input::fromAttrs(std::move(attrs));
|
||||
|
||||
// FIXME: use name
|
||||
auto [tree, input2] = input->fetchTree(state.store);
|
||||
auto [tree, input2] = input.fetch(state.store);
|
||||
|
||||
state.mkAttrs(v, 8);
|
||||
auto storePath = state.store->printStorePath(tree.storePath);
|
||||
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
|
||||
if (input2->getRef())
|
||||
mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2->getRef());
|
||||
if (input2.getRef())
|
||||
mkString(*state.allocAttr(v, state.symbols.create("branch")), *input2.getRef());
|
||||
// Backward compatibility: set 'rev' to
|
||||
// 0000000000000000000000000000000000000000 for a dirty tree.
|
||||
auto rev2 = input2->getRev().value_or(Hash(htSHA1));
|
||||
auto rev2 = input2.getRev().value_or(Hash(htSHA1));
|
||||
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev2.gitRev());
|
||||
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), std::string(rev2.gitRev(), 0, 12));
|
||||
if (tree.info.revCount)
|
||||
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
|
||||
if (auto revCount = input2.getRevCount())
|
||||
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
|
||||
v.attrs->sort();
|
||||
|
||||
if (state.allowedPaths)
|
||||
|
@ -3,6 +3,7 @@
|
||||
#include "store-api.hh"
|
||||
#include "fetchers.hh"
|
||||
#include "filetransfer.hh"
|
||||
#include "registry.hh"
|
||||
|
||||
#include <ctime>
|
||||
#include <iomanip>
|
||||
@ -12,39 +13,73 @@ namespace nix {
|
||||
void emitTreeAttrs(
|
||||
EvalState & state,
|
||||
const fetchers::Tree & tree,
|
||||
std::shared_ptr<const fetchers::Input> input,
|
||||
Value & v)
|
||||
const fetchers::Input & input,
|
||||
Value & v,
|
||||
bool emptyRevFallback)
|
||||
{
|
||||
assert(input.isImmutable());
|
||||
|
||||
state.mkAttrs(v, 8);
|
||||
|
||||
auto storePath = state.store->printStorePath(tree.storePath);
|
||||
|
||||
mkString(*state.allocAttr(v, state.sOutPath), storePath, PathSet({storePath}));
|
||||
|
||||
assert(tree.info.narHash);
|
||||
mkString(*state.allocAttr(v, state.symbols.create("narHash")),
|
||||
tree.info.narHash.to_string(SRI, true));
|
||||
// FIXME: support arbitrary input attributes.
|
||||
|
||||
if (input->getRev()) {
|
||||
mkString(*state.allocAttr(v, state.symbols.create("rev")), input->getRev()->gitRev());
|
||||
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), input->getRev()->gitShortRev());
|
||||
auto narHash = input.getNarHash();
|
||||
assert(narHash);
|
||||
mkString(*state.allocAttr(v, state.symbols.create("narHash")),
|
||||
narHash->to_string(SRI, true));
|
||||
|
||||
if (auto rev = input.getRev()) {
|
||||
mkString(*state.allocAttr(v, state.symbols.create("rev")), rev->gitRev());
|
||||
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), rev->gitShortRev());
|
||||
} else if (emptyRevFallback) {
|
||||
// Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
|
||||
auto emptyHash = Hash(htSHA1);
|
||||
mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
|
||||
mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitRev());
|
||||
}
|
||||
|
||||
if (tree.info.revCount)
|
||||
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount);
|
||||
if (input.getType() == "git")
|
||||
mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
|
||||
|
||||
if (tree.info.lastModified)
|
||||
mkString(*state.allocAttr(v, state.symbols.create("lastModified")),
|
||||
fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S")));
|
||||
if (auto revCount = input.getRevCount())
|
||||
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
|
||||
else if (emptyRevFallback)
|
||||
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), 0);
|
||||
|
||||
if (auto lastModified = input.getLastModified()) {
|
||||
mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *lastModified);
|
||||
mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")),
|
||||
fmt("%s", std::put_time(std::gmtime(&*lastModified), "%Y%m%d%H%M%S")));
|
||||
}
|
||||
|
||||
v.attrs->sort();
|
||||
}
|
||||
|
||||
static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
||||
std::string fixURI(std::string uri, EvalState &state)
|
||||
{
|
||||
settings.requireExperimentalFeature("flakes");
|
||||
state.checkURI(uri);
|
||||
return uri.find("://") != std::string::npos ? uri : "file://" + uri;
|
||||
}
|
||||
|
||||
std::shared_ptr<const fetchers::Input> input;
|
||||
void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
|
||||
{
|
||||
string n(name);
|
||||
attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
|
||||
}
|
||||
|
||||
static void fetchTree(
|
||||
EvalState &state,
|
||||
const Pos &pos,
|
||||
Value **args,
|
||||
Value &v,
|
||||
const std::optional<std::string> type,
|
||||
bool emptyRevFallback = false
|
||||
) {
|
||||
fetchers::Input input;
|
||||
PathSet context;
|
||||
|
||||
state.forceValue(*args[0]);
|
||||
@ -56,35 +91,65 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
|
||||
|
||||
for (auto & attr : *args[0]->attrs) {
|
||||
state.forceValue(*attr.value);
|
||||
if (attr.value->type == tString)
|
||||
attrs.emplace(attr.name, attr.value->string.s);
|
||||
if (attr.value->type == tPath || attr.value->type == tString)
|
||||
addURI(
|
||||
state,
|
||||
attrs,
|
||||
attr.name,
|
||||
state.coerceToString(*attr.pos, *attr.value, context, false, false)
|
||||
);
|
||||
else if (attr.value->type == tString)
|
||||
addURI(state, attrs, attr.name, attr.value->string.s);
|
||||
else if (attr.value->type == tBool)
|
||||
attrs.emplace(attr.name, attr.value->boolean);
|
||||
attrs.emplace(attr.name, fetchers::Explicit<bool>{attr.value->boolean});
|
||||
else if (attr.value->type == tInt)
|
||||
attrs.emplace(attr.name, attr.value->integer);
|
||||
else
|
||||
throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected",
|
||||
throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
|
||||
attr.name, showType(*attr.value));
|
||||
}
|
||||
|
||||
if (type)
|
||||
attrs.emplace("type", type.value());
|
||||
|
||||
if (!attrs.count("type"))
|
||||
throw Error({
|
||||
.hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
|
||||
input = fetchers::inputFromAttrs(attrs);
|
||||
} else
|
||||
input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false));
|
||||
input = fetchers::Input::fromAttrs(std::move(attrs));
|
||||
} else {
|
||||
auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state);
|
||||
|
||||
if (evalSettings.pureEval && !input->isImmutable())
|
||||
throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input");
|
||||
if (type == "git") {
|
||||
fetchers::Attrs attrs;
|
||||
attrs.emplace("type", "git");
|
||||
attrs.emplace("url", url);
|
||||
input = fetchers::Input::fromAttrs(std::move(attrs));
|
||||
} else {
|
||||
input = fetchers::Input::fromURL(url);
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME: use fetchOrSubstituteTree
|
||||
auto [tree, input2] = input->fetchTree(state.store);
|
||||
if (!evalSettings.pureEval && !input.isDirect())
|
||||
input = lookupInRegistries(state.store, input).first;
|
||||
|
||||
if (evalSettings.pureEval && !input.isImmutable())
|
||||
throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos);
|
||||
|
||||
auto [tree, input2] = input.fetch(state.store);
|
||||
|
||||
if (state.allowedPaths)
|
||||
state.allowedPaths->insert(tree.actualPath);
|
||||
|
||||
emitTreeAttrs(state, tree, input2, v);
|
||||
emitTreeAttrs(state, tree, input2, v, emptyRevFallback);
|
||||
}
|
||||
|
||||
static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v)
|
||||
{
|
||||
settings.requireExperimentalFeature("flakes");
|
||||
fetchTree(state, pos, args, v, std::nullopt);
|
||||
}
|
||||
|
||||
static RegisterPrimOp r("fetchTree", 1, prim_fetchTree);
|
||||
@ -112,14 +177,14 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
|
||||
else
|
||||
throw EvalError({
|
||||
.hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
|
||||
.nixCode = NixCode { .errPos = *attr.pos }
|
||||
.errPos = *attr.pos
|
||||
});
|
||||
}
|
||||
|
||||
if (!url)
|
||||
throw EvalError({
|
||||
.hint = hintfmt("'url' argument required"),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
} else
|
||||
url = state.forceStringNoCtx(*args[0], pos);
|
||||
@ -136,7 +201,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
|
||||
|
||||
auto storePath =
|
||||
unpack
|
||||
? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).storePath
|
||||
? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
|
||||
: fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
|
||||
|
||||
auto path = state.store->toRealPath(storePath);
|
||||
@ -147,7 +212,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
|
||||
: hashFile(htSHA256, path);
|
||||
if (hash != *expectedHash)
|
||||
throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
|
||||
*url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
|
||||
*url, expectedHash->to_string(Base32, true), hash->to_string(Base32, true));
|
||||
}
|
||||
|
||||
if (state.allowedPaths)
|
||||
@ -166,7 +231,13 @@ static void prim_fetchTarball(EvalState & state, const Pos & pos, Value * * args
|
||||
fetch(state, pos, args, v, "fetchTarball", true, "source");
|
||||
}
|
||||
|
||||
static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
|
||||
{
|
||||
fetchTree(state, pos, args, v, "git", true);
|
||||
}
|
||||
|
||||
static RegisterPrimOp r2("__fetchurl", 1, prim_fetchurl);
|
||||
static RegisterPrimOp r3("fetchTarball", 1, prim_fetchTarball);
|
||||
static RegisterPrimOp r4("fetchGit", 1, prim_fetchGit);
|
||||
|
||||
}
|
||||
|
@ -83,7 +83,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
|
||||
} catch (std::runtime_error & e) {
|
||||
throw EvalError({
|
||||
.hint = hintfmt("while parsing a TOML string: %s", e.what()),
|
||||
.nixCode = NixCode { .errPos = pos }
|
||||
.errPos = pos
|
||||
});
|
||||
}
|
||||
}
|
||||
|
@ -28,6 +28,12 @@ public:
|
||||
return s == s2.s;
|
||||
}
|
||||
|
||||
// FIXME: remove
|
||||
bool operator == (std::string_view s2) const
|
||||
{
|
||||
return s->compare(s2) == 0;
|
||||
}
|
||||
|
||||
bool operator != (const Symbol & s2) const
|
||||
{
|
||||
return s != s2.s;
|
||||
@ -68,9 +74,10 @@ private:
|
||||
Symbols symbols;
|
||||
|
||||
public:
|
||||
Symbol create(const string & s)
|
||||
Symbol create(std::string_view s)
|
||||
{
|
||||
std::pair<Symbols::iterator, bool> res = symbols.insert(s);
|
||||
// FIXME: avoid allocation if 's' already exists in the symbol table.
|
||||
std::pair<Symbols::iterator, bool> res = symbols.emplace(std::string(s));
|
||||
return Symbol(&*res.first);
|
||||
}
|
||||
|
||||
|
@ -166,6 +166,13 @@ struct Value
|
||||
{
|
||||
return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
|
||||
}
|
||||
|
||||
/* Check whether forcing this value requires a trivial amount of
|
||||
computation. In particular, function applications are
|
||||
non-trivial. */
|
||||
bool isTrivial() const;
|
||||
|
||||
std::vector<std::pair<Path, std::string>> getContext();
|
||||
};
|
||||
|
||||
|
||||
|
@ -27,7 +27,7 @@ nlohmann::json attrsToJson(const Attrs & attrs)
|
||||
{
|
||||
nlohmann::json json;
|
||||
for (auto & attr : attrs) {
|
||||
if (auto v = std::get_if<int64_t>(&attr.second)) {
|
||||
if (auto v = std::get_if<uint64_t>(&attr.second)) {
|
||||
json[attr.first] = *v;
|
||||
} else if (auto v = std::get_if<std::string>(&attr.second)) {
|
||||
json[attr.first] = *v;
|
||||
@ -55,16 +55,16 @@ std::string getStrAttr(const Attrs & attrs, const std::string & name)
|
||||
return *s;
|
||||
}
|
||||
|
||||
std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
|
||||
std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name)
|
||||
{
|
||||
auto i = attrs.find(name);
|
||||
if (i == attrs.end()) return {};
|
||||
if (auto v = std::get_if<int64_t>(&i->second))
|
||||
if (auto v = std::get_if<uint64_t>(&i->second))
|
||||
return *v;
|
||||
throw Error("input attribute '%s' is not an integer", name);
|
||||
}
|
||||
|
||||
int64_t getIntAttr(const Attrs & attrs, const std::string & name)
|
||||
uint64_t getIntAttr(const Attrs & attrs, const std::string & name)
|
||||
{
|
||||
auto s = maybeGetIntAttr(attrs, name);
|
||||
if (!s)
|
||||
@ -76,8 +76,8 @@ std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & na
|
||||
{
|
||||
auto i = attrs.find(name);
|
||||
if (i == attrs.end()) return {};
|
||||
if (auto v = std::get_if<int64_t>(&i->second))
|
||||
return *v;
|
||||
if (auto v = std::get_if<Explicit<bool>>(&i->second))
|
||||
return v->t;
|
||||
throw Error("input attribute '%s' is not a Boolean", name);
|
||||
}
|
||||
|
||||
@ -93,7 +93,7 @@ std::map<std::string, std::string> attrsToQuery(const Attrs & attrs)
|
||||
{
|
||||
std::map<std::string, std::string> query;
|
||||
for (auto & attr : attrs) {
|
||||
if (auto v = std::get_if<int64_t>(&attr.second)) {
|
||||
if (auto v = std::get_if<uint64_t>(&attr.second)) {
|
||||
query.insert_or_assign(attr.first, fmt("%d", *v));
|
||||
} else if (auto v = std::get_if<std::string>(&attr.second)) {
|
||||
query.insert_or_assign(attr.first, *v);
|
||||
|
@ -13,9 +13,14 @@ namespace nix::fetchers {
|
||||
template<typename T>
|
||||
struct Explicit {
|
||||
T t;
|
||||
|
||||
bool operator ==(const Explicit<T> & other) const
|
||||
{
|
||||
return t == other.t;
|
||||
}
|
||||
};
|
||||
|
||||
typedef std::variant<std::string, int64_t, Explicit<bool>> Attr;
|
||||
typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr;
|
||||
typedef std::map<std::string, Attr> Attrs;
|
||||
|
||||
Attrs jsonToAttrs(const nlohmann::json & json);
|
||||
@ -26,9 +31,9 @@ std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::strin
|
||||
|
||||
std::string getStrAttr(const Attrs & attrs, const std::string & name);
|
||||
|
||||
std::optional<int64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
|
||||
std::optional<uint64_t> maybeGetIntAttr(const Attrs & attrs, const std::string & name);
|
||||
|
||||
int64_t getIntAttr(const Attrs & attrs, const std::string & name);
|
||||
uint64_t getIntAttr(const Attrs & attrs, const std::string & name);
|
||||
|
||||
std::optional<bool> maybeGetBoolAttr(const Attrs & attrs, const std::string & name);
|
||||
|
||||
|
@ -6,6 +6,8 @@ namespace nix::fetchers {
|
||||
|
||||
struct Cache
|
||||
{
|
||||
virtual ~Cache() { }
|
||||
|
||||
virtual void add(
|
||||
ref<Store> store,
|
||||
const Attrs & inAttrs,
|
||||
|
@ -5,71 +5,265 @@
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
std::unique_ptr<std::vector<std::unique_ptr<InputScheme>>> inputSchemes = nullptr;
|
||||
std::unique_ptr<std::vector<std::shared_ptr<InputScheme>>> inputSchemes = nullptr;
|
||||
|
||||
void registerInputScheme(std::unique_ptr<InputScheme> && inputScheme)
|
||||
void registerInputScheme(std::shared_ptr<InputScheme> && inputScheme)
|
||||
{
|
||||
if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::unique_ptr<InputScheme>>>();
|
||||
if (!inputSchemes) inputSchemes = std::make_unique<std::vector<std::shared_ptr<InputScheme>>>();
|
||||
inputSchemes->push_back(std::move(inputScheme));
|
||||
}
|
||||
|
||||
std::unique_ptr<Input> inputFromURL(const ParsedURL & url)
|
||||
Input Input::fromURL(const std::string & url)
|
||||
{
|
||||
return fromURL(parseURL(url));
|
||||
}
|
||||
|
||||
static void fixupInput(Input & input)
|
||||
{
|
||||
// Check common attributes.
|
||||
input.getType();
|
||||
input.getRef();
|
||||
if (input.getRev())
|
||||
input.immutable = true;
|
||||
input.getRevCount();
|
||||
input.getLastModified();
|
||||
if (input.getNarHash())
|
||||
input.immutable = true;
|
||||
}
|
||||
|
||||
Input Input::fromURL(const ParsedURL & url)
|
||||
{
|
||||
for (auto & inputScheme : *inputSchemes) {
|
||||
auto res = inputScheme->inputFromURL(url);
|
||||
if (res) return res;
|
||||
if (res) {
|
||||
res->scheme = inputScheme;
|
||||
fixupInput(*res);
|
||||
return std::move(*res);
|
||||
}
|
||||
}
|
||||
|
||||
throw Error("input '%s' is unsupported", url.url);
|
||||
}
|
||||
|
||||
std::unique_ptr<Input> inputFromURL(const std::string & url)
|
||||
Input Input::fromAttrs(Attrs && attrs)
|
||||
{
|
||||
return inputFromURL(parseURL(url));
|
||||
for (auto & inputScheme : *inputSchemes) {
|
||||
auto res = inputScheme->inputFromAttrs(attrs);
|
||||
if (res) {
|
||||
res->scheme = inputScheme;
|
||||
fixupInput(*res);
|
||||
return std::move(*res);
|
||||
}
|
||||
}
|
||||
|
||||
Input input;
|
||||
input.attrs = attrs;
|
||||
fixupInput(input);
|
||||
return input;
|
||||
}
|
||||
|
||||
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs)
|
||||
ParsedURL Input::toURL() const
|
||||
{
|
||||
auto attrs2(attrs);
|
||||
attrs2.erase("narHash");
|
||||
for (auto & inputScheme : *inputSchemes) {
|
||||
auto res = inputScheme->inputFromAttrs(attrs2);
|
||||
if (res) {
|
||||
if (auto narHash = maybeGetStrAttr(attrs, "narHash"))
|
||||
// FIXME: require SRI hash.
|
||||
res->narHash = newHashAllowEmpty(*narHash, {});
|
||||
return res;
|
||||
}
|
||||
}
|
||||
throw Error("input '%s' is unsupported", attrsToJson(attrs));
|
||||
if (!scheme)
|
||||
throw Error("cannot show unsupported input '%s'", attrsToJson(attrs));
|
||||
return scheme->toURL(*this);
|
||||
}
|
||||
|
||||
std::string Input::to_string() const
|
||||
{
|
||||
return toURL().to_string();
|
||||
}
|
||||
|
||||
Attrs Input::toAttrs() const
|
||||
{
|
||||
auto attrs = toAttrsInternal();
|
||||
if (narHash)
|
||||
attrs.emplace("narHash", narHash->to_string(SRI, true));
|
||||
attrs.emplace("type", type());
|
||||
return attrs;
|
||||
}
|
||||
|
||||
std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) const
|
||||
bool Input::hasAllInfo() const
|
||||
{
|
||||
auto [tree, input] = fetchTreeInternal(store);
|
||||
return getNarHash() && scheme && scheme->hasAllInfo(*this);
|
||||
}
|
||||
|
||||
bool Input::operator ==(const Input & other) const
|
||||
{
|
||||
return attrs == other.attrs;
|
||||
}
|
||||
|
||||
bool Input::contains(const Input & other) const
|
||||
{
|
||||
if (*this == other) return true;
|
||||
auto other2(other);
|
||||
other2.attrs.erase("ref");
|
||||
other2.attrs.erase("rev");
|
||||
if (*this == other2) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
std::pair<Tree, Input> Input::fetch(ref<Store> store) const
|
||||
{
|
||||
if (!scheme)
|
||||
throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs()));
|
||||
|
||||
/* The tree may already be in the Nix store, or it could be
|
||||
substituted (which is often faster than fetching from the
|
||||
original source). So check that. */
|
||||
if (hasAllInfo()) {
|
||||
try {
|
||||
auto storePath = computeStorePath(*store);
|
||||
|
||||
store->ensurePath(storePath);
|
||||
|
||||
debug("using substituted/cached input '%s' in '%s'",
|
||||
to_string(), store->printStorePath(storePath));
|
||||
|
||||
auto actualPath = store->toRealPath(storePath);
|
||||
|
||||
return {fetchers::Tree(std::move(actualPath), std::move(storePath)), *this};
|
||||
} catch (Error & e) {
|
||||
debug("substitution of input '%s' failed: %s", to_string(), e.what());
|
||||
}
|
||||
}
|
||||
|
||||
auto [tree, input] = scheme->fetch(store, *this);
|
||||
|
||||
if (tree.actualPath == "")
|
||||
tree.actualPath = store->toRealPath(tree.storePath);
|
||||
|
||||
if (!tree.info.narHash)
|
||||
tree.info.narHash = store->queryPathInfo(tree.storePath)->narHash;
|
||||
auto narHash = store->queryPathInfo(tree.storePath)->narHash;
|
||||
input.attrs.insert_or_assign("narHash", narHash->to_string(SRI, true));
|
||||
|
||||
if (input->narHash)
|
||||
assert(input->narHash == tree.info.narHash);
|
||||
if (auto prevNarHash = getNarHash()) {
|
||||
if (narHash != *prevNarHash)
|
||||
throw Error((unsigned int) 102, "NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
|
||||
to_string(), tree.actualPath, prevNarHash->to_string(SRI, true), narHash->to_string(SRI, true));
|
||||
}
|
||||
|
||||
if (narHash && narHash != input->narHash)
|
||||
throw Error("NAR hash mismatch in input '%s' (%s), expected '%s', got '%s'",
|
||||
to_string(), tree.actualPath, narHash->to_string(SRI, true), input->narHash->to_string(SRI, true));
|
||||
if (auto prevLastModified = getLastModified()) {
|
||||
if (input.getLastModified() != prevLastModified)
|
||||
throw Error("'lastModified' attribute mismatch in input '%s', expected %d",
|
||||
input.to_string(), *prevLastModified);
|
||||
}
|
||||
|
||||
if (auto prevRevCount = getRevCount()) {
|
||||
if (input.getRevCount() != prevRevCount)
|
||||
throw Error("'revCount' attribute mismatch in input '%s', expected %d",
|
||||
input.to_string(), *prevRevCount);
|
||||
}
|
||||
|
||||
input.immutable = true;
|
||||
|
||||
assert(input.hasAllInfo());
|
||||
|
||||
return {std::move(tree), input};
|
||||
}
|
||||
|
||||
Input Input::applyOverrides(
|
||||
std::optional<std::string> ref,
|
||||
std::optional<Hash> rev) const
|
||||
{
|
||||
if (!scheme) return *this;
|
||||
return scheme->applyOverrides(*this, ref, rev);
|
||||
}
|
||||
|
||||
void Input::clone(const Path & destDir) const
|
||||
{
|
||||
assert(scheme);
|
||||
scheme->clone(*this, destDir);
|
||||
}
|
||||
|
||||
std::optional<Path> Input::getSourcePath() const
|
||||
{
|
||||
assert(scheme);
|
||||
return scheme->getSourcePath(*this);
|
||||
}
|
||||
|
||||
void Input::markChangedFile(
|
||||
std::string_view file,
|
||||
std::optional<std::string> commitMsg) const
|
||||
{
|
||||
assert(scheme);
|
||||
return scheme->markChangedFile(*this, file, commitMsg);
|
||||
}
|
||||
|
||||
StorePath Input::computeStorePath(Store & store) const
|
||||
{
|
||||
auto narHash = getNarHash();
|
||||
if (!narHash)
|
||||
throw Error("cannot compute store path for mutable input '%s'", to_string());
|
||||
return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source");
|
||||
}
|
||||
|
||||
std::string Input::getType() const
|
||||
{
|
||||
return getStrAttr(attrs, "type");
|
||||
}
|
||||
|
||||
std::optional<Hash> Input::getNarHash() const
|
||||
{
|
||||
if (auto s = maybeGetStrAttr(attrs, "narHash"))
|
||||
// FIXME: require SRI hash.
|
||||
return newHashAllowEmpty(*s, htSHA256);
|
||||
return {};
|
||||
}
|
||||
|
||||
std::optional<std::string> Input::getRef() const
|
||||
{
|
||||
if (auto s = maybeGetStrAttr(attrs, "ref"))
|
||||
return *s;
|
||||
return {};
|
||||
}
|
||||
|
||||
std::optional<Hash> Input::getRev() const
|
||||
{
|
||||
if (auto s = maybeGetStrAttr(attrs, "rev"))
|
||||
return Hash(*s, htSHA1);
|
||||
return {};
|
||||
}
|
||||
|
||||
std::optional<uint64_t> Input::getRevCount() const
|
||||
{
|
||||
if (auto n = maybeGetIntAttr(attrs, "revCount"))
|
||||
return *n;
|
||||
return {};
|
||||
}
|
||||
|
||||
std::optional<time_t> Input::getLastModified() const
|
||||
{
|
||||
if (auto n = maybeGetIntAttr(attrs, "lastModified"))
|
||||
return *n;
|
||||
return {};
|
||||
}
|
||||
|
||||
ParsedURL InputScheme::toURL(const Input & input)
|
||||
{
|
||||
throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs));
|
||||
}
|
||||
|
||||
Input InputScheme::applyOverrides(
|
||||
const Input & input,
|
||||
std::optional<std::string> ref,
|
||||
std::optional<Hash> rev)
|
||||
{
|
||||
if (ref)
|
||||
throw Error("don't know how to set branch/tag name of input '%s' to '%s'", input.to_string(), *ref);
|
||||
if (rev)
|
||||
throw Error("don't know how to set revision of input '%s' to '%s'", input.to_string(), rev->gitRev());
|
||||
return input;
|
||||
}
|
||||
|
||||
std::optional<Path> InputScheme::getSourcePath(const Input & input)
|
||||
{
|
||||
return {};
|
||||
}
|
||||
|
||||
void InputScheme::markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg)
|
||||
{
|
||||
assert(false);
|
||||
}
|
||||
|
||||
void InputScheme::clone(const Input & input, const Path & destDir)
|
||||
{
|
||||
throw Error("do not know how to clone input '%s'", input.to_string());
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -3,7 +3,6 @@
|
||||
#include "types.hh"
|
||||
#include "hash.hh"
|
||||
#include "path.hh"
|
||||
#include "tree-info.hh"
|
||||
#include "attrs.hh"
|
||||
#include "url.hh"
|
||||
|
||||
@ -13,73 +12,101 @@ namespace nix { class Store; }
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
struct Input;
|
||||
|
||||
struct Tree
|
||||
{
|
||||
Path actualPath;
|
||||
StorePath storePath;
|
||||
TreeInfo info;
|
||||
Tree(Path && actualPath, StorePath && storePath) : actualPath(actualPath), storePath(std::move(storePath)) {}
|
||||
};
|
||||
|
||||
struct Input : std::enable_shared_from_this<Input>
|
||||
struct InputScheme;
|
||||
|
||||
struct Input
|
||||
{
|
||||
std::optional<Hash> narHash; // FIXME: implement
|
||||
friend class InputScheme;
|
||||
|
||||
virtual std::string type() const = 0;
|
||||
std::shared_ptr<InputScheme> scheme; // note: can be null
|
||||
Attrs attrs;
|
||||
bool immutable = false;
|
||||
bool direct = true;
|
||||
|
||||
virtual ~Input() { }
|
||||
public:
|
||||
static Input fromURL(const std::string & url);
|
||||
|
||||
virtual bool operator ==(const Input & other) const { return false; }
|
||||
static Input fromURL(const ParsedURL & url);
|
||||
|
||||
/* Check whether this is a "direct" input, that is, not
|
||||
one that goes through a registry. */
|
||||
virtual bool isDirect() const { return true; }
|
||||
static Input fromAttrs(Attrs && attrs);
|
||||
|
||||
/* Check whether this is an "immutable" input, that is,
|
||||
one that contains a commit hash or content hash. */
|
||||
virtual bool isImmutable() const { return (bool) narHash; }
|
||||
ParsedURL toURL() const;
|
||||
|
||||
virtual bool contains(const Input & other) const { return false; }
|
||||
|
||||
virtual std::optional<std::string> getRef() const { return {}; }
|
||||
|
||||
virtual std::optional<Hash> getRev() const { return {}; }
|
||||
|
||||
virtual ParsedURL toURL() const = 0;
|
||||
|
||||
std::string to_string() const
|
||||
{
|
||||
return toURL().to_string();
|
||||
}
|
||||
std::string to_string() const;
|
||||
|
||||
Attrs toAttrs() const;
|
||||
|
||||
std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const;
|
||||
/* Check whether this is a "direct" input, that is, not
|
||||
one that goes through a registry. */
|
||||
bool isDirect() const { return direct; }
|
||||
|
||||
private:
|
||||
/* Check whether this is an "immutable" input, that is,
|
||||
one that contains a commit hash or content hash. */
|
||||
bool isImmutable() const { return immutable; }
|
||||
|
||||
virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0;
|
||||
bool hasAllInfo() const;
|
||||
|
||||
virtual Attrs toAttrsInternal() const = 0;
|
||||
bool operator ==(const Input & other) const;
|
||||
|
||||
bool contains(const Input & other) const;
|
||||
|
||||
std::pair<Tree, Input> fetch(ref<Store> store) const;
|
||||
|
||||
Input applyOverrides(
|
||||
std::optional<std::string> ref,
|
||||
std::optional<Hash> rev) const;
|
||||
|
||||
void clone(const Path & destDir) const;
|
||||
|
||||
std::optional<Path> getSourcePath() const;
|
||||
|
||||
void markChangedFile(
|
||||
std::string_view file,
|
||||
std::optional<std::string> commitMsg) const;
|
||||
|
||||
StorePath computeStorePath(Store & store) const;
|
||||
|
||||
// Convience functions for common attributes.
|
||||
std::string getType() const;
|
||||
std::optional<Hash> getNarHash() const;
|
||||
std::optional<std::string> getRef() const;
|
||||
std::optional<Hash> getRev() const;
|
||||
std::optional<uint64_t> getRevCount() const;
|
||||
std::optional<time_t> getLastModified() const;
|
||||
};
|
||||
|
||||
struct InputScheme
|
||||
{
|
||||
virtual ~InputScheme() { }
|
||||
virtual std::optional<Input> inputFromURL(const ParsedURL & url) = 0;
|
||||
|
||||
virtual std::unique_ptr<Input> inputFromURL(const ParsedURL & url) = 0;
|
||||
virtual std::optional<Input> inputFromAttrs(const Attrs & attrs) = 0;
|
||||
|
||||
virtual std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) = 0;
|
||||
virtual ParsedURL toURL(const Input & input);
|
||||
|
||||
virtual bool hasAllInfo(const Input & input) = 0;
|
||||
|
||||
virtual Input applyOverrides(
|
||||
const Input & input,
|
||||
std::optional<std::string> ref,
|
||||
std::optional<Hash> rev);
|
||||
|
||||
virtual void clone(const Input & input, const Path & destDir);
|
||||
|
||||
virtual std::optional<Path> getSourcePath(const Input & input);
|
||||
|
||||
virtual void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg);
|
||||
|
||||
virtual std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) = 0;
|
||||
};
|
||||
|
||||
std::unique_ptr<Input> inputFromURL(const ParsedURL & url);
|
||||
|
||||
std::unique_ptr<Input> inputFromURL(const std::string & url);
|
||||
|
||||
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs);
|
||||
|
||||
void registerInputScheme(std::unique_ptr<InputScheme> && fetcher);
|
||||
void registerInputScheme(std::shared_ptr<InputScheme> && fetcher);
|
||||
|
||||
struct DownloadFileResult
|
||||
{
|
||||
@ -94,7 +121,7 @@ DownloadFileResult downloadFile(
|
||||
const std::string & name,
|
||||
bool immutable);
|
||||
|
||||
Tree downloadTarball(
|
||||
std::pair<Tree, time_t> downloadTarball(
|
||||
ref<Store> store,
|
||||
const std::string & url,
|
||||
const std::string & name,
|
||||
|
@ -22,80 +22,152 @@ static bool isNotDotGitDirectory(const Path & path)
|
||||
return not std::regex_match(path, gitDirRegex);
|
||||
}
|
||||
|
||||
struct GitInput : Input
|
||||
struct GitInputScheme : InputScheme
|
||||
{
|
||||
ParsedURL url;
|
||||
std::optional<std::string> ref;
|
||||
std::optional<Hash> rev;
|
||||
bool shallow = false;
|
||||
bool submodules = false;
|
||||
|
||||
GitInput(const ParsedURL & url) : url(url)
|
||||
{ }
|
||||
|
||||
std::string type() const override { return "git"; }
|
||||
|
||||
bool operator ==(const Input & other) const override
|
||||
std::optional<Input> inputFromURL(const ParsedURL & url) override
|
||||
{
|
||||
auto other2 = dynamic_cast<const GitInput *>(&other);
|
||||
return
|
||||
other2
|
||||
&& url == other2->url
|
||||
&& rev == other2->rev
|
||||
&& ref == other2->ref;
|
||||
}
|
||||
if (url.scheme != "git" &&
|
||||
url.scheme != "git+http" &&
|
||||
url.scheme != "git+https" &&
|
||||
url.scheme != "git+ssh" &&
|
||||
url.scheme != "git+file") return {};
|
||||
|
||||
bool isImmutable() const override
|
||||
{
|
||||
return (bool) rev || narHash;
|
||||
}
|
||||
auto url2(url);
|
||||
if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
|
||||
url2.query.clear();
|
||||
|
||||
std::optional<std::string> getRef() const override { return ref; }
|
||||
|
||||
std::optional<Hash> getRev() const override { return rev; }
|
||||
|
||||
ParsedURL toURL() const override
|
||||
{
|
||||
ParsedURL url2(url);
|
||||
if (url2.scheme != "git") url2.scheme = "git+" + url2.scheme;
|
||||
if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
|
||||
if (ref) url2.query.insert_or_assign("ref", *ref);
|
||||
if (shallow) url2.query.insert_or_assign("shallow", "1");
|
||||
return url2;
|
||||
}
|
||||
|
||||
Attrs toAttrsInternal() const override
|
||||
{
|
||||
Attrs attrs;
|
||||
attrs.emplace("url", url.to_string());
|
||||
if (ref)
|
||||
attrs.emplace("ref", *ref);
|
||||
if (rev)
|
||||
attrs.emplace("rev", rev->gitRev());
|
||||
if (shallow)
|
||||
attrs.emplace("shallow", true);
|
||||
if (submodules)
|
||||
attrs.emplace("submodules", true);
|
||||
return attrs;
|
||||
attrs.emplace("type", "git");
|
||||
|
||||
for (auto &[name, value] : url.query) {
|
||||
if (name == "rev" || name == "ref")
|
||||
attrs.emplace(name, value);
|
||||
else if (name == "shallow")
|
||||
attrs.emplace(name, Explicit<bool> { value == "1" });
|
||||
else
|
||||
url2.query.emplace(name, value);
|
||||
}
|
||||
|
||||
std::pair<bool, std::string> getActualUrl() const
|
||||
attrs.emplace("url", url2.to_string());
|
||||
|
||||
return inputFromAttrs(attrs);
|
||||
}
|
||||
|
||||
std::optional<Input> inputFromAttrs(const Attrs & attrs) override
|
||||
{
|
||||
if (maybeGetStrAttr(attrs, "type") != "git") return {};
|
||||
|
||||
for (auto & [name, value] : attrs)
|
||||
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash")
|
||||
throw Error("unsupported Git input attribute '%s'", name);
|
||||
|
||||
parseURL(getStrAttr(attrs, "url"));
|
||||
maybeGetBoolAttr(attrs, "shallow");
|
||||
maybeGetBoolAttr(attrs, "submodules");
|
||||
|
||||
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
|
||||
if (std::regex_search(*ref, badGitRefRegex))
|
||||
throw BadURL("invalid Git branch/tag name '%s'", *ref);
|
||||
}
|
||||
|
||||
Input input;
|
||||
input.attrs = attrs;
|
||||
return input;
|
||||
}
|
||||
|
||||
ParsedURL toURL(const Input & input) override
|
||||
{
|
||||
auto url = parseURL(getStrAttr(input.attrs, "url"));
|
||||
if (url.scheme != "git") url.scheme = "git+" + url.scheme;
|
||||
if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
|
||||
if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
|
||||
if (maybeGetBoolAttr(input.attrs, "shallow").value_or(false))
|
||||
url.query.insert_or_assign("shallow", "1");
|
||||
return url;
|
||||
}
|
||||
|
||||
bool hasAllInfo(const Input & input) override
|
||||
{
|
||||
bool maybeDirty = !input.getRef();
|
||||
bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
|
||||
return
|
||||
maybeGetIntAttr(input.attrs, "lastModified")
|
||||
&& (shallow || maybeDirty || maybeGetIntAttr(input.attrs, "revCount"));
|
||||
}
|
||||
|
||||
Input applyOverrides(
|
||||
const Input & input,
|
||||
std::optional<std::string> ref,
|
||||
std::optional<Hash> rev) override
|
||||
{
|
||||
auto res(input);
|
||||
if (rev) res.attrs.insert_or_assign("rev", rev->gitRev());
|
||||
if (ref) res.attrs.insert_or_assign("ref", *ref);
|
||||
if (!res.getRef() && res.getRev())
|
||||
throw Error("Git input '%s' has a commit hash but no branch/tag name", res.to_string());
|
||||
return res;
|
||||
}
|
||||
|
||||
void clone(const Input & input, const Path & destDir) override
|
||||
{
|
||||
auto [isLocal, actualUrl] = getActualUrl(input);
|
||||
|
||||
Strings args = {"clone"};
|
||||
|
||||
args.push_back(actualUrl);
|
||||
|
||||
if (auto ref = input.getRef()) {
|
||||
args.push_back("--branch");
|
||||
args.push_back(*ref);
|
||||
}
|
||||
|
||||
if (input.getRev()) throw Error("cloning a specific revision is not implemented");
|
||||
|
||||
args.push_back(destDir);
|
||||
|
||||
runProgram("git", true, args);
|
||||
}
|
||||
|
||||
std::optional<Path> getSourcePath(const Input & input) override
|
||||
{
|
||||
auto url = parseURL(getStrAttr(input.attrs, "url"));
|
||||
if (url.scheme == "file" && !input.getRef() && !input.getRev())
|
||||
return url.path;
|
||||
return {};
|
||||
}
|
||||
|
||||
void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
|
||||
{
|
||||
auto sourcePath = getSourcePath(input);
|
||||
assert(sourcePath);
|
||||
|
||||
runProgram("git", true,
|
||||
{ "-C", *sourcePath, "add", "--force", "--intent-to-add", "--", std::string(file) });
|
||||
|
||||
if (commitMsg)
|
||||
runProgram("git", true,
|
||||
{ "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg });
|
||||
}
|
||||
|
||||
std::pair<bool, std::string> getActualUrl(const Input & input) const
|
||||
{
|
||||
// Don't clone file:// URIs (but otherwise treat them the
|
||||
// same as remote URIs, i.e. don't use the working tree or
|
||||
// HEAD).
|
||||
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
|
||||
auto url = parseURL(getStrAttr(input.attrs, "url"));
|
||||
bool isLocal = url.scheme == "file" && !forceHttp;
|
||||
return {isLocal, isLocal ? url.path : url.base};
|
||||
}
|
||||
|
||||
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
|
||||
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
|
||||
{
|
||||
auto name = "source";
|
||||
|
||||
auto input = std::make_shared<GitInput>(*this);
|
||||
Input input(_input);
|
||||
|
||||
assert(!rev || rev->type == htSHA1);
|
||||
bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
|
||||
bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
|
||||
|
||||
std::string cacheType = "git";
|
||||
if (shallow) cacheType += "-shallow";
|
||||
@ -106,39 +178,35 @@ struct GitInput : Input
|
||||
return Attrs({
|
||||
{"type", cacheType},
|
||||
{"name", name},
|
||||
{"rev", input->rev->gitRev()},
|
||||
{"rev", input.getRev()->gitRev()},
|
||||
});
|
||||
};
|
||||
|
||||
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
|
||||
-> std::pair<Tree, std::shared_ptr<const Input>>
|
||||
-> std::pair<Tree, Input>
|
||||
{
|
||||
assert(input->rev);
|
||||
assert(!rev || rev == input->rev);
|
||||
assert(input.getRev());
|
||||
assert(!_input.getRev() || _input.getRev() == input.getRev());
|
||||
if (!shallow)
|
||||
input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
|
||||
input.attrs.insert_or_assign("lastModified", getIntAttr(infoAttrs, "lastModified"));
|
||||
return {
|
||||
Tree {
|
||||
.actualPath = store->toRealPath(storePath),
|
||||
.storePath = std::move(storePath),
|
||||
.info = TreeInfo {
|
||||
.revCount = shallow ? std::nullopt : std::optional(getIntAttr(infoAttrs, "revCount")),
|
||||
.lastModified = getIntAttr(infoAttrs, "lastModified"),
|
||||
},
|
||||
},
|
||||
Tree(store->toRealPath(storePath), std::move(storePath)),
|
||||
input
|
||||
};
|
||||
};
|
||||
|
||||
if (rev) {
|
||||
if (input.getRev()) {
|
||||
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
|
||||
return makeResult(res->first, std::move(res->second));
|
||||
}
|
||||
|
||||
auto [isLocal, actualUrl_] = getActualUrl();
|
||||
auto [isLocal, actualUrl_] = getActualUrl(input);
|
||||
auto actualUrl = actualUrl_; // work around clang bug
|
||||
|
||||
// If this is a local directory and no ref or revision is
|
||||
// given, then allow the use of an unclean working tree.
|
||||
if (!input->ref && !input->rev && isLocal) {
|
||||
if (!input.getRef() && !input.getRev() && isLocal) {
|
||||
bool clean = false;
|
||||
|
||||
/* Check whether this repo has any commits. There are
|
||||
@ -197,35 +265,35 @@ struct GitInput : Input
|
||||
|
||||
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
|
||||
|
||||
auto tree = Tree {
|
||||
.actualPath = store->printStorePath(storePath),
|
||||
.storePath = std::move(storePath),
|
||||
.info = TreeInfo {
|
||||
// FIXME: maybe we should use the timestamp of the last
|
||||
// modified dirty file?
|
||||
.lastModified = haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0,
|
||||
}
|
||||
input.attrs.insert_or_assign(
|
||||
"lastModified",
|
||||
haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "HEAD" })) : 0);
|
||||
|
||||
return {
|
||||
Tree(store->printStorePath(storePath), std::move(storePath)),
|
||||
input
|
||||
};
|
||||
|
||||
return {std::move(tree), input};
|
||||
}
|
||||
}
|
||||
|
||||
if (!input->ref) input->ref = isLocal ? readHead(actualUrl) : "master";
|
||||
if (!input.getRef()) input.attrs.insert_or_assign("ref", isLocal ? readHead(actualUrl) : "master");
|
||||
|
||||
Attrs mutableAttrs({
|
||||
{"type", cacheType},
|
||||
{"name", name},
|
||||
{"url", actualUrl},
|
||||
{"ref", *input->ref},
|
||||
{"ref", *input.getRef()},
|
||||
});
|
||||
|
||||
Path repoDir;
|
||||
|
||||
if (isLocal) {
|
||||
|
||||
if (!input->rev)
|
||||
input->rev = Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input->ref })), htSHA1);
|
||||
if (!input.getRev())
|
||||
input.attrs.insert_or_assign("rev",
|
||||
Hash(chomp(runProgram("git", true, { "-C", actualUrl, "rev-parse", *input.getRef() })), htSHA1).gitRev());
|
||||
|
||||
repoDir = actualUrl;
|
||||
|
||||
@ -233,8 +301,8 @@ struct GitInput : Input
|
||||
|
||||
if (auto res = getCache()->lookup(store, mutableAttrs)) {
|
||||
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
|
||||
if (!rev || rev == rev2) {
|
||||
input->rev = rev2;
|
||||
if (!input.getRev() || input.getRev() == rev2) {
|
||||
input.attrs.insert_or_assign("rev", rev2.gitRev());
|
||||
return makeResult(res->first, std::move(res->second));
|
||||
}
|
||||
}
|
||||
@ -248,18 +316,18 @@ struct GitInput : Input
|
||||
}
|
||||
|
||||
Path localRefFile =
|
||||
input->ref->compare(0, 5, "refs/") == 0
|
||||
? cacheDir + "/" + *input->ref
|
||||
: cacheDir + "/refs/heads/" + *input->ref;
|
||||
input.getRef()->compare(0, 5, "refs/") == 0
|
||||
? cacheDir + "/" + *input.getRef()
|
||||
: cacheDir + "/refs/heads/" + *input.getRef();
|
||||
|
||||
bool doFetch;
|
||||
time_t now = time(0);
|
||||
|
||||
/* If a rev was specified, we need to fetch if it's not in the
|
||||
repo. */
|
||||
if (input->rev) {
|
||||
if (input.getRev()) {
|
||||
try {
|
||||
runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input->rev->gitRev() });
|
||||
runProgram("git", true, { "-C", repoDir, "cat-file", "-e", input.getRev()->gitRev() });
|
||||
doFetch = false;
|
||||
} catch (ExecError & e) {
|
||||
if (WIFEXITED(e.status)) {
|
||||
@ -282,9 +350,10 @@ struct GitInput : Input
|
||||
// FIXME: git stderr messes up our progress indicator, so
|
||||
// we're using --quiet for now. Should process its stderr.
|
||||
try {
|
||||
auto fetchRef = input->ref->compare(0, 5, "refs/") == 0
|
||||
? *input->ref
|
||||
: "refs/heads/" + *input->ref;
|
||||
auto ref = input.getRef();
|
||||
auto fetchRef = ref->compare(0, 5, "refs/") == 0
|
||||
? *ref
|
||||
: "refs/heads/" + *ref;
|
||||
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
|
||||
} catch (Error & e) {
|
||||
if (!pathExists(localRefFile)) throw;
|
||||
@ -300,8 +369,8 @@ struct GitInput : Input
|
||||
utimes(localRefFile.c_str(), times);
|
||||
}
|
||||
|
||||
if (!input->rev)
|
||||
input->rev = Hash(chomp(readFile(localRefFile)), htSHA1);
|
||||
if (!input.getRev())
|
||||
input.attrs.insert_or_assign("rev", Hash(chomp(readFile(localRefFile)), htSHA1).gitRev());
|
||||
}
|
||||
|
||||
bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true";
|
||||
@ -311,7 +380,7 @@ struct GitInput : Input
|
||||
|
||||
// FIXME: check whether rev is an ancestor of ref.
|
||||
|
||||
printTalkative("using revision %s of repo '%s'", input->rev->gitRev(), actualUrl);
|
||||
printTalkative("using revision %s of repo '%s'", input.getRev()->gitRev(), actualUrl);
|
||||
|
||||
/* Now that we know the ref, check again whether we have it in
|
||||
the store. */
|
||||
@ -333,7 +402,7 @@ struct GitInput : Input
|
||||
runProgram("git", true, { "-C", tmpDir, "fetch", "--quiet", "--force",
|
||||
"--update-head-ok", "--", repoDir, "refs/*:refs/*" });
|
||||
|
||||
runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input->rev->gitRev() });
|
||||
runProgram("git", true, { "-C", tmpDir, "checkout", "--quiet", input.getRev()->gitRev() });
|
||||
runProgram("git", true, { "-C", tmpDir, "remote", "add", "origin", actualUrl });
|
||||
runProgram("git", true, { "-C", tmpDir, "submodule", "--quiet", "update", "--init", "--recursive" });
|
||||
|
||||
@ -342,7 +411,7 @@ struct GitInput : Input
|
||||
// FIXME: should pipe this, or find some better way to extract a
|
||||
// revision.
|
||||
auto source = sinkToSource([&](Sink & sink) {
|
||||
RunOptions gitOptions("git", { "-C", repoDir, "archive", input->rev->gitRev() });
|
||||
RunOptions gitOptions("git", { "-C", repoDir, "archive", input.getRev()->gitRev() });
|
||||
gitOptions.standardOut = &sink;
|
||||
runProgram2(gitOptions);
|
||||
});
|
||||
@ -352,18 +421,18 @@ struct GitInput : Input
|
||||
|
||||
auto storePath = store->addToStore(name, tmpDir, FileIngestionMethod::Recursive, htSHA256, filter);
|
||||
|
||||
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input->rev->gitRev() }));
|
||||
auto lastModified = std::stoull(runProgram("git", true, { "-C", repoDir, "log", "-1", "--format=%ct", input.getRev()->gitRev() }));
|
||||
|
||||
Attrs infoAttrs({
|
||||
{"rev", input->rev->gitRev()},
|
||||
{"rev", input.getRev()->gitRev()},
|
||||
{"lastModified", lastModified},
|
||||
});
|
||||
|
||||
if (!shallow)
|
||||
infoAttrs.insert_or_assign("revCount",
|
||||
std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input->rev->gitRev() })));
|
||||
std::stoull(runProgram("git", true, { "-C", repoDir, "rev-list", "--count", input.getRev()->gitRev() })));
|
||||
|
||||
if (!this->rev)
|
||||
if (!_input.getRev())
|
||||
getCache()->add(
|
||||
store,
|
||||
mutableAttrs,
|
||||
@ -382,60 +451,6 @@ struct GitInput : Input
|
||||
}
|
||||
};
|
||||
|
||||
struct GitInputScheme : InputScheme
|
||||
{
|
||||
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
|
||||
{
|
||||
if (url.scheme != "git" &&
|
||||
url.scheme != "git+http" &&
|
||||
url.scheme != "git+https" &&
|
||||
url.scheme != "git+ssh" &&
|
||||
url.scheme != "git+file") return nullptr;
|
||||
|
||||
auto url2(url);
|
||||
if (hasPrefix(url2.scheme, "git+")) url2.scheme = std::string(url2.scheme, 4);
|
||||
url2.query.clear();
|
||||
|
||||
Attrs attrs;
|
||||
attrs.emplace("type", "git");
|
||||
|
||||
for (auto &[name, value] : url.query) {
|
||||
if (name == "rev" || name == "ref")
|
||||
attrs.emplace(name, value);
|
||||
else
|
||||
url2.query.emplace(name, value);
|
||||
}
|
||||
|
||||
attrs.emplace("url", url2.to_string());
|
||||
|
||||
return inputFromAttrs(attrs);
|
||||
}
|
||||
|
||||
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
|
||||
{
|
||||
if (maybeGetStrAttr(attrs, "type") != "git") return {};
|
||||
|
||||
for (auto & [name, value] : attrs)
|
||||
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules")
|
||||
throw Error("unsupported Git input attribute '%s'", name);
|
||||
|
||||
auto input = std::make_unique<GitInput>(parseURL(getStrAttr(attrs, "url")));
|
||||
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
|
||||
if (std::regex_search(*ref, badGitRefRegex))
|
||||
throw BadURL("invalid Git branch/tag name '%s'", *ref);
|
||||
input->ref = *ref;
|
||||
}
|
||||
if (auto rev = maybeGetStrAttr(attrs, "rev"))
|
||||
input->rev = Hash(*rev, htSHA1);
|
||||
|
||||
input->shallow = maybeGetBoolAttr(attrs, "shallow").value_or(false);
|
||||
|
||||
input->submodules = maybeGetBoolAttr(attrs, "submodules").value_or(false);
|
||||
|
||||
return input;
|
||||
}
|
||||
};
|
||||
|
||||
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitInputScheme>()); });
|
||||
|
||||
}
|
||||
|
@ -8,81 +8,142 @@
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
std::regex ownerRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
|
||||
std::regex repoRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
|
||||
// A github or gitlab url
|
||||
const static std::string urlRegexS = "[a-zA-Z0-9.]*"; // FIXME: check
|
||||
std::regex urlRegex(urlRegexS, std::regex::ECMAScript);
|
||||
|
||||
struct GitHubInput : Input
|
||||
struct GitArchiveInputScheme : InputScheme
|
||||
{
|
||||
std::string owner;
|
||||
std::string repo;
|
||||
std::optional<std::string> ref;
|
||||
virtual std::string type() = 0;
|
||||
|
||||
std::optional<Input> inputFromURL(const ParsedURL & url) override
|
||||
{
|
||||
if (url.scheme != type()) return {};
|
||||
|
||||
auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
|
||||
|
||||
std::optional<Hash> rev;
|
||||
std::optional<std::string> ref;
|
||||
std::optional<std::string> host_url;
|
||||
|
||||
std::string type() const override { return "github"; }
|
||||
if (path.size() == 2) {
|
||||
} else if (path.size() == 3) {
|
||||
if (std::regex_match(path[2], revRegex))
|
||||
rev = Hash(path[2], htSHA1);
|
||||
else if (std::regex_match(path[2], refRegex))
|
||||
ref = path[2];
|
||||
else
|
||||
throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
|
||||
} else
|
||||
throw BadURL("URL '%s' is invalid", url.url);
|
||||
|
||||
bool operator ==(const Input & other) const override
|
||||
{
|
||||
auto other2 = dynamic_cast<const GitHubInput *>(&other);
|
||||
return
|
||||
other2
|
||||
&& owner == other2->owner
|
||||
&& repo == other2->repo
|
||||
&& rev == other2->rev
|
||||
&& ref == other2->ref;
|
||||
for (auto &[name, value] : url.query) {
|
||||
if (name == "rev") {
|
||||
if (rev)
|
||||
throw BadURL("URL '%s' contains multiple commit hashes", url.url);
|
||||
rev = Hash(value, htSHA1);
|
||||
}
|
||||
else if (name == "ref") {
|
||||
if (!std::regex_match(value, refRegex))
|
||||
throw BadURL("URL '%s' contains an invalid branch/tag name", url.url);
|
||||
if (ref)
|
||||
throw BadURL("URL '%s' contains multiple branch/tag names", url.url);
|
||||
ref = value;
|
||||
}
|
||||
else if (name == "url") {
|
||||
if (!std::regex_match(value, urlRegex))
|
||||
throw BadURL("URL '%s' contains an invalid instance url", url.url);
|
||||
host_url = value;
|
||||
}
|
||||
// FIXME: barf on unsupported attributes
|
||||
}
|
||||
|
||||
bool isImmutable() const override
|
||||
{
|
||||
return (bool) rev || narHash;
|
||||
if (ref && rev)
|
||||
throw BadURL("URL '%s' contains both a commit hash and a branch/tag name %s %s", url.url, *ref, rev->gitRev());
|
||||
|
||||
Input input;
|
||||
input.attrs.insert_or_assign("type", type());
|
||||
input.attrs.insert_or_assign("owner", path[0]);
|
||||
input.attrs.insert_or_assign("repo", path[1]);
|
||||
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
|
||||
if (ref) input.attrs.insert_or_assign("ref", *ref);
|
||||
if (host_url) input.attrs.insert_or_assign("url", *host_url);
|
||||
|
||||
return input;
|
||||
}
|
||||
|
||||
std::optional<std::string> getRef() const override { return ref; }
|
||||
|
||||
std::optional<Hash> getRev() const override { return rev; }
|
||||
|
||||
ParsedURL toURL() const override
|
||||
std::optional<Input> inputFromAttrs(const Attrs & attrs) override
|
||||
{
|
||||
if (maybeGetStrAttr(attrs, "type") != type()) return {};
|
||||
|
||||
for (auto & [name, value] : attrs)
|
||||
if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev" && name != "narHash" && name != "lastModified")
|
||||
throw Error("unsupported input attribute '%s'", name);
|
||||
|
||||
getStrAttr(attrs, "owner");
|
||||
getStrAttr(attrs, "repo");
|
||||
|
||||
Input input;
|
||||
input.attrs = attrs;
|
||||
return input;
|
||||
}
|
||||
|
||||
ParsedURL toURL(const Input & input) override
|
||||
{
|
||||
auto owner = getStrAttr(input.attrs, "owner");
|
||||
auto repo = getStrAttr(input.attrs, "repo");
|
||||
auto ref = input.getRef();
|
||||
auto rev = input.getRev();
|
||||
auto path = owner + "/" + repo;
|
||||
assert(!(ref && rev));
|
||||
if (ref) path += "/" + *ref;
|
||||
if (rev) path += "/" + rev->to_string(Base16, false);
|
||||
return ParsedURL {
|
||||
.scheme = "github",
|
||||
.scheme = type(),
|
||||
.path = path,
|
||||
};
|
||||
}
|
||||
|
||||
Attrs toAttrsInternal() const override
|
||||
bool hasAllInfo(const Input & input) override
|
||||
{
|
||||
Attrs attrs;
|
||||
attrs.emplace("owner", owner);
|
||||
attrs.emplace("repo", repo);
|
||||
if (ref)
|
||||
attrs.emplace("ref", *ref);
|
||||
if (rev)
|
||||
attrs.emplace("rev", rev->gitRev());
|
||||
return attrs;
|
||||
return input.getRev() && maybeGetIntAttr(input.attrs, "lastModified");
|
||||
}
|
||||
|
||||
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
|
||||
Input applyOverrides(
|
||||
const Input & _input,
|
||||
std::optional<std::string> ref,
|
||||
std::optional<Hash> rev) override
|
||||
{
|
||||
auto rev = this->rev;
|
||||
auto ref = this->ref.value_or("master");
|
||||
|
||||
if (!rev) {
|
||||
auto url = fmt("https://api.github.com/repos/%s/%s/commits/%s",
|
||||
owner, repo, ref);
|
||||
auto json = nlohmann::json::parse(
|
||||
readFile(
|
||||
store->toRealPath(
|
||||
downloadFile(store, url, "source", false).storePath)));
|
||||
rev = Hash(std::string { json["sha"] }, htSHA1);
|
||||
debug("HEAD revision for '%s' is %s", url, rev->gitRev());
|
||||
auto input(_input);
|
||||
if (rev && ref)
|
||||
throw BadURL("cannot apply both a commit hash (%s) and a branch/tag name ('%s') to input '%s'",
|
||||
rev->gitRev(), *ref, input.to_string());
|
||||
if (rev) {
|
||||
input.attrs.insert_or_assign("rev", rev->gitRev());
|
||||
input.attrs.erase("ref");
|
||||
}
|
||||
if (ref) {
|
||||
input.attrs.insert_or_assign("ref", *ref);
|
||||
input.attrs.erase("rev");
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
auto input = std::make_shared<GitHubInput>(*this);
|
||||
input->ref = {};
|
||||
input->rev = *rev;
|
||||
virtual Hash getRevFromRef(nix::ref<Store> store, const Input & input) const = 0;
|
||||
|
||||
virtual std::string getDownloadUrl(const Input & input) const = 0;
|
||||
|
||||
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
|
||||
{
|
||||
Input input(_input);
|
||||
|
||||
if (!maybeGetStrAttr(input.attrs, "ref")) input.attrs.insert_or_assign("ref", "HEAD");
|
||||
|
||||
auto rev = input.getRev();
|
||||
if (!rev) rev = getRevFromRef(store, input);
|
||||
|
||||
input.attrs.erase("ref");
|
||||
input.attrs.insert_or_assign("rev", rev->gitRev());
|
||||
|
||||
Attrs immutableAttrs({
|
||||
{"type", "git-tarball"},
|
||||
@ -90,36 +151,25 @@ struct GitHubInput : Input
|
||||
});
|
||||
|
||||
if (auto res = getCache()->lookup(store, immutableAttrs)) {
|
||||
input.attrs.insert_or_assign("lastModified", getIntAttr(res->first, "lastModified"));
|
||||
return {
|
||||
Tree{
|
||||
.actualPath = store->toRealPath(res->second),
|
||||
.storePath = std::move(res->second),
|
||||
.info = TreeInfo {
|
||||
.lastModified = getIntAttr(res->first, "lastModified"),
|
||||
},
|
||||
},
|
||||
Tree(store->toRealPath(res->second), std::move(res->second)),
|
||||
input
|
||||
};
|
||||
}
|
||||
|
||||
// FIXME: use regular /archive URLs instead? api.github.com
|
||||
// might have stricter rate limits.
|
||||
auto url = getDownloadUrl(input);
|
||||
|
||||
auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s",
|
||||
owner, repo, rev->to_string(Base16, false));
|
||||
auto [tree, lastModified] = downloadTarball(store, url, "source", true);
|
||||
|
||||
std::string accessToken = settings.githubAccessToken.get();
|
||||
if (accessToken != "")
|
||||
url += "?access_token=" + accessToken;
|
||||
|
||||
auto tree = downloadTarball(store, url, "source", true);
|
||||
input.attrs.insert_or_assign("lastModified", lastModified);
|
||||
|
||||
getCache()->add(
|
||||
store,
|
||||
immutableAttrs,
|
||||
{
|
||||
{"rev", rev->gitRev()},
|
||||
{"lastModified", *tree.info.lastModified}
|
||||
{"lastModified", lastModified}
|
||||
},
|
||||
tree.storePath,
|
||||
true);
|
||||
@ -128,68 +178,96 @@ struct GitHubInput : Input
|
||||
}
|
||||
};
|
||||
|
||||
struct GitHubInputScheme : InputScheme
|
||||
struct GitHubInputScheme : GitArchiveInputScheme
|
||||
{
|
||||
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
|
||||
std::string type() override { return "github"; }
|
||||
|
||||
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
|
||||
{
|
||||
if (url.scheme != "github") return nullptr;
|
||||
|
||||
auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
|
||||
auto input = std::make_unique<GitHubInput>();
|
||||
|
||||
if (path.size() == 2) {
|
||||
} else if (path.size() == 3) {
|
||||
if (std::regex_match(path[2], revRegex))
|
||||
input->rev = Hash(path[2], htSHA1);
|
||||
else if (std::regex_match(path[2], refRegex))
|
||||
input->ref = path[2];
|
||||
else
|
||||
throw BadURL("in GitHub URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
|
||||
} else
|
||||
throw BadURL("GitHub URL '%s' is invalid", url.url);
|
||||
|
||||
for (auto &[name, value] : url.query) {
|
||||
if (name == "rev") {
|
||||
if (input->rev)
|
||||
throw BadURL("GitHub URL '%s' contains multiple commit hashes", url.url);
|
||||
input->rev = Hash(value, htSHA1);
|
||||
}
|
||||
else if (name == "ref") {
|
||||
if (!std::regex_match(value, refRegex))
|
||||
throw BadURL("GitHub URL '%s' contains an invalid branch/tag name", url.url);
|
||||
if (input->ref)
|
||||
throw BadURL("GitHub URL '%s' contains multiple branch/tag names", url.url);
|
||||
input->ref = value;
|
||||
}
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
|
||||
auto url = fmt("https://api.%s/repos/%s/%s/commits/%s", // FIXME: check
|
||||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
|
||||
auto json = nlohmann::json::parse(
|
||||
readFile(
|
||||
store->toRealPath(
|
||||
downloadFile(store, url, "source", false).storePath)));
|
||||
auto rev = Hash(std::string { json["sha"] }, htSHA1);
|
||||
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
|
||||
return rev;
|
||||
}
|
||||
|
||||
if (input->ref && input->rev)
|
||||
throw BadURL("GitHub URL '%s' contains both a commit hash and a branch/tag name", url.url);
|
||||
|
||||
input->owner = path[0];
|
||||
input->repo = path[1];
|
||||
|
||||
return input;
|
||||
}
|
||||
|
||||
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
|
||||
std::string getDownloadUrl(const Input & input) const override
|
||||
{
|
||||
if (maybeGetStrAttr(attrs, "type") != "github") return {};
|
||||
// FIXME: use regular /archive URLs instead? api.github.com
|
||||
// might have stricter rate limits.
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
|
||||
auto url = fmt("https://api.%s/repos/%s/%s/tarball/%s", // FIXME: check if this is correct for self hosted instances
|
||||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
|
||||
input.getRev()->to_string(Base16, false));
|
||||
|
||||
for (auto & [name, value] : attrs)
|
||||
if (name != "type" && name != "owner" && name != "repo" && name != "ref" && name != "rev")
|
||||
throw Error("unsupported GitHub input attribute '%s'", name);
|
||||
std::string accessToken = settings.githubAccessToken.get();
|
||||
if (accessToken != "")
|
||||
url += "?access_token=" + accessToken;
|
||||
|
||||
auto input = std::make_unique<GitHubInput>();
|
||||
input->owner = getStrAttr(attrs, "owner");
|
||||
input->repo = getStrAttr(attrs, "repo");
|
||||
input->ref = maybeGetStrAttr(attrs, "ref");
|
||||
if (auto rev = maybeGetStrAttr(attrs, "rev"))
|
||||
input->rev = Hash(*rev, htSHA1);
|
||||
return input;
|
||||
return url;
|
||||
}
|
||||
|
||||
void clone(const Input & input, const Path & destDir) override
|
||||
{
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("github.com");
|
||||
Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
|
||||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
|
||||
.applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
|
||||
.clone(destDir);
|
||||
}
|
||||
};
|
||||
|
||||
struct GitLabInputScheme : GitArchiveInputScheme
|
||||
{
|
||||
std::string type() override { return "gitlab"; }
|
||||
|
||||
Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override
|
||||
{
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
|
||||
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/commits?ref_name=%s",
|
||||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"), *input.getRef());
|
||||
auto json = nlohmann::json::parse(
|
||||
readFile(
|
||||
store->toRealPath(
|
||||
downloadFile(store, url, "source", false).storePath)));
|
||||
auto rev = Hash(std::string(json[0]["id"]), htSHA1);
|
||||
debug("HEAD revision for '%s' is %s", url, rev.gitRev());
|
||||
return rev;
|
||||
}
|
||||
|
||||
std::string getDownloadUrl(const Input & input) const override
|
||||
{
|
||||
// FIXME: This endpoint has a rate limit threshold of 5 requests per minute
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
|
||||
auto url = fmt("https://%s/api/v4/projects/%s%%2F%s/repository/archive.tar.gz?sha=%s",
|
||||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"),
|
||||
input.getRev()->to_string(Base16, false));
|
||||
|
||||
/* # FIXME: add privat token auth (`curl --header "PRIVATE-TOKEN: <your_access_token>"`)
|
||||
std::string accessToken = settings.githubAccessToken.get();
|
||||
if (accessToken != "")
|
||||
url += "?access_token=" + accessToken;*/
|
||||
|
||||
return url;
|
||||
}
|
||||
|
||||
void clone(const Input & input, const Path & destDir) override
|
||||
{
|
||||
auto host_url = maybeGetStrAttr(input.attrs, "url").value_or("gitlab.com");
|
||||
// FIXME: get username somewhere
|
||||
Input::fromURL(fmt("git+ssh://git@%s/%s/%s.git",
|
||||
host_url, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo")))
|
||||
.applyOverrides(input.getRef().value_or("HEAD"), input.getRev())
|
||||
.clone(destDir);
|
||||
}
|
||||
};
|
||||
|
||||
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<GitHubInputScheme>()); });
|
||||
static auto r2 = OnStartup([] { registerInputScheme(std::make_unique<GitLabInputScheme>()); });
|
||||
|
||||
}
|
||||
|
104
src/libfetchers/indirect.cc
Normal file
104
src/libfetchers/indirect.cc
Normal file
@ -0,0 +1,104 @@
|
||||
#include "fetchers.hh"
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript);
|
||||
|
||||
struct IndirectInputScheme : InputScheme
|
||||
{
|
||||
std::optional<Input> inputFromURL(const ParsedURL & url) override
|
||||
{
|
||||
if (url.scheme != "flake") return {};
|
||||
|
||||
auto path = tokenizeString<std::vector<std::string>>(url.path, "/");
|
||||
|
||||
std::optional<Hash> rev;
|
||||
std::optional<std::string> ref;
|
||||
|
||||
if (path.size() == 1) {
|
||||
} else if (path.size() == 2) {
|
||||
if (std::regex_match(path[1], revRegex))
|
||||
rev = Hash(path[1], htSHA1);
|
||||
else if (std::regex_match(path[1], refRegex))
|
||||
ref = path[1];
|
||||
else
|
||||
throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]);
|
||||
} else if (path.size() == 3) {
|
||||
if (!std::regex_match(path[1], refRegex))
|
||||
throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]);
|
||||
ref = path[1];
|
||||
if (!std::regex_match(path[2], revRegex))
|
||||
throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]);
|
||||
rev = Hash(path[2], htSHA1);
|
||||
} else
|
||||
throw BadURL("GitHub URL '%s' is invalid", url.url);
|
||||
|
||||
std::string id = path[0];
|
||||
if (!std::regex_match(id, flakeRegex))
|
||||
throw BadURL("'%s' is not a valid flake ID", id);
|
||||
|
||||
// FIXME: forbid query params?
|
||||
|
||||
Input input;
|
||||
input.direct = false;
|
||||
input.attrs.insert_or_assign("type", "indirect");
|
||||
input.attrs.insert_or_assign("id", id);
|
||||
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
|
||||
if (ref) input.attrs.insert_or_assign("ref", *ref);
|
||||
|
||||
return input;
|
||||
}
|
||||
|
||||
std::optional<Input> inputFromAttrs(const Attrs & attrs) override
|
||||
{
|
||||
if (maybeGetStrAttr(attrs, "type") != "indirect") return {};
|
||||
|
||||
for (auto & [name, value] : attrs)
|
||||
if (name != "type" && name != "id" && name != "ref" && name != "rev" && name != "narHash")
|
||||
throw Error("unsupported indirect input attribute '%s'", name);
|
||||
|
||||
auto id = getStrAttr(attrs, "id");
|
||||
if (!std::regex_match(id, flakeRegex))
|
||||
throw BadURL("'%s' is not a valid flake ID", id);
|
||||
|
||||
Input input;
|
||||
input.direct = false;
|
||||
input.attrs = attrs;
|
||||
return input;
|
||||
}
|
||||
|
||||
ParsedURL toURL(const Input & input) override
|
||||
{
|
||||
ParsedURL url;
|
||||
url.scheme = "flake";
|
||||
url.path = getStrAttr(input.attrs, "id");
|
||||
if (auto ref = input.getRef()) { url.path += '/'; url.path += *ref; };
|
||||
if (auto rev = input.getRev()) { url.path += '/'; url.path += rev->gitRev(); };
|
||||
return url;
|
||||
}
|
||||
|
||||
bool hasAllInfo(const Input & input) override
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
Input applyOverrides(
|
||||
const Input & _input,
|
||||
std::optional<std::string> ref,
|
||||
std::optional<Hash> rev) override
|
||||
{
|
||||
auto input(_input);
|
||||
if (rev) input.attrs.insert_or_assign("rev", rev->gitRev());
|
||||
if (ref) input.attrs.insert_or_assign("ref", *ref);
|
||||
return input;
|
||||
}
|
||||
|
||||
std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
|
||||
{
|
||||
throw Error("indirect input '%s' cannot be fetched directly", input.to_string());
|
||||
}
|
||||
};
|
||||
|
||||
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); });
|
||||
|
||||
}
|
@ -10,76 +10,124 @@ using namespace std::string_literals;
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
struct MercurialInput : Input
|
||||
struct MercurialInputScheme : InputScheme
|
||||
{
|
||||
ParsedURL url;
|
||||
std::optional<std::string> ref;
|
||||
std::optional<Hash> rev;
|
||||
|
||||
MercurialInput(const ParsedURL & url) : url(url)
|
||||
{ }
|
||||
|
||||
std::string type() const override { return "hg"; }
|
||||
|
||||
bool operator ==(const Input & other) const override
|
||||
std::optional<Input> inputFromURL(const ParsedURL & url) override
|
||||
{
|
||||
auto other2 = dynamic_cast<const MercurialInput *>(&other);
|
||||
return
|
||||
other2
|
||||
&& url == other2->url
|
||||
&& rev == other2->rev
|
||||
&& ref == other2->ref;
|
||||
if (url.scheme != "hg+http" &&
|
||||
url.scheme != "hg+https" &&
|
||||
url.scheme != "hg+ssh" &&
|
||||
url.scheme != "hg+file") return {};
|
||||
|
||||
auto url2(url);
|
||||
url2.scheme = std::string(url2.scheme, 3);
|
||||
url2.query.clear();
|
||||
|
||||
Attrs attrs;
|
||||
attrs.emplace("type", "hg");
|
||||
|
||||
for (auto &[name, value] : url.query) {
|
||||
if (name == "rev" || name == "ref")
|
||||
attrs.emplace(name, value);
|
||||
else
|
||||
url2.query.emplace(name, value);
|
||||
}
|
||||
|
||||
bool isImmutable() const override
|
||||
{
|
||||
return (bool) rev || narHash;
|
||||
attrs.emplace("url", url2.to_string());
|
||||
|
||||
return inputFromAttrs(attrs);
|
||||
}
|
||||
|
||||
std::optional<std::string> getRef() const override { return ref; }
|
||||
|
||||
std::optional<Hash> getRev() const override { return rev; }
|
||||
|
||||
ParsedURL toURL() const override
|
||||
std::optional<Input> inputFromAttrs(const Attrs & attrs) override
|
||||
{
|
||||
ParsedURL url2(url);
|
||||
url2.scheme = "hg+" + url2.scheme;
|
||||
if (rev) url2.query.insert_or_assign("rev", rev->gitRev());
|
||||
if (ref) url2.query.insert_or_assign("ref", *ref);
|
||||
if (maybeGetStrAttr(attrs, "type") != "hg") return {};
|
||||
|
||||
for (auto & [name, value] : attrs)
|
||||
if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash")
|
||||
throw Error("unsupported Mercurial input attribute '%s'", name);
|
||||
|
||||
parseURL(getStrAttr(attrs, "url"));
|
||||
|
||||
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
|
||||
if (!std::regex_match(*ref, refRegex))
|
||||
throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
|
||||
}
|
||||
|
||||
Input input;
|
||||
input.attrs = attrs;
|
||||
return input;
|
||||
}
|
||||
|
||||
ParsedURL toURL(const Input & input) override
|
||||
{
|
||||
auto url = parseURL(getStrAttr(input.attrs, "url"));
|
||||
url.scheme = "hg+" + url.scheme;
|
||||
if (auto rev = input.getRev()) url.query.insert_or_assign("rev", rev->gitRev());
|
||||
if (auto ref = input.getRef()) url.query.insert_or_assign("ref", *ref);
|
||||
return url;
|
||||
}
|
||||
|
||||
Attrs toAttrsInternal() const override
|
||||
bool hasAllInfo(const Input & input) override
|
||||
{
|
||||
Attrs attrs;
|
||||
attrs.emplace("url", url.to_string());
|
||||
if (ref)
|
||||
attrs.emplace("ref", *ref);
|
||||
if (rev)
|
||||
attrs.emplace("rev", rev->gitRev());
|
||||
return attrs;
|
||||
// FIXME: ugly, need to distinguish between dirty and clean
|
||||
// default trees.
|
||||
return input.getRef() == "default" || maybeGetIntAttr(input.attrs, "revCount");
|
||||
}
|
||||
|
||||
std::pair<bool, std::string> getActualUrl() const
|
||||
Input applyOverrides(
|
||||
const Input & input,
|
||||
std::optional<std::string> ref,
|
||||
std::optional<Hash> rev) override
|
||||
{
|
||||
auto res(input);
|
||||
if (rev) res.attrs.insert_or_assign("rev", rev->gitRev());
|
||||
if (ref) res.attrs.insert_or_assign("ref", *ref);
|
||||
return res;
|
||||
}
|
||||
|
||||
std::optional<Path> getSourcePath(const Input & input) override
|
||||
{
|
||||
auto url = parseURL(getStrAttr(input.attrs, "url"));
|
||||
if (url.scheme == "file" && !input.getRef() && !input.getRev())
|
||||
return url.path;
|
||||
return {};
|
||||
}
|
||||
|
||||
void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
|
||||
{
|
||||
auto sourcePath = getSourcePath(input);
|
||||
assert(sourcePath);
|
||||
|
||||
// FIXME: shut up if file is already tracked.
|
||||
runProgram("hg", true,
|
||||
{ "add", *sourcePath + "/" + std::string(file) });
|
||||
|
||||
if (commitMsg)
|
||||
runProgram("hg", true,
|
||||
{ "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg });
|
||||
}
|
||||
|
||||
std::pair<bool, std::string> getActualUrl(const Input & input) const
|
||||
{
|
||||
auto url = parseURL(getStrAttr(input.attrs, "url"));
|
||||
bool isLocal = url.scheme == "file";
|
||||
return {isLocal, isLocal ? url.path : url.base};
|
||||
}
|
||||
|
||||
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
|
||||
std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
|
||||
{
|
||||
auto name = "source";
|
||||
|
||||
auto input = std::make_shared<MercurialInput>(*this);
|
||||
Input input(_input);
|
||||
|
||||
auto [isLocal, actualUrl_] = getActualUrl();
|
||||
auto [isLocal, actualUrl_] = getActualUrl(input);
|
||||
auto actualUrl = actualUrl_; // work around clang bug
|
||||
|
||||
// FIXME: return lastModified.
|
||||
|
||||
// FIXME: don't clone local repositories.
|
||||
|
||||
if (!input->ref && !input->rev && isLocal && pathExists(actualUrl + "/.hg")) {
|
||||
if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) {
|
||||
|
||||
bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
|
||||
|
||||
@ -94,7 +142,7 @@ struct MercurialInput : Input
|
||||
if (settings.warnDirty)
|
||||
warn("Mercurial tree '%s' is unclean", actualUrl);
|
||||
|
||||
input->ref = chomp(runProgram("hg", true, { "branch", "-R", actualUrl }));
|
||||
input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl })));
|
||||
|
||||
auto files = tokenizeString<std::set<std::string>>(
|
||||
runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
|
||||
@ -116,60 +164,54 @@ struct MercurialInput : Input
|
||||
|
||||
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
|
||||
|
||||
return {Tree {
|
||||
.actualPath = store->printStorePath(storePath),
|
||||
.storePath = std::move(storePath),
|
||||
}, input};
|
||||
return {
|
||||
Tree(store->printStorePath(storePath), std::move(storePath)),
|
||||
input
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (!input->ref) input->ref = "default";
|
||||
if (!input.getRef()) input.attrs.insert_or_assign("ref", "default");
|
||||
|
||||
auto getImmutableAttrs = [&]()
|
||||
{
|
||||
return Attrs({
|
||||
{"type", "hg"},
|
||||
{"name", name},
|
||||
{"rev", input->rev->gitRev()},
|
||||
{"rev", input.getRev()->gitRev()},
|
||||
});
|
||||
};
|
||||
|
||||
auto makeResult = [&](const Attrs & infoAttrs, StorePath && storePath)
|
||||
-> std::pair<Tree, std::shared_ptr<const Input>>
|
||||
-> std::pair<Tree, Input>
|
||||
{
|
||||
assert(input->rev);
|
||||
assert(!rev || rev == input->rev);
|
||||
assert(input.getRev());
|
||||
assert(!_input.getRev() || _input.getRev() == input.getRev());
|
||||
input.attrs.insert_or_assign("revCount", getIntAttr(infoAttrs, "revCount"));
|
||||
return {
|
||||
Tree{
|
||||
.actualPath = store->toRealPath(storePath),
|
||||
.storePath = std::move(storePath),
|
||||
.info = TreeInfo {
|
||||
.revCount = getIntAttr(infoAttrs, "revCount"),
|
||||
},
|
||||
},
|
||||
Tree(store->toRealPath(storePath), std::move(storePath)),
|
||||
input
|
||||
};
|
||||
};
|
||||
|
||||
if (input->rev) {
|
||||
if (input.getRev()) {
|
||||
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
|
||||
return makeResult(res->first, std::move(res->second));
|
||||
}
|
||||
|
||||
assert(input->rev || input->ref);
|
||||
auto revOrRef = input->rev ? input->rev->gitRev() : *input->ref;
|
||||
auto revOrRef = input.getRev() ? input.getRev()->gitRev() : *input.getRef();
|
||||
|
||||
Attrs mutableAttrs({
|
||||
{"type", "hg"},
|
||||
{"name", name},
|
||||
{"url", actualUrl},
|
||||
{"ref", *input->ref},
|
||||
{"ref", *input.getRef()},
|
||||
});
|
||||
|
||||
if (auto res = getCache()->lookup(store, mutableAttrs)) {
|
||||
auto rev2 = Hash(getStrAttr(res->first, "rev"), htSHA1);
|
||||
if (!rev || rev == rev2) {
|
||||
input->rev = rev2;
|
||||
if (!input.getRev() || input.getRev() == rev2) {
|
||||
input.attrs.insert_or_assign("rev", rev2.gitRev());
|
||||
return makeResult(res->first, std::move(res->second));
|
||||
}
|
||||
}
|
||||
@ -178,10 +220,10 @@ struct MercurialInput : Input
|
||||
|
||||
/* If this is a commit hash that we already have, we don't
|
||||
have to pull again. */
|
||||
if (!(input->rev
|
||||
if (!(input.getRev()
|
||||
&& pathExists(cacheDir)
|
||||
&& runProgram(
|
||||
RunOptions("hg", { "log", "-R", cacheDir, "-r", input->rev->gitRev(), "--template", "1" })
|
||||
RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
|
||||
.killStderr(true)).second == "1"))
|
||||
{
|
||||
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
|
||||
@ -210,9 +252,9 @@ struct MercurialInput : Input
|
||||
runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
|
||||
assert(tokens.size() == 3);
|
||||
|
||||
input->rev = Hash(tokens[0], htSHA1);
|
||||
input.attrs.insert_or_assign("rev", Hash(tokens[0], htSHA1).gitRev());
|
||||
auto revCount = std::stoull(tokens[1]);
|
||||
input->ref = tokens[2];
|
||||
input.attrs.insert_or_assign("ref", tokens[2]);
|
||||
|
||||
if (auto res = getCache()->lookup(store, getImmutableAttrs()))
|
||||
return makeResult(res->first, std::move(res->second));
|
||||
@ -220,18 +262,18 @@ struct MercurialInput : Input
|
||||
Path tmpDir = createTempDir();
|
||||
AutoDelete delTmpDir(tmpDir, true);
|
||||
|
||||
runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input->rev->gitRev(), tmpDir });
|
||||
runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir });
|
||||
|
||||
deletePath(tmpDir + "/.hg_archival.txt");
|
||||
|
||||
auto storePath = store->addToStore(name, tmpDir);
|
||||
|
||||
Attrs infoAttrs({
|
||||
{"rev", input->rev->gitRev()},
|
||||
{"rev", input.getRev()->gitRev()},
|
||||
{"revCount", (int64_t) revCount},
|
||||
});
|
||||
|
||||
if (!this->rev)
|
||||
if (!_input.getRev())
|
||||
getCache()->add(
|
||||
store,
|
||||
mutableAttrs,
|
||||
@ -250,54 +292,6 @@ struct MercurialInput : Input
|
||||
}
|
||||
};
|
||||
|
||||
struct MercurialInputScheme : InputScheme
|
||||
{
|
||||
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
|
||||
{
|
||||
if (url.scheme != "hg+http" &&
|
||||
url.scheme != "hg+https" &&
|
||||
url.scheme != "hg+ssh" &&
|
||||
url.scheme != "hg+file") return nullptr;
|
||||
|
||||
auto url2(url);
|
||||
url2.scheme = std::string(url2.scheme, 3);
|
||||
url2.query.clear();
|
||||
|
||||
Attrs attrs;
|
||||
attrs.emplace("type", "hg");
|
||||
|
||||
for (auto &[name, value] : url.query) {
|
||||
if (name == "rev" || name == "ref")
|
||||
attrs.emplace(name, value);
|
||||
else
|
||||
url2.query.emplace(name, value);
|
||||
}
|
||||
|
||||
attrs.emplace("url", url2.to_string());
|
||||
|
||||
return inputFromAttrs(attrs);
|
||||
}
|
||||
|
||||
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
|
||||
{
|
||||
if (maybeGetStrAttr(attrs, "type") != "hg") return {};
|
||||
|
||||
for (auto & [name, value] : attrs)
|
||||
if (name != "type" && name != "url" && name != "ref" && name != "rev")
|
||||
throw Error("unsupported Mercurial input attribute '%s'", name);
|
||||
|
||||
auto input = std::make_unique<MercurialInput>(parseURL(getStrAttr(attrs, "url")));
|
||||
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
|
||||
if (!std::regex_match(*ref, refRegex))
|
||||
throw BadURL("invalid Mercurial branch/tag name '%s'", *ref);
|
||||
input->ref = *ref;
|
||||
}
|
||||
if (auto rev = maybeGetStrAttr(attrs, "rev"))
|
||||
input->rev = Hash(*rev, htSHA1);
|
||||
return input;
|
||||
}
|
||||
};
|
||||
|
||||
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<MercurialInputScheme>()); });
|
||||
|
||||
}
|
||||
|
@ -3,65 +3,86 @@
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
struct PathInput : Input
|
||||
struct PathInputScheme : InputScheme
|
||||
{
|
||||
Path path;
|
||||
std::optional<Input> inputFromURL(const ParsedURL & url) override
|
||||
{
|
||||
if (url.scheme != "path") return {};
|
||||
|
||||
/* Allow the user to pass in "fake" tree info attributes. This is
|
||||
useful for making a pinned tree work the same as the repository
|
||||
from which is exported
|
||||
if (url.authority && *url.authority != "")
|
||||
throw Error("path URL '%s' should not have an authority ('%s')", url.url, *url.authority);
|
||||
|
||||
Input input;
|
||||
input.attrs.insert_or_assign("type", "path");
|
||||
input.attrs.insert_or_assign("path", url.path);
|
||||
|
||||
for (auto & [name, value] : url.query)
|
||||
if (name == "rev" || name == "narHash")
|
||||
input.attrs.insert_or_assign(name, value);
|
||||
else if (name == "revCount" || name == "lastModified") {
|
||||
uint64_t n;
|
||||
if (!string2Int(value, n))
|
||||
throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
|
||||
input.attrs.insert_or_assign(name, n);
|
||||
}
|
||||
else
|
||||
throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
|
||||
|
||||
return input;
|
||||
}
|
||||
|
||||
std::optional<Input> inputFromAttrs(const Attrs & attrs) override
|
||||
{
|
||||
if (maybeGetStrAttr(attrs, "type") != "path") return {};
|
||||
|
||||
getStrAttr(attrs, "path");
|
||||
|
||||
for (auto & [name, value] : attrs)
|
||||
/* Allow the user to pass in "fake" tree info
|
||||
attributes. This is useful for making a pinned tree
|
||||
work the same as the repository from which is exported
|
||||
(e.g. path:/nix/store/...-source?lastModified=1585388205&rev=b0c285...). */
|
||||
std::optional<Hash> rev;
|
||||
std::optional<uint64_t> revCount;
|
||||
std::optional<time_t> lastModified;
|
||||
if (name == "type" || name == "rev" || name == "revCount" || name == "lastModified" || name == "narHash" || name == "path")
|
||||
// checked in Input::fromAttrs
|
||||
;
|
||||
else
|
||||
throw Error("unsupported path input attribute '%s'", name);
|
||||
|
||||
std::string type() const override { return "path"; }
|
||||
|
||||
std::optional<Hash> getRev() const override { return rev; }
|
||||
|
||||
bool operator ==(const Input & other) const override
|
||||
{
|
||||
auto other2 = dynamic_cast<const PathInput *>(&other);
|
||||
return
|
||||
other2
|
||||
&& path == other2->path
|
||||
&& rev == other2->rev
|
||||
&& revCount == other2->revCount
|
||||
&& lastModified == other2->lastModified;
|
||||
Input input;
|
||||
input.attrs = attrs;
|
||||
return input;
|
||||
}
|
||||
|
||||
bool isImmutable() const override
|
||||
ParsedURL toURL(const Input & input) override
|
||||
{
|
||||
return (bool) narHash;
|
||||
}
|
||||
|
||||
ParsedURL toURL() const override
|
||||
{
|
||||
auto query = attrsToQuery(toAttrsInternal());
|
||||
auto query = attrsToQuery(input.attrs);
|
||||
query.erase("path");
|
||||
query.erase("type");
|
||||
return ParsedURL {
|
||||
.scheme = "path",
|
||||
.path = path,
|
||||
.path = getStrAttr(input.attrs, "path"),
|
||||
.query = query,
|
||||
};
|
||||
}
|
||||
|
||||
Attrs toAttrsInternal() const override
|
||||
bool hasAllInfo(const Input & input) override
|
||||
{
|
||||
Attrs attrs;
|
||||
attrs.emplace("path", path);
|
||||
if (rev)
|
||||
attrs.emplace("rev", rev->gitRev());
|
||||
if (revCount)
|
||||
attrs.emplace("revCount", *revCount);
|
||||
if (lastModified)
|
||||
attrs.emplace("lastModified", *lastModified);
|
||||
return attrs;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
|
||||
std::optional<Path> getSourcePath(const Input & input) override
|
||||
{
|
||||
auto input = std::make_shared<PathInput>(*this);
|
||||
return getStrAttr(input.attrs, "path");
|
||||
}
|
||||
|
||||
void markChangedFile(const Input & input, std::string_view file, std::optional<std::string> commitMsg) override
|
||||
{
|
||||
// nothing to do
|
||||
}
|
||||
|
||||
std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
|
||||
{
|
||||
auto path = getStrAttr(input.attrs, "path");
|
||||
|
||||
// FIXME: check whether access to 'path' is allowed.
|
||||
|
||||
@ -74,73 +95,11 @@ struct PathInput : Input
|
||||
// FIXME: try to substitute storePath.
|
||||
storePath = store->addToStore("source", path);
|
||||
|
||||
return
|
||||
{
|
||||
Tree {
|
||||
.actualPath = store->toRealPath(*storePath),
|
||||
.storePath = std::move(*storePath),
|
||||
.info = TreeInfo {
|
||||
.revCount = revCount,
|
||||
.lastModified = lastModified
|
||||
}
|
||||
},
|
||||
return {
|
||||
Tree(store->toRealPath(*storePath), std::move(*storePath)),
|
||||
input
|
||||
};
|
||||
}
|
||||
|
||||
};
|
||||
|
||||
struct PathInputScheme : InputScheme
|
||||
{
|
||||
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
|
||||
{
|
||||
if (url.scheme != "path") return nullptr;
|
||||
|
||||
auto input = std::make_unique<PathInput>();
|
||||
input->path = url.path;
|
||||
|
||||
for (auto & [name, value] : url.query)
|
||||
if (name == "rev")
|
||||
input->rev = Hash(value, htSHA1);
|
||||
else if (name == "revCount") {
|
||||
uint64_t revCount;
|
||||
if (!string2Int(value, revCount))
|
||||
throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
|
||||
input->revCount = revCount;
|
||||
}
|
||||
else if (name == "lastModified") {
|
||||
time_t lastModified;
|
||||
if (!string2Int(value, lastModified))
|
||||
throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
|
||||
input->lastModified = lastModified;
|
||||
}
|
||||
else
|
||||
throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
|
||||
|
||||
return input;
|
||||
}
|
||||
|
||||
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
|
||||
{
|
||||
if (maybeGetStrAttr(attrs, "type") != "path") return {};
|
||||
|
||||
auto input = std::make_unique<PathInput>();
|
||||
input->path = getStrAttr(attrs, "path");
|
||||
|
||||
for (auto & [name, value] : attrs)
|
||||
if (name == "rev")
|
||||
input->rev = Hash(getStrAttr(attrs, "rev"), htSHA1);
|
||||
else if (name == "revCount")
|
||||
input->revCount = getIntAttr(attrs, "revCount");
|
||||
else if (name == "lastModified")
|
||||
input->lastModified = getIntAttr(attrs, "lastModified");
|
||||
else if (name == "type" || name == "path")
|
||||
;
|
||||
else
|
||||
throw Error("unsupported path input attribute '%s'", name);
|
||||
|
||||
return input;
|
||||
}
|
||||
};
|
||||
|
||||
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<PathInputScheme>()); });
|
||||
|
212
src/libfetchers/registry.cc
Normal file
212
src/libfetchers/registry.cc
Normal file
@ -0,0 +1,212 @@
|
||||
#include "registry.hh"
|
||||
#include "fetchers.hh"
|
||||
#include "util.hh"
|
||||
#include "globals.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
std::shared_ptr<Registry> Registry::read(
|
||||
const Path & path, RegistryType type)
|
||||
{
|
||||
auto registry = std::make_shared<Registry>(type);
|
||||
|
||||
if (!pathExists(path))
|
||||
return std::make_shared<Registry>(type);
|
||||
|
||||
try {
|
||||
|
||||
auto json = nlohmann::json::parse(readFile(path));
|
||||
|
||||
auto version = json.value("version", 0);
|
||||
|
||||
if (version == 2) {
|
||||
for (auto & i : json["flakes"]) {
|
||||
auto toAttrs = jsonToAttrs(i["to"]);
|
||||
Attrs extraAttrs;
|
||||
auto j = toAttrs.find("dir");
|
||||
if (j != toAttrs.end()) {
|
||||
extraAttrs.insert(*j);
|
||||
toAttrs.erase(j);
|
||||
}
|
||||
auto exact = i.find("exact");
|
||||
registry->entries.push_back(
|
||||
Entry {
|
||||
.from = Input::fromAttrs(jsonToAttrs(i["from"])),
|
||||
.to = Input::fromAttrs(std::move(toAttrs)),
|
||||
.extraAttrs = extraAttrs,
|
||||
.exact = exact != i.end() && exact.value()
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
else
|
||||
throw Error("flake registry '%s' has unsupported version %d", path, version);
|
||||
|
||||
} catch (nlohmann::json::exception & e) {
|
||||
warn("cannot parse flake registry '%s': %s", path, e.what());
|
||||
} catch (Error & e) {
|
||||
warn("cannot read flake registry '%s': %s", path, e.what());
|
||||
}
|
||||
|
||||
return registry;
|
||||
}
|
||||
|
||||
void Registry::write(const Path & path)
|
||||
{
|
||||
nlohmann::json arr;
|
||||
for (auto & entry : entries) {
|
||||
nlohmann::json obj;
|
||||
obj["from"] = attrsToJson(entry.from.toAttrs());
|
||||
obj["to"] = attrsToJson(entry.to.toAttrs());
|
||||
if (!entry.extraAttrs.empty())
|
||||
obj["to"].update(attrsToJson(entry.extraAttrs));
|
||||
if (entry.exact)
|
||||
obj["exact"] = true;
|
||||
arr.emplace_back(std::move(obj));
|
||||
}
|
||||
|
||||
nlohmann::json json;
|
||||
json["version"] = 2;
|
||||
json["flakes"] = std::move(arr);
|
||||
|
||||
createDirs(dirOf(path));
|
||||
writeFile(path, json.dump(2));
|
||||
}
|
||||
|
||||
void Registry::add(
|
||||
const Input & from,
|
||||
const Input & to,
|
||||
const Attrs & extraAttrs)
|
||||
{
|
||||
entries.emplace_back(
|
||||
Entry {
|
||||
.from = from,
|
||||
.to = to,
|
||||
.extraAttrs = extraAttrs
|
||||
});
|
||||
}
|
||||
|
||||
void Registry::remove(const Input & input)
|
||||
{
|
||||
// FIXME: use C++20 std::erase.
|
||||
for (auto i = entries.begin(); i != entries.end(); )
|
||||
if (i->from == input)
|
||||
i = entries.erase(i);
|
||||
else
|
||||
++i;
|
||||
}
|
||||
|
||||
static Path getSystemRegistryPath()
|
||||
{
|
||||
return settings.nixConfDir + "/registry.json";
|
||||
}
|
||||
|
||||
static std::shared_ptr<Registry> getSystemRegistry()
|
||||
{
|
||||
static auto systemRegistry =
|
||||
Registry::read(getSystemRegistryPath(), Registry::System);
|
||||
return systemRegistry;
|
||||
}
|
||||
|
||||
Path getUserRegistryPath()
|
||||
{
|
||||
return getHome() + "/.config/nix/registry.json";
|
||||
}
|
||||
|
||||
std::shared_ptr<Registry> getUserRegistry()
|
||||
{
|
||||
static auto userRegistry =
|
||||
Registry::read(getUserRegistryPath(), Registry::User);
|
||||
return userRegistry;
|
||||
}
|
||||
|
||||
static std::shared_ptr<Registry> flagRegistry =
|
||||
std::make_shared<Registry>(Registry::Flag);
|
||||
|
||||
std::shared_ptr<Registry> getFlagRegistry()
|
||||
{
|
||||
return flagRegistry;
|
||||
}
|
||||
|
||||
void overrideRegistry(
|
||||
const Input & from,
|
||||
const Input & to,
|
||||
const Attrs & extraAttrs)
|
||||
{
|
||||
flagRegistry->add(from, to, extraAttrs);
|
||||
}
|
||||
|
||||
static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store)
|
||||
{
|
||||
static auto reg = [&]() {
|
||||
auto path = settings.flakeRegistry.get();
|
||||
|
||||
if (!hasPrefix(path, "/")) {
|
||||
auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath;
|
||||
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
|
||||
store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json", true);
|
||||
path = store->toRealPath(storePath);
|
||||
}
|
||||
|
||||
return Registry::read(path, Registry::Global);
|
||||
}();
|
||||
|
||||
return reg;
|
||||
}
|
||||
|
||||
Registries getRegistries(ref<Store> store)
|
||||
{
|
||||
Registries registries;
|
||||
registries.push_back(getFlagRegistry());
|
||||
registries.push_back(getUserRegistry());
|
||||
registries.push_back(getSystemRegistry());
|
||||
registries.push_back(getGlobalRegistry(store));
|
||||
return registries;
|
||||
}
|
||||
|
||||
std::pair<Input, Attrs> lookupInRegistries(
|
||||
ref<Store> store,
|
||||
const Input & _input)
|
||||
{
|
||||
Attrs extraAttrs;
|
||||
int n = 0;
|
||||
Input input(_input);
|
||||
|
||||
restart:
|
||||
|
||||
n++;
|
||||
if (n > 100) throw Error("cycle detected in flake registry for '%s'", input.to_string());
|
||||
|
||||
for (auto & registry : getRegistries(store)) {
|
||||
// FIXME: O(n)
|
||||
for (auto & entry : registry->entries) {
|
||||
if (entry.exact) {
|
||||
if (entry.from == input) {
|
||||
input = entry.to;
|
||||
extraAttrs = entry.extraAttrs;
|
||||
goto restart;
|
||||
}
|
||||
} else {
|
||||
if (entry.from.contains(input)) {
|
||||
input = entry.to.applyOverrides(
|
||||
!entry.from.getRef() && input.getRef() ? input.getRef() : std::optional<std::string>(),
|
||||
!entry.from.getRev() && input.getRev() ? input.getRev() : std::optional<Hash>());
|
||||
extraAttrs = entry.extraAttrs;
|
||||
goto restart;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!input.isDirect())
|
||||
throw Error("cannot find flake '%s' in the flake registries", input.to_string());
|
||||
|
||||
debug("looked up '%s' -> '%s'", _input.to_string(), input.to_string());
|
||||
|
||||
return {input, extraAttrs};
|
||||
}
|
||||
|
||||
}
|
64
src/libfetchers/registry.hh
Normal file
64
src/libfetchers/registry.hh
Normal file
@ -0,0 +1,64 @@
|
||||
#pragma once
|
||||
|
||||
#include "types.hh"
|
||||
#include "fetchers.hh"
|
||||
|
||||
namespace nix { class Store; }
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
struct Registry
|
||||
{
|
||||
enum RegistryType {
|
||||
Flag = 0,
|
||||
User = 1,
|
||||
System = 2,
|
||||
Global = 3,
|
||||
};
|
||||
|
||||
RegistryType type;
|
||||
|
||||
struct Entry
|
||||
{
|
||||
Input from, to;
|
||||
Attrs extraAttrs;
|
||||
bool exact = false;
|
||||
};
|
||||
|
||||
std::vector<Entry> entries;
|
||||
|
||||
Registry(RegistryType type)
|
||||
: type(type)
|
||||
{ }
|
||||
|
||||
static std::shared_ptr<Registry> read(
|
||||
const Path & path, RegistryType type);
|
||||
|
||||
void write(const Path & path);
|
||||
|
||||
void add(
|
||||
const Input & from,
|
||||
const Input & to,
|
||||
const Attrs & extraAttrs);
|
||||
|
||||
void remove(const Input & input);
|
||||
};
|
||||
|
||||
typedef std::vector<std::shared_ptr<Registry>> Registries;
|
||||
|
||||
std::shared_ptr<Registry> getUserRegistry();
|
||||
|
||||
Path getUserRegistryPath();
|
||||
|
||||
Registries getRegistries(ref<Store> store);
|
||||
|
||||
void overrideRegistry(
|
||||
const Input & from,
|
||||
const Input & to,
|
||||
const Attrs & extraAttrs);
|
||||
|
||||
std::pair<Input, Attrs> lookupInRegistries(
|
||||
ref<Store> store,
|
||||
const Input & input);
|
||||
|
||||
}
|
@ -105,7 +105,7 @@ DownloadFileResult downloadFile(
|
||||
};
|
||||
}
|
||||
|
||||
Tree downloadTarball(
|
||||
std::pair<Tree, time_t> downloadTarball(
|
||||
ref<Store> store,
|
||||
const std::string & url,
|
||||
const std::string & name,
|
||||
@ -120,12 +120,9 @@ Tree downloadTarball(
|
||||
auto cached = getCache()->lookupExpired(store, inAttrs);
|
||||
|
||||
if (cached && !cached->expired)
|
||||
return Tree {
|
||||
.actualPath = store->toRealPath(cached->storePath),
|
||||
.storePath = std::move(cached->storePath),
|
||||
.info = TreeInfo {
|
||||
.lastModified = getIntAttr(cached->infoAttrs, "lastModified"),
|
||||
},
|
||||
return {
|
||||
Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)),
|
||||
getIntAttr(cached->infoAttrs, "lastModified")
|
||||
};
|
||||
|
||||
auto res = downloadFile(store, url, name, immutable);
|
||||
@ -160,117 +157,72 @@ Tree downloadTarball(
|
||||
*unpackedStorePath,
|
||||
immutable);
|
||||
|
||||
return Tree {
|
||||
.actualPath = store->toRealPath(*unpackedStorePath),
|
||||
.storePath = std::move(*unpackedStorePath),
|
||||
.info = TreeInfo {
|
||||
.lastModified = lastModified,
|
||||
},
|
||||
return {
|
||||
Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)),
|
||||
lastModified,
|
||||
};
|
||||
}
|
||||
|
||||
struct TarballInput : Input
|
||||
{
|
||||
ParsedURL url;
|
||||
std::optional<Hash> hash;
|
||||
|
||||
TarballInput(const ParsedURL & url) : url(url)
|
||||
{ }
|
||||
|
||||
std::string type() const override { return "tarball"; }
|
||||
|
||||
bool operator ==(const Input & other) const override
|
||||
{
|
||||
auto other2 = dynamic_cast<const TarballInput *>(&other);
|
||||
return
|
||||
other2
|
||||
&& to_string() == other2->to_string()
|
||||
&& hash == other2->hash;
|
||||
}
|
||||
|
||||
bool isImmutable() const override
|
||||
{
|
||||
return hash || narHash;
|
||||
}
|
||||
|
||||
ParsedURL toURL() const override
|
||||
{
|
||||
auto url2(url);
|
||||
// NAR hashes are preferred over file hashes since tar/zip files
|
||||
// don't have a canonical representation.
|
||||
if (narHash)
|
||||
url2.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
|
||||
else if (hash)
|
||||
url2.query.insert_or_assign("hash", hash->to_string(SRI, true));
|
||||
return url2;
|
||||
}
|
||||
|
||||
Attrs toAttrsInternal() const override
|
||||
{
|
||||
Attrs attrs;
|
||||
attrs.emplace("url", url.to_string());
|
||||
if (hash)
|
||||
attrs.emplace("hash", hash->to_string(SRI, true));
|
||||
return attrs;
|
||||
}
|
||||
|
||||
std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override
|
||||
{
|
||||
auto tree = downloadTarball(store, url.to_string(), "source", false);
|
||||
|
||||
auto input = std::make_shared<TarballInput>(*this);
|
||||
input->narHash = store->queryPathInfo(tree.storePath)->narHash;
|
||||
|
||||
return {std::move(tree), input};
|
||||
}
|
||||
};
|
||||
|
||||
struct TarballInputScheme : InputScheme
|
||||
{
|
||||
std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override
|
||||
std::optional<Input> inputFromURL(const ParsedURL & url) override
|
||||
{
|
||||
if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return nullptr;
|
||||
if (url.scheme != "file" && url.scheme != "http" && url.scheme != "https") return {};
|
||||
|
||||
if (!hasSuffix(url.path, ".zip")
|
||||
&& !hasSuffix(url.path, ".tar")
|
||||
&& !hasSuffix(url.path, ".tar.gz")
|
||||
&& !hasSuffix(url.path, ".tar.xz")
|
||||
&& !hasSuffix(url.path, ".tar.bz2"))
|
||||
return nullptr;
|
||||
|
||||
auto input = std::make_unique<TarballInput>(url);
|
||||
|
||||
auto hash = input->url.query.find("hash");
|
||||
if (hash != input->url.query.end()) {
|
||||
// FIXME: require SRI hash.
|
||||
input->hash = Hash(hash->second);
|
||||
input->url.query.erase(hash);
|
||||
}
|
||||
|
||||
auto narHash = input->url.query.find("narHash");
|
||||
if (narHash != input->url.query.end()) {
|
||||
// FIXME: require SRI hash.
|
||||
input->narHash = Hash(narHash->second);
|
||||
input->url.query.erase(narHash);
|
||||
}
|
||||
return {};
|
||||
|
||||
Input input;
|
||||
input.attrs.insert_or_assign("type", "tarball");
|
||||
input.attrs.insert_or_assign("url", url.to_string());
|
||||
auto narHash = url.query.find("narHash");
|
||||
if (narHash != url.query.end())
|
||||
input.attrs.insert_or_assign("narHash", narHash->second);
|
||||
return input;
|
||||
}
|
||||
|
||||
std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override
|
||||
std::optional<Input> inputFromAttrs(const Attrs & attrs) override
|
||||
{
|
||||
if (maybeGetStrAttr(attrs, "type") != "tarball") return {};
|
||||
|
||||
for (auto & [name, value] : attrs)
|
||||
if (name != "type" && name != "url" && name != "hash")
|
||||
if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash")
|
||||
throw Error("unsupported tarball input attribute '%s'", name);
|
||||
|
||||
auto input = std::make_unique<TarballInput>(parseURL(getStrAttr(attrs, "url")));
|
||||
if (auto hash = maybeGetStrAttr(attrs, "hash"))
|
||||
input->hash = newHashAllowEmpty(*hash, {});
|
||||
|
||||
Input input;
|
||||
input.attrs = attrs;
|
||||
//input.immutable = (bool) maybeGetStrAttr(input.attrs, "hash");
|
||||
return input;
|
||||
}
|
||||
|
||||
ParsedURL toURL(const Input & input) override
|
||||
{
|
||||
auto url = parseURL(getStrAttr(input.attrs, "url"));
|
||||
// NAR hashes are preferred over file hashes since tar/zip files
|
||||
// don't have a canonical representation.
|
||||
if (auto narHash = input.getNarHash())
|
||||
url.query.insert_or_assign("narHash", narHash->to_string(SRI, true));
|
||||
/*
|
||||
else if (auto hash = maybeGetStrAttr(input.attrs, "hash"))
|
||||
url.query.insert_or_assign("hash", Hash(*hash).to_string(SRI, true));
|
||||
*/
|
||||
return url;
|
||||
}
|
||||
|
||||
bool hasAllInfo(const Input & input) override
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
|
||||
{
|
||||
auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
|
||||
return {std::move(tree), input};
|
||||
}
|
||||
};
|
||||
|
||||
static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<TarballInputScheme>()); });
|
||||
|
@ -1,14 +0,0 @@
|
||||
#include "tree-info.hh"
|
||||
#include "store-api.hh"
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
StorePath TreeInfo::computeStorePath(Store & store) const
|
||||
{
|
||||
assert(narHash);
|
||||
return store.makeFixedOutputPath(FileIngestionMethod::Recursive, narHash, "source");
|
||||
}
|
||||
|
||||
}
|
@ -1,29 +0,0 @@
|
||||
#pragma once
|
||||
|
||||
#include "path.hh"
|
||||
#include "hash.hh"
|
||||
|
||||
#include <nlohmann/json_fwd.hpp>
|
||||
|
||||
namespace nix { class Store; }
|
||||
|
||||
namespace nix::fetchers {
|
||||
|
||||
struct TreeInfo
|
||||
{
|
||||
Hash narHash;
|
||||
std::optional<uint64_t> revCount;
|
||||
std::optional<time_t> lastModified;
|
||||
|
||||
bool operator ==(const TreeInfo & other) const
|
||||
{
|
||||
return
|
||||
narHash == other.narHash
|
||||
&& revCount == other.revCount
|
||||
&& lastModified == other.lastModified;
|
||||
}
|
||||
|
||||
StorePath computeStorePath(Store & store) const;
|
||||
};
|
||||
|
||||
}
|
@ -34,9 +34,19 @@ MixCommonArgs::MixCommonArgs(const string & programName)
|
||||
try {
|
||||
globalConfig.set(name, value);
|
||||
} catch (UsageError & e) {
|
||||
if (!completions)
|
||||
warn(e.what());
|
||||
}
|
||||
}},
|
||||
.completer = [](size_t index, std::string_view prefix) {
|
||||
if (index == 0) {
|
||||
std::map<std::string, Config::SettingInfo> settings;
|
||||
globalConfig.getSettings(settings);
|
||||
for (auto & s : settings)
|
||||
if (hasPrefix(s.first, prefix))
|
||||
completions->insert(s.first);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
addFlag({
|
||||
|
@ -1,12 +1,13 @@
|
||||
#include "loggers.hh"
|
||||
#include "progress-bar.hh"
|
||||
#include "util.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
LogFormat defaultLogFormat = LogFormat::raw;
|
||||
|
||||
LogFormat parseLogFormat(const std::string & logFormatStr) {
|
||||
if (logFormatStr == "raw")
|
||||
if (logFormatStr == "raw" || getEnv("NIX_GET_COMPLETIONS"))
|
||||
return LogFormat::raw;
|
||||
else if (logFormatStr == "raw-with-logs")
|
||||
return LogFormat::rawWithLogs;
|
||||
@ -26,7 +27,7 @@ Logger * makeDefaultLogger() {
|
||||
case LogFormat::rawWithLogs:
|
||||
return makeSimpleLogger(true);
|
||||
case LogFormat::internalJson:
|
||||
return makeJSONLogger(*makeSimpleLogger());
|
||||
return makeJSONLogger(*makeSimpleLogger(true));
|
||||
case LogFormat::bar:
|
||||
return makeProgressBar();
|
||||
case LogFormat::barWithLogs:
|
||||
|
@ -131,7 +131,7 @@ public:
|
||||
auto state(state_.lock());
|
||||
|
||||
std::stringstream oss;
|
||||
oss << ei;
|
||||
showErrorInfo(oss, ei, loggerSettings.showTrace.get());
|
||||
|
||||
log(*state, ei.level, oss.str());
|
||||
}
|
||||
|
@ -36,7 +36,7 @@ void printGCWarning()
|
||||
|
||||
void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & paths, Verbosity lvl)
|
||||
{
|
||||
unsigned long long downloadSize, narSize;
|
||||
uint64_t downloadSize, narSize;
|
||||
StorePathSet willBuild, willSubstitute, unknown;
|
||||
store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
printMissing(store, willBuild, willSubstitute, unknown, downloadSize, narSize, lvl);
|
||||
@ -45,7 +45,7 @@ void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & pa
|
||||
|
||||
void printMissing(ref<Store> store, const StorePathSet & willBuild,
|
||||
const StorePathSet & willSubstitute, const StorePathSet & unknown,
|
||||
unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl)
|
||||
uint64_t downloadSize, uint64_t narSize, Verbosity lvl)
|
||||
{
|
||||
if (!willBuild.empty()) {
|
||||
if (willBuild.size() == 1)
|
||||
@ -323,10 +323,8 @@ int handleExceptions(const string & programName, std::function<void()> fun)
|
||||
printError("Try '%1% --help' for more information.", programName);
|
||||
return 1;
|
||||
} catch (BaseError & e) {
|
||||
if (settings.showTrace && e.prefix() != "")
|
||||
printError(e.prefix());
|
||||
logError(e.info());
|
||||
if (e.prefix() != "" && !settings.showTrace)
|
||||
if (e.hasTrace() && !loggerSettings.showTrace.get())
|
||||
printError("(use '--show-trace' to show detailed location information)");
|
||||
return e.status;
|
||||
} catch (std::bad_alloc & e) {
|
||||
@ -386,7 +384,7 @@ RunPager::~RunPager()
|
||||
}
|
||||
|
||||
|
||||
string showBytes(unsigned long long bytes)
|
||||
string showBytes(uint64_t bytes)
|
||||
{
|
||||
return (format("%.2f MiB") % (bytes / (1024.0 * 1024.0))).str();
|
||||
}
|
||||
|
@ -47,7 +47,7 @@ void printMissing(
|
||||
|
||||
void printMissing(ref<Store> store, const StorePathSet & willBuild,
|
||||
const StorePathSet & willSubstitute, const StorePathSet & unknown,
|
||||
unsigned long long downloadSize, unsigned long long narSize, Verbosity lvl = lvlInfo);
|
||||
uint64_t downloadSize, uint64_t narSize, Verbosity lvl = lvlInfo);
|
||||
|
||||
string getArg(const string & opt,
|
||||
Strings::iterator & i, const Strings::iterator & end);
|
||||
@ -110,7 +110,7 @@ extern volatile ::sig_atomic_t blockInt;
|
||||
|
||||
/* GC helpers. */
|
||||
|
||||
string showBytes(unsigned long long bytes);
|
||||
string showBytes(uint64_t bytes);
|
||||
|
||||
struct GCResults;
|
||||
|
||||
|
@ -15,6 +15,7 @@
|
||||
#include <chrono>
|
||||
#include <future>
|
||||
#include <regex>
|
||||
#include <fstream>
|
||||
|
||||
#include <nlohmann/json.hpp>
|
||||
|
||||
@ -57,6 +58,13 @@ void BinaryCacheStore::init()
|
||||
}
|
||||
}
|
||||
|
||||
void BinaryCacheStore::upsertFile(const std::string & path,
|
||||
std::string && data,
|
||||
const std::string & mimeType)
|
||||
{
|
||||
upsertFile(path, std::make_shared<std::stringstream>(std::move(data)), mimeType);
|
||||
}
|
||||
|
||||
void BinaryCacheStore::getFile(const std::string & path,
|
||||
Callback<std::shared_ptr<std::string>> callback) noexcept
|
||||
{
|
||||
@ -113,13 +121,74 @@ void BinaryCacheStore::writeNarInfo(ref<NarInfo> narInfo)
|
||||
diskCache->upsertNarInfo(getUri(), hashPart, std::shared_ptr<NarInfo>(narInfo));
|
||||
}
|
||||
|
||||
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
|
||||
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
|
||||
AutoCloseFD openFile(const Path & path)
|
||||
{
|
||||
// FIXME: See if we can use the original source to reduce memory usage.
|
||||
auto nar = make_ref<std::string>(narSource.drain());
|
||||
auto fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
|
||||
if (!fd)
|
||||
throw SysError("opening file '%1%'", path);
|
||||
return fd;
|
||||
}
|
||||
|
||||
if (!repair && isValidPath(info.path)) return;
|
||||
struct FileSource : FdSource
|
||||
{
|
||||
AutoCloseFD fd2;
|
||||
|
||||
FileSource(const Path & path)
|
||||
: fd2(openFile(path))
|
||||
{
|
||||
fd = fd2.get();
|
||||
}
|
||||
};
|
||||
|
||||
void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource,
|
||||
RepairFlag repair, CheckSigsFlag checkSigs)
|
||||
{
|
||||
assert(info.narHash && info.narSize);
|
||||
|
||||
if (!repair && isValidPath(info.path)) {
|
||||
// FIXME: copyNAR -> null sink
|
||||
narSource.drain();
|
||||
return;
|
||||
}
|
||||
|
||||
auto [fdTemp, fnTemp] = createTempFile();
|
||||
|
||||
auto now1 = std::chrono::steady_clock::now();
|
||||
|
||||
/* Read the NAR simultaneously into a CompressionSink+FileSink (to
|
||||
write the compressed NAR to disk), into a HashSink (to get the
|
||||
NAR hash), and into a NarAccessor (to get the NAR listing). */
|
||||
HashSink fileHashSink(htSHA256);
|
||||
std::shared_ptr<FSAccessor> narAccessor;
|
||||
{
|
||||
FdSink fileSink(fdTemp.get());
|
||||
TeeSink teeSink(fileSink, fileHashSink);
|
||||
auto compressionSink = makeCompressionSink(compression, teeSink);
|
||||
TeeSource teeSource(narSource, *compressionSink);
|
||||
narAccessor = makeNarAccessor(teeSource);
|
||||
compressionSink->finish();
|
||||
}
|
||||
|
||||
auto now2 = std::chrono::steady_clock::now();
|
||||
|
||||
auto narInfo = make_ref<NarInfo>(info);
|
||||
narInfo->narSize = info.narSize;
|
||||
narInfo->narHash = info.narHash;
|
||||
narInfo->compression = compression;
|
||||
auto [fileHash, fileSize] = fileHashSink.finish();
|
||||
narInfo->fileHash = fileHash;
|
||||
narInfo->fileSize = fileSize;
|
||||
narInfo->url = "nar/" + narInfo->fileHash->to_string(Base32, false) + ".nar"
|
||||
+ (compression == "xz" ? ".xz" :
|
||||
compression == "bzip2" ? ".bz2" :
|
||||
compression == "br" ? ".br" :
|
||||
"");
|
||||
|
||||
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
|
||||
printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
|
||||
printStorePath(narInfo->path), info.narSize,
|
||||
((1.0 - (double) fileSize / info.narSize) * 100.0),
|
||||
duration);
|
||||
|
||||
/* Verify that all references are valid. This may do some .narinfo
|
||||
reads, but typically they'll already be cached. */
|
||||
@ -132,23 +201,6 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
|
||||
printStorePath(info.path), printStorePath(ref));
|
||||
}
|
||||
|
||||
assert(nar->compare(0, narMagic.size(), narMagic) == 0);
|
||||
|
||||
auto narInfo = make_ref<NarInfo>(info);
|
||||
|
||||
narInfo->narSize = nar->size();
|
||||
narInfo->narHash = hashString(htSHA256, *nar);
|
||||
|
||||
if (info.narHash && info.narHash != narInfo->narHash)
|
||||
throw Error("refusing to copy corrupted path '%1%' to binary cache", printStorePath(info.path));
|
||||
|
||||
auto accessor_ = std::dynamic_pointer_cast<RemoteFSAccessor>(accessor);
|
||||
|
||||
auto narAccessor = makeNarAccessor(nar);
|
||||
|
||||
if (accessor_)
|
||||
accessor_->addToCache(printStorePath(info.path), *nar, narAccessor);
|
||||
|
||||
/* Optionally write a JSON file containing a listing of the
|
||||
contents of the NAR. */
|
||||
if (writeNARListing) {
|
||||
@ -160,33 +212,13 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
|
||||
|
||||
{
|
||||
auto res = jsonRoot.placeholder("root");
|
||||
listNar(res, narAccessor, "", true);
|
||||
listNar(res, ref<FSAccessor>(narAccessor), "", true);
|
||||
}
|
||||
}
|
||||
|
||||
upsertFile(std::string(info.path.to_string()) + ".ls", jsonOut.str(), "application/json");
|
||||
}
|
||||
|
||||
/* Compress the NAR. */
|
||||
narInfo->compression = compression;
|
||||
auto now1 = std::chrono::steady_clock::now();
|
||||
auto narCompressed = compress(compression, *nar, parallelCompression);
|
||||
auto now2 = std::chrono::steady_clock::now();
|
||||
narInfo->fileHash = hashString(htSHA256, *narCompressed);
|
||||
narInfo->fileSize = narCompressed->size();
|
||||
|
||||
auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(now2 - now1).count();
|
||||
printMsg(lvlTalkative, "copying path '%1%' (%2% bytes, compressed %3$.1f%% in %4% ms) to binary cache",
|
||||
printStorePath(narInfo->path), narInfo->narSize,
|
||||
((1.0 - (double) narCompressed->size() / nar->size()) * 100.0),
|
||||
duration);
|
||||
|
||||
narInfo->url = "nar/" + narInfo->fileHash.to_string(Base32, false) + ".nar"
|
||||
+ (compression == "xz" ? ".xz" :
|
||||
compression == "bzip2" ? ".bz2" :
|
||||
compression == "br" ? ".br" :
|
||||
"");
|
||||
|
||||
/* Optionally maintain an index of DWARF debug info files
|
||||
consisting of JSON files named 'debuginfo/<build-id>' that
|
||||
specify the NAR file and member containing the debug info. */
|
||||
@ -247,12 +279,14 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, Source & narSource
|
||||
/* Atomically write the NAR file. */
|
||||
if (repair || !fileExists(narInfo->url)) {
|
||||
stats.narWrite++;
|
||||
upsertFile(narInfo->url, *narCompressed, "application/x-nix-nar");
|
||||
upsertFile(narInfo->url,
|
||||
std::make_shared<std::fstream>(fnTemp, std::ios_base::in),
|
||||
"application/x-nix-nar");
|
||||
} else
|
||||
stats.narWriteAverted++;
|
||||
|
||||
stats.narWriteBytes += nar->size();
|
||||
stats.narWriteCompressedBytes += narCompressed->size();
|
||||
stats.narWriteBytes += info.narSize;
|
||||
stats.narWriteCompressedBytes += fileSize;
|
||||
stats.narWriteCompressionTimeMs += duration;
|
||||
|
||||
/* Atomically write the NAR info file.*/
|
||||
@ -338,7 +372,7 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
|
||||
method for very large paths, but `copyPath' is mainly used for
|
||||
small files. */
|
||||
StringSink sink;
|
||||
Hash h;
|
||||
std::optional<Hash> h;
|
||||
if (method == FileIngestionMethod::Recursive) {
|
||||
dumpPath(srcPath, sink, filter);
|
||||
h = hashString(hashAlgo, *sink.s);
|
||||
@ -348,10 +382,10 @@ StorePath BinaryCacheStore::addToStore(const string & name, const Path & srcPath
|
||||
h = hashString(hashAlgo, s);
|
||||
}
|
||||
|
||||
ValidPathInfo info(makeFixedOutputPath(method, h, name));
|
||||
ValidPathInfo info(makeFixedOutputPath(method, *h, name));
|
||||
|
||||
auto source = StringSource { *sink.s };
|
||||
addToStore(info, source, repair, CheckSigs, nullptr);
|
||||
addToStore(info, source, repair, CheckSigs);
|
||||
|
||||
return std::move(info.path);
|
||||
}
|
||||
@ -366,7 +400,7 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
|
||||
StringSink sink;
|
||||
dumpString(s, sink);
|
||||
auto source = StringSource { *sink.s };
|
||||
addToStore(info, source, repair, CheckSigs, nullptr);
|
||||
addToStore(info, source, repair, CheckSigs);
|
||||
}
|
||||
|
||||
return std::move(info.path);
|
||||
|
@ -36,9 +36,13 @@ public:
|
||||
virtual bool fileExists(const std::string & path) = 0;
|
||||
|
||||
virtual void upsertFile(const std::string & path,
|
||||
const std::string & data,
|
||||
std::shared_ptr<std::basic_iostream<char>> istream,
|
||||
const std::string & mimeType) = 0;
|
||||
|
||||
void upsertFile(const std::string & path,
|
||||
std::string && data,
|
||||
const std::string & mimeType);
|
||||
|
||||
/* Note: subclasses must implement at least one of the two
|
||||
following getFile() methods. */
|
||||
|
||||
@ -75,8 +79,7 @@ public:
|
||||
{ unsupported("queryPathFromHashPart"); }
|
||||
|
||||
void addToStore(const ValidPathInfo & info, Source & narSource,
|
||||
RepairFlag repair, CheckSigsFlag checkSigs,
|
||||
std::shared_ptr<FSAccessor> accessor) override;
|
||||
RepairFlag repair, CheckSigsFlag checkSigs) override;
|
||||
|
||||
StorePath addToStore(const string & name, const Path & srcPath,
|
||||
FileIngestionMethod method, HashType hashAlgo,
|
||||
|
@ -1047,7 +1047,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
|
||||
{
|
||||
this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
|
||||
state = &DerivationGoal::haveDerivation;
|
||||
name = fmt("building of %s", worker.store.showPaths(drv.outputPaths()));
|
||||
name = fmt("building of %s", worker.store.showPaths(drv.outputPaths(worker.store)));
|
||||
trace("created");
|
||||
|
||||
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
|
||||
@ -1182,7 +1182,7 @@ void DerivationGoal::haveDerivation()
|
||||
retrySubstitution = false;
|
||||
|
||||
for (auto & i : drv->outputs)
|
||||
worker.store.addTempRoot(i.second.path);
|
||||
worker.store.addTempRoot(i.second.path(worker.store, drv->name));
|
||||
|
||||
/* Check what outputs paths are not already valid. */
|
||||
auto invalidOutputs = checkPathValidity(false, buildMode == bmRepair);
|
||||
@ -1290,12 +1290,12 @@ void DerivationGoal::repairClosure()
|
||||
StorePathSet outputClosure;
|
||||
for (auto & i : drv->outputs) {
|
||||
if (!wantOutput(i.first, wantedOutputs)) continue;
|
||||
worker.store.computeFSClosure(i.second.path, outputClosure);
|
||||
worker.store.computeFSClosure(i.second.path(worker.store, drv->name), outputClosure);
|
||||
}
|
||||
|
||||
/* Filter out our own outputs (which we have already checked). */
|
||||
for (auto & i : drv->outputs)
|
||||
outputClosure.erase(i.second.path);
|
||||
outputClosure.erase(i.second.path(worker.store, drv->name));
|
||||
|
||||
/* Get all dependencies of this derivation so that we know which
|
||||
derivation is responsible for which path in the output
|
||||
@ -1307,7 +1307,7 @@ void DerivationGoal::repairClosure()
|
||||
if (i.isDerivation()) {
|
||||
Derivation drv = worker.store.derivationFromPath(i);
|
||||
for (auto & j : drv.outputs)
|
||||
outputsToDrv.insert_or_assign(j.second.path, i);
|
||||
outputsToDrv.insert_or_assign(j.second.path(worker.store, drv.name), i);
|
||||
}
|
||||
|
||||
/* Check each path (slow!). */
|
||||
@ -1379,7 +1379,7 @@ void DerivationGoal::inputsRealised()
|
||||
for (auto & j : i.second) {
|
||||
auto k = inDrv.outputs.find(j);
|
||||
if (k != inDrv.outputs.end())
|
||||
worker.store.computeFSClosure(k->second.path, inputPaths);
|
||||
worker.store.computeFSClosure(k->second.path(worker.store, inDrv.name), inputPaths);
|
||||
else
|
||||
throw Error(
|
||||
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
|
||||
@ -1432,7 +1432,7 @@ void DerivationGoal::tryToBuild()
|
||||
goal can start a build, and if not, the main loop will sleep a
|
||||
few seconds and then retry this goal. */
|
||||
PathSet lockFiles;
|
||||
for (auto & outPath : drv->outputPaths())
|
||||
for (auto & outPath : drv->outputPaths(worker.store))
|
||||
lockFiles.insert(worker.store.Store::toRealPath(outPath));
|
||||
|
||||
if (!outputLocks.lockPaths(lockFiles, "", false)) {
|
||||
@ -1460,16 +1460,16 @@ void DerivationGoal::tryToBuild()
|
||||
return;
|
||||
}
|
||||
|
||||
missingPaths = drv->outputPaths();
|
||||
missingPaths = drv->outputPaths(worker.store);
|
||||
if (buildMode != bmCheck)
|
||||
for (auto & i : validPaths) missingPaths.erase(i);
|
||||
|
||||
/* If any of the outputs already exist but are not valid, delete
|
||||
them. */
|
||||
for (auto & i : drv->outputs) {
|
||||
if (worker.store.isValidPath(i.second.path)) continue;
|
||||
debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path));
|
||||
deletePath(worker.store.Store::toRealPath(i.second.path));
|
||||
if (worker.store.isValidPath(i.second.path(worker.store, drv->name))) continue;
|
||||
debug("removing invalid path '%s'", worker.store.printStorePath(i.second.path(worker.store, drv->name)));
|
||||
deletePath(worker.store.Store::toRealPath(i.second.path(worker.store, drv->name)));
|
||||
}
|
||||
|
||||
/* Don't do a remote build if the derivation has the attribute
|
||||
@ -1646,13 +1646,13 @@ void DerivationGoal::buildDone()
|
||||
So instead, check if the disk is (nearly) full now. If
|
||||
so, we don't mark this build as a permanent failure. */
|
||||
#if HAVE_STATVFS
|
||||
unsigned long long required = 8ULL * 1024 * 1024; // FIXME: make configurable
|
||||
uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
|
||||
struct statvfs st;
|
||||
if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 &&
|
||||
(unsigned long long) st.f_bavail * st.f_bsize < required)
|
||||
(uint64_t) st.f_bavail * st.f_bsize < required)
|
||||
diskFull = true;
|
||||
if (statvfs(tmpDir.c_str(), &st) == 0 &&
|
||||
(unsigned long long) st.f_bavail * st.f_bsize < required)
|
||||
(uint64_t) st.f_bavail * st.f_bsize < required)
|
||||
diskFull = true;
|
||||
#endif
|
||||
|
||||
@ -1692,7 +1692,7 @@ void DerivationGoal::buildDone()
|
||||
fmt("running post-build-hook '%s'", settings.postBuildHook),
|
||||
Logger::Fields{worker.store.printStorePath(drvPath)});
|
||||
PushActivity pact(act.id);
|
||||
auto outputPaths = drv->outputPaths();
|
||||
auto outputPaths = drv->outputPaths(worker.store);
|
||||
std::map<std::string, std::string> hookEnvironment = getEnv();
|
||||
|
||||
hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath));
|
||||
@ -1920,7 +1920,7 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
|
||||
if (j.isDerivation()) {
|
||||
Derivation drv = worker.store.derivationFromPath(j);
|
||||
for (auto & k : drv.outputs)
|
||||
worker.store.computeFSClosure(k.second.path, paths);
|
||||
worker.store.computeFSClosure(k.second.path(worker.store, drv.name), paths);
|
||||
}
|
||||
}
|
||||
|
||||
@ -2015,7 +2015,7 @@ void DerivationGoal::startBuilder()
|
||||
|
||||
/* Substitute output placeholders with the actual output paths. */
|
||||
for (auto & output : drv->outputs)
|
||||
inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path);
|
||||
inputRewrites[hashPlaceholder(output.first)] = worker.store.printStorePath(output.second.path(worker.store, drv->name));
|
||||
|
||||
/* Construct the environment passed to the builder. */
|
||||
initEnv();
|
||||
@ -2041,7 +2041,10 @@ void DerivationGoal::startBuilder()
|
||||
if (!std::regex_match(fileName, regex))
|
||||
throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
|
||||
|
||||
auto storePath = worker.store.parseStorePath(*i++);
|
||||
auto storePathS = *i++;
|
||||
if (!worker.store.isInStore(storePathS))
|
||||
throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
|
||||
auto storePath = worker.store.toStorePath(storePathS).first;
|
||||
|
||||
/* Write closure info to <fileName>. */
|
||||
writeFile(tmpDir + "/" + fileName,
|
||||
@ -2080,7 +2083,7 @@ void DerivationGoal::startBuilder()
|
||||
for (auto & i : dirsInChroot)
|
||||
try {
|
||||
if (worker.store.isInStore(i.second.source))
|
||||
worker.store.computeFSClosure(worker.store.parseStorePath(worker.store.toStorePath(i.second.source)), closure);
|
||||
worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
|
||||
} catch (InvalidPath & e) {
|
||||
} catch (Error & e) {
|
||||
throw Error("while processing 'sandbox-paths': %s", e.what());
|
||||
@ -2197,7 +2200,7 @@ void DerivationGoal::startBuilder()
|
||||
(typically the dependencies of /bin/sh). Throw them
|
||||
out. */
|
||||
for (auto & i : drv->outputs)
|
||||
dirsInChroot.erase(worker.store.printStorePath(i.second.path));
|
||||
dirsInChroot.erase(worker.store.printStorePath(i.second.path(worker.store, drv->name)));
|
||||
|
||||
#elif __APPLE__
|
||||
/* We don't really have any parent prep work to do (yet?)
|
||||
@ -2610,7 +2613,7 @@ void DerivationGoal::writeStructuredAttrs()
|
||||
/* Add an "outputs" object containing the output paths. */
|
||||
nlohmann::json outputs;
|
||||
for (auto & i : drv->outputs)
|
||||
outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path), inputRewrites);
|
||||
outputs[i.first] = rewriteStrings(worker.store.printStorePath(i.second.path(worker.store, drv->name)), inputRewrites);
|
||||
json["outputs"] = outputs;
|
||||
|
||||
/* Handle exportReferencesGraph. */
|
||||
@ -2753,8 +2756,8 @@ struct RestrictedStore : public LocalFSStore
|
||||
void queryReferrers(const StorePath & path, StorePathSet & referrers) override
|
||||
{ }
|
||||
|
||||
StorePathSet queryDerivationOutputs(const StorePath & path) override
|
||||
{ throw Error("queryDerivationOutputs"); }
|
||||
OutputPathMap queryDerivationOutputMap(const StorePath & path) override
|
||||
{ throw Error("queryDerivationOutputMap"); }
|
||||
|
||||
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
|
||||
{ throw Error("queryPathFromHashPart"); }
|
||||
@ -2765,14 +2768,13 @@ struct RestrictedStore : public LocalFSStore
|
||||
{ throw Error("addToStore"); }
|
||||
|
||||
void addToStore(const ValidPathInfo & info, Source & narSource,
|
||||
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs,
|
||||
std::shared_ptr<FSAccessor> accessor = 0) override
|
||||
RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
|
||||
{
|
||||
next->addToStore(info, narSource, repair, checkSigs, accessor);
|
||||
next->addToStore(info, narSource, repair, checkSigs);
|
||||
goal.addDependency(info.path);
|
||||
}
|
||||
|
||||
StorePath addToStoreFromDump(const string & dump, const string & name,
|
||||
StorePath addToStoreFromDump(Source & dump, const string & name,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
|
||||
{
|
||||
auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
|
||||
@ -2815,7 +2817,7 @@ struct RestrictedStore : public LocalFSStore
|
||||
auto drv = derivationFromPath(path.path);
|
||||
for (auto & output : drv.outputs)
|
||||
if (wantOutput(output.first, path.outputs))
|
||||
newPaths.insert(output.second.path);
|
||||
newPaths.insert(output.second.path(*this, drv.name));
|
||||
} else if (!goal.isAllowed(path.path))
|
||||
throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
|
||||
}
|
||||
@ -2849,7 +2851,7 @@ struct RestrictedStore : public LocalFSStore
|
||||
|
||||
void queryMissing(const std::vector<StorePathWithOutputs> & targets,
|
||||
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
|
||||
unsigned long long & downloadSize, unsigned long long & narSize) override
|
||||
uint64_t & downloadSize, uint64_t & narSize) override
|
||||
{
|
||||
/* This is slightly impure since it leaks information to the
|
||||
client about what paths will be built/substituted or are
|
||||
@ -3577,7 +3579,7 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv
|
||||
if (store.isStorePath(i))
|
||||
result.insert(store.parseStorePath(i));
|
||||
else if (drv.outputs.count(i))
|
||||
result.insert(drv.outputs.find(i)->second.path);
|
||||
result.insert(drv.outputs.find(i)->second.path(store, drv.name));
|
||||
else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
|
||||
}
|
||||
return result;
|
||||
@ -3615,7 +3617,7 @@ void DerivationGoal::registerOutputs()
|
||||
if (hook) {
|
||||
bool allValid = true;
|
||||
for (auto & i : drv->outputs)
|
||||
if (!worker.store.isValidPath(i.second.path)) allValid = false;
|
||||
if (!worker.store.isValidPath(i.second.path(worker.store, drv->name))) allValid = false;
|
||||
if (allValid) return;
|
||||
}
|
||||
|
||||
@ -3636,23 +3638,23 @@ void DerivationGoal::registerOutputs()
|
||||
Nix calls. */
|
||||
StorePathSet referenceablePaths;
|
||||
for (auto & p : inputPaths) referenceablePaths.insert(p);
|
||||
for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path);
|
||||
for (auto & i : drv->outputs) referenceablePaths.insert(i.second.path(worker.store, drv->name));
|
||||
for (auto & p : addedPaths) referenceablePaths.insert(p);
|
||||
|
||||
/* Check whether the output paths were created, and grep each
|
||||
output path to determine what other paths it references. Also make all
|
||||
output paths read-only. */
|
||||
for (auto & i : drv->outputs) {
|
||||
auto path = worker.store.printStorePath(i.second.path);
|
||||
if (!missingPaths.count(i.second.path)) continue;
|
||||
auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
|
||||
if (!missingPaths.count(i.second.path(worker.store, drv->name))) continue;
|
||||
|
||||
Path actualPath = path;
|
||||
if (needsHashRewrite()) {
|
||||
auto r = redirectedOutputs.find(i.second.path);
|
||||
auto r = redirectedOutputs.find(i.second.path(worker.store, drv->name));
|
||||
if (r != redirectedOutputs.end()) {
|
||||
auto redirected = worker.store.Store::toRealPath(r->second);
|
||||
if (buildMode == bmRepair
|
||||
&& redirectedBadOutputs.count(i.second.path)
|
||||
&& redirectedBadOutputs.count(i.second.path(worker.store, drv->name))
|
||||
&& pathExists(redirected))
|
||||
replaceValidPath(path, redirected);
|
||||
if (buildMode == bmCheck)
|
||||
@ -3721,7 +3723,9 @@ void DerivationGoal::registerOutputs()
|
||||
|
||||
if (fixedOutput) {
|
||||
|
||||
if (i.second.hash->method == FileIngestionMethod::Flat) {
|
||||
FixedOutputHash outputHash = std::get<DerivationOutputFixed>(i.second.output).hash;
|
||||
|
||||
if (outputHash.method == FileIngestionMethod::Flat) {
|
||||
/* The output path should be a regular file without execute permission. */
|
||||
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
|
||||
throw BuildError(
|
||||
@ -3732,13 +3736,13 @@ void DerivationGoal::registerOutputs()
|
||||
|
||||
/* Check the hash. In hash mode, move the path produced by
|
||||
the derivation to its content-addressed location. */
|
||||
Hash h2 = i.second.hash->method == FileIngestionMethod::Recursive
|
||||
? hashPath(*i.second.hash->hash.type, actualPath).first
|
||||
: hashFile(*i.second.hash->hash.type, actualPath);
|
||||
Hash h2 = outputHash.method == FileIngestionMethod::Recursive
|
||||
? hashPath(outputHash.hash.type, actualPath).first
|
||||
: hashFile(outputHash.hash.type, actualPath);
|
||||
|
||||
auto dest = worker.store.makeFixedOutputPath(i.second.hash->method, h2, i.second.path.name());
|
||||
auto dest = worker.store.makeFixedOutputPath(outputHash.method, h2, i.second.path(worker.store, drv->name).name());
|
||||
|
||||
if (i.second.hash->hash != h2) {
|
||||
if (outputHash.hash != h2) {
|
||||
|
||||
/* Throw an error after registering the path as
|
||||
valid. */
|
||||
@ -3746,7 +3750,7 @@ void DerivationGoal::registerOutputs()
|
||||
delayedException = std::make_exception_ptr(
|
||||
BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
|
||||
worker.store.printStorePath(dest),
|
||||
i.second.hash->hash.to_string(SRI, true),
|
||||
outputHash.hash.to_string(SRI, true),
|
||||
h2.to_string(SRI, true)));
|
||||
|
||||
Path actualDest = worker.store.Store::toRealPath(dest);
|
||||
@ -3768,7 +3772,7 @@ void DerivationGoal::registerOutputs()
|
||||
assert(worker.store.parseStorePath(path) == dest);
|
||||
|
||||
ca = FixedOutputHash {
|
||||
.method = i.second.hash->method,
|
||||
.method = outputHash.method,
|
||||
.hash = h2,
|
||||
};
|
||||
}
|
||||
@ -3783,8 +3787,10 @@ void DerivationGoal::registerOutputs()
|
||||
time. The hash is stored in the database so that we can
|
||||
verify later on whether nobody has messed with the store. */
|
||||
debug("scanning for references inside '%1%'", path);
|
||||
HashResult hash;
|
||||
auto references = worker.store.parseStorePathSet(scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths), hash));
|
||||
// HashResult hash;
|
||||
auto pathSetAndHash = scanForReferences(actualPath, worker.store.printStorePathSet(referenceablePaths));
|
||||
auto references = worker.store.parseStorePathSet(pathSetAndHash.first);
|
||||
HashResult hash = pathSetAndHash.second;
|
||||
|
||||
if (buildMode == bmCheck) {
|
||||
if (!worker.store.isValidPath(worker.store.parseStorePath(path))) continue;
|
||||
@ -3892,7 +3898,7 @@ void DerivationGoal::registerOutputs()
|
||||
/* If this is the first round of several, then move the output out of the way. */
|
||||
if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
|
||||
for (auto & i : drv->outputs) {
|
||||
auto path = worker.store.printStorePath(i.second.path);
|
||||
auto path = worker.store.printStorePath(i.second.path(worker.store, drv->name));
|
||||
Path prev = path + checkSuffix;
|
||||
deletePath(prev);
|
||||
Path dst = path + checkSuffix;
|
||||
@ -3910,7 +3916,7 @@ void DerivationGoal::registerOutputs()
|
||||
if the result was not determistic? */
|
||||
if (curRound == nrRounds) {
|
||||
for (auto & i : drv->outputs) {
|
||||
Path prev = worker.store.printStorePath(i.second.path) + checkSuffix;
|
||||
Path prev = worker.store.printStorePath(i.second.path(worker.store, drv->name)) + checkSuffix;
|
||||
deletePath(prev);
|
||||
}
|
||||
}
|
||||
@ -4211,9 +4217,9 @@ StorePathSet DerivationGoal::checkPathValidity(bool returnValid, bool checkHash)
|
||||
for (auto & i : drv->outputs) {
|
||||
if (!wantOutput(i.first, wantedOutputs)) continue;
|
||||
bool good =
|
||||
worker.store.isValidPath(i.second.path) &&
|
||||
(!checkHash || worker.pathContentsGood(i.second.path));
|
||||
if (good == returnValid) result.insert(i.second.path);
|
||||
worker.store.isValidPath(i.second.path(worker.store, drv->name)) &&
|
||||
(!checkHash || worker.pathContentsGood(i.second.path(worker.store, drv->name)));
|
||||
if (good == returnValid) result.insert(i.second.path(worker.store, drv->name));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
@ -5006,7 +5012,7 @@ bool Worker::pathContentsGood(const StorePath & path)
|
||||
if (!pathExists(store.printStorePath(path)))
|
||||
res = false;
|
||||
else {
|
||||
HashResult current = hashPath(*info->narHash.type, store.printStorePath(path));
|
||||
HashResult current = hashPath(info->narHash->type, store.printStorePath(path));
|
||||
Hash nullHash(htSHA256);
|
||||
res = info->narHash == nullHash || info->narHash == current.first;
|
||||
}
|
||||
@ -5032,7 +5038,7 @@ void Worker::markContentsGood(const StorePath & path)
|
||||
static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
|
||||
{
|
||||
StorePathSet willBuild, willSubstitute, unknown;
|
||||
unsigned long long downloadSize, narSize;
|
||||
uint64_t downloadSize, narSize;
|
||||
store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
|
||||
if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
|
||||
|
@ -9,7 +9,7 @@ struct Package {
|
||||
Path path;
|
||||
bool active;
|
||||
int priority;
|
||||
Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
|
||||
Package(const Path & path, bool active, int priority) : path{path}, active{active}, priority{priority} {}
|
||||
};
|
||||
|
||||
typedef std::vector<Package> Packages;
|
||||
|
@ -63,16 +63,19 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
|
||||
auto & output = drv.outputs.begin()->second;
|
||||
|
||||
/* Try the hashed mirrors first. */
|
||||
if (output.hash && output.hash->method == FileIngestionMethod::Flat)
|
||||
for (auto hashedMirror : settings.hashedMirrors.get())
|
||||
if (auto hash = std::get_if<DerivationOutputFixed>(&output.output)) {
|
||||
if (hash->hash.method == FileIngestionMethod::Flat) {
|
||||
for (auto hashedMirror : settings.hashedMirrors.get()) {
|
||||
try {
|
||||
if (!hasSuffix(hashedMirror, "/")) hashedMirror += '/';
|
||||
auto & h = output.hash->hash;
|
||||
fetch(hashedMirror + printHashType(*h.type) + "/" + h.to_string(Base16, false));
|
||||
fetch(hashedMirror + printHashType(hash->hash.hash.type) + "/" + hash->hash.hash.to_string(Base16, false));
|
||||
return;
|
||||
} catch (Error & e) {
|
||||
debug(e.what());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Otherwise try the specified URL. */
|
||||
fetch(mainUrl);
|
||||
|
@ -3,7 +3,7 @@
|
||||
namespace nix {
|
||||
|
||||
std::string FixedOutputHash::printMethodAlgo() const {
|
||||
return makeFileIngestionPrefix(method) + printHashType(*hash.type);
|
||||
return makeFileIngestionPrefix(method) + printHashType(hash.type);
|
||||
}
|
||||
|
||||
std::string makeFileIngestionPrefix(const FileIngestionMethod m) {
|
||||
@ -46,7 +46,7 @@ ContentAddress parseContentAddress(std::string_view rawCa) {
|
||||
if (prefix == "text") {
|
||||
auto hashTypeAndHash = rawCa.substr(prefixSeparator+1, string::npos);
|
||||
Hash hash = Hash(string(hashTypeAndHash));
|
||||
if (*hash.type != htSHA256) {
|
||||
if (hash.type != htSHA256) {
|
||||
throw Error("parseContentAddress: the text hash should have type SHA256");
|
||||
}
|
||||
return TextHash { hash };
|
||||
@ -82,4 +82,16 @@ std::string renderContentAddress(std::optional<ContentAddress> ca) {
|
||||
return ca ? renderContentAddress(*ca) : "";
|
||||
}
|
||||
|
||||
Hash getContentAddressHash(const ContentAddress & ca)
|
||||
{
|
||||
return std::visit(overloaded {
|
||||
[](TextHash th) {
|
||||
return th.hash;
|
||||
},
|
||||
[](FixedOutputHash fsh) {
|
||||
return fsh.hash;
|
||||
}
|
||||
}, ca);
|
||||
}
|
||||
|
||||
}
|
||||
|
@ -53,4 +53,6 @@ ContentAddress parseContentAddress(std::string_view rawCa);
|
||||
|
||||
std::optional<ContentAddress> parseContentAddressOpt(std::string_view rawCaOpt);
|
||||
|
||||
Hash getContentAddressHash(const ContentAddress & ca);
|
||||
|
||||
}
|
||||
|
@ -78,10 +78,10 @@ struct TunnelLogger : public Logger
|
||||
if (ei.level > verbosity) return;
|
||||
|
||||
std::stringstream oss;
|
||||
oss << ei;
|
||||
showErrorInfo(oss, ei, false);
|
||||
|
||||
StringSink buf;
|
||||
buf << STDERR_NEXT << oss.str() << "\n"; // (fs.s + "\n");
|
||||
buf << STDERR_NEXT << oss.str();
|
||||
enqueueMsg(*buf.s);
|
||||
}
|
||||
|
||||
@ -173,31 +173,6 @@ struct TunnelSource : BufferedSource
|
||||
}
|
||||
};
|
||||
|
||||
/* If the NAR archive contains a single file at top-level, then save
|
||||
the contents of the file to `s'. Otherwise barf. */
|
||||
struct RetrieveRegularNARSink : ParseSink
|
||||
{
|
||||
bool regular;
|
||||
string s;
|
||||
|
||||
RetrieveRegularNARSink() : regular(true) { }
|
||||
|
||||
void createDirectory(const Path & path)
|
||||
{
|
||||
regular = false;
|
||||
}
|
||||
|
||||
void receiveContents(unsigned char * data, unsigned int len)
|
||||
{
|
||||
s.append((const char *) data, len);
|
||||
}
|
||||
|
||||
void createSymlink(const Path & path, const string & target)
|
||||
{
|
||||
regular = false;
|
||||
}
|
||||
};
|
||||
|
||||
struct ClientSettings
|
||||
{
|
||||
bool keepFailed;
|
||||
@ -314,7 +289,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||
logger->startWork();
|
||||
auto hash = store->queryPathInfo(path)->narHash;
|
||||
logger->stopWork();
|
||||
to << hash.to_string(Base16, false);
|
||||
to << hash->to_string(Base16, false);
|
||||
break;
|
||||
}
|
||||
|
||||
@ -347,6 +322,15 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||
break;
|
||||
}
|
||||
|
||||
case wopQueryDerivationOutputMap: {
|
||||
auto path = store->parseStorePath(readString(from));
|
||||
logger->startWork();
|
||||
OutputPathMap outputs = store->queryDerivationOutputMap(path);
|
||||
logger->stopWork();
|
||||
writeOutputPathMap(*store, to, outputs);
|
||||
break;
|
||||
}
|
||||
|
||||
case wopQueryDeriver: {
|
||||
auto path = store->parseStorePath(readString(from));
|
||||
logger->startWork();
|
||||
@ -366,42 +350,44 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||
}
|
||||
|
||||
case wopAddToStore: {
|
||||
std::string s, baseName;
|
||||
HashType hashAlgo;
|
||||
std::string baseName;
|
||||
FileIngestionMethod method;
|
||||
{
|
||||
bool fixed; uint8_t recursive;
|
||||
from >> baseName >> fixed /* obsolete */ >> recursive >> s;
|
||||
bool fixed;
|
||||
uint8_t recursive;
|
||||
std::string hashAlgoRaw;
|
||||
from >> baseName >> fixed /* obsolete */ >> recursive >> hashAlgoRaw;
|
||||
if (recursive > (uint8_t) FileIngestionMethod::Recursive)
|
||||
throw Error("unsupported FileIngestionMethod with value of %i; you may need to upgrade nix-daemon", recursive);
|
||||
method = FileIngestionMethod { recursive };
|
||||
/* Compatibility hack. */
|
||||
if (!fixed) {
|
||||
s = "sha256";
|
||||
hashAlgoRaw = "sha256";
|
||||
method = FileIngestionMethod::Recursive;
|
||||
}
|
||||
hashAlgo = parseHashType(hashAlgoRaw);
|
||||
}
|
||||
HashType hashAlgo = parseHashType(s);
|
||||
|
||||
TeeSource savedNAR(from);
|
||||
RetrieveRegularNARSink savedRegular;
|
||||
StringSink saved;
|
||||
TeeSource savedNARSource(from, saved);
|
||||
RetrieveRegularNARSink savedRegular { saved };
|
||||
|
||||
if (method == FileIngestionMethod::Recursive) {
|
||||
/* Get the entire NAR dump from the client and save it to
|
||||
a string so that we can pass it to
|
||||
addToStoreFromDump(). */
|
||||
ParseSink sink; /* null sink; just parse the NAR */
|
||||
parseDump(sink, savedNAR);
|
||||
parseDump(sink, savedNARSource);
|
||||
} else
|
||||
parseDump(savedRegular, from);
|
||||
|
||||
logger->startWork();
|
||||
if (!savedRegular.regular) throw Error("regular file expected");
|
||||
|
||||
auto path = store->addToStoreFromDump(
|
||||
method == FileIngestionMethod::Recursive ? *savedNAR.data : savedRegular.s,
|
||||
baseName,
|
||||
method,
|
||||
hashAlgo);
|
||||
// FIXME: try to stream directly from `from`.
|
||||
StringSource dumpSource { *saved.s };
|
||||
auto path = store->addToStoreFromDump(dumpSource, baseName, method, hashAlgo);
|
||||
logger->stopWork();
|
||||
|
||||
to << store->printStorePath(path);
|
||||
@ -433,7 +419,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||
case wopImportPaths: {
|
||||
logger->startWork();
|
||||
TunnelSource source(from, to);
|
||||
auto paths = store->importPaths(source, nullptr,
|
||||
auto paths = store->importPaths(source,
|
||||
trusted ? NoCheckSigs : CheckSigs);
|
||||
logger->stopWork();
|
||||
Strings paths2;
|
||||
@ -465,7 +451,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||
case wopBuildDerivation: {
|
||||
auto drvPath = store->parseStorePath(readString(from));
|
||||
BasicDerivation drv;
|
||||
readDerivation(from, *store, drv);
|
||||
readDerivation(from, *store, drv, Derivation::nameFromPath(drvPath));
|
||||
BuildMode buildMode = (BuildMode) readInt(from);
|
||||
logger->startWork();
|
||||
if (!trusted)
|
||||
@ -646,7 +632,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 17)
|
||||
to << 1;
|
||||
to << (info->deriver ? store->printStorePath(*info->deriver) : "")
|
||||
<< info->narHash.to_string(Base16, false);
|
||||
<< info->narHash->to_string(Base16, false);
|
||||
writeStorePaths(*store, to, info->references);
|
||||
to << info->registrationTime << info->narSize;
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 16) {
|
||||
@ -717,24 +703,84 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||
if (!trusted)
|
||||
info.ultimate = false;
|
||||
|
||||
std::string saved;
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 23) {
|
||||
|
||||
struct FramedSource : Source
|
||||
{
|
||||
Source & from;
|
||||
bool eof = false;
|
||||
std::vector<unsigned char> pending;
|
||||
size_t pos = 0;
|
||||
|
||||
FramedSource(Source & from) : from(from)
|
||||
{ }
|
||||
|
||||
~FramedSource()
|
||||
{
|
||||
if (!eof) {
|
||||
while (true) {
|
||||
auto n = readInt(from);
|
||||
if (!n) break;
|
||||
std::vector<unsigned char> data(n);
|
||||
from(data.data(), n);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
size_t read(unsigned char * data, size_t len) override
|
||||
{
|
||||
if (eof) throw EndOfFile("reached end of FramedSource");
|
||||
|
||||
if (pos >= pending.size()) {
|
||||
size_t len = readInt(from);
|
||||
if (!len) {
|
||||
eof = true;
|
||||
return 0;
|
||||
}
|
||||
pending = std::vector<unsigned char>(len);
|
||||
pos = 0;
|
||||
from(pending.data(), len);
|
||||
}
|
||||
|
||||
auto n = std::min(len, pending.size() - pos);
|
||||
memcpy(data, pending.data() + pos, n);
|
||||
pos += n;
|
||||
return n;
|
||||
}
|
||||
};
|
||||
|
||||
logger->startWork();
|
||||
|
||||
{
|
||||
FramedSource source(from);
|
||||
store->addToStore(info, source, (RepairFlag) repair,
|
||||
dontCheckSigs ? NoCheckSigs : CheckSigs);
|
||||
}
|
||||
|
||||
logger->stopWork();
|
||||
}
|
||||
|
||||
else {
|
||||
std::unique_ptr<Source> source;
|
||||
if (GET_PROTOCOL_MINOR(clientVersion) >= 21)
|
||||
source = std::make_unique<TunnelSource>(from, to);
|
||||
else {
|
||||
TeeSink tee(from);
|
||||
parseDump(tee, tee.source);
|
||||
saved = std::move(*tee.source.data);
|
||||
source = std::make_unique<StringSource>(saved);
|
||||
StringSink saved;
|
||||
TeeSource tee { from, saved };
|
||||
ParseSink ether;
|
||||
parseDump(ether, tee);
|
||||
source = std::make_unique<StringSource>(std::move(*saved.s));
|
||||
}
|
||||
|
||||
logger->startWork();
|
||||
|
||||
// FIXME: race if addToStore doesn't read source?
|
||||
store->addToStore(info, *source, (RepairFlag) repair,
|
||||
dontCheckSigs ? NoCheckSigs : CheckSigs, nullptr);
|
||||
dontCheckSigs ? NoCheckSigs : CheckSigs);
|
||||
|
||||
logger->stopWork();
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
@ -744,7 +790,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
|
||||
targets.push_back(store->parsePathWithOutputs(s));
|
||||
logger->startWork();
|
||||
StorePathSet willBuild, willSubstitute, unknown;
|
||||
unsigned long long downloadSize, narSize;
|
||||
uint64_t downloadSize, narSize;
|
||||
store->queryMissing(targets, willBuild, willSubstitute, unknown, downloadSize, narSize);
|
||||
logger->stopWork();
|
||||
writeStorePaths(*store, to, willBuild);
|
||||
|
@ -4,16 +4,23 @@
|
||||
#include "util.hh"
|
||||
#include "worker-protocol.hh"
|
||||
#include "fs-accessor.hh"
|
||||
#include "istringstream_nocopy.hh"
|
||||
|
||||
namespace nix {
|
||||
|
||||
const StorePath & BasicDerivation::findOutput(const string & id) const
|
||||
// FIXME Put this somewhere?
|
||||
template<class... Ts> struct overloaded : Ts... { using Ts::operator()...; };
|
||||
template<class... Ts> overloaded(Ts...) -> overloaded<Ts...>;
|
||||
|
||||
StorePath DerivationOutput::path(const Store & store, std::string_view drvName) const
|
||||
{
|
||||
auto i = outputs.find(id);
|
||||
if (i == outputs.end())
|
||||
throw Error("derivation has no output '%s'", id);
|
||||
return i->second.path;
|
||||
return std::visit(overloaded {
|
||||
[](DerivationOutputInputAddressed doi) {
|
||||
return doi.path;
|
||||
},
|
||||
[&](DerivationOutputFixed dof) {
|
||||
return store.makeFixedOutputPath(dof.hash.method, dof.hash.hash, drvName);
|
||||
}
|
||||
}, output);
|
||||
}
|
||||
|
||||
|
||||
@ -101,14 +108,13 @@ static StringSet parseStrings(std::istream & str, bool arePaths)
|
||||
}
|
||||
|
||||
|
||||
static DerivationOutput parseDerivationOutput(const Store & store, istringstream_nocopy & str)
|
||||
static DerivationOutput parseDerivationOutput(const Store & store, std::istringstream & str)
|
||||
{
|
||||
expect(str, ","); auto path = store.parseStorePath(parsePath(str));
|
||||
expect(str, ","); auto hashAlgo = parseString(str);
|
||||
expect(str, ","); const auto hash = parseString(str);
|
||||
expect(str, ")");
|
||||
|
||||
std::optional<FixedOutputHash> fsh;
|
||||
if (hashAlgo != "") {
|
||||
auto method = FileIngestionMethod::Flat;
|
||||
if (string(hashAlgo, 0, 2) == "r:") {
|
||||
@ -116,23 +122,30 @@ static DerivationOutput parseDerivationOutput(const Store & store, istringstream
|
||||
hashAlgo = string(hashAlgo, 2);
|
||||
}
|
||||
const HashType hashType = parseHashType(hashAlgo);
|
||||
fsh = FixedOutputHash {
|
||||
.method = std::move(method),
|
||||
.hash = Hash(hash, hashType),
|
||||
};
|
||||
}
|
||||
|
||||
return DerivationOutput {
|
||||
.output = DerivationOutputFixed {
|
||||
.hash = FixedOutputHash {
|
||||
.method = std::move(method),
|
||||
.hash = Hash(hash, hashType),
|
||||
},
|
||||
}
|
||||
};
|
||||
} else
|
||||
return DerivationOutput {
|
||||
.output = DerivationOutputInputAddressed {
|
||||
.path = std::move(path),
|
||||
.hash = std::move(fsh),
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
static Derivation parseDerivation(const Store & store, const string & s)
|
||||
static Derivation parseDerivation(const Store & store, std::string && s, std::string_view name)
|
||||
{
|
||||
Derivation drv;
|
||||
istringstream_nocopy str(s);
|
||||
drv.name = name;
|
||||
|
||||
std::istringstream str(std::move(s));
|
||||
expect(str, "Derive([");
|
||||
|
||||
/* Parse the list of outputs. */
|
||||
@ -175,10 +188,10 @@ static Derivation parseDerivation(const Store & store, const string & s)
|
||||
}
|
||||
|
||||
|
||||
Derivation readDerivation(const Store & store, const Path & drvPath)
|
||||
Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name)
|
||||
{
|
||||
try {
|
||||
return parseDerivation(store, readFile(drvPath));
|
||||
return parseDerivation(store, readFile(drvPath), name);
|
||||
} catch (FormatError & e) {
|
||||
throw Error("error parsing derivation '%1%': %2%", drvPath, e.msg());
|
||||
}
|
||||
@ -196,7 +209,7 @@ Derivation Store::readDerivation(const StorePath & drvPath)
|
||||
{
|
||||
auto accessor = getFSAccessor();
|
||||
try {
|
||||
return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)));
|
||||
return parseDerivation(*this, accessor->readFile(printStorePath(drvPath)), Derivation::nameFromPath(drvPath));
|
||||
} catch (FormatError & e) {
|
||||
throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
|
||||
}
|
||||
@ -264,10 +277,14 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
|
||||
for (auto & i : outputs) {
|
||||
if (first) first = false; else s += ',';
|
||||
s += '('; printUnquotedString(s, i.first);
|
||||
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path));
|
||||
s += ','; printUnquotedString(s, i.second.hash ? i.second.hash->printMethodAlgo() : "");
|
||||
s += ','; printUnquotedString(s,
|
||||
i.second.hash ? i.second.hash->hash.to_string(Base16, false) : "");
|
||||
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(i.second.path(store, name)));
|
||||
if (auto hash = std::get_if<DerivationOutputFixed>(&i.second.output)) {
|
||||
s += ','; printUnquotedString(s, hash->hash.printMethodAlgo());
|
||||
s += ','; printUnquotedString(s, hash->hash.hash.to_string(Base16, false));
|
||||
} else {
|
||||
s += ','; printUnquotedString(s, "");
|
||||
s += ','; printUnquotedString(s, "");
|
||||
}
|
||||
s += ')';
|
||||
}
|
||||
|
||||
@ -323,7 +340,7 @@ bool BasicDerivation::isFixedOutput() const
|
||||
{
|
||||
return outputs.size() == 1 &&
|
||||
outputs.begin()->first == "out" &&
|
||||
outputs.begin()->second.hash;
|
||||
std::holds_alternative<DerivationOutputFixed>(outputs.begin()->second.output);
|
||||
}
|
||||
|
||||
|
||||
@ -355,10 +372,11 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
|
||||
/* Return a fixed hash for fixed-output derivations. */
|
||||
if (drv.isFixedOutput()) {
|
||||
DerivationOutputs::const_iterator i = drv.outputs.begin();
|
||||
auto hash = std::get<DerivationOutputFixed>(i->second.output);
|
||||
return hashString(htSHA256, "fixed:out:"
|
||||
+ i->second.hash->printMethodAlgo() + ":"
|
||||
+ i->second.hash->hash.to_string(Base16, false) + ":"
|
||||
+ store.printStorePath(i->second.path));
|
||||
+ hash.hash.printMethodAlgo() + ":"
|
||||
+ hash.hash.hash.to_string(Base16, false) + ":"
|
||||
+ store.printStorePath(i->second.path(store, drv.name)));
|
||||
}
|
||||
|
||||
/* For other derivations, replace the inputs paths with recursive
|
||||
@ -392,11 +410,11 @@ bool wantOutput(const string & output, const std::set<string> & wanted)
|
||||
}
|
||||
|
||||
|
||||
StorePathSet BasicDerivation::outputPaths() const
|
||||
StorePathSet BasicDerivation::outputPaths(const Store & store) const
|
||||
{
|
||||
StorePathSet paths;
|
||||
for (auto & i : outputs)
|
||||
paths.insert(i.second.path);
|
||||
paths.insert(i.second.path(store, name));
|
||||
return paths;
|
||||
}
|
||||
|
||||
@ -406,7 +424,6 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
|
||||
auto hashAlgo = readString(in);
|
||||
auto hash = readString(in);
|
||||
|
||||
std::optional<FixedOutputHash> fsh;
|
||||
if (hashAlgo != "") {
|
||||
auto method = FileIngestionMethod::Flat;
|
||||
if (string(hashAlgo, 0, 2) == "r:") {
|
||||
@ -414,15 +431,19 @@ static DerivationOutput readDerivationOutput(Source & in, const Store & store)
|
||||
hashAlgo = string(hashAlgo, 2);
|
||||
}
|
||||
auto hashType = parseHashType(hashAlgo);
|
||||
fsh = FixedOutputHash {
|
||||
return DerivationOutput {
|
||||
.output = DerivationOutputFixed {
|
||||
.hash = FixedOutputHash {
|
||||
.method = std::move(method),
|
||||
.hash = Hash(hash, hashType),
|
||||
};
|
||||
},
|
||||
}
|
||||
|
||||
};
|
||||
} else
|
||||
return DerivationOutput {
|
||||
.output = DerivationOutputInputAddressed {
|
||||
.path = std::move(path),
|
||||
.hash = std::move(fsh),
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
@ -435,8 +456,19 @@ StringSet BasicDerivation::outputNames() const
|
||||
}
|
||||
|
||||
|
||||
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv)
|
||||
std::string_view BasicDerivation::nameFromPath(const StorePath & drvPath) {
|
||||
auto nameWithSuffix = drvPath.name();
|
||||
constexpr std::string_view extension = ".drv";
|
||||
assert(hasSuffix(nameWithSuffix, extension));
|
||||
nameWithSuffix.remove_suffix(extension.size());
|
||||
return nameWithSuffix;
|
||||
}
|
||||
|
||||
|
||||
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name)
|
||||
{
|
||||
drv.name = name;
|
||||
|
||||
drv.outputs.clear();
|
||||
auto nr = readNum<size_t>(in);
|
||||
for (size_t n = 0; n < nr; n++) {
|
||||
@ -465,10 +497,10 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
|
||||
out << drv.outputs.size();
|
||||
for (auto & i : drv.outputs) {
|
||||
out << i.first
|
||||
<< store.printStorePath(i.second.path);
|
||||
if (i.second.hash) {
|
||||
out << i.second.hash->printMethodAlgo()
|
||||
<< i.second.hash->hash.to_string(Base16, false);
|
||||
<< store.printStorePath(i.second.path(store, drv.name));
|
||||
if (auto hash = std::get_if<DerivationOutputFixed>(&i.second.output)) {
|
||||
out << hash->hash.printMethodAlgo()
|
||||
<< hash->hash.hash.to_string(Base16, false);
|
||||
} else {
|
||||
out << "" << "";
|
||||
}
|
||||
|
@ -13,10 +13,20 @@ namespace nix {
|
||||
|
||||
/* Abstract syntax of derivations. */
|
||||
|
||||
struct DerivationOutput
|
||||
struct DerivationOutputInputAddressed
|
||||
{
|
||||
StorePath path;
|
||||
std::optional<FixedOutputHash> hash; /* hash used for expected hash computation */
|
||||
};
|
||||
|
||||
struct DerivationOutputFixed
|
||||
{
|
||||
FixedOutputHash hash; /* hash used for expected hash computation */
|
||||
};
|
||||
|
||||
struct DerivationOutput
|
||||
{
|
||||
std::variant<DerivationOutputInputAddressed, DerivationOutputFixed> output;
|
||||
StorePath path(const Store & store, std::string_view drvName) const;
|
||||
};
|
||||
|
||||
typedef std::map<string, DerivationOutput> DerivationOutputs;
|
||||
@ -35,24 +45,23 @@ struct BasicDerivation
|
||||
Path builder;
|
||||
Strings args;
|
||||
StringPairs env;
|
||||
std::string name;
|
||||
|
||||
BasicDerivation() { }
|
||||
virtual ~BasicDerivation() { };
|
||||
|
||||
/* Return the path corresponding to the output identifier `id' in
|
||||
the given derivation. */
|
||||
const StorePath & findOutput(const std::string & id) const;
|
||||
|
||||
bool isBuiltin() const;
|
||||
|
||||
/* Return true iff this is a fixed-output derivation. */
|
||||
bool isFixedOutput() const;
|
||||
|
||||
/* Return the output paths of a derivation. */
|
||||
StorePathSet outputPaths() const;
|
||||
StorePathSet outputPaths(const Store & store) const;
|
||||
|
||||
/* Return the output names of a derivation. */
|
||||
StringSet outputNames() const;
|
||||
|
||||
static std::string_view nameFromPath(const StorePath & storePath);
|
||||
};
|
||||
|
||||
struct Derivation : BasicDerivation
|
||||
@ -76,7 +85,7 @@ StorePath writeDerivation(ref<Store> store,
|
||||
const Derivation & drv, std::string_view name, RepairFlag repair = NoRepair);
|
||||
|
||||
/* Read a derivation from a file. */
|
||||
Derivation readDerivation(const Store & store, const Path & drvPath);
|
||||
Derivation readDerivation(const Store & store, const Path & drvPath, std::string_view name);
|
||||
|
||||
// FIXME: remove
|
||||
bool isDerivation(const string & fileName);
|
||||
@ -93,7 +102,7 @@ bool wantOutput(const string & output, const std::set<string> & wanted);
|
||||
struct Source;
|
||||
struct Sink;
|
||||
|
||||
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv);
|
||||
Source & readDerivation(Source & in, const Store & store, BasicDerivation & drv, std::string_view name);
|
||||
void writeDerivation(Sink & out, const Store & store, const BasicDerivation & drv);
|
||||
|
||||
std::string hashPlaceholder(const std::string & outputName);
|
||||
|
@ -7,24 +7,6 @@
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct HashAndWriteSink : Sink
|
||||
{
|
||||
Sink & writeSink;
|
||||
HashSink hashSink;
|
||||
HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256)
|
||||
{
|
||||
}
|
||||
virtual void operator () (const unsigned char * data, size_t len)
|
||||
{
|
||||
writeSink(data, len);
|
||||
hashSink(data, len);
|
||||
}
|
||||
Hash currentHash()
|
||||
{
|
||||
return hashSink.currentHash().first;
|
||||
}
|
||||
};
|
||||
|
||||
void Store::exportPaths(const StorePathSet & paths, Sink & sink)
|
||||
{
|
||||
auto sorted = topoSortPaths(paths);
|
||||
@ -47,28 +29,29 @@ void Store::exportPath(const StorePath & path, Sink & sink)
|
||||
{
|
||||
auto info = queryPathInfo(path);
|
||||
|
||||
HashAndWriteSink hashAndWriteSink(sink);
|
||||
HashSink hashSink(htSHA256);
|
||||
TeeSink teeSink(sink, hashSink);
|
||||
|
||||
narFromPath(path, hashAndWriteSink);
|
||||
narFromPath(path, teeSink);
|
||||
|
||||
/* Refuse to export paths that have changed. This prevents
|
||||
filesystem corruption from spreading to other machines.
|
||||
Don't complain if the stored hash is zero (unknown). */
|
||||
Hash hash = hashAndWriteSink.currentHash();
|
||||
if (hash != info->narHash && info->narHash != Hash(*info->narHash.type))
|
||||
Hash hash = hashSink.currentHash().first;
|
||||
if (hash != info->narHash && info->narHash != Hash(info->narHash->type))
|
||||
throw Error("hash of path '%s' has changed from '%s' to '%s'!",
|
||||
printStorePath(path), info->narHash.to_string(Base32, true), hash.to_string(Base32, true));
|
||||
printStorePath(path), info->narHash->to_string(Base32, true), hash.to_string(Base32, true));
|
||||
|
||||
hashAndWriteSink
|
||||
teeSink
|
||||
<< exportMagic
|
||||
<< printStorePath(path);
|
||||
writeStorePaths(*this, hashAndWriteSink, info->references);
|
||||
hashAndWriteSink
|
||||
writeStorePaths(*this, teeSink, info->references);
|
||||
teeSink
|
||||
<< (info->deriver ? printStorePath(*info->deriver) : "")
|
||||
<< 0;
|
||||
}
|
||||
|
||||
StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> accessor, CheckSigsFlag checkSigs)
|
||||
StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs)
|
||||
{
|
||||
StorePaths res;
|
||||
while (true) {
|
||||
@ -77,8 +60,10 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
|
||||
if (n != 1) throw Error("input doesn't look like something created by 'nix-store --export'");
|
||||
|
||||
/* Extract the NAR from the source. */
|
||||
TeeSink tee(source);
|
||||
parseDump(tee, tee.source);
|
||||
StringSink saved;
|
||||
TeeSource tee { source, saved };
|
||||
ParseSink ether;
|
||||
parseDump(ether, tee);
|
||||
|
||||
uint32_t magic = readInt(source);
|
||||
if (magic != exportMagic)
|
||||
@ -94,16 +79,16 @@ StorePaths Store::importPaths(Source & source, std::shared_ptr<FSAccessor> acces
|
||||
if (deriver != "")
|
||||
info.deriver = parseStorePath(deriver);
|
||||
|
||||
info.narHash = hashString(htSHA256, *tee.source.data);
|
||||
info.narSize = tee.source.data->size();
|
||||
info.narHash = hashString(htSHA256, *saved.s);
|
||||
info.narSize = saved.s->size();
|
||||
|
||||
// Ignore optional legacy signature.
|
||||
if (readInt(source) == 1)
|
||||
readString(source);
|
||||
|
||||
// Can't use underlying source, which would have been exhausted
|
||||
auto source = StringSource { *tee.source.data };
|
||||
addToStore(info, source, NoRepair, checkSigs, accessor);
|
||||
auto source = StringSource { *saved.s };
|
||||
addToStore(info, source, NoRepair, checkSigs);
|
||||
|
||||
res.push_back(info.path);
|
||||
}
|
||||
|
@ -22,6 +22,7 @@
|
||||
#include <queue>
|
||||
#include <random>
|
||||
#include <thread>
|
||||
#include <regex>
|
||||
|
||||
using namespace std::string_literals;
|
||||
|
||||
@ -56,7 +57,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
Callback<FileTransferResult> callback;
|
||||
CURL * req = 0;
|
||||
bool active = false; // whether the handle has been added to the multi object
|
||||
std::string status;
|
||||
std::string statusMsg;
|
||||
|
||||
unsigned int attempt = 0;
|
||||
|
||||
@ -123,7 +124,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
if (requestHeaders) curl_slist_free_all(requestHeaders);
|
||||
try {
|
||||
if (!done)
|
||||
fail(FileTransferError(Interrupted, "download of '%s' was interrupted", request.uri));
|
||||
fail(FileTransferError(Interrupted, nullptr, "download of '%s' was interrupted", request.uri));
|
||||
} catch (...) {
|
||||
ignoreException();
|
||||
}
|
||||
@ -144,6 +145,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
|
||||
LambdaSink finalSink;
|
||||
std::shared_ptr<CompressionSink> decompressionSink;
|
||||
std::optional<StringSink> errorSink;
|
||||
|
||||
std::exception_ptr writeException;
|
||||
|
||||
@ -153,9 +155,19 @@ struct curlFileTransfer : public FileTransfer
|
||||
size_t realSize = size * nmemb;
|
||||
result.bodySize += realSize;
|
||||
|
||||
if (!decompressionSink)
|
||||
if (!decompressionSink) {
|
||||
decompressionSink = makeDecompressionSink(encoding, finalSink);
|
||||
if (! successfulStatuses.count(getHTTPStatus())) {
|
||||
// In this case we want to construct a TeeSink, to keep
|
||||
// the response around (which we figure won't be big
|
||||
// like an actual download should be) to improve error
|
||||
// messages.
|
||||
errorSink = StringSink { };
|
||||
}
|
||||
}
|
||||
|
||||
if (errorSink)
|
||||
(*errorSink)((unsigned char *) contents, realSize);
|
||||
(*decompressionSink)((unsigned char *) contents, realSize);
|
||||
|
||||
return realSize;
|
||||
@ -175,12 +187,13 @@ struct curlFileTransfer : public FileTransfer
|
||||
size_t realSize = size * nmemb;
|
||||
std::string line((char *) contents, realSize);
|
||||
printMsg(lvlVomit, format("got header for '%s': %s") % request.uri % trim(line));
|
||||
if (line.compare(0, 5, "HTTP/") == 0) { // new response starts
|
||||
static std::regex statusLine("HTTP/[^ ]+ +[0-9]+(.*)", std::regex::extended | std::regex::icase);
|
||||
std::smatch match;
|
||||
if (std::regex_match(line, match, statusLine)) {
|
||||
result.etag = "";
|
||||
auto ss = tokenizeString<vector<string>>(line, " ");
|
||||
status = ss.size() >= 2 ? ss[1] : "";
|
||||
result.data = std::make_shared<std::string>();
|
||||
result.bodySize = 0;
|
||||
statusMsg = trim(match[1]);
|
||||
acceptRanges = false;
|
||||
encoding = "";
|
||||
} else {
|
||||
@ -194,7 +207,9 @@ struct curlFileTransfer : public FileTransfer
|
||||
the expected ETag on a 200 response, then shut
|
||||
down the connection because we already have the
|
||||
data. */
|
||||
if (result.etag == request.expectedETag && status == "200") {
|
||||
long httpStatus = 0;
|
||||
curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
|
||||
if (result.etag == request.expectedETag && httpStatus == 200) {
|
||||
debug(format("shutting down on 200 HTTP response with expected ETag"));
|
||||
return 0;
|
||||
}
|
||||
@ -408,16 +423,21 @@ struct curlFileTransfer : public FileTransfer
|
||||
|
||||
attempt++;
|
||||
|
||||
std::shared_ptr<std::string> response;
|
||||
if (errorSink)
|
||||
response = errorSink->s;
|
||||
auto exc =
|
||||
code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
|
||||
? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
|
||||
? FileTransferError(Interrupted, response, "%s of '%s' was interrupted", request.verb(), request.uri)
|
||||
: httpStatus != 0
|
||||
? FileTransferError(err,
|
||||
fmt("unable to %s '%s': HTTP error %d",
|
||||
request.verb(), request.uri, httpStatus)
|
||||
response,
|
||||
fmt("unable to %s '%s': HTTP error %d ('%s')",
|
||||
request.verb(), request.uri, httpStatus, statusMsg)
|
||||
+ (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
|
||||
)
|
||||
: FileTransferError(err,
|
||||
response,
|
||||
fmt("unable to %s '%s': %s (%d)",
|
||||
request.verb(), request.uri, curl_easy_strerror(code), code));
|
||||
|
||||
@ -675,7 +695,7 @@ struct curlFileTransfer : public FileTransfer
|
||||
auto s3Res = s3Helper.getObject(bucketName, key);
|
||||
FileTransferResult res;
|
||||
if (!s3Res.data)
|
||||
throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
|
||||
throw FileTransferError(NotFound, nullptr, "S3 object '%s' does not exist", request.uri);
|
||||
res.data = s3Res.data;
|
||||
callback(std::move(res));
|
||||
#else
|
||||
@ -820,6 +840,21 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
|
||||
}
|
||||
}
|
||||
|
||||
template<typename... Args>
|
||||
FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args)
|
||||
: Error(args...), error(error), response(response)
|
||||
{
|
||||
const auto hf = hintfmt(args...);
|
||||
// FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how
|
||||
// to print different messages for different verbosity levels. For now
|
||||
// we add some heuristics for detecting when we want to show the response.
|
||||
if (response && (response->size() < 1024 || response->find("<html>") != string::npos)) {
|
||||
err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response);
|
||||
} else {
|
||||
err.hint = hf;
|
||||
}
|
||||
}
|
||||
|
||||
bool isUri(const string & s)
|
||||
{
|
||||
if (s.compare(0, 8, "channel:") == 0) return true;
|
||||
|
@ -103,10 +103,12 @@ class FileTransferError : public Error
|
||||
{
|
||||
public:
|
||||
FileTransfer::Error error;
|
||||
std::shared_ptr<string> response; // intentionally optional
|
||||
|
||||
template<typename... Args>
|
||||
FileTransferError(FileTransfer::Error error, const Args & ... args)
|
||||
: Error(args...), error(error)
|
||||
{ }
|
||||
FileTransferError(FileTransfer::Error error, std::shared_ptr<string> response, const Args & ... args);
|
||||
|
||||
virtual const char* sname() const override { return "FileTransferError"; }
|
||||
};
|
||||
|
||||
bool isUri(const string & s);
|
||||
|
@ -262,11 +262,13 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
|
||||
void LocalStore::findRoots(const Path & path, unsigned char type, Roots & roots)
|
||||
{
|
||||
auto foundRoot = [&](const Path & path, const Path & target) {
|
||||
auto storePath = maybeParseStorePath(toStorePath(target));
|
||||
if (storePath && isValidPath(*storePath))
|
||||
roots[std::move(*storePath)].emplace(path);
|
||||
try {
|
||||
auto storePath = toStorePath(target).first;
|
||||
if (isValidPath(storePath))
|
||||
roots[std::move(storePath)].emplace(path);
|
||||
else
|
||||
printInfo("skipping invalid root from '%1%' to '%2%'", path, target);
|
||||
} catch (BadStorePath &) { }
|
||||
};
|
||||
|
||||
try {
|
||||
@ -472,15 +474,15 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
|
||||
|
||||
for (auto & [target, links] : unchecked) {
|
||||
if (!isInStore(target)) continue;
|
||||
Path pathS = toStorePath(target);
|
||||
if (!isStorePath(pathS)) continue;
|
||||
auto path = parseStorePath(pathS);
|
||||
try {
|
||||
auto path = toStorePath(target).first;
|
||||
if (!isValidPath(path)) continue;
|
||||
debug("got additional root '%1%'", pathS);
|
||||
debug("got additional root '%1%'", printStorePath(path));
|
||||
if (censor)
|
||||
roots[path].insert(censored);
|
||||
else
|
||||
roots[path].insert(links.begin(), links.end());
|
||||
} catch (BadStorePath &) { }
|
||||
}
|
||||
}
|
||||
|
||||
@ -498,7 +500,7 @@ struct LocalStore::GCState
|
||||
StorePathSet alive;
|
||||
bool gcKeepOutputs;
|
||||
bool gcKeepDerivations;
|
||||
unsigned long long bytesInvalidated;
|
||||
uint64_t bytesInvalidated;
|
||||
bool moveToTrash = true;
|
||||
bool shouldDelete;
|
||||
GCState(const GCOptions & options, GCResults & results)
|
||||
@ -516,7 +518,7 @@ bool LocalStore::isActiveTempFile(const GCState & state,
|
||||
|
||||
void LocalStore::deleteGarbage(GCState & state, const Path & path)
|
||||
{
|
||||
unsigned long long bytesFreed;
|
||||
uint64_t bytesFreed;
|
||||
deletePath(path, bytesFreed);
|
||||
state.results.bytesFreed += bytesFreed;
|
||||
}
|
||||
@ -526,7 +528,7 @@ void LocalStore::deletePathRecursive(GCState & state, const Path & path)
|
||||
{
|
||||
checkInterrupt();
|
||||
|
||||
unsigned long long size = 0;
|
||||
uint64_t size = 0;
|
||||
|
||||
auto storePath = maybeParseStorePath(path);
|
||||
if (storePath && isValidPath(*storePath)) {
|
||||
@ -685,7 +687,7 @@ void LocalStore::removeUnusedLinks(const GCState & state)
|
||||
AutoCloseDir dir(opendir(linksDir.c_str()));
|
||||
if (!dir) throw SysError("opening directory '%1%'", linksDir);
|
||||
|
||||
long long actualSize = 0, unsharedSize = 0;
|
||||
int64_t actualSize = 0, unsharedSize = 0;
|
||||
|
||||
struct dirent * dirent;
|
||||
while (errno = 0, dirent = readdir(dir.get())) {
|
||||
@ -715,10 +717,10 @@ void LocalStore::removeUnusedLinks(const GCState & state)
|
||||
struct stat st;
|
||||
if (stat(linksDir.c_str(), &st) == -1)
|
||||
throw SysError("statting '%1%'", linksDir);
|
||||
long long overhead = st.st_blocks * 512ULL;
|
||||
auto overhead = st.st_blocks * 512ULL;
|
||||
|
||||
printInfo(format("note: currently hard linking saves %.2f MiB")
|
||||
% ((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
|
||||
printInfo("note: currently hard linking saves %.2f MiB",
|
||||
((unsharedSize - actualSize - overhead) / (1024.0 * 1024.0)));
|
||||
}
|
||||
|
||||
|
||||
|
@ -35,7 +35,7 @@ Settings::Settings()
|
||||
, nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR)))
|
||||
, nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
|
||||
, nixManDir(canonPath(NIX_MAN_DIR))
|
||||
, nixDaemonSocketFile(canonPath(nixStateDir + DEFAULT_SOCKET_PATH))
|
||||
, nixDaemonSocketFile(canonPath(getEnv("NIX_DAEMON_SOCKET_PATH").value_or(nixStateDir + DEFAULT_SOCKET_PATH)))
|
||||
{
|
||||
buildUsersGroup = getuid() == 0 ? "nixbld" : "";
|
||||
lockCPU = getEnv("NIX_AFFINITY_HACK") == "1";
|
||||
|
@ -196,10 +196,6 @@ public:
|
||||
/* Whether to lock the Nix client and worker to the same CPU. */
|
||||
bool lockCPU;
|
||||
|
||||
/* Whether to show a stack trace if Nix evaluation fails. */
|
||||
Setting<bool> showTrace{this, false, "show-trace",
|
||||
"Whether to show a stack trace on evaluation errors."};
|
||||
|
||||
Setting<SandboxMode> sandboxMode{this,
|
||||
#if __linux__
|
||||
smEnabled
|
||||
@ -369,6 +365,12 @@ public:
|
||||
|
||||
Setting<bool> warnDirty{this, true, "warn-dirty",
|
||||
"Whether to warn about dirty Git/Mercurial trees."};
|
||||
|
||||
Setting<size_t> narBufferSize{this, 32 * 1024 * 1024, "nar-buffer-size",
|
||||
"Maximum size of NARs before spilling them to disk."};
|
||||
|
||||
Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry",
|
||||
"Path or URI of the global flake registry."};
|
||||
};
|
||||
|
||||
|
||||
|
@ -100,11 +100,11 @@ protected:
|
||||
}
|
||||
|
||||
void upsertFile(const std::string & path,
|
||||
const std::string & data,
|
||||
std::shared_ptr<std::basic_iostream<char>> istream,
|
||||
const std::string & mimeType) override
|
||||
{
|
||||
auto req = FileTransferRequest(cacheUri + "/" + path);
|
||||
req.data = std::make_shared<string>(data); // FIXME: inefficient
|
||||
req.data = std::make_shared<string>(StreamToSourceAdapter(istream).drain());
|
||||
req.mimeType = mimeType;
|
||||
try {
|
||||
getFileTransfer()->upload(req);
|
||||
|
@ -113,7 +113,7 @@ struct LegacySSHStore : public Store
|
||||
|
||||
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4) {
|
||||
auto s = readString(conn->from);
|
||||
info->narHash = s.empty() ? Hash() : Hash(s);
|
||||
info->narHash = s.empty() ? std::optional<Hash>{} : Hash{s};
|
||||
info->ca = parseContentAddressOpt(readString(conn->from));
|
||||
info->sigs = readStrings<StringSet>(conn->from);
|
||||
}
|
||||
@ -126,8 +126,7 @@ struct LegacySSHStore : public Store
|
||||
}
|
||||
|
||||
void addToStore(const ValidPathInfo & info, Source & source,
|
||||
RepairFlag repair, CheckSigsFlag checkSigs,
|
||||
std::shared_ptr<FSAccessor> accessor) override
|
||||
RepairFlag repair, CheckSigsFlag checkSigs) override
|
||||
{
|
||||
debug("adding path '%s' to remote host '%s'", printStorePath(info.path), host);
|
||||
|
||||
@ -139,7 +138,7 @@ struct LegacySSHStore : public Store
|
||||
<< cmdAddToStoreNar
|
||||
<< printStorePath(info.path)
|
||||
<< (info.deriver ? printStorePath(*info.deriver) : "")
|
||||
<< info.narHash.to_string(Base16, false);
|
||||
<< info.narHash->to_string(Base16, false);
|
||||
writeStorePaths(*this, conn->to, info.references);
|
||||
conn->to
|
||||
<< info.registrationTime
|
||||
|
@ -31,8 +31,18 @@ protected:
|
||||
bool fileExists(const std::string & path) override;
|
||||
|
||||
void upsertFile(const std::string & path,
|
||||
const std::string & data,
|
||||
const std::string & mimeType) override;
|
||||
std::shared_ptr<std::basic_iostream<char>> istream,
|
||||
const std::string & mimeType) override
|
||||
{
|
||||
auto path2 = binaryCacheDir + "/" + path;
|
||||
Path tmp = path2 + ".tmp." + std::to_string(getpid());
|
||||
AutoDelete del(tmp, false);
|
||||
StreamToSourceAdapter source(istream);
|
||||
writeFile(tmp, source);
|
||||
if (rename(tmp.c_str(), path2.c_str()))
|
||||
throw SysError("renaming '%1%' to '%2%'", tmp, path2);
|
||||
del.cancel();
|
||||
}
|
||||
|
||||
void getFile(const std::string & path, Sink & sink) override
|
||||
{
|
||||
@ -52,7 +62,9 @@ protected:
|
||||
if (entry.name.size() != 40 ||
|
||||
!hasSuffix(entry.name, ".narinfo"))
|
||||
continue;
|
||||
paths.insert(parseStorePath(storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)));
|
||||
paths.insert(parseStorePath(
|
||||
storeDir + "/" + entry.name.substr(0, entry.name.size() - 8)
|
||||
+ "-" + MissingName));
|
||||
}
|
||||
|
||||
return paths;
|
||||
@ -68,28 +80,11 @@ void LocalBinaryCacheStore::init()
|
||||
BinaryCacheStore::init();
|
||||
}
|
||||
|
||||
static void atomicWrite(const Path & path, const std::string & s)
|
||||
{
|
||||
Path tmp = path + ".tmp." + std::to_string(getpid());
|
||||
AutoDelete del(tmp, false);
|
||||
writeFile(tmp, s);
|
||||
if (rename(tmp.c_str(), path.c_str()))
|
||||
throw SysError("renaming '%1%' to '%2%'", tmp, path);
|
||||
del.cancel();
|
||||
}
|
||||
|
||||
bool LocalBinaryCacheStore::fileExists(const std::string & path)
|
||||
{
|
||||
return pathExists(binaryCacheDir + "/" + path);
|
||||
}
|
||||
|
||||
void LocalBinaryCacheStore::upsertFile(const std::string & path,
|
||||
const std::string & data,
|
||||
const std::string & mimeType)
|
||||
{
|
||||
atomicWrite(binaryCacheDir + "/" + path, data);
|
||||
}
|
||||
|
||||
static RegisterStoreImplementation regStore([](
|
||||
const std::string & uri, const Store::Params & params)
|
||||
-> std::shared_ptr<Store>
|
||||
|
@ -20,9 +20,9 @@ struct LocalStoreAccessor : public FSAccessor
|
||||
|
||||
Path toRealPath(const Path & path)
|
||||
{
|
||||
Path storePath = store->toStorePath(path);
|
||||
if (!store->isValidPath(store->parseStorePath(storePath)))
|
||||
throw InvalidPath("path '%1%' is not a valid store path", storePath);
|
||||
auto storePath = store->toStorePath(path).first;
|
||||
if (!store->isValidPath(storePath))
|
||||
throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
|
||||
return store->getRealStoreDir() + std::string(path, store->storeDir.size());
|
||||
}
|
||||
|
||||
|
@ -560,19 +560,12 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
|
||||
DerivationOutputs::const_iterator out = drv.outputs.find("out");
|
||||
if (out == drv.outputs.end())
|
||||
throw Error("derivation '%s' does not have an output named 'out'", printStorePath(drvPath));
|
||||
|
||||
check(
|
||||
makeFixedOutputPath(
|
||||
out->second.hash->method,
|
||||
out->second.hash->hash,
|
||||
drvName),
|
||||
out->second.path, "out");
|
||||
}
|
||||
|
||||
else {
|
||||
Hash h = hashDerivationModulo(*this, drv, true);
|
||||
for (auto & i : drv.outputs)
|
||||
check(makeOutputPath(i.first, h, drvName), i.second.path, i.first);
|
||||
check(makeOutputPath(i.first, h, drvName), i.second.path(*this, drv.name), i.first);
|
||||
}
|
||||
}
|
||||
|
||||
@ -586,7 +579,7 @@ uint64_t LocalStore::addValidPath(State & state,
|
||||
|
||||
state.stmtRegisterValidPath.use()
|
||||
(printStorePath(info.path))
|
||||
(info.narHash.to_string(Base16, true))
|
||||
(info.narHash->to_string(Base16, true))
|
||||
(info.registrationTime == 0 ? time(0) : info.registrationTime)
|
||||
(info.deriver ? printStorePath(*info.deriver) : "", (bool) info.deriver)
|
||||
(info.narSize, info.narSize != 0)
|
||||
@ -594,7 +587,7 @@ uint64_t LocalStore::addValidPath(State & state,
|
||||
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
|
||||
(renderContentAddress(info.ca), (bool) info.ca)
|
||||
.exec();
|
||||
uint64_t id = sqlite3_last_insert_rowid(state.db);
|
||||
uint64_t id = state.db.getLastInsertedRowId();
|
||||
|
||||
/* If this is a derivation, then store the derivation outputs in
|
||||
the database. This is useful for the garbage collector: it can
|
||||
@ -614,7 +607,7 @@ uint64_t LocalStore::addValidPath(State & state,
|
||||
state.stmtAddDerivationOutput.use()
|
||||
(id)
|
||||
(i.first)
|
||||
(printStorePath(i.second.path))
|
||||
(printStorePath(i.second.path(*this, drv.name)))
|
||||
.exec();
|
||||
}
|
||||
}
|
||||
@ -686,7 +679,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
|
||||
{
|
||||
state.stmtUpdatePathInfo.use()
|
||||
(info.narSize, info.narSize != 0)
|
||||
(info.narHash.to_string(Base16, true))
|
||||
(info.narHash->to_string(Base16, true))
|
||||
(info.ultimate ? 1 : 0, info.ultimate)
|
||||
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
|
||||
(renderContentAddress(info.ca), (bool) info.ca)
|
||||
@ -774,17 +767,20 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
|
||||
}
|
||||
|
||||
|
||||
StorePathSet LocalStore::queryDerivationOutputs(const StorePath & path)
|
||||
OutputPathMap LocalStore::queryDerivationOutputMap(const StorePath & path)
|
||||
{
|
||||
return retrySQLite<StorePathSet>([&]() {
|
||||
return retrySQLite<OutputPathMap>([&]() {
|
||||
auto state(_state.lock());
|
||||
|
||||
auto useQueryDerivationOutputs(state->stmtQueryDerivationOutputs.use()
|
||||
(queryValidPathId(*state, path)));
|
||||
|
||||
StorePathSet outputs;
|
||||
OutputPathMap outputs;
|
||||
while (useQueryDerivationOutputs.next())
|
||||
outputs.insert(parseStorePath(useQueryDerivationOutputs.getStr(1)));
|
||||
outputs.emplace(
|
||||
useQueryDerivationOutputs.getStr(0),
|
||||
parseStorePath(useQueryDerivationOutputs.getStr(1))
|
||||
);
|
||||
|
||||
return outputs;
|
||||
});
|
||||
@ -897,7 +893,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
|
||||
StorePathSet paths;
|
||||
|
||||
for (auto & i : infos) {
|
||||
assert(i.narHash.type == htSHA256);
|
||||
assert(i.narHash && i.narHash->type == htSHA256);
|
||||
if (isValidPath_(*state, i.path))
|
||||
updatePathInfo(*state, i);
|
||||
else
|
||||
@ -959,7 +955,7 @@ const PublicKeys & LocalStore::getPublicKeys()
|
||||
|
||||
|
||||
void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
|
||||
RepairFlag repair, CheckSigsFlag checkSigs, std::shared_ptr<FSAccessor> accessor)
|
||||
RepairFlag repair, CheckSigsFlag checkSigs)
|
||||
{
|
||||
if (!info.narHash)
|
||||
throw Error("cannot add path '%s' because it lacks a hash", printStorePath(info.path));
|
||||
@ -973,7 +969,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
|
||||
|
||||
PathLocks outputLock;
|
||||
|
||||
Path realPath = realStoreDir + "/" + std::string(info.path.to_string());
|
||||
auto realPath = Store::toRealPath(info.path);
|
||||
|
||||
/* Lock the output path. But don't lock if we're being called
|
||||
from a build hook (whose parent process already acquired a
|
||||
@ -1010,7 +1006,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
|
||||
|
||||
if (hashResult.first != info.narHash)
|
||||
throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s",
|
||||
printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true));
|
||||
printStorePath(info.path), info.narHash->to_string(Base32, true), hashResult.first.to_string(Base32, true));
|
||||
|
||||
if (hashResult.second != info.narSize)
|
||||
throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s",
|
||||
@ -1030,12 +1026,78 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
|
||||
}
|
||||
|
||||
|
||||
StorePath LocalStore::addToStoreFromDump(const string & dump, const string & name,
|
||||
StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
|
||||
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
||||
{
|
||||
Path srcPath(absPath(_srcPath));
|
||||
auto source = sinkToSource([&](Sink & sink) {
|
||||
if (method == FileIngestionMethod::Recursive)
|
||||
dumpPath(srcPath, sink, filter);
|
||||
else
|
||||
readFile(srcPath, sink);
|
||||
});
|
||||
return addToStoreFromDump(*source, name, method, hashAlgo, repair);
|
||||
}
|
||||
|
||||
|
||||
StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
|
||||
FileIngestionMethod method, HashType hashAlgo, RepairFlag repair)
|
||||
{
|
||||
Hash h = hashString(hashAlgo, dump);
|
||||
/* For computing the store path. */
|
||||
auto hashSink = std::make_unique<HashSink>(hashAlgo);
|
||||
TeeSource source { source0, *hashSink };
|
||||
|
||||
auto dstPath = makeFixedOutputPath(method, h, name);
|
||||
/* Read the source path into memory, but only if it's up to
|
||||
narBufferSize bytes. If it's larger, write it to a temporary
|
||||
location in the Nix store. If the subsequently computed
|
||||
destination store path is already valid, we just delete the
|
||||
temporary path. Otherwise, we move it to the destination store
|
||||
path. */
|
||||
bool inMemory = false;
|
||||
|
||||
std::string dump;
|
||||
|
||||
/* Fill out buffer, and decide whether we are working strictly in
|
||||
memory based on whether we break out because the buffer is full
|
||||
or the original source is empty */
|
||||
while (dump.size() < settings.narBufferSize) {
|
||||
auto oldSize = dump.size();
|
||||
constexpr size_t chunkSize = 65536;
|
||||
auto want = std::min(chunkSize, settings.narBufferSize - oldSize);
|
||||
dump.resize(oldSize + want);
|
||||
auto got = 0;
|
||||
try {
|
||||
got = source.read((uint8_t *) dump.data() + oldSize, want);
|
||||
} catch (EndOfFile &) {
|
||||
inMemory = true;
|
||||
break;
|
||||
}
|
||||
dump.resize(oldSize + got);
|
||||
}
|
||||
|
||||
std::unique_ptr<AutoDelete> delTempDir;
|
||||
Path tempPath;
|
||||
|
||||
if (!inMemory) {
|
||||
/* Drain what we pulled so far, and then keep on pulling */
|
||||
StringSource dumpSource { dump };
|
||||
ChainSource bothSource { dumpSource, source };
|
||||
|
||||
auto tempDir = createTempDir(realStoreDir, "add");
|
||||
delTempDir = std::make_unique<AutoDelete>(tempDir);
|
||||
tempPath = tempDir + "/x";
|
||||
|
||||
if (method == FileIngestionMethod::Recursive)
|
||||
restorePath(tempPath, bothSource);
|
||||
else
|
||||
writeFile(tempPath, bothSource);
|
||||
|
||||
dump.clear();
|
||||
}
|
||||
|
||||
auto [hash, size] = hashSink->finish();
|
||||
|
||||
auto dstPath = makeFixedOutputPath(method, hash, name);
|
||||
|
||||
addTempRoot(dstPath);
|
||||
|
||||
@ -1044,8 +1106,7 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
|
||||
/* The first check above is an optimisation to prevent
|
||||
unnecessary lock acquisition. */
|
||||
|
||||
Path realPath = realStoreDir + "/";
|
||||
realPath += dstPath.to_string();
|
||||
auto realPath = Store::toRealPath(dstPath);
|
||||
|
||||
PathLocks outputLock({realPath});
|
||||
|
||||
@ -1055,31 +1116,36 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
|
||||
|
||||
autoGC();
|
||||
|
||||
if (method == FileIngestionMethod::Recursive) {
|
||||
StringSource source(dump);
|
||||
restorePath(realPath, source);
|
||||
} else
|
||||
writeFile(realPath, dump);
|
||||
if (inMemory) {
|
||||
StringSource dumpSource { dump };
|
||||
/* Restore from the NAR in memory. */
|
||||
if (method == FileIngestionMethod::Recursive)
|
||||
restorePath(realPath, dumpSource);
|
||||
else
|
||||
writeFile(realPath, dumpSource);
|
||||
} else {
|
||||
/* Move the temporary path we restored above. */
|
||||
if (rename(tempPath.c_str(), realPath.c_str()))
|
||||
throw Error("renaming '%s' to '%s'", tempPath, realPath);
|
||||
}
|
||||
|
||||
canonicalisePathMetaData(realPath, -1);
|
||||
/* For computing the nar hash. In recursive SHA-256 mode, this
|
||||
is the same as the store hash, so no need to do it again. */
|
||||
auto narHash = std::pair { hash, size };
|
||||
if (method != FileIngestionMethod::Recursive || hashAlgo != htSHA256) {
|
||||
HashSink narSink { htSHA256 };
|
||||
dumpPath(realPath, narSink);
|
||||
narHash = narSink.finish();
|
||||
}
|
||||
|
||||
/* Register the SHA-256 hash of the NAR serialisation of
|
||||
the path in the database. We may just have computed it
|
||||
above (if called with recursive == true and hashAlgo ==
|
||||
sha256); otherwise, compute it here. */
|
||||
HashResult hash;
|
||||
if (method == FileIngestionMethod::Recursive) {
|
||||
hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump);
|
||||
hash.second = dump.size();
|
||||
} else
|
||||
hash = hashPath(htSHA256, realPath);
|
||||
canonicalisePathMetaData(realPath, -1); // FIXME: merge into restorePath
|
||||
|
||||
optimisePath(realPath); // FIXME: combine with hashPath()
|
||||
optimisePath(realPath);
|
||||
|
||||
ValidPathInfo info(dstPath);
|
||||
info.narHash = hash.first;
|
||||
info.narSize = hash.second;
|
||||
info.ca = FixedOutputHash { .method = method, .hash = h };
|
||||
info.narHash = narHash.first;
|
||||
info.narSize = narHash.second;
|
||||
info.ca = FixedOutputHash { .method = method, .hash = hash };
|
||||
registerValidPath(info);
|
||||
}
|
||||
|
||||
@ -1090,24 +1156,6 @@ StorePath LocalStore::addToStoreFromDump(const string & dump, const string & nam
|
||||
}
|
||||
|
||||
|
||||
StorePath LocalStore::addToStore(const string & name, const Path & _srcPath,
|
||||
FileIngestionMethod method, HashType hashAlgo, PathFilter & filter, RepairFlag repair)
|
||||
{
|
||||
Path srcPath(absPath(_srcPath));
|
||||
|
||||
/* Read the whole path into memory. This is not a very scalable
|
||||
method for very large paths, but `copyPath' is mainly used for
|
||||
small files. */
|
||||
StringSink sink;
|
||||
if (method == FileIngestionMethod::Recursive)
|
||||
dumpPath(srcPath, sink, filter);
|
||||
else
|
||||
sink.s = make_ref<std::string>(readFile(srcPath));
|
||||
|
||||
return addToStoreFromDump(*sink.s, name, method, hashAlgo, repair);
|
||||
}
|
||||
|
||||
|
||||
StorePath LocalStore::addTextToStore(const string & name, const string & s,
|
||||
const StorePathSet & references, RepairFlag repair)
|
||||
{
|
||||
@ -1118,8 +1166,7 @@ StorePath LocalStore::addTextToStore(const string & name, const string & s,
|
||||
|
||||
if (repair || !isValidPath(dstPath)) {
|
||||
|
||||
Path realPath = realStoreDir + "/";
|
||||
realPath += dstPath.to_string();
|
||||
auto realPath = Store::toRealPath(dstPath);
|
||||
|
||||
PathLocks outputLock({realPath});
|
||||
|
||||
@ -1255,9 +1302,9 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
|
||||
|
||||
std::unique_ptr<AbstractHashSink> hashSink;
|
||||
if (!info->ca || !info->references.count(info->path))
|
||||
hashSink = std::make_unique<HashSink>(*info->narHash.type);
|
||||
hashSink = std::make_unique<HashSink>(info->narHash->type);
|
||||
else
|
||||
hashSink = std::make_unique<HashModuloSink>(*info->narHash.type, std::string(info->path.hashPart()));
|
||||
hashSink = std::make_unique<HashModuloSink>(info->narHash->type, std::string(info->path.hashPart()));
|
||||
|
||||
dumpPath(Store::toRealPath(i), *hashSink);
|
||||
auto current = hashSink->finish();
|
||||
@ -1266,7 +1313,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
|
||||
logError({
|
||||
.name = "Invalid hash - path modified",
|
||||
.hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'",
|
||||
printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true))
|
||||
printStorePath(i), info->narHash->to_string(Base32, true), current.first.to_string(Base32, true))
|
||||
});
|
||||
if (repair) repairPath(i); else errors = true;
|
||||
} else {
|
||||
|
@ -29,8 +29,8 @@ struct Derivation;
|
||||
struct OptimiseStats
|
||||
{
|
||||
unsigned long filesLinked = 0;
|
||||
unsigned long long bytesFreed = 0;
|
||||
unsigned long long blocksFreed = 0;
|
||||
uint64_t bytesFreed = 0;
|
||||
uint64_t blocksFreed = 0;
|
||||
};
|
||||
|
||||
|
||||
@ -133,7 +133,7 @@ public:
|
||||
|
||||
StorePathSet queryValidDerivers(const StorePath & path) override;
|
||||
|
||||
StorePathSet queryDerivationOutputs(const StorePath & path) override;
|
||||
OutputPathMap queryDerivationOutputMap(const StorePath & path) override;
|
||||
|
||||
std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override;
|
||||
|
||||
@ -143,8 +143,7 @@ public:
|
||||
SubstitutablePathInfos & infos) override;
|
||||
|
||||
void addToStore(const ValidPathInfo & info, Source & source,
|
||||
RepairFlag repair, CheckSigsFlag checkSigs,
|
||||
std::shared_ptr<FSAccessor> accessor) override;
|
||||
RepairFlag repair, CheckSigsFlag checkSigs) override;
|
||||
|
||||
StorePath addToStore(const string & name, const Path & srcPath,
|
||||
FileIngestionMethod method, HashType hashAlgo,
|
||||
@ -154,7 +153,7 @@ public:
|
||||
in `dump', which is either a NAR serialisation (if recursive ==
|
||||
true) or simply the contents of a regular file (if recursive ==
|
||||
false). */
|
||||
StorePath addToStoreFromDump(const string & dump, const string & name,
|
||||
StorePath addToStoreFromDump(Source & dump, const string & name,
|
||||
FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override;
|
||||
|
||||
StorePath addTextToStore(const string & name, const string & s,
|
||||
|
@ -61,3 +61,6 @@ $(d)/build.cc:
|
||||
clean-files += $(d)/schema.sql.gen.hh
|
||||
|
||||
$(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644))
|
||||
|
||||
$(foreach i, $(wildcard src/libstore/builtins/*.hh), \
|
||||
$(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644)))
|
||||
|
@ -110,7 +110,7 @@ void Store::computeFSClosure(const StorePath & startPath,
|
||||
|
||||
void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
|
||||
StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
|
||||
unsigned long long & downloadSize_, unsigned long long & narSize_)
|
||||
uint64_t & downloadSize_, uint64_t & narSize_)
|
||||
{
|
||||
Activity act(*logger, lvlDebug, actUnknown, "querying info about missing paths");
|
||||
|
||||
@ -122,8 +122,8 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
|
||||
{
|
||||
std::unordered_set<std::string> done;
|
||||
StorePathSet & unknown, & willSubstitute, & willBuild;
|
||||
unsigned long long & downloadSize;
|
||||
unsigned long long & narSize;
|
||||
uint64_t & downloadSize;
|
||||
uint64_t & narSize;
|
||||
};
|
||||
|
||||
struct DrvState
|
||||
@ -198,8 +198,8 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
|
||||
PathSet invalid;
|
||||
for (auto & j : drv->outputs)
|
||||
if (wantOutput(j.first, path.outputs)
|
||||
&& !isValidPath(j.second.path))
|
||||
invalid.insert(printStorePath(j.second.path));
|
||||
&& !isValidPath(j.second.path(*this, drv->name)))
|
||||
invalid.insert(printStorePath(j.second.path(*this, drv->name)));
|
||||
if (invalid.empty()) return;
|
||||
|
||||
if (settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
|
||||
|
@ -18,7 +18,7 @@ struct NarMember
|
||||
|
||||
/* If this is a regular file, position of the contents of this
|
||||
file in the NAR. */
|
||||
size_t start = 0, size = 0;
|
||||
uint64_t start = 0, size = 0;
|
||||
|
||||
std::string target;
|
||||
|
||||
@ -34,17 +34,19 @@ struct NarAccessor : public FSAccessor
|
||||
|
||||
NarMember root;
|
||||
|
||||
struct NarIndexer : ParseSink, StringSource
|
||||
struct NarIndexer : ParseSink, Source
|
||||
{
|
||||
NarAccessor & acc;
|
||||
Source & source;
|
||||
|
||||
std::stack<NarMember *> parents;
|
||||
|
||||
std::string currentStart;
|
||||
bool isExec = false;
|
||||
|
||||
NarIndexer(NarAccessor & acc, const std::string & nar)
|
||||
: StringSource(nar), acc(acc)
|
||||
uint64_t pos = 0;
|
||||
|
||||
NarIndexer(NarAccessor & acc, Source & source)
|
||||
: acc(acc), source(source)
|
||||
{ }
|
||||
|
||||
void createMember(const Path & path, NarMember member) {
|
||||
@ -77,33 +79,40 @@ struct NarAccessor : public FSAccessor
|
||||
parents.top()->isExecutable = true;
|
||||
}
|
||||
|
||||
void preallocateContents(unsigned long long size) override
|
||||
void preallocateContents(uint64_t size) override
|
||||
{
|
||||
currentStart = string(s, pos, 16);
|
||||
assert(size <= std::numeric_limits<size_t>::max());
|
||||
parents.top()->size = (size_t)size;
|
||||
assert(size <= std::numeric_limits<uint64_t>::max());
|
||||
parents.top()->size = (uint64_t) size;
|
||||
parents.top()->start = pos;
|
||||
}
|
||||
|
||||
void receiveContents(unsigned char * data, unsigned int len) override
|
||||
{
|
||||
// Sanity check
|
||||
if (!currentStart.empty()) {
|
||||
assert(len < 16 || currentStart == string((char *) data, 16));
|
||||
currentStart.clear();
|
||||
}
|
||||
}
|
||||
void receiveContents(unsigned char * data, size_t len) override
|
||||
{ }
|
||||
|
||||
void createSymlink(const Path & path, const string & target) override
|
||||
{
|
||||
createMember(path,
|
||||
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
|
||||
}
|
||||
|
||||
size_t read(unsigned char * data, size_t len) override
|
||||
{
|
||||
auto n = source.read(data, len);
|
||||
pos += n;
|
||||
return n;
|
||||
}
|
||||
};
|
||||
|
||||
NarAccessor(ref<const std::string> nar) : nar(nar)
|
||||
{
|
||||
NarIndexer indexer(*this, *nar);
|
||||
StringSource source(*nar);
|
||||
NarIndexer indexer(*this, source);
|
||||
parseDump(indexer, indexer);
|
||||
}
|
||||
|
||||
NarAccessor(Source & source)
|
||||
{
|
||||
NarIndexer indexer(*this, source);
|
||||
parseDump(indexer, indexer);
|
||||
}
|
||||
|
||||
@ -219,6 +228,11 @@ ref<FSAccessor> makeNarAccessor(ref<const std::string> nar)
|
||||
return make_ref<NarAccessor>(nar);
|
||||
}
|
||||
|
||||
ref<FSAccessor> makeNarAccessor(Source & source)
|
||||
{
|
||||
return make_ref<NarAccessor>(source);
|
||||
}
|
||||
|
||||
ref<FSAccessor> makeLazyNarAccessor(const std::string & listing,
|
||||
GetNarBytes getNarBytes)
|
||||
{
|
||||
|
@ -6,10 +6,14 @@
|
||||
|
||||
namespace nix {
|
||||
|
||||
struct Source;
|
||||
|
||||
/* Return an object that provides access to the contents of a NAR
|
||||
file. */
|
||||
ref<FSAccessor> makeNarAccessor(ref<const std::string> nar);
|
||||
|
||||
ref<FSAccessor> makeNarAccessor(Source & source);
|
||||
|
||||
/* Create a NAR accessor from a NAR listing (in the format produced by
|
||||
listNar()). The callback getNarBytes(offset, length) is used by the
|
||||
readFile() method of the accessor to get the contents of files
|
||||
|
@ -230,9 +230,9 @@ public:
|
||||
(std::string(info->path.name()))
|
||||
(narInfo ? narInfo->url : "", narInfo != 0)
|
||||
(narInfo ? narInfo->compression : "", narInfo != 0)
|
||||
(narInfo && narInfo->fileHash ? narInfo->fileHash.to_string(Base32, true) : "", narInfo && narInfo->fileHash)
|
||||
(narInfo && narInfo->fileHash ? narInfo->fileHash->to_string(Base32, true) : "", narInfo && narInfo->fileHash)
|
||||
(narInfo ? narInfo->fileSize : 0, narInfo != 0 && narInfo->fileSize)
|
||||
(info->narHash.to_string(Base32, true))
|
||||
(info->narHash->to_string(Base32, true))
|
||||
(info->narSize)
|
||||
(concatStringsSep(" ", info->shortRefs()))
|
||||
(info->deriver ? std::string(info->deriver->to_string()) : "", (bool) info->deriver)
|
||||
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user