Merged R9105

Makefile.am

@@ -35,11 +35,11 @@ init-state:
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/temproots
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/tmp
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/gcroots/channels
-	rm -f $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
+	ln -sfn $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
-	ln -s $(localstatedir)/nix/profiles $(DESTDIR)$(localstatedir)/nix/gcroots/profiles
 	$(INSTALL) $(INIT_FLAGS) -d $(DESTDIR)$(localstatedir)/nix/userpool
 	$(INSTALL) $(INIT_FLAGS) -m 1777 -d $(DESTDIR)$(prefix)/store
 	$(INSTALL) $(INIT_FLAGS) $(GROUP_WRITABLE) -d $(DESTDIR)$(localstatedir)/nix/manifests
+	ln -sfn $(localstatedir)/nix/manifests $(DESTDIR)$(localstatedir)/nix/gcroots/manifests
 #	$(bindir)/nix-store --init
 
 else

doc/manual/nix-store.xml

@@ -649,36 +649,6 @@ $ gv graph.ps</screen>
 
 
 
-<!--######################################################################-->
-
-<!--
-<refsection><title>Operation <option>-XXX-substitute</option></title>
-
-<refsection><title>Synopsis</title>
-
-<cmdsynopsis>
-  <command>nix-store</command>
-  <arg choice='plain'><option>-XXX-substitute</option></arg>
-  <arg choice='plain'
-       rep='repeat'><replaceable>srcpath</replaceable> <replaceable>subpath</replaceable></arg>
-</cmdsynopsis>
-</refsection>
-
-<refsection><title>Description</title>
-            
-<para>The operation <option>-XXX-substitute</option> registers that the
-store path <replaceable>srcpath</replaceable> can be built by
-realising the derivation expression in
-<replaceable>subpath</replaceable>.  This is used to implement binary
-deployment.</para>
-
-</refsection>
-            
-</refsection>
--->
-
-
-
 <!--######################################################################-->
 
 <refsection xml:id='refsec-nix-store-verify'><title>Operation <option>--verify</option></title>

externals/Makefile.am

@@ -35,12 +35,12 @@ endif
 
 # CWI ATerm
 
-ATERM = aterm-2.4.2-fixes
+ATERM = aterm-2.4.2-fixes-r2
 
 $(ATERM).tar.bz2:
 	@echo "Nix requires the CWI ATerm library to build."
-	@echo "Please download version 2.4.2-fixes from"
+	@echo "Please download version 2.4.2-fixes-r2 from"
-	@echo "  http://losser.st-lab.cs.uu.nl/~eelco/dist/aterm-2.4.2-fixes.tar.bz2"
+	@echo "  http://losser.st-lab.cs.uu.nl/~eelco/dist/aterm-2.4.2-fixes-r2.tar.bz2"
 	@echo "and place it in the externals/ directory."
 	false
 

scripts/download-using-manifests.pl.in

@@ -22,16 +22,6 @@ my $tmpNar2 = "$tmpDir/nar2";
 END { unlink $tmpNar; unlink $tmpNar2; rmdir $tmpDir; }
 
 
-# Check the arguments.
-die unless scalar @ARGV == 1;
-my $targetPath = $ARGV[0];
-
-my $date = strftime ("%F %H:%M:%S UTC", gmtime (time));
-print LOGFILE "$$ get $targetPath $date\n";
-
-print "\n*** Trying to download/patch `$targetPath'\n";
-
-
 # Load all manifests.
 my %narFiles;
 my %localPaths;
@@ -46,6 +36,54 @@ for my $manifest (glob "$manifestDir/*.nixmanifest") {
 }
 
 
+# Parse the arguments.
+
+if ($ARGV[0] eq "--query-paths") {
+    foreach my $storePath (keys %narFiles) { print "$storePath\n"; }
+    foreach my $storePath (keys %localPaths) { print "$storePath\n"; }
+    exit 0;
+}
+
+elsif ($ARGV[0] eq "--query-info") {
+    shift @ARGV;
+    foreach my $storePath (@ARGV) {
+        my $info;
+        if (defined $narFiles{$storePath}) {
+            $info = @{$narFiles{$storePath}}[0];
+        }
+        elsif (defined $localPaths{$storePath}) {
+            $info = @{$localPaths{$storePath}}[0];
+        }
+        else {
+            next; # not an error
+        }
+        print "$storePath\n";
+        print "$info->{deriver}\n";
+        my @references = split " ", $info->{references};
+        my $count = scalar @references;
+        print "$count\n";
+        foreach my $reference (@references) {
+            print "$reference\n";
+        }
+    }
+    exit 0;
+}
+
+elsif ($ARGV[0] ne "--substitute") {
+    die "syntax: $0 [--query-paths | --query-info PATHS... | --substitute PATH]\n";
+}
+
+
+die unless scalar @ARGV == 2;
+my $targetPath = $ARGV[1];
+
+
+my $date = strftime ("%F %H:%M:%S UTC", gmtime (time));
+print LOGFILE "$$ get $targetPath $date\n";
+
+print "\n*** Trying to download/patch `$targetPath'\n";
+
+
 # If we can copy from a local path, do that.
 my $localPathList = $localPaths{$targetPath};
 foreach my $localPath (@{$localPathList}) {

scripts/nix-channel.in

@@ -8,6 +8,12 @@ my $stateDir = $ENV{"NIX_STATE_DIR"};
 $stateDir = "@localstatedir@/nix" unless defined $stateDir;
 
 
+# Turn on caching in nix-prefetch-url.
+my $channelCache = "$stateDir/channel-cache";
+$ENV{'NIX_DOWNLOAD_CACHE'} = $channelCache;
+mkdir $channelCache, 0755 unless -e $channelCache;
+
+
 # Figure out the name of the `.nix-channels' file to use.
 my $home = $ENV{"HOME"};
 die '$HOME not set' unless defined $home;
@@ -70,10 +76,6 @@ sub removeChannel {
 sub update {
     readChannels;
 
-    # Get rid of all the old substitutes.
-    system("@bindir@/nix-store", "--clear-substitutes") == 0
-        or die "cannot clear substitutes";
-
     # Remove all the old manifests.
     for my $manifest (glob "$stateDir/manifests/*.nixmanifest") {
         unlink $manifest or die "cannot remove `$manifest': $!";
@@ -98,7 +100,8 @@ sub update {
         my $fullURL = "$url/nixexprs.tar.bz2";
         print "downloading Nix expressions from `$fullURL'...\n";
         $ENV{"PRINT_PATH"} = 1;
-        my ($hash, $path) = `@bindir@/nix-prefetch-url '$fullURL' 2> /dev/null`;
+        $ENV{"QUIET"} = 1;
+        my ($hash, $path) = `@bindir@/nix-prefetch-url '$fullURL'`;
         die "cannot fetch `$fullURL'" if $? != 0;
         chomp $path;
         $inputs .= '"' . $channelName . '"' . " " . $path . " ";

scripts/nix-pack-closure.in

@@ -17,9 +17,9 @@ $binDir = "@bindir@" unless defined $binDir;
 my $tmpDir = tempdir("nix-pack-closure.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
 
-mkdir "$tmpDir/contents", 0777 or die;
+mkdir "$tmpDir/contents", 0755 or die;
-mkdir "$tmpDir/references", 0777 or die;
+mkdir "$tmpDir/references", 0755 or die;
-mkdir "$tmpDir/derivers", 0777 or die;
+mkdir "$tmpDir/derivers", 0755 or die;
 
 open TOPLEVEL, ">$tmpDir/top-level" or die;
 

scripts/nix-prefetch-url.in

@@ -36,6 +36,12 @@ if test -n "$expHash"; then
 fi
 
 
+doDownload() {
+    @curl@ $cacheFlags --fail -# --show-error --location --max-redirs 20 --disable-epsv \
+        --cookie-jar $tmpPath/cookies "$url" -o $tmpFile
+}
+
+
 # If we don't know the hash or a file with that hash doesn't exist,
 # download the file and add it to the store.
 if test -z "$finalPath"; then
@@ -44,22 +50,61 @@ if test -z "$finalPath"; then
     tmpFile=$tmpPath/$name
     mkdir $tmpPath
 
+    # Optionally do timestamp-based caching of the download.
+    # Actually, the only thing that we cache in $NIX_DOWNLOAD_CACHE is
+    # the hash and the timestamp of the file at $url.  The caching of
+    # the file *contents* is done in Nix store, where it can be
+    # garbage-collected independently.
+    if test -n "$NIX_DOWNLOAD_CACHE"; then
+        echo -n "$url" > $tmpPath/url
+        urlHash=$(nix-hash --type sha256 --base32 --flat $tmpPath/url)
+        echo "$url" > "$NIX_DOWNLOAD_CACHE/$urlHash.url"
+        cachedHashFN="$NIX_DOWNLOAD_CACHE/$urlHash.$hashType"
+        cachedTimestampFN="$NIX_DOWNLOAD_CACHE/$urlHash.stamp"
+        cacheFlags="--remote-time"
+        if test -e "$cachedTimestampFN" -a -e "$cachedHashFN"; then
+            # Only download the file if it is newer than the cached version.
+            cacheFlags="$cacheFlags --time-cond $cachedTimestampFN"
+        fi
+    fi
+
     # Perform the download.
-    @curl@ --fail --location --max-redirs 20 --disable-epsv \
+    doDownload
-        --cookie-jar $tmpPath/cookies "$url" > $tmpFile
 
-    # Compute the hash.
+    if test -n "$NIX_DOWNLOAD_CACHE" -a ! -e $tmpFile; then
-    hash=$(@bindir@/nix-hash --type "$hashType" $hashFormat --flat $tmpFile)
+        # Curl didn't create $tmpFile, so apparently there's no newer
-    if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi
+        # file on the server.
+        hash=$(cat $cachedHashFN)
+        finalPath=$(@bindir@/nix-store --print-fixed-path "$hashType" "$hash" "$name") 
+        if ! @bindir@/nix-store --check-validity "$finalPath" 2> /dev/null; then
+            echo "cached contents of \`$url' disappeared, redownloading..." >&2
+            finalPath=
+            cacheFlags="--remote-time"
+            doDownload
+        fi
+    fi
 
-    # Add the downloaded file to the Nix store.
+    if test -z "$finalPath"; then
-    finalPath=$(@bindir@/nix-store --add-fixed "$hashType" $tmpFile)
 
-    if test -n "$tmpPath"; then rm -rf $tmpPath || true; fi
+        # Compute the hash.
+        hash=$(@bindir@/nix-hash --type "$hashType" $hashFormat --flat $tmpFile)
+        if ! test -n "$QUIET"; then echo "hash is $hash" >&2; fi
 
-    if test -n "$expHash" -a "$expHash" != "$hash"; then
+        if test -n "$NIX_DOWNLOAD_CACHE"; then
-        echo "hash mismatch for URL \`$url'" >&2
+            echo $hash > $cachedHashFN
-        exit 1
+            touch -r $tmpFile $cachedTimestampFN
+        fi
+
+        # Add the downloaded file to the Nix store.
+        finalPath=$(@bindir@/nix-store --add-fixed "$hashType" $tmpFile)
+
+        if test -n "$tmpPath"; then rm -rf $tmpPath || true; fi
+
+        if test -n "$expHash" -a "$expHash" != "$hash"; then
+            echo "hash mismatch for URL \`$url'" >&2
+            exit 1
+        fi
+        
     fi
 fi
 

scripts/nix-pull.in

@@ -7,8 +7,6 @@ use readmanifest;
 my $tmpDir = tempdir("nix-pull.XXXXXX", CLEANUP => 1, TMPDIR => 1)
     or die "cannot create a temporary directory";
 
-my $manifest = "$tmpDir/manifest";
-
 my $binDir = $ENV{"NIX_BIN_DIR"};
 $binDir = "@bindir@" unless defined $binDir;
 
@@ -38,16 +36,43 @@ my %patches;
 
 my $skipWrongStore = 0;
 
+sub downloadFile {
+    my $url = shift;
+    $ENV{"PRINT_PATH"} = 1;
+    $ENV{"QUIET"} = 1;
+    my ($dummy, $path) = `@bindir@/nix-prefetch-url '$url'`;
+    chomp $path;
+    return $path;
+}
+
 sub processURL {
     my $url = shift;
 
     $url =~ s/\/$//;
-    print "obtaining list of Nix archives at $url...\n";
 
-    system("@curl@ --fail --silent --show-error --location --max-redirs 20 " .
+    my $manifest;
-           "'$url' > '$manifest'") == 0
-           or die "curl failed: $?";
 
+    # First see if a bzipped manifest is available.
+    if (system("@curl@ --fail --silent --head '$url'.bz2 > /dev/null") == 0) {
+        print "obtaining list of Nix archives at `$url.bz2'...\n";
+        my $bzipped = downloadFile "$url.bz2";
+
+        $manifest = "$tmpDir/MANIFEST";
+
+        system("@bunzip2@ < $bzipped > $manifest") == 0
+            or die "cannot decompress manifest";
+
+        $manifest = (`$binDir/nix-store --add $manifest`
+                     or die "cannot copy $manifest to the store");
+        chomp $manifest;
+    }
+
+    # Otherwise, just get the uncompressed manifest.
+    else {
+        print "obtaining list of Nix archives at `$url'...\n";
+        $manifest = downloadFile $url;
+    }
+    
     if (readManifest($manifest, \%narFiles, \%localPaths, \%patches) < 3) {
         die "manifest `$url' is too old (i.e., for Nix <= 0.7)\n";
     }
@@ -72,8 +97,8 @@ sub processURL {
     
     my $finalPath = "$stateDir/manifests/$baseName-$hash.nixmanifest";
     
-    system ("@coreutils@/mv", "-f", "$manifest", "$finalPath") == 0
+    system("@coreutils@/ln", "-sfn", "$manifest", "$finalPath") == 0
-        or die "cannot move `$manifest' to `$finalPath";
+        or die "cannot link `$finalPath to `$manifest'";
 }
 
 while (@ARGV) {
@@ -88,41 +113,3 @@ while (@ARGV) {
 
 my $size = scalar (keys %narFiles) + scalar (keys %localPaths);
 print "$size store paths in manifest\n";
-
-
-# Register all substitutes.
-print STDERR "registering substitutes...\n";
-
-my $pid = open(WRITE, "|$binDir/nix-store --register-substitutes")
-    or die "cannot run nix-store";
-
-sub writeRegistration {
-    my $storePath = shift;
-    my $object = shift;
-    print WRITE "$storePath\n";
-    print WRITE "$object->{deriver}\n";
-    print WRITE "$libexecDir/nix/download-using-manifests.pl\n";
-    print WRITE "0\n";
-    my @references = split " ", $object->{references};
-    my $count = scalar @references;
-    print WRITE "$count\n";
-    foreach my $reference (@references) {
-        print WRITE "$reference\n";
-    }
-}
-
-foreach my $storePath (keys %narFiles) {
-    my $narFileList = $narFiles{$storePath};
-    foreach my $narFile (@{$narFileList}) {
-        writeRegistration $storePath, $narFile;
-    }
-}
-
-foreach my $storePath (keys %localPaths) {
-    my $localPathList = $localPaths{$storePath};
-    foreach my $localPath (@{$localPathList}) {
-        writeRegistration $storePath, $localPath;
-    }
-}
-
-close WRITE or die "nix-store failed: $?";

src/libmain/shared.cc

@@ -116,6 +116,12 @@ static void initAndRun(int argc, char * * argv)
     nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
     nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
 
+    string subs = getEnv("NIX_SUBSTITUTERS", "default");
+    if (subs == "default")
+        substituters.push_back(nixLibexecDir + "/nix/download-using-manifests.pl");
+    else
+        substituters = tokenizeString(subs, ":");
+
     /* Get some settings from the configuration file. */
     thisSystem = querySetting("system", SYSTEM);
     maxBuildJobs = queryIntSetting("build-max-jobs", 1);
@@ -331,7 +337,7 @@ int main(int argc, char * * argv)
                 "Try `%2% --help' for more information.")
             % e.what() % programId);
         return 1;
-    } catch (Error & e) {
+    } catch (BaseError & e) {
         printMsg(lvlError, format("error: %1%") % e.msg());
         return 1;
     } catch (std::exception & e) {

src/libstore/build.cc

@@ -166,6 +166,11 @@ private:
     /* Goals waiting for a build slot. */
     WeakGoals wantingToBuild;
 
+    /* Goals waiting for info from substituters (using --query-info),
+       and the info they're (collectively) waiting for. */
+    WeakGoals waitingForInfo;
+    map<Path, PathSet> requestedInfo;
+
     /* Child processes currently running. */
     Children children;
 
@@ -214,12 +219,24 @@ public:
     /* Put `goal' to sleep until a child process terminates, i.e., a
        call is made to childTerminate(..., true).  */
     void waitForChildTermination(GoalPtr goal);
+
+    /* Put `goal' to sleep until the top-level loop has run `sub' to
+       get info about `storePath' (with --query-info).  We combine
+       substituter invocations to reduce overhead. */
+    void waitForInfo(GoalPtr goal, Path sub, Path storePath);
     
     /* Loop until the specified top-level goals have finished. */
     void run(const Goals & topGoals);
 
     /* Wait for input to become available. */
     void waitForInput();
+
+private:
+
+    /* Process the pending paths in requestedInfo and wake up the
+       goals in waitingForInfo. */
+    void getInfo();
+    
 };
 
 
@@ -797,7 +814,7 @@ void DerivationGoal::haveDerivation()
            substitutes. */
         if (store->hasSubstitutes(*i))
             addWaitee(worker.makeSubstitutionGoal(*i));
-
+    
     if (waitees.empty()) /* to prevent hang (no wake-up event) */
         outputsSubstituted();
     else
@@ -1952,21 +1969,23 @@ PathSet DerivationGoal::checkPathValidity(bool returnValid)
 
 class SubstitutionGoal : public Goal
 {
+    friend class Worker;
+    
 private:
     /* The store path that should be realised through a substitute. */
     Path storePath;															//TODO !!!!!!!!!!!!!!!!!!!!! add statePath?
 
-    /* The remaining substitutes for this path. */
+    /* The remaining substituters. */
-    Substitutes subs;
+    Paths subs;
 
-    /* The current substitute. */
+    /* The current substituter. */
-    Substitute sub;
+    Path sub;
 
-    /* Outgoing references for this path. */
+    /* Path info returned by the substituter's --query-info operation. */
+    bool infoOkay;
     PathSet references;
-    
+    PathSet stateReferences;	/* Outgoing state references for this path. */
-    /* Outgoing state references for this path. */
+    Path deriver;
-    PathSet stateReferences;
 
     /* Pipe for the substitute's standard output/error. */
     Pipe logPipe;
@@ -1990,8 +2009,9 @@ public:
 
     /* The states. */
     void init();
-    void referencesValid();
     void tryNext();
+    void gotInfo();
+    void referencesValid();
     void tryToRun();
     void finished();
 
@@ -2049,17 +2069,46 @@ void SubstitutionGoal::init()
         return;
     }
 
-    /* !!! race condition; should get the substitutes and the
+    subs = substituters;
-       references in a transaction (in case a clearSubstitutes() is
-       done simultaneously). */
 
-    /* Read the substitutes. */
+    tryNext();
-    subs = store->querySubstitutes(storePath);
+}
 
+
+void SubstitutionGoal::tryNext()
+{
+    trace("trying next substituter");
+
+    if (subs.size() == 0) {
+        /* None left.  Terminate this goal and let someone else deal
+           with it. */
+        printMsg(lvlError,
+            format("path `%1%' is required, but there is no substituter that can build it")
+            % storePath);
+        amDone(ecFailed);
+        return;
+    }
+
+    sub = subs.front();
+    subs.pop_front();
+
+    infoOkay = false;
+    state = &SubstitutionGoal::gotInfo;
+    worker.waitForInfo(shared_from_this(), sub, storePath);
+}
+
+
+void SubstitutionGoal::gotInfo()
+{
+    trace("got info");
+
+    if (!infoOkay) {
+        tryNext();
+        return;
+    }
+    
     /* To maintain the closure invariant, we first have to realise the
        paths referenced by this one. */
-    store->queryStoreReferences(storePath, references, 0);
-
     for (PathSet::iterator i = references.begin();
          i != references.end(); ++i)
         if (*i != storePath) /* ignore self-references */
@@ -2074,7 +2123,7 @@ void SubstitutionGoal::init()
 
 void SubstitutionGoal::referencesValid()
 {
-    trace("all referenced realised");
+    trace("all references realised");
 
     if (nrFailed > 0) {
         printMsg(lvlError,
@@ -2087,28 +2136,7 @@ void SubstitutionGoal::referencesValid()
          i != references.end(); ++i)
         if (*i != storePath) /* ignore self-references */
             assert(store->isValidPath(*i));
-    
-    tryNext();
-}
 
-
-void SubstitutionGoal::tryNext()
-{
-    trace("trying next substitute");
-
-    if (subs.size() == 0) {
-        /* None left.  Terminate this goal and let someone else deal
-           with it. */
-        printMsg(lvlError,
-            format("path `%1%' is required, but it has no (remaining) substitutes")
-            % storePath);
-        amDone(ecFailed);
-        return;
-    }
-    sub = subs.front();
-    subs.pop_front();
-
-    /* Wait until we can run the substitute program. */
     state = &SubstitutionGoal::tryToRun;
     worker.waitForBuildSlot(shared_from_this());
 }
@@ -2139,7 +2167,7 @@ void SubstitutionGoal::tryToRun()
 
     printMsg(lvlInfo,
         format("substituting path `%1%' using substituter `%2%'")
-        % storePath % sub.program);
+        % storePath % sub);
     
     logPipe.create();
 
@@ -2170,14 +2198,15 @@ void SubstitutionGoal::tryToRun()
             commonChildInit(logPipe);
 
             /* Fill in the arguments. */
-            Strings args(sub.args);
+            Strings args;
-            args.push_front(storePath);
+            args.push_back(baseNameOf(sub));
-            args.push_front(baseNameOf(sub.program));
+            args.push_back("--substitute");
+            args.push_back(storePath);
             const char * * argArr = strings2CharPtrs(args);
 
-            execv(sub.program.c_str(), (char * *) argArr);
+            execv(sub.c_str(), (char * *) argArr);
             
-            throw SysError(format("executing `%1%'") % sub.program);
+            throw SysError(format("executing `%1%'") % sub);
             
         } catch (std::exception & e) {
             std::cerr << format("substitute error: %1%\n") % e.what();
@@ -2230,7 +2259,7 @@ void SubstitutionGoal::finished()
 
         printMsg(lvlInfo,
             format("substitution of path `%1%' using substituter `%2%' failed: %3%")
-            % storePath % sub.program % e.msg());
+            % storePath % sub % e.msg());
         
         /* Try the next substitute. */
         state = &SubstitutionGoal::tryNext;
@@ -2245,7 +2274,7 @@ void SubstitutionGoal::finished()
     Transaction txn;
     createStoreTransaction(txn);
     registerValidPath(txn, storePath, contentHash,					//TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! how about substituing a state path ?????
-        references, stateReferences, sub.deriver, 0);
+        references, stateReferences, deriver, 0);
     txn.commit();
 
     outputLock->setDeletion(true);
@@ -2430,6 +2459,76 @@ void Worker::waitForChildTermination(GoalPtr goal)
 }
 
 
+void Worker::waitForInfo(GoalPtr goal, Path sub, Path storePath)
+{
+    debug("wait for info");
+    requestedInfo[sub].insert(storePath);
+    waitingForInfo.insert(goal);
+}
+
+
+void Worker::getInfo()
+{
+    for (map<Path, PathSet>::iterator i = requestedInfo.begin();
+         i != requestedInfo.end(); ++i)
+    {
+        Path sub = i->first;
+        PathSet paths = i->second;
+
+        while (!paths.empty()) {
+
+            /* Run the substituter for at most 100 paths at a time to
+               prevent command line overflows. */
+            PathSet paths2;
+            while (!paths.empty() && paths2.size() < 100) {
+                paths2.insert(*paths.begin());
+                paths.erase(paths.begin());
+            }
+
+            /* Ask the substituter for the references and deriver of
+               the paths. */
+            debug(format("running `%1%' to get info about `%2%'") % sub % showPaths(paths2));
+            Strings args;
+            args.push_back("--query-info");
+            args.insert(args.end(), paths2.begin(), paths2.end());
+            string res = runProgram(sub, false, args);
+            std::istringstream str(res);
+
+            while (true) {
+                ValidPathInfo info = decodeValidPathInfo(str);
+                if (info.path == "") break;
+
+                /* !!! inefficient */
+                for (WeakGoals::iterator k = waitingForInfo.begin();
+                     k != waitingForInfo.end(); ++k)
+                {
+                    GoalPtr goal = k->lock();
+                    if (goal) {
+                        SubstitutionGoal * goal2 = dynamic_cast<SubstitutionGoal *>(goal.get());
+                        if (goal2->storePath == info.path) {
+                            goal2->references = info.references;
+                            goal2->deriver = info.deriver;
+                            goal2->infoOkay = true;
+                            wakeUp(goal);
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    for (WeakGoals::iterator k = waitingForInfo.begin();
+         k != waitingForInfo.end(); ++k)
+    {
+        GoalPtr goal = k->lock();
+        if (goal) wakeUp(goal);
+    }
+    
+    requestedInfo.clear();
+    waitingForInfo.clear(); // !!! have we done them all?
+}
+
+
 void Worker::run(const Goals & _topGoals)
 {
     for (Goals::iterator i = _topGoals.begin();
@@ -2456,11 +2555,14 @@ void Worker::run(const Goals & _topGoals)
 
         if (topGoals.empty()) break;
 
-        /* !!! not when we're polling */
+        getInfo();
-        assert(!children.empty());
+
-        
         /* Wait for input. */
-        waitForInput();
+        if (!children.empty())
+            waitForInput();
+        else
+            /* !!! not when we're polling */
+            assert(!awake.empty());
     }
 
     /* If --keep-going is not set, it's possible that the main goal

src/libstore/db.cc

@@ -197,7 +197,8 @@ void Database::open2(const string & path, bool removeOldEnv)
 
     env->set_errcall(errorPrinter);
     env->set_msgcall(messagePrinter);
-    //env->set_verbose(DB_VERB_REGISTER, 1);
+    if (getEnv("NIX_DEBUG_DB_REGISTER") == "1")
+        env->set_verbose(DB_VERB_REGISTER, 1);
     env->set_verbose(DB_VERB_RECOVERY, 1);
     
     /* Smaller log files. */
@@ -458,4 +459,14 @@ void Database::enumTable(const Transaction & txn, TableId table,
 
 
  
+void Database::clearTable(const Transaction & txn, TableId table)
+{
+    try {
+        Db * db = getDb(table);
+        u_int32_t count;
+        db->truncate(txn.txn, &count, 0);
+    } catch (DbException e) { rethrow(e); }
+}
+
+
 }

src/libstore/db.hh

@@ -91,7 +91,8 @@ public:
 
     void enumTable(const Transaction & txn, TableId table,
         Strings & keys, const string & keyPrefix = "");
-    
+
+    void clearTable(const Transaction & txn, TableId table);
 };
 
 

src/libstore/gc.cc

@@ -607,6 +607,8 @@ void LocalStore::collectGarbage(GCAction action, const PathSet & pathsToDelete,
             }
 #endif
 
+            if (!pathExists(*i)) continue;
+                
             printMsg(lvlInfo, format("deleting `%1%'") % *i);
             
             /* Okay, it's safe to delete. */

src/libstore/globals.cc

@@ -28,9 +28,12 @@ unsigned int maxBuildJobs = 1;
 bool readOnlyMode = false;
 string thisSystem = "unset";
 unsigned int maxSilentTime = 0;
-static bool settingsRead = false;
+Paths substituters;
-uid_t callingUID = 0;			//A root user will not set this value, so the default uid is 0
 
+
+static bool settingsRead = false;
+
+uid_t callingUID = 0;			//A root user will not set this value, so the default uid is 0
 bool singleThreaded = false;	//TODO Gives an error: cannot start worker (environment already open) / waiting for process 7487: No child processes
 bool sendOutput = true;
 

src/libstore/globals.hh

@@ -75,6 +75,11 @@ extern string thisSystem;
    infinity. */
 extern unsigned int maxSilentTime;
 
+/* The substituters.  There are programs that can somehow realise a
+   store path without building, e.g., by downloading it or copying it
+   from a CD. */
+extern Paths substituters;
+
 Strings querySetting(const string & name, const Strings & def);
 
 string querySetting(const string & name, const string & def);

src/libstore/local-store.cc

@@ -79,22 +79,6 @@ static TableId dbStateStateReferences = 0;
  */
 static TableId dbSolidStateReferences = 0;
 
-/* dbSubstitutes :: Path -> [[Path]]
-
-   Each pair $(p, subs)$ tells Nix that it can use any of the
-   substitutes in $subs$ to build path $p$.  Each substitute defines a
-   command-line invocation of a program (i.e., the first list element
-   is the full path to the program, the remaining elements are
-   arguments).
-
-   The main purpose of this is for distributed caching of derivates.
-   One system can compute a derivate and put it on a website (as a Nix
-   archive), for instance, and then another system can register a
-   substitute for that derivate.  The substitute in this case might be
-   a Nix derivation that fetches the Nix archive.
-*/
-static TableId dbSubstitutes = 0;
-
 /* dbDerivers :: Path -> [Path]
 
    This table lists the derivation used to build a path.  There can
@@ -174,12 +158,6 @@ static TableId dbStateSnapshots = 0;
  */
 static TableId dbSharedState = 0;
 
-bool Substitute::operator == (const Substitute & sub) const
-{
-    return program == sub.program
-        && args == sub.args;
-}
-
 
 static void upgradeStore07();
 static void upgradeStore09();
@@ -205,6 +183,8 @@ void checkStoreNotSymlink()
 
 LocalStore::LocalStore(bool reserveSpace)
 {
+    substitutablePathsLoaded = false;
+    
     if (readOnlyMode) return;
 
     checkStoreNotSymlink();
@@ -234,7 +214,6 @@ LocalStore::LocalStore(bool reserveSpace)
     }
     dbValidPaths = nixDB.openTable("validpaths");
     dbValidStatePaths = nixDB.openTable("validpaths_state");
-    dbSubstitutes = nixDB.openTable("substitutes");
     dbDerivers = nixDB.openTable("derivers");
 
 	dbStateInfo = nixDB.openTable("stateinfo");
@@ -397,6 +376,7 @@ bool isValidStatePathTxn(const Transaction & txn, const Path & path)
     return nixDB.queryString(txn, dbValidStatePaths, path, s);
 }
 
+
 bool LocalStore::isValidStatePath(const Path & path)
 {
     return isValidStatePathTxn(noTxn, path);
@@ -412,11 +392,8 @@ bool LocalStore::isValidComponentOrStatePath(const Path & path)
     return isValidComponentOrStatePathTxn(noTxn, path);
 }
 
-
+//TODO REMOVE BLOCK
-static Substitutes readSubstitutes(const Transaction & txn,
+/*
-    const Path & srcPath);
-
-
 static bool isRealisablePath(const Transaction & txn, const Path & path)
 {
     return isValidPathTxn(txn, path) || readSubstitutes(txn, path).size() > 0;
@@ -427,10 +404,12 @@ static bool isRealisableStatePath(const Transaction & txn, const Path & path)
     return isValidStatePathTxn(txn, path) || readSubstitutes(txn, path).size() > 0;
 }
 
+
 static bool isRealisableComponentOrStatePath(const Transaction & txn, const Path & path)
 {
     return isValidComponentOrStatePathTxn(txn, path) || readSubstitutes(txn, path).size() > 0;					//TODO State paths are not yet in substitutes !!!!!!!!!!!!!! ??
 }
+*/
 
 /*
 static string addPrefix(const string & prefix, const string & s)
@@ -456,8 +435,9 @@ static string stripPrefix(const string & prefix, const string & s)
 void setReferences(const Transaction & txn, const Path & store_or_statePath,
     const PathSet & references, const PathSet & stateReferences, const unsigned int revision)		
 {
+
     /* For unrealisable paths, we can only clear the references. */
-    if (references.size() > 0 && !isRealisableComponentOrStatePath(txn, store_or_statePath))
+    if (references.size() > 0 && !isValidComponentOrStatePathTxn(txn, store_or_statePath))
         throw Error(format("cannot set references for path `%1%' which is invalid and has no substitutes") % store_or_statePath);
 	
 	
@@ -468,7 +448,7 @@ void setReferences(const Transaction & txn, const Path & store_or_statePath,
     	printMsg(lvlError, format("'%2%' has stateReferences: %1%") % *i % store_or_statePath);
 	*/
 	
-    if(isRealisablePath(txn, store_or_statePath))
+    if(isValidPathTxn(txn, store_or_statePath))
     {
 		printMsg(lvlError, format("Setting references for storepath '%1%'") % store_or_statePath);
 		
@@ -486,7 +466,7 @@ void setReferences(const Transaction & txn, const Path & store_or_statePath,
     	nixDB.setStrings(txn, dbComponentComponentReferences, store_or_statePath, Paths(references.begin(), references.end()));
 		nixDB.setStrings(txn, dbComponentStateReferences, store_or_statePath, Paths(stateReferences.begin(), stateReferences.end()));
     }
-    else if(isRealisableStatePath(txn, store_or_statePath))
+    else if(isValidStatePathTxn(txn, store_or_statePath))
     {
 		
 		printMsg(lvlError, format("Setting references for statepath '%1%' (revision:%2%)") % store_or_statePath % unsignedInt2String(revision));
@@ -525,9 +505,9 @@ void queryXReferencesTxn(const Transaction & txn, const Path & store_or_statePat
     	table2 = dbStateStateReferences;
     }
     
-    if(isRealisablePath(txn, store_or_statePath))
+    if(isValidPathTxn(txn, store_or_statePath))
     	nixDB.queryStrings(txn, table1, store_or_statePath, references2);
-    else if(isRealisableStatePath(txn, store_or_statePath)){
+    else if(isValidStatePathTxn(txn, store_or_statePath)){
     	Path statePath_ns = toNonSharedPathTxn(txn, store_or_statePath);	    //Lookup its where it points to if its shared
     	queryStateReferences(nixDB, txn, table2, dbStateRevisions, statePath_ns, references2, revision, timestamp);
     }
@@ -655,7 +635,7 @@ static PathSet getStateReferrersTxn(const Transaction & txn, const Path & store_
 void queryStoreReferrersTxn(const Transaction & txn,
     const Path & storePath, PathSet & referrers, const unsigned int revision)
 {
-    if (!isRealisableComponentOrStatePath(txn, storePath))
+    if (!isValidComponentOrStatePathTxn(txn, storePath))	
 		throw Error(format("path `%1%' is not valid") % storePath);
     PathSet referrers2 = getStoreReferrersTxn(txn, storePath, revision);
     referrers.insert(referrers2.begin(), referrers2.end());
@@ -669,7 +649,7 @@ void LocalStore::queryStoreReferrers(const Path & storePath,
 
 void queryStateReferrersTxn(const Transaction & txn, const Path & storePath, PathSet & stateReferrers, const unsigned int revision)
 {
-    if (!isRealisableComponentOrStatePath(txn, storePath))
+    if (!isValidComponentOrStatePathTxn(txn, storePath))
         throw Error(format("path `%1%' is not valid") % storePath);
     PathSet stateReferrers2 = getStateReferrersTxn(txn, storePath, revision);
     stateReferrers.insert(stateReferrers2.begin(), stateReferrers2.end());
@@ -686,7 +666,7 @@ void setDeriver(const Transaction & txn, const Path & storePath, const Path & de
     if (deriver == "") return;
     assertStorePath(deriver);
     
-    if (!isRealisablePath(txn, storePath))
+    if (!isValidPathTxn(txn, storePath))
         throw Error(format("path `%1%' is not valid") % storePath);
 
     if (isStateDrvPathTxn(txn, deriver)){								//Redirect if its a state component					
@@ -740,7 +720,7 @@ void addStateDeriver(const Transaction & txn, const Path & storePath, const Path
 		return;
 	assertStorePath(deriver);
 	
-    if (!isRealisablePath(txn, storePath))
+    if (!isValidPathTxn(txn, storePath))
         throw Error(format("path `%1%' is not valid") % storePath);
 
 	Derivation drv = derivationFromPathTxn(txn, deriver);
@@ -793,7 +773,7 @@ bool isStateDrv(const Derivation & drv)
 
 static Path queryDeriver(const Transaction & txn, const Path & storePath)  
 {
-    if (!isRealisablePath(txn, storePath))
+    if (!isValidPathTxn(txn, storePath))
         throw Error(format("path `%1%' is not valid") % storePath);
     Path deriver;
     
@@ -817,7 +797,7 @@ Path LocalStore::queryDeriver(const Path & path)
 //A '*' as argument stands for all identifiers or all users
 PathSet queryDerivers(const Transaction & txn, const Path & storePath, const string & identifier, const string & user)
 {
-	if (!isRealisablePath(txn, storePath))
+	if (!isValidPathTxn(txn, storePath))
         throw Error(format("path `%1%' is not valid") % storePath);
 	
 	if(user == "")
@@ -845,7 +825,7 @@ PathSet queryDerivers(const Transaction & txn, const Path & storePath, const str
 	
 	return filtereddata;
 }
-
+	
 PathSet LocalStore::queryDerivers(const Path & storePath, const string & identifier, const string & user)
 {
 	return nix::queryDerivers(noTxn, storePath, identifier, user);
@@ -865,120 +845,33 @@ PathSet queryDeriversStatePath(const Transaction & txn, const Path & storePath,
 }
 */
 
-
+PathSet LocalStore::querySubstitutablePaths()
-const int substituteVersion = 2;
-
-
-static Substitutes readSubstitutes(const Transaction & txn,
-    const Path & srcPath)
 {
-    Strings ss;
+    if (!substitutablePathsLoaded) {
-    nixDB.queryStrings(txn, dbSubstitutes, srcPath, ss);
+        for (Paths::iterator i = substituters.begin(); i != substituters.end(); ++i) {
-
+            debug(format("running `%1%' to find out substitutable paths") % *i);
-    Substitutes subs;
+            Strings args;
-    
+            args.push_back("--query-paths");
-    for (Strings::iterator i = ss.begin(); i != ss.end(); ++i) {
+            Strings ss = tokenizeString(runProgram(*i, false, args), "\n");
-        if (i->size() < 4 || (*i)[3] != 0) {
+            for (Strings::iterator j = ss.begin(); j != ss.end(); ++j) {
-            /* Old-style substitute.  !!! remove this code
+                if (!isStorePath(*j))
-               eventually? */
+                    throw Error(format("`%1%' returned a bad substitutable path `%2%'")
-            break;
+                        % *i % *j);
+                substitutablePaths.insert(*j);
+            }
         }
-        Strings ss2 = unpackStrings(*i);
+        substitutablePathsLoaded = true;
-        if (ss2.size() == 0) continue;
-        int version;
-        if (!string2Int(ss2.front(), version)) continue;
-        if (version != substituteVersion) continue;
-        if (ss2.size() != 4) throw Error("malformed substitute");
-        Strings::iterator j = ss2.begin();
-        j++;
-        Substitute sub;
-        sub.deriver = *j++;
-        sub.program = *j++;
-        sub.args = unpackStrings(*j++);
-        subs.push_back(sub);
     }
 
-    return subs;
+    return substitutablePaths;
 }
 
 
-static void writeSubstitutes(const Transaction & txn,
+bool LocalStore::hasSubstitutes(const Path & path)
-    const Path & srcPath, const Substitutes & subs)
 {
-    Strings ss;
+    if (!substitutablePathsLoaded)
-
+        querySubstitutablePaths(); 
-    for (Substitutes::const_iterator i = subs.begin();
+    return substitutablePaths.find(path) != substitutablePaths.end();
-         i != subs.end(); ++i)
-    {
-        Strings ss2;
-        ss2.push_back((format("%1%") % substituteVersion).str());
-        ss2.push_back(i->deriver);
-        ss2.push_back(i->program);
-        ss2.push_back(packStrings(i->args));
-        ss.push_back(packStrings(ss2));
-    }
-
-    nixDB.setStrings(txn, dbSubstitutes, srcPath, ss);
-}
-
-
-void registerSubstitute(const Transaction & txn,
-    const Path & srcPath, const Substitute & sub)
-{
-    assertStorePath(srcPath);
-    
-    Substitutes subs = readSubstitutes(txn, srcPath);
-
-    if (find(subs.begin(), subs.end(), sub) != subs.end())
-        return;
-
-    /* New substitutes take precedence over old ones.  If the
-       substitute is already present, it's moved to the front. */
-    remove(subs.begin(), subs.end(), sub);
-    subs.push_front(sub);
-        
-    writeSubstitutes(txn, srcPath, subs);
-}
-
-
-Substitutes querySubstitutes(const Transaction & txn, const Path & path)
-{
-    return readSubstitutes(txn, path);
-}
-
-
-Substitutes LocalStore::querySubstitutes(const Path & path)
-{
-    return nix::querySubstitutes(noTxn, path);
-}
-
-
-static void invalidateStorePath(Transaction & txn, const Path & path);
-
-
-void clearSubstitutes()								//TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! ALSO FOR STATE
-{
-    Transaction txn(nixDB);
-    
-    /* Iterate over all paths for which there are substitutes. */
-    Paths subKeys;
-    nixDB.enumTable(txn, dbSubstitutes, subKeys);
-    for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) {
-        
-        /* Delete all substitutes for path *i. */
-        nixDB.delPair(txn, dbSubstitutes, *i);
-        
-        /* Maintain the cleanup invariant. */
-        if (!isValidPathTxn(txn, *i))
-            invalidateStorePath(txn, *i);
-    }
-
-    /* !!! there should be no referrers to any of the invalid
-       substitutable paths.  This should be the case by construction
-       (the only referrers can be other invalid substitutable paths,
-       which have all been removed now). */
-    
-    txn.commit();
 }
 
 
@@ -1510,38 +1403,9 @@ void verifyStore(bool checkContents)
         }
     }
 
-
-    printMsg(lvlInfo, "checking path realisability");
-    
-    /* "Realisable" paths are those that are valid or have a
-       substitute. */
-    PathSet realisablePaths(validPaths);
-    
-    
-    //TODO !!!!!!!!!!!!!!!!!!!!!!!!! Do also for validStatePaths
-    
-
-    /* Check that the values of the substitute mappings are valid
-       paths. */ 
-    Paths subKeys;
-    nixDB.enumTable(txn, dbSubstitutes, subKeys);
-    for (Paths::iterator i = subKeys.begin(); i != subKeys.end(); ++i) {
-        Substitutes subs = readSubstitutes(txn, *i);
-        if (!isStorePath(*i)) {
-            printMsg(lvlError, format("removing substitutes for non-store path `%1%'") % *i);
-            nixDB.delPair(txn, dbSubstitutes, *i);
-        }
-        else if (subs.size() == 0)
-            nixDB.delPair(txn, dbSubstitutes, *i);
-        else
-			realisablePaths.insert(*i);
-    }
-    
-
     /* Check the cleanup invariant: only realisable paths can have
        `references', `referrers', or `derivers' entries. */
 
-
     /* Check the `derivers' table. */
     printMsg(lvlInfo, "checking the derivers table");
     Paths deriversKeys;
@@ -1549,8 +1413,8 @@ void verifyStore(bool checkContents)
     for (Paths::iterator i = deriversKeys.begin();
          i != deriversKeys.end(); ++i)
     {
-        if (realisablePaths.find(*i) == realisablePaths.end()) {
+        if (validPaths.find(*i) == validPaths.end()) {
-            printMsg(lvlError, format("removing deriver entry for unrealisable path `%1%'")
+            printMsg(lvlError, format("removing deriver entry for invalid path `%1%'")
                 % *i);
             nixDB.delPair(txn, dbDerivers, *i);
         }
@@ -1572,13 +1436,12 @@ void verifyStore(bool checkContents)
     for (Paths::iterator i = referencesKeys.begin();
          i != referencesKeys.end(); ++i)
     {
-        if (realisablePaths.find(*i) == realisablePaths.end()) {
+        if (validPaths.find(*i) == validPaths.end()) {
-            printMsg(lvlError, format("removing references entry for unrealisable path `%1%'")
+            printMsg(lvlError, format("removing references entry for invalid path `%1%'")
                 % *i);
             setReferences(txn, *i, PathSet(), PathSet(), 0);		//TODO?
         }
         else {
-            bool isValid = validPaths.find(*i) != validPaths.end();
             PathSet references;
             queryXReferencesTxn(txn, *i, references, true, -1);				//TODO
             for (PathSet::iterator j = references.begin();
@@ -1594,7 +1457,7 @@ void verifyStore(bool checkContents)
                 }
                 */
                 
-                if (isValid && validPaths.find(*j) == validPaths.end()) {
+                if (validPaths.find(*j) == validPaths.end()) {
                     printMsg(lvlError, format("incomplete closure: `%1%' needs missing `%2%'")
                         % *i % *j);
                 }
@@ -1629,6 +1492,7 @@ void setStatePathsIntervalTxn(const Transaction & txn, const PathSet & statePath
     }
 }
 
+
 void LocalStore::setStatePathsInterval(const PathSet & statePaths, const IntVector & intervals, bool allZero)
 {
 	Transaction txn(nixDB);

src/libstore/local-store.hh

@@ -23,6 +23,10 @@ extern string drvsLogDir;
 
 class LocalStore : public StoreAPI
 {
+private:
+    bool substitutablePathsLoaded;
+    PathSet substitutablePaths;
+    
 public:
 
     /* Open the database environment.  If `reserveSpace' is true, make
@@ -45,8 +49,6 @@ public:
     
     bool isValidComponentOrStatePath(const Path & path);
     
-    Substitutes querySubstitutes(const Path & srcPath);
-
     Hash queryPathHash(const Path & path);
     
     Path queryStatePathDrv(const Path & statePath);
@@ -61,6 +63,10 @@ public:
 
     Path queryDeriver(const Path & path);
     
+    PathSet querySubstitutablePaths();
+    
+    bool hasSubstitutes(const Path & path);
+    
     Path addToStore(const Path & srcPath, bool fixed = false,
         bool recursive = false, string hashAlgo = "",
         PathFilter & filter = defaultPathFilter);
@@ -128,13 +134,6 @@ void createStoreTransaction(Transaction & txn);
 /* Copy a path recursively. */
 void copyPath(const Path & src, const Path & dst);
 
-/* Register a substitute. */
-void registerSubstitute(const Transaction & txn,
-    const Path & srcPath, const Substitute & sub);
-
-/* Deregister all substitutes. */
-void clearSubstitutes();
-
 /* Register the validity of a path, i.e., that `path' exists, that the
    paths referenced by it exists, and in the case of an output path of
    a derivation, that it has been produced by a succesful execution of
@@ -146,16 +145,6 @@ void registerValidPath(const Transaction & txn,
     const PathSet & references, const PathSet & stateReferences,
     const Path & deriver, const unsigned int revision);
 
-struct ValidPathInfo 
-{
-    Path path;
-    Path deriver;
-    Hash hash;
-    PathSet references;
-    PathSet stateReferences;
-    int unsigned revision;
-};
-
 typedef list<ValidPathInfo> ValidPathInfos;
 
 void registerValidPaths(const Transaction & txn,

src/libstore/misc.cc

@@ -124,8 +124,7 @@ void queryMissing(const PathSet & targets,
             bool mustBuild = false;
             for (DerivationOutputs::iterator i = drv.outputs.begin();
                  i != drv.outputs.end(); ++i)
-                if (!store->isValidPath(i->second.path) &&
+                if (!store->isValidPath(i->second.path) && !store->hasSubstitutes(i->second.path))
-                    !store->hasSubstitutes(i->second.path))
                     mustBuild = true;
 
             if (mustBuild) {
@@ -144,8 +143,8 @@ void queryMissing(const PathSet & targets,
             if (store->isValidPath(p)) continue;
             if (store->hasSubstitutes(p))
                 willSubstitute.insert(p);
-            PathSet refs;
+            // XXX call the substituters
-            store->queryStoreReferences(p, todo, 0);		//TODO?
+            // store->queryReferences(p, todo);
         }
     }
 }

src/libstore/remote-store.cc

@@ -182,12 +182,6 @@ bool RemoteStore::isValidComponentOrStatePath(const Path & path)
     return reply != 0;
 }
 
-Substitutes RemoteStore::querySubstitutes(const Path & path)
-{
-    throw Error("not implemented 2");
-}
-
-
 bool RemoteStore::hasSubstitutes(const Path & path)
 {
     writeInt(wopHasSubstitutes, to);
@@ -270,6 +264,12 @@ Path RemoteStore::queryDeriver(const Path & path)
 }
 
 
+PathSet RemoteStore::querySubstitutablePaths()
+{
+    throw Error("not implemented");
+}
+
+
 Path RemoteStore::addToStore(const Path & _srcPath, bool fixed,
     bool recursive, string hashAlgo, PathFilter & filter)
 {
@@ -283,8 +283,6 @@ Path RemoteStore::addToStore(const Path & _srcPath, bool fixed,
     dumpPath(srcPath, to, filter);
     processStderr();
     return readStorePath(from);
-    //Path path = readStorePath(from);		//TODO REMOVE CODE		
-    //return path;
 }
 
 

src/libstore/remote-store.hh

@@ -31,10 +31,6 @@ public:
     
     bool isValidComponentOrStatePath(const Path & path);
 
-    Substitutes querySubstitutes(const Path & path);
-
-    bool hasSubstitutes(const Path & path);
-    
     Hash queryPathHash(const Path & path);
     
     Path queryStatePathDrv(const Path & statePath);
@@ -49,6 +45,10 @@ public:
 
     Path queryDeriver(const Path & path);
     
+    PathSet querySubstitutablePaths();
+    
+    bool hasSubstitutes(const Path & path);
+    
     Path addToStore(const Path & srcPath, bool fixed = false,
         bool recursive = false, string hashAlgo = "",
         PathFilter & filter = defaultPathFilter);

src/libstore/store-api.cc

@@ -8,7 +8,8 @@ namespace nix {
 
 bool StoreAPI::hasSubstitutes(const Path & path)
 {
-    return !querySubstitutes(path).empty();
+    PathSet paths = querySubstitutablePaths();
+    return paths.find(path) != paths.end();
 }
 
 
@@ -207,6 +208,45 @@ Path computeStorePathForText(const string & suffix, const string & s,
     return makeStorePath(type, hash, suffix);
 }
 
+//TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+ValidPathInfo decodeValidPathInfo(std::istream & str)
+{
+    ValidPathInfo info;
+    
+    getline(str, info.path);
+    if (str.eof()) { info.path = ""; return info; }
+    
+    getline(str, info.deriver);
+    
+    string s; int n;
+    
+    getline(str, s);
+    if (!string2Int(s, n)) 
+    	throw Error("number expected");
+    while (n--) {
+        getline(str, s);
+        info.references.insert(s);
+    }
+    
+    getline(str, s);
+    if (!string2Int(s, n)) 
+    	throw Error("number expected");
+    while (n--) {
+        getline(str, s);
+        info.stateReferences.insert(s);
+    }
+    
+    unsigned int u;
+	getline(str, s);
+    if (!string2UnsignedInt(s, u)) 
+    	throw Error("number expected");
+   	info.revision = u;
+    
+    if (!str || str.eof()) throw Error("missing input");
+    return info;
+}
+//TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
 
 }
 

src/libstore/store-api.hh

@@ -15,28 +15,6 @@
 namespace nix {
 
 
-/* A substitute is a program invocation that constructs some store
-   path (typically by fetching it from somewhere, e.g., from the
-   network). */
-struct Substitute
-{       
-    /* The derivation that built this store path (empty if none). */
-    Path deriver;
-    
-    /* Program to be executed to create the store path.  Must be in
-       the output path of `storeExpr'. */
-    Path program;
-
-    /* Extra arguments to be passed to the program (the first argument
-       is the store path to be substituted). */
-    Strings args;
-
-    bool operator == (const Substitute & sub) const;
-};
-
-typedef list<Substitute> Substitutes;
-
-
 typedef std::map<Path, Path> Roots;
 
 
@@ -65,13 +43,6 @@ public:
     /* TODO */
     virtual bool isValidComponentOrStatePath(const Path & path) = 0;
 
-    /* Return the substitutes for the given path. */
-    virtual Substitutes querySubstitutes(const Path & path) = 0;
-
-    /* More efficient variant if we just want to know if a path has
-       substitutes. */
-    virtual bool hasSubstitutes(const Path & path);
-
     /* Queries the hash of a valid path. */ 
     virtual Hash queryPathHash(const Path & path) = 0;
 
@@ -100,6 +71,13 @@ public:
        no deriver has been set. */
     virtual Path queryDeriver(const Path & path) = 0;
 
+    /* Query the set of substitutable paths. */
+    virtual PathSet querySubstitutablePaths() = 0;
+
+    /* More efficient variant if we just want to know if a path has
+       substitutes. */
+    virtual bool hasSubstitutes(const Path & path);
+    
     /* Copy the contents of a path to the store and register the
        validity the resulting path.  The resulting path is returned.
        If `fixed' is true, then the output of a fixed-output
@@ -128,10 +106,10 @@ public:
 
     /* Ensure that the output paths of the derivation are valid.  If
        they are already valid, this is a no-op.  Otherwise, validity
-       can be reached in two ways.  First, if the output paths have
+       can be reached in two ways.  First, if the output paths is
-       substitutes, then those can be used.  Second, the output paths
+       substitutable, then build the path that way.  Second, the
-       can be created by running the builder, after recursively
+       output paths can be created by running the builder, after
-       building any sub-derivations. */
+       recursively building any sub-derivations. */
     virtual void buildDerivations(const PathSet & drvPaths) = 0;
 
     /* Ensure that a path is valid.  If it is not currently valid, it
@@ -339,6 +317,29 @@ extern boost::shared_ptr<StoreAPI> store;
 boost::shared_ptr<StoreAPI> openStore(bool reserveSpace = true);
 
 
+
+/* OLD TODO REMOVE
+struct ValidPathInfo 
+{
+    Path path;
+    Path deriver;
+    Hash hash;
+    PathSet references;
+};
+*/
+struct ValidPathInfo 
+{
+    Path path;
+    Path deriver;
+    Hash hash;
+    PathSet references;
+    PathSet stateReferences;
+    int unsigned revision;
+};
+
+ValidPathInfo decodeValidPathInfo(std::istream & str);
+
+
 }
 
 

src/libstore/worker-protocol.hh

@@ -14,7 +14,6 @@ typedef enum {
     wopIsValidPath,							
     wopIsValidStatePath,
     wopIsValidComponentOrStatePath,
-    wopQuerySubstitutes,
     wopHasSubstitutes,
     wopQueryPathHash,
     wopQueryStatePathDrv,

src/libutil/types.hh

@@ -21,23 +21,18 @@ using std::vector;
 using boost::format;
 
 
-class Error : public std::exception
+/* BaseError should generally not be caught, as it has Interrupted as
+   a subclass. Catch Error instead. */
+class BaseError : public std::exception 
 {
 protected:
     string err;
 public:
-    Error(const format & f);
+    BaseError(const format & f);
-    ~Error() throw () { };
+    ~BaseError() throw () { };
     const char * what() const throw () { return err.c_str(); }
     const string & msg() const throw () { return err; }
-    Error & addPrefix(const format & f);
+    BaseError & addPrefix(const format & f);
-};
-
-class SysError : public Error
-{
-public:
-    int errNo;
-    SysError(const format & f);
 };
 
 #define MakeError(newClass, superClass) \
@@ -47,6 +42,15 @@ public:
         newClass(const format & f) : superClass(f) { }; \
     };
 
+MakeError(Error, BaseError)
+
+class SysError : public Error
+{
+public:
+    int errNo;
+    SysError(const format & f);
+};
+
 
 typedef list<string> Strings;
 typedef list<Strings> StringsList;

src/libutil/util.cc

@@ -24,13 +24,13 @@ extern char * * environ;
 namespace nix {
 
 
-Error::Error(const format & f)
+BaseError::BaseError(const format & f)
 {
     err = f.str();
 }
 
 
-Error & Error::addPrefix(const format & f)
+BaseError & BaseError::addPrefix(const format & f)
 {
     err = f.str() + err;
     return *this;
@@ -494,6 +494,7 @@ string drainFD(int fd)
     string result;
     unsigned char buffer[4096];
     while (1) {
+        checkInterrupt();
         ssize_t rd = read(fd, buffer, sizeof buffer);
         if (rd == -1) {
             if (errno != EINTR)
@@ -777,6 +778,8 @@ void killUser(uid_t uid)
 
 string runProgram(Path program, bool searchPath, const Strings & args)
 {
+    checkInterrupt();
+    
     /* Create a pipe. */
     Pipe pipe;
     pipe.create();

src/libutil/util.hh

@@ -265,7 +265,7 @@ void inline checkInterrupt()
     if (_isInterrupted) _interrupted();
 }
 
-MakeError(Interrupted, Error)
+MakeError(Interrupted, BaseError)
 
 
 /* String packing / unpacking. */

src/nix-store/help.txt

@@ -11,8 +11,6 @@ Operations:
   --query / -q: query information
   --read-log / -l: print build log of given store paths
 
-  --register-substitutes: register a substitute expression (dangerous!)
-  --clear-substitutes: clear all substitutes
   --register-validity: register path validity (dangerous!)
   --check-validity: check path validity
 

src/nix-store/nix-store.cc

@@ -417,59 +417,6 @@ static void opReadLog(Strings opFlags, Strings opArgs)
     }
 }
 
-
-static void opRegisterSubstitutes(Strings opFlags, Strings opArgs)
-{
-    if (!opFlags.empty()) throw UsageError("unknown flag");
-    if (!opArgs.empty()) throw UsageError("no arguments expected");
-
-    Transaction txn;
-    createStoreTransaction(txn);
-
-    while (1) {
-        Path srcPath;
-        Substitute sub;
-        PathSet references;
-        
-        //TODO TODO TODO !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
-        PathSet stateReferences;
-        
-        getline(cin, srcPath);
-        if (cin.eof()) break;
-        getline(cin, sub.deriver);
-        getline(cin, sub.program);
-        string s; int n;
-        getline(cin, s);
-        if (!string2Int(s, n)) throw Error("number expected");
-        while (n--) {
-            getline(cin, s);
-            sub.args.push_back(s);
-        }
-        getline(cin, s);
-        if (!string2Int(s, n)) throw Error("number expected");
-        while (n--) {
-            getline(cin, s);
-            references.insert(s);
-        }
-        if (!cin || cin.eof()) throw Error("missing input");
-        registerSubstitute(txn, srcPath, sub);
-        setReferences(txn, srcPath, references, stateReferences, 1);	//state revision 1, e.g. first commit
-    }
-
-    txn.commit();
-}
-
-
-static void opClearSubstitutes(Strings opFlags, Strings opArgs)
-{
-    if (!opFlags.empty()) throw UsageError("unknown flag");
-    if (!opArgs.empty())
-        throw UsageError("no arguments expected");
-
-    clearSubstitutes();
-}
-
-
 static void opRegisterValidity(Strings opFlags, Strings opArgs)
 {
     bool reregister = false; // !!! maybe this should be the default
@@ -484,18 +431,8 @@ static void opRegisterValidity(Strings opFlags, Strings opArgs)
     ValidPathInfos infos;
     
     while (1) {
-        ValidPathInfo info;
+        ValidPathInfo info = decodeValidPathInfo(cin);
-        getline(cin, info.path);
+        if (info.path == "") break;
-        if (cin.eof()) break;
-        getline(cin, info.deriver);
-        string s; int n;
-        getline(cin, s);
-        if (!string2Int(s, n)) throw Error("number expected");
-        while (n--) {
-            getline(cin, s);
-            info.references.insert(s);
-        }
-        if (!cin || cin.eof()) throw Error("missing input");
         if (!store->isValidPath(info.path) || reregister) {
             /* !!! races */
             canonicalisePathMetaData(info.path);
@@ -708,10 +645,6 @@ void run(Strings args)
             op = opQuery;
         else if (arg == "--read-log" || arg == "-l")
             op = opReadLog;
-        else if (arg == "--register-substitutes")
-            op = opRegisterSubstitutes;
-        else if (arg == "--clear-substitutes")
-            op = opClearSubstitutes;
         else if (arg == "--register-validity")
             op = opRegisterValidity;
         else if (arg == "--check-validity")