From 9416202465c56e4cd2f7f2cf2fde6e6c6bc3dffa Mon Sep 17 00:00:00 2001 From: Nicolas Pierron Date: Wed, 8 Jul 2009 09:27:35 +0000 Subject: [PATCH] Add a way to cache result for future evaluations. --- src/libexpr/Makefile.am | 2 +- src/libexpr/eval.cc | 23 ++++++-- src/libexpr/eval.hh | 6 +- src/libexpr/normal-forms.cc | 64 +++++++++++++++++++++ src/libstore/globals.cc | 1 + src/libstore/globals.hh | 3 + src/libutil/aterm-map.cc | 2 +- src/libutil/aterm-map.hh | 2 +- src/libutil/serialise.cc | 107 +++++++++++++++++++++++++++++++++++- src/libutil/serialise.hh | 6 +- 10 files changed, 206 insertions(+), 10 deletions(-) create mode 100644 src/libexpr/normal-forms.cc diff --git a/src/libexpr/Makefile.am b/src/libexpr/Makefile.am index 09be6f327..548f00ee1 100644 --- a/src/libexpr/Makefile.am +++ b/src/libexpr/Makefile.am @@ -3,7 +3,7 @@ pkglib_LTLIBRARIES = libexpr.la libexpr_la_SOURCES = \ nixexpr.cc eval.cc primops.cc lexer-tab.cc parser-tab.cc \ get-drvs.cc attr-path.cc expr-to-xml.cc common-opts.cc \ - names.cc + names.cc normal-forms.cc pkginclude_HEADERS = \ nixexpr.hh eval.hh parser.hh lexer-tab.hh parser-tab.hh \ diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 02a9ea701..26f078c8f 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -25,6 +25,14 @@ EvalState::EvalState() addPrimOps(); allowUnsafeEquality = getEnv("NIX_NO_UNSAFE_EQ", "") == ""; + safeCache = true; + + loadNormalForms(); +} + +EvalState::~EvalState() +{ + saveNormalForms(); } @@ -468,8 +476,10 @@ LocalNoInline(Expr evalVar(EvalState & state, ATerm name)) if (arity == 0) /* !!! backtrace for primop call */ return ((PrimOp) ATgetBlobData(fun)) (state, ATermVector()); - else + else { + state.safeCache = false; return makePrimOp(arity, fun, ATempty); + } } @@ -495,6 +505,7 @@ LocalNoInline(Expr evalCall(EvalState & state, Expr fun, Expr arg)) for (ATermIterator i(args); i; ++i) args2[--arity] = *i; /* !!! backtrace for primop call */ + state.safeCache = true; return ((PrimOp) ATgetBlobData(funBlob)) (state, args2); } else @@ -802,7 +813,8 @@ Expr evalExpr(EvalState & state, Expr e) format("evaluating expression: %1%") % e); #endif - state.nrEvaluated++; + const unsigned int hugeEvalExpr = 100; + unsigned int nrEvaluated = state.nrEvaluated++; state.nrDephtAfterReset++; /* Consult the memo table to quickly get the normal form of @@ -830,7 +842,10 @@ Expr evalExpr(EvalState & state, Expr e) throw; } - if (state.nrDephtAfterReset) { + // session independent condition + if (state.nrDephtAfterReset && state.safeCache && + // heuristic condition + state.nrEvaluated - nrEvaluated > hugeEvalExpr) { state.sessionNormalForms.remove(e); state.normalForms.set(e, nf); state.nrDephtAfterReset--; @@ -917,7 +932,7 @@ void printEvalStats(EvalState & state) char x; bool showStats = getEnv("NIX_SHOW_STATS", "0") != "0"; printMsg(showStats ? lvlInfo : lvlDebug, - format("evaluated %1% expressions, %2% cache hits, %3%%% efficiency, used %4% ATerm bytes, used %5% bytes of stack space, %6% normal reduction, %7% session dependent reduction.") + format("evaluated %1% expressions, %2% cache hits, %3%%% efficiency, used %4% ATerm bytes, used %5% bytes of stack space, %6% cached normal reduction, %7% session dependent reduction.") % state.nrEvaluated % state.nrCached % ((float) state.nrCached / (float) state.nrEvaluated * 100) % AT_calcAllocatedSize() diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index c8557a4b3..87f7d6fba 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -29,8 +29,8 @@ typedef Expr (* PrimOp) (EvalState &, const ATermVector & args); struct EvalState { - ATermMap sessionNormalForms; ATermMap normalForms; + ATermMap sessionNormalForms; ATermMap primOps; DrvRoots drvRoots; DrvHashes drvHashes; /* normalised derivation hashes */ @@ -41,12 +41,16 @@ struct EvalState unsigned int nrDephtAfterReset; bool allowUnsafeEquality; + bool safeCache; EvalState(); + ~EvalState(); void addPrimOps(); void addPrimOp(const string & name, unsigned int arity, PrimOp primOp); + void loadNormalForms(); + void saveNormalForms(); }; diff --git a/src/libexpr/normal-forms.cc b/src/libexpr/normal-forms.cc new file mode 100644 index 000000000..7d99de4f3 --- /dev/null +++ b/src/libexpr/normal-forms.cc @@ -0,0 +1,64 @@ +#include "eval.hh" +#include "util.hh" +#include "globals.hh" +#include "serialise.hh" + +#include +#include +#include + +namespace nix { + + +void EvalState::loadNormalForms() +{ + // nixCacheFile = getEnv("NIX_CACHE_FILE", nixStateDir + "/reduce-cache"); + nixCacheFile = getEnv("NIX_CACHE_FILE"); + string loadFlag = getEnv("NIX_CACHE_FILE_LOAD", "1"); + + if(nixCacheFile == "" || loadFlag == "") + return; + if(!pathExists(nixCacheFile)) + return; + + printMsg(lvlInfo, format("Load cache: ...")); + try { + int fd = open(nixCacheFile.c_str(), O_RDONLY); + if (fd == -1) + throw SysError(format("opening file `%1%'") % nixCacheFile); + AutoCloseFD auto_fd = fd; + + FdSource source = fd; + normalForms = readATermMap(source); + } catch (Error & e) { + e.addPrefix(format("Cannot load cached reduce operations from %1%:\n") % nixCacheFile); + throw; + } + printMsg(lvlInfo, format("Load cache: end")); +} + +void EvalState::saveNormalForms() +{ + string saveFlag = getEnv("NIX_CACHE_FILE_SAVE", ""); + + if(nixCacheFile == "" || saveFlag == "") + return; + + printMsg(lvlInfo, format("Save cache: ...")); + try { + int fd = open(nixCacheFile.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666); + if (fd == -1) + throw SysError(format("opening file `%1%'") % nixCacheFile); + AutoCloseFD auto_fd = fd; + + FdSink sink = fd; + writeATermMap(normalForms, sink); + } catch (Error & e) { + e.addPrefix(format("Cannot save cached reduce operations to %1%:\n") % nixCacheFile); + throw; + } + printMsg(lvlInfo, format("Save cache: end")); +} + + +} diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index cc0e44e8e..fbc813b1c 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -13,6 +13,7 @@ string nixDataDir = "/UNINIT"; string nixLogDir = "/UNINIT"; string nixStateDir = "/UNINIT"; string nixDBPath = "/UNINIT"; +string nixCacheFile = "/UNINIT"; string nixConfDir = "/UNINIT"; string nixLibexecDir = "/UNINIT"; string nixBinDir = "/UNINIT"; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index d3388e309..692fb7747 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -24,6 +24,9 @@ extern string nixStateDir; /* nixDBPath is the path name of our Berkeley DB environment. */ extern string nixDBPath; +/* nixCacheFile is the path name of the normal form cache. */ +extern string nixCacheFile; + /* nixConfDir is the directory where configuration files are stored. */ extern string nixConfDir; diff --git a/src/libutil/aterm-map.cc b/src/libutil/aterm-map.cc index c31fcdba3..8783d1a9a 100644 --- a/src/libutil/aterm-map.cc +++ b/src/libutil/aterm-map.cc @@ -215,7 +215,7 @@ void ATermMap::remove(ATerm key) } -unsigned int ATermMap::size() +unsigned int ATermMap::size() const { return count; /* STL nomenclature */ } diff --git a/src/libutil/aterm-map.hh b/src/libutil/aterm-map.hh index db36da377..f42bdf54a 100644 --- a/src/libutil/aterm-map.hh +++ b/src/libutil/aterm-map.hh @@ -59,7 +59,7 @@ public: void remove(ATerm key); - unsigned int size(); + unsigned int size() const; struct const_iterator { diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 9b4222713..88ff257dc 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -1,9 +1,9 @@ #include "serialise.hh" #include "util.hh" +#include "aterm.hh" #include - namespace nix { @@ -73,6 +73,67 @@ void writeStringSet(const StringSet & ss, Sink & sink) } +void writeATerm(ATerm t, Sink & sink) +{ + int len; + unsigned char *buf = (unsigned char *) ATwriteToBinaryString(t, &len); + AutoDeleteArray d(buf); + writeInt(len, sink); + sink(buf, len); +} + + +/* convert the ATermMap to a list of couple because many terms are shared + between the keys and between the values. Thus the BAF stored by + writeATerm consume less memory space. The list of couples is saved + inside a tree structure of /treedepth/ height because the serialiasation + of ATerm cause a tail recurssion on list tails. */ +void writeATermMap(const ATermMap & tm, Sink & sink) +{ + const unsigned int treedepth = 5; + const unsigned int maxarity = 128; // 2 < maxarity < MAX_ARITY (= 255) + const unsigned int bufsize = treedepth * maxarity; + + AFun map = ATmakeAFun("map", 2, ATfalse); + AFun node = ATmakeAFun("node", maxarity, ATfalse); + ATerm empty = (ATerm) ATmakeAppl0(ATmakeAFun("empty", 0, ATfalse)); + + unsigned int c[treedepth]; + ATerm *buf = new ATerm[bufsize]; + AutoDeleteArray d(buf); + + memset(buf, 0, bufsize * sizeof(ATerm)); + ATprotectArray(buf, bufsize); + + for (unsigned int j = 0; j < treedepth; j++) + c[j] = 0; + + for (ATermMap::const_iterator i = tm.begin(); i != tm.end(); ++i) { + unsigned int depth = treedepth - 1; + ATerm term = (ATerm) ATmakeAppl2(map, i->key, i->value); + buf[depth * maxarity + c[depth]++] = term; + while (c[depth] % maxarity == 0) { + c[depth] = 0; + term = (ATerm) ATmakeApplArray(node, &buf[depth * maxarity]); + depth--; + buf[depth * maxarity + c[depth]++] = term; + } + } + + unsigned int depth = treedepth; + ATerm last_node = empty; + while (depth--) { + buf[depth * maxarity + c[depth]++] = last_node; + while (c[depth] % maxarity) + buf[depth * maxarity + c[depth]++] = empty; + last_node = (ATerm) ATmakeApplArray(node, &buf[depth * maxarity]); + } + + writeATerm(last_node, sink); + ATunprotectArray(buf); +} + + void readPadding(unsigned int len, Source & source) { if (len % 8) { @@ -136,4 +197,48 @@ StringSet readStringSet(Source & source) } +ATerm readATerm(Source & source) +{ + unsigned int len = readInt(source); + unsigned char * buf = new unsigned char[len]; + AutoDeleteArray d(buf); + source(buf, len); + ATerm t = ATreadFromBinaryString((char *) buf, len); + if (t == 0) + throw SerialisationError("cannot read a valid ATerm."); + return t; +} + + +static void recursiveInitATermMap(ATermMap &tm, bool &stop, ATermAppl node) +{ + const unsigned int arity = ATgetArity(ATgetAFun(node)); + ATerm key, value; + + switch (arity) { + case 0: + stop = true; + return; + case 2: + key = ATgetArgument(node, 0); + value = ATgetArgument(node, 1); + tm.set(key, value); + return; + default: + for (unsigned int i = 0; i < arity && !stop; i++) + recursiveInitATermMap(tm, stop, (ATermAppl) ATgetArgument(node, i)); + return; + } +} + +ATermMap readATermMap(Source & source) +{ + ATermMap tm; + bool stop = false; + + recursiveInitATermMap(tm, stop, (ATermAppl) readATerm(source)); + return tm; +} + + } diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh index 0e797d63b..9d0088477 100644 --- a/src/libutil/serialise.hh +++ b/src/libutil/serialise.hh @@ -2,7 +2,7 @@ #define __SERIALISE_H #include "types.hh" - +#include "aterm-map.hh" namespace nix { @@ -98,12 +98,16 @@ void writeInt(unsigned int n, Sink & sink); void writeLongLong(unsigned long long n, Sink & sink); void writeString(const string & s, Sink & sink); void writeStringSet(const StringSet & ss, Sink & sink); +void writeATerm(ATerm t, Sink & sink); +void writeATermMap(const ATermMap & tm, Sink & sink); void readPadding(unsigned int len, Source & source); unsigned int readInt(Source & source); unsigned long long readLongLong(Source & source); string readString(Source & source); StringSet readStringSet(Source & source); +ATerm readATerm(Source & source); +ATermMap readATermMap(Source & source); MakeError(SerialisationError, Error)