diff options
Diffstat (limited to 'src/libexpr')
32 files changed, 786 insertions, 492 deletions
diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 7c0705091..ab654c1b0 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -106,7 +106,7 @@ std::pair<Value *, PosIdx> findAlongAttrPath(EvalState & state, const std::strin } -std::pair<std::string, uint32_t> findPackageFilename(EvalState & state, Value & v, std::string what) +std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v, std::string what) { Value * v2; try { @@ -118,21 +118,25 @@ std::pair<std::string, uint32_t> findPackageFilename(EvalState & state, Value & // FIXME: is it possible to extract the Pos object instead of doing this // toString + parsing? - auto pos = state.forceString(*v2, noPos, "while evaluating the 'meta.position' attribute of a derivation"); + NixStringContext context; + auto path = state.coerceToPath(noPos, *v2, context, "while evaluating the 'meta.position' attribute of a derivation"); - auto colon = pos.rfind(':'); - if (colon == std::string::npos) - throw ParseError("cannot parse meta.position attribute '%s'", pos); + auto fn = path.path.abs(); + + auto fail = [fn]() { + throw ParseError("cannot parse 'meta.position' attribute '%s'", fn); + }; - std::string filename(pos, 0, colon); - unsigned int lineno; try { - lineno = std::stoi(std::string(pos, colon + 1, std::string::npos)); + auto colon = fn.rfind(':'); + if (colon == std::string::npos) fail(); + std::string filename(fn, 0, colon); + auto lineno = std::stoi(std::string(fn, colon + 1, std::string::npos)); + return {CanonPath(fn.substr(0, colon)), lineno}; } catch (std::invalid_argument & e) { - throw ParseError("cannot parse line number '%s'", pos); + fail(); + abort(); } - - return { std::move(filename), lineno }; } diff --git a/src/libexpr/attr-path.hh b/src/libexpr/attr-path.hh index b2bfb5d04..eb00ffb93 100644 --- a/src/libexpr/attr-path.hh +++ b/src/libexpr/attr-path.hh @@ -20,7 +20,7 @@ std::pair<Value *, PosIdx> findAlongAttrPath( /** * Heuristic to find the filename and lineno or a nix value. */ -std::pair<std::string, uint32_t> findPackageFilename(EvalState & state, Value & v, std::string what); +std::pair<SourcePath, uint32_t> findPackageFilename(EvalState & state, Value & v, std::string what); std::vector<Symbol> parseAttrPath(EvalState & state, std::string_view s); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 1219b2471..9e734e654 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -47,7 +47,7 @@ struct AttrDb { auto state(_state->lock()); - Path cacheDir = getCacheDir() + "/nix/eval-cache-v4"; + Path cacheDir = getCacheDir() + "/nix/eval-cache-v5"; createDirs(cacheDir); Path dbPath = cacheDir + "/" + fingerprint.to_string(Base16, false) + ".sqlite"; @@ -300,7 +300,7 @@ struct AttrDb NixStringContext context; if (!queryAttribute.isNull(3)) for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";")) - context.push_back(NixStringContextElem::parse(cfg, s)); + context.insert(NixStringContextElem::parse(s)); return {{rowId, string_t{queryAttribute.getStr(2), context}}}; } case AttrType::Bool: @@ -442,8 +442,10 @@ Value & AttrCursor::forceValue() if (v.type() == nString) cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), string_t{v.string.s, {}}}; - else if (v.type() == nPath) - cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}}; + else if (v.type() == nPath) { + auto path = v.path().path; + cachedValue = {root->db->setString(getKey(), path.abs()), string_t{path.abs(), {}}}; + } else if (v.type() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; else if (v.type() == nInt) @@ -580,7 +582,7 @@ std::string AttrCursor::getString() if (v.type() != nString && v.type() != nPath) root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow<TypeError>(); - return v.type() == nString ? v.string.s : v.path; + return v.type() == nString ? v.string.s : v.path().to_string(); } string_t AttrCursor::getStringWithContext() @@ -619,10 +621,13 @@ string_t AttrCursor::getStringWithContext() auto & v = forceValue(); - if (v.type() == nString) - return {v.string.s, v.getContext(*root->state.store)}; + if (v.type() == nString) { + NixStringContext context; + copyContext(v, context); + return {v.string.s, std::move(context)}; + } else if (v.type() == nPath) - return {v.path, {}}; + return {v.path().to_string(), {}}; else root->state.error("'%s' is not a string but %s", getAttrPathStr()).debugThrow<TypeError>(); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 6668add8c..71fd6e6e4 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -4,6 +4,7 @@ #include "util.hh" #include "store-api.hh" #include "derivations.hh" +#include "downstream-placeholder.hh" #include "globals.hh" #include "eval-inline.hh" #include "filetransfer.hh" @@ -94,7 +95,6 @@ RootValue allocRootValue(Value * v) #endif } - void Value::print(const SymbolTable & symbols, std::ostream & str, std::set<const void *> * seen) const { @@ -111,7 +111,7 @@ void Value::print(const SymbolTable & symbols, std::ostream & str, printLiteralString(str, string.s); break; case tPath: - str << path; // !!! escaping? + str << path().to_string(); // !!! escaping? break; case tNull: str << "null"; @@ -535,6 +535,7 @@ EvalState::EvalState( , sOutputSpecified(symbols.create("outputSpecified")) , repair(NoRepair) , emptyBindings(0) + , derivationInternal(rootPath(CanonPath("/builtin/derivation.nix"))) , store(store) , buildStore(buildStore ? buildStore : store) , debugRepl(nullptr) @@ -609,15 +610,14 @@ void EvalState::allowAndSetStorePathString(const StorePath & storePath, Value & { allowPath(storePath); - auto path = store->printStorePath(storePath); - v.mkString(path, PathSet({path})); + mkStorePathString(storePath, v); } -Path EvalState::checkSourcePath(const Path & path_) +SourcePath EvalState::checkSourcePath(const SourcePath & path_) { if (!allowedPaths) return path_; - auto i = resolvedPaths.find(path_); + auto i = resolvedPaths.find(path_.path.abs()); if (i != resolvedPaths.end()) return i->second; @@ -627,9 +627,9 @@ Path EvalState::checkSourcePath(const Path & path_) * attacker can't append ../../... to a path that would be in allowedPaths * and thus leak symlink targets. */ - Path abspath = canonPath(path_); + Path abspath = canonPath(path_.path.abs()); - if (hasPrefix(abspath, corepkgsPrefix)) return abspath; + if (hasPrefix(abspath, corepkgsPrefix)) return CanonPath(abspath); for (auto & i : *allowedPaths) { if (isDirOrInDir(abspath, i)) { @@ -647,11 +647,11 @@ Path EvalState::checkSourcePath(const Path & path_) /* Resolve symlinks. */ debug("checking access to '%s'", abspath); - Path path = canonPath(abspath, true); + SourcePath path = CanonPath(canonPath(abspath, true)); for (auto & i : *allowedPaths) { - if (isDirOrInDir(path, i)) { - resolvedPaths[path_] = path; + if (isDirOrInDir(path.path.abs(), i)) { + resolvedPaths.insert_or_assign(path_.path.abs(), path); return path; } } @@ -679,12 +679,12 @@ void EvalState::checkURI(const std::string & uri) /* If the URI is a path, then check it against allowedPaths as well. */ if (hasPrefix(uri, "/")) { - checkSourcePath(uri); + checkSourcePath(CanonPath(uri)); return; } if (hasPrefix(uri, "file://")) { - checkSourcePath(std::string(uri, 7)); + checkSourcePath(CanonPath(std::string(uri, 7))); return; } @@ -692,7 +692,7 @@ void EvalState::checkURI(const std::string & uri) } -Path EvalState::toRealPath(const Path & path, const PathSet & context) +Path EvalState::toRealPath(const Path & path, const NixStringContext & context) { // FIXME: check whether 'path' is in 'context'. return @@ -944,34 +944,34 @@ void Value::mkString(std::string_view s) } -static void copyContextToValue(Value & v, const PathSet & context) +static void copyContextToValue(Value & v, const NixStringContext & context) { if (!context.empty()) { size_t n = 0; v.string.context = (const char * *) allocBytes((context.size() + 1) * sizeof(char *)); for (auto & i : context) - v.string.context[n++] = dupString(i.c_str()); + v.string.context[n++] = dupString(i.to_string().c_str()); v.string.context[n] = 0; } } -void Value::mkString(std::string_view s, const PathSet & context) +void Value::mkString(std::string_view s, const NixStringContext & context) { mkString(s); copyContextToValue(*this, context); } -void Value::mkStringMove(const char * s, const PathSet & context) +void Value::mkStringMove(const char * s, const NixStringContext & context) { mkString(s); copyContextToValue(*this, context); } -void Value::mkPath(std::string_view s) +void Value::mkPath(const SourcePath & path) { - mkPath(makeImmutableString(s)); + mkPath(makeImmutableString(path.path.abs())); } @@ -1027,9 +1027,9 @@ void EvalState::mkThunk_(Value & v, Expr * expr) void EvalState::mkPos(Value & v, PosIdx p) { auto pos = positions[p]; - if (auto path = std::get_if<Path>(&pos.origin)) { + if (auto path = std::get_if<SourcePath>(&pos.origin)) { auto attrs = buildBindings(3); - attrs.alloc(sFile).mkString(*path); + attrs.alloc(sFile).mkString(path->path.abs()); attrs.alloc(sLine).mkInt(pos.line); attrs.alloc(sColumn).mkInt(pos.column); v.mkAttrs(attrs); @@ -1038,6 +1038,37 @@ void EvalState::mkPos(Value & v, PosIdx p) } +void EvalState::mkStorePathString(const StorePath & p, Value & v) +{ + v.mkString( + store->printStorePath(p), + NixStringContext { + NixStringContextElem::Opaque { .path = p }, + }); +} + + +void EvalState::mkOutputString( + Value & value, + const StorePath & drvPath, + const std::string outputName, + std::optional<StorePath> optOutputPath) +{ + value.mkString( + optOutputPath + ? store->printStorePath(*std::move(optOutputPath)) + /* Downstream we would substitute this for an actual path once + we build the floating CA derivation */ + : DownstreamPlaceholder::unknownCaOutput(drvPath, outputName).render(), + NixStringContext { + NixStringContextElem::Built { + .drvPath = drvPath, + .output = outputName, + } + }); +} + + /* Create a thunk for the delayed computation of the given expression in the given environment. But if the expression is a variable, then look it up right away. This significantly reduces the number @@ -1085,7 +1116,7 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env) } -void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial) +void EvalState::evalFile(const SourcePath & path_, Value & v, bool mustBeTrivial) { auto path = checkSourcePath(path_); @@ -1095,7 +1126,7 @@ void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial) return; } - Path resolvedPath = resolveExprPath(path); + auto resolvedPath = resolveExprPath(path); if ((i = fileEvalCache.find(resolvedPath)) != fileEvalCache.end()) { v = i->second; return; @@ -1123,8 +1154,8 @@ void EvalState::resetFileCache() void EvalState::cacheFile( - const Path & path, - const Path & resolvedPath, + const SourcePath & path, + const SourcePath & resolvedPath, Expr * e, Value & v, bool mustBeTrivial) @@ -1138,7 +1169,7 @@ void EvalState::cacheFile( *e, this->baseEnv, e->getPos() ? static_cast<std::shared_ptr<AbstractPos>>(positions[e->getPos()]) : nullptr, - "while evaluating the file '%1%':", resolvedPath) + "while evaluating the file '%1%':", resolvedPath.to_string()) : nullptr; // Enforce that 'flake.nix' is a direct attrset, not a @@ -1148,7 +1179,7 @@ void EvalState::cacheFile( error("file '%s' must be an attribute set", path).debugThrow<EvalError>(); eval(e, v); } catch (Error & e) { - addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath); + addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath.to_string()); throw; } @@ -1409,8 +1440,8 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) } catch (Error & e) { if (pos2) { auto pos2r = state.positions[pos2]; - auto origin = std::get_if<Path>(&pos2r.origin); - if (!(origin && *origin == state.derivationNixPath)) + auto origin = std::get_if<SourcePath>(&pos2r.origin); + if (!(origin && *origin == state.derivationInternal)) state.addErrorTrace(e, pos2, "while evaluating the attribute '%1%'", showAttrPath(state, env, attrPath)); } @@ -1900,7 +1931,7 @@ void EvalState::concatLists(Value & v, size_t nrLists, Value * * lists, const Po void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) { - PathSet context; + NixStringContext context; std::vector<BackedStringView> s; size_t sSize = 0; NixInt n = 0; @@ -1983,7 +2014,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) else if (firstType == nPath) { if (!context.empty()) state.error("a string that refers to a store path cannot be appended to a path").atPos(pos).withFrame(env, *this).debugThrow<EvalError>(); - v.mkPath(canonPath(str())); + v.mkPath(CanonPath(canonPath(str()))); } else v.mkStringMove(c_str(), context); } @@ -2109,26 +2140,15 @@ std::string_view EvalState::forceString(Value & v, const PosIdx pos, std::string } -void copyContext(const Value & v, PathSet & context) +void copyContext(const Value & v, NixStringContext & context) { if (v.string.context) for (const char * * p = v.string.context; *p; ++p) - context.insert(*p); -} - - -NixStringContext Value::getContext(const Store & store) -{ - NixStringContext res; - assert(internalType == tString); - if (string.context) - for (const char * * p = string.context; *p; ++p) - res.push_back(NixStringContextElem::parse(store, *p)); - return res; + context.insert(NixStringContextElem::parse(*p)); } -std::string_view EvalState::forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx) +std::string_view EvalState::forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx) { auto s = forceString(v, pos, errorCtx); copyContext(v, context); @@ -2158,7 +2178,7 @@ bool EvalState::isDerivation(Value & v) std::optional<std::string> EvalState::tryAttrsToString(const PosIdx pos, Value & v, - PathSet & context, bool coerceMore, bool copyToStore) + NixStringContext & context, bool coerceMore, bool copyToStore) { auto i = v.attrs->find(sToString); if (i != v.attrs->end()) { @@ -2172,8 +2192,14 @@ std::optional<std::string> EvalState::tryAttrsToString(const PosIdx pos, Value & return {}; } -BackedStringView EvalState::coerceToString(const PosIdx pos, Value &v, PathSet &context, - std::string_view errorCtx, bool coerceMore, bool copyToStore, bool canonicalizePath) +BackedStringView EvalState::coerceToString( + const PosIdx pos, + Value & v, + NixStringContext & context, + std::string_view errorCtx, + bool coerceMore, + bool copyToStore, + bool canonicalizePath) { forceValue(v, pos); @@ -2183,12 +2209,14 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value &v, PathSet & } if (v.type() == nPath) { - BackedStringView path(PathView(v.path)); - if (canonicalizePath) - path = canonPath(*path); - if (copyToStore) - path = store->printStorePath(copyPathToStore(context, std::move(path).toOwned())); - return path; + return + !canonicalizePath && !copyToStore + ? // FIXME: hack to preserve path literals that end in a + // slash, as in /foo/${x}. + v._path + : copyToStore + ? store->printStorePath(copyPathToStore(context, v.path())) + : std::string(v.path().path.abs()); } if (v.type() == nAttrs) { @@ -2249,40 +2277,40 @@ BackedStringView EvalState::coerceToString(const PosIdx pos, Value &v, PathSet & } -StorePath EvalState::copyPathToStore(PathSet & context, const Path & path) +StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePath & path) { - if (nix::isDerivation(path)) + if (nix::isDerivation(path.path.abs())) error("file names are not allowed to end in '%1%'", drvExtension).debugThrow<EvalError>(); - auto dstPath = [&]() -> StorePath - { - auto i = srcToStore.find(path); - if (i != srcToStore.end()) return i->second; - - auto dstPath = settings.readOnlyMode - ? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first - : store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair); - allowPath(dstPath); - srcToStore.insert_or_assign(path, dstPath); - printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); - return dstPath; - }(); - - context.insert(store->printStorePath(dstPath)); + auto i = srcToStore.find(path); + + auto dstPath = i != srcToStore.end() + ? i->second + : [&]() { + auto dstPath = path.fetchToStore(store, path.baseName(), nullptr, repair); + allowPath(dstPath); + srcToStore.insert_or_assign(path, dstPath); + printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); + return dstPath; + }(); + + context.insert(NixStringContextElem::Opaque { + .path = dstPath + }); return dstPath; } -Path EvalState::coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx) +SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) { auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (path == "" || path[0] != '/') error("string '%1%' doesn't represent an absolute path", path).withTrace(pos, errorCtx).debugThrow<EvalError>(); - return path; + return CanonPath(path); } -StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx) +StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx) { auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (auto storePath = store->maybeParseStorePath(path)) @@ -2291,6 +2319,80 @@ StorePath EvalState::coerceToStorePath(const PosIdx pos, Value & v, PathSet & co } +std::pair<DerivedPath, std::string_view> EvalState::coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx) +{ + NixStringContext context; + auto s = forceString(v, context, pos, errorCtx); + auto csize = context.size(); + if (csize != 1) + error( + "string '%s' has %d entries in its context. It should only have exactly one entry", + s, csize) + .withTrace(pos, errorCtx).debugThrow<EvalError>(); + auto derivedPath = std::visit(overloaded { + [&](NixStringContextElem::Opaque && o) -> DerivedPath { + return DerivedPath::Opaque { + .path = std::move(o.path), + }; + }, + [&](NixStringContextElem::DrvDeep &&) -> DerivedPath { + error( + "string '%s' has a context which refers to a complete source and binary closure. This is not supported at this time", + s).withTrace(pos, errorCtx).debugThrow<EvalError>(); + }, + [&](NixStringContextElem::Built && b) -> DerivedPath { + return DerivedPath::Built { + .drvPath = std::move(b.drvPath), + .outputs = OutputsSpec::Names { std::move(b.output) }, + }; + }, + }, ((NixStringContextElem &&) *context.begin()).raw()); + return { + std::move(derivedPath), + std::move(s), + }; +} + + +DerivedPath EvalState::coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx) +{ + auto [derivedPath, s_] = coerceToDerivedPathUnchecked(pos, v, errorCtx); + auto s = s_; + std::visit(overloaded { + [&](const DerivedPath::Opaque & o) { + auto sExpected = store->printStorePath(o.path); + if (s != sExpected) + error( + "path string '%s' has context with the different path '%s'", + s, sExpected) + .withTrace(pos, errorCtx).debugThrow<EvalError>(); + }, + [&](const DerivedPath::Built & b) { + // TODO need derived path with single output to make this + // total. Will add as part of RFC 92 work and then this is + // cleaned up. + auto output = *std::get<OutputsSpec::Names>(b.outputs).begin(); + + auto drv = store->readDerivation(b.drvPath); + auto i = drv.outputs.find(output); + if (i == drv.outputs.end()) + throw Error("derivation '%s' does not have output '%s'", store->printStorePath(b.drvPath), output); + auto optOutputPath = i->second.path(*store, drv.name, output); + // This is testing for the case of CA derivations + auto sExpected = optOutputPath + ? store->printStorePath(*optOutputPath) + : DownstreamPlaceholder::unknownCaOutput(b.drvPath, output).render(); + if (s != sExpected) + error( + "string '%s' has context with the output '%s' from derivation '%s', but the string is not the right placeholder for this derivation output. It should be '%s'", + s, output, store->printStorePath(b.drvPath), sExpected) + .withTrace(pos, errorCtx).debugThrow<EvalError>(); + } + }, derivedPath.raw()); + return derivedPath; +} + + bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_view errorCtx) { forceValue(v1, noPos); @@ -2321,7 +2423,7 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v return strcmp(v1.string.s, v2.string.s) == 0; case nPath: - return strcmp(v1.path, v2.path) == 0; + return strcmp(v1._path, v2._path) == 0; case nNull: return true; @@ -2448,8 +2550,8 @@ void EvalState::printStats() else obj["name"] = nullptr; if (auto pos = positions[fun->pos]) { - if (auto path = std::get_if<Path>(&pos.origin)) - obj["file"] = *path; + if (auto path = std::get_if<SourcePath>(&pos.origin)) + obj["file"] = path->to_string(); obj["line"] = pos.line; obj["column"] = pos.column; } @@ -2463,8 +2565,8 @@ void EvalState::printStats() for (auto & i : attrSelects) { json obj = json::object(); if (auto pos = positions[i.first]) { - if (auto path = std::get_if<Path>(&pos.origin)) - obj["file"] = *path; + if (auto path = std::get_if<SourcePath>(&pos.origin)) + obj["file"] = path->to_string(); obj["line"] = pos.line; obj["column"] = pos.column; } @@ -2489,7 +2591,7 @@ void EvalState::printStats() } -std::string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const +std::string ExternalValueBase::coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const { throw TypeError({ .msg = hintfmt("cannot coerce %1% to a string", showType()) @@ -2518,7 +2620,7 @@ Strings EvalSettings::getDefaultNixPath() { Strings res; auto add = [&](const Path & p, const std::string & s = std::string()) { - if (pathExists(p)) { + if (pathAccessible(p)) { if (s.empty()) { res.push_back(p); } else { diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index b3b985683..d6f4560a5 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -8,6 +8,7 @@ #include "symbol-table.hh" #include "config.hh" #include "experimental-features.hh" +#include "input-accessor.hh" #include <map> #include <optional> @@ -20,6 +21,7 @@ namespace nix { class Store; class EvalState; class StorePath; +struct DerivedPath; enum RepairFlag : bool; @@ -56,20 +58,14 @@ void printEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & std::unique_ptr<ValMap> mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const Env & env); -void copyContext(const Value & v, PathSet & context); - - -/** - * Cache for calls to addToStore(); maps source paths to the store - * paths. - */ -typedef std::map<Path, StorePath> SrcToStore; +void copyContext(const Value & v, NixStringContext & context); std::string printValue(const EvalState & state, const Value & v); std::ostream & operator << (std::ostream & os, const ValueType t); +// FIXME: maybe change this to an std::variant<SourcePath, URL>. typedef std::pair<std::string, std::string> SearchPathElem; typedef std::list<SearchPathElem> SearchPath; @@ -137,8 +133,6 @@ public: SymbolTable symbols; PosTable positions; - static inline std::string derivationNixPath = "//builtin/derivation.nix"; - const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, @@ -149,7 +143,6 @@ public: sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath, sPrefix, sOutputSpecified; - Symbol sDerivationNix; /** * If set, force copying files to the Nix store even if they @@ -165,6 +158,8 @@ public: Bindings emptyBindings; + const SourcePath derivationInternal; + /** * Store used to materialise .drv files. */ @@ -234,15 +229,18 @@ public: } private: - SrcToStore srcToStore; + + /* Cache for calls to addToStore(); maps source paths to the store + paths. */ + std::map<SourcePath, StorePath> srcToStore; /** * A cache from path names to parse trees. */ #if HAVE_BOEHMGC - typedef std::map<Path, Expr *, std::less<Path>, traceable_allocator<std::pair<const Path, Expr *>>> FileParseCache; + typedef std::map<SourcePath, Expr *, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Expr *>>> FileParseCache; #else - typedef std::map<Path, Expr *> FileParseCache; + typedef std::map<SourcePath, Expr *> FileParseCache; #endif FileParseCache fileParseCache; @@ -250,9 +248,9 @@ private: * A cache from path names to values. */ #if HAVE_BOEHMGC - typedef std::map<Path, Value, std::less<Path>, traceable_allocator<std::pair<const Path, Value>>> FileEvalCache; + typedef std::map<SourcePath, Value, std::less<SourcePath>, traceable_allocator<std::pair<const SourcePath, Value>>> FileEvalCache; #else - typedef std::map<Path, Value> FileEvalCache; + typedef std::map<SourcePath, Value> FileEvalCache; #endif FileEvalCache fileEvalCache; @@ -263,7 +261,7 @@ private: /** * Cache used by checkSourcePath(). */ - std::unordered_map<Path, Path> resolvedPaths; + std::unordered_map<Path, SourcePath> resolvedPaths; /** * Cache used by prim_match(). @@ -295,6 +293,12 @@ public: SearchPath getSearchPath() { return searchPath; } /** + * Return a `SourcePath` that refers to `path` in the root + * filesystem. + */ + SourcePath rootPath(CanonPath path); + + /** * Allow access to a path. */ void allowPath(const Path & path); @@ -314,7 +318,7 @@ public: * Check whether access to a path is allowed and throw an error if * not. Otherwise return the canonicalised path. */ - Path checkSourcePath(const Path & path); + SourcePath checkSourcePath(const SourcePath & path); void checkURI(const std::string & uri); @@ -327,19 +331,19 @@ public: * intended to distinguish between import-from-derivation and * sources stored in the actual /nix/store. */ - Path toRealPath(const Path & path, const PathSet & context); + Path toRealPath(const Path & path, const NixStringContext & context); /** * Parse a Nix expression from the specified file. */ - Expr * parseExprFromFile(const Path & path); - Expr * parseExprFromFile(const Path & path, std::shared_ptr<StaticEnv> & staticEnv); + Expr * parseExprFromFile(const SourcePath & path); + Expr * parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv); /** * Parse a Nix expression from the specified string. */ - Expr * parseExprFromString(std::string s, const Path & basePath, std::shared_ptr<StaticEnv> & staticEnv); - Expr * parseExprFromString(std::string s, const Path & basePath); + Expr * parseExprFromString(std::string s, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv); + Expr * parseExprFromString(std::string s, const SourcePath & basePath); Expr * parseStdin(); @@ -348,14 +352,14 @@ public: * form. Optionally enforce that the top-level expression is * trivial (i.e. doesn't require arbitrary computation). */ - void evalFile(const Path & path, Value & v, bool mustBeTrivial = false); + void evalFile(const SourcePath & path, Value & v, bool mustBeTrivial = false); /** * Like `evalFile`, but with an already parsed expression. */ void cacheFile( - const Path & path, - const Path & resolvedPath, + const SourcePath & path, + const SourcePath & resolvedPath, Expr * e, Value & v, bool mustBeTrivial = false); @@ -365,8 +369,8 @@ public: /** * Look up a file in the search path. */ - Path findFile(const std::string_view path); - Path findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); + SourcePath findFile(const std::string_view path); + SourcePath findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos = noPos); /** * If the specified search path element is a URI, download it. @@ -423,7 +427,7 @@ public: */ void forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx); std::string_view forceString(Value & v, const PosIdx pos, std::string_view errorCtx); - std::string_view forceString(Value & v, PathSet & context, const PosIdx pos, std::string_view errorCtx); + std::string_view forceString(Value & v, NixStringContext & context, const PosIdx pos, std::string_view errorCtx); std::string_view forceStringNoCtx(Value & v, const PosIdx pos, std::string_view errorCtx); [[gnu::noinline]] @@ -439,7 +443,7 @@ public: bool isDerivation(Value & v); std::optional<std::string> tryAttrsToString(const PosIdx pos, Value & v, - PathSet & context, bool coerceMore = false, bool copyToStore = true); + NixStringContext & context, bool coerceMore = false, bool copyToStore = true); /** * String coercion. @@ -449,12 +453,12 @@ public: * booleans and lists to a string. If `copyToStore` is set, * referenced paths are copied to the Nix store as a side effect. */ - BackedStringView coerceToString(const PosIdx pos, Value & v, PathSet & context, + BackedStringView coerceToString(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx, bool coerceMore = false, bool copyToStore = true, bool canonicalizePath = true); - StorePath copyPathToStore(PathSet & context, const Path & path); + StorePath copyPathToStore(NixStringContext & context, const SourcePath & path); /** * Path coercion. @@ -463,12 +467,34 @@ public: * path. The result is guaranteed to be a canonicalised, absolute * path. Nothing is copied to the store. */ - Path coerceToPath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); + SourcePath coerceToPath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); /** * Like coerceToPath, but the result must be a store path. */ - StorePath coerceToStorePath(const PosIdx pos, Value & v, PathSet & context, std::string_view errorCtx); + StorePath coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & context, std::string_view errorCtx); + + /** + * Part of `coerceToDerivedPath()` without any store IO which is exposed for unit testing only. + */ + std::pair<DerivedPath, std::string_view> coerceToDerivedPathUnchecked(const PosIdx pos, Value & v, std::string_view errorCtx); + + /** + * Coerce to `DerivedPath`. + * + * Must be a string which is either a literal store path or a + * "placeholder (see `DownstreamPlaceholder`). + * + * Even more importantly, the string context must be exactly one + * element, which is either a `NixStringContextElem::Opaque` or + * `NixStringContextElem::Built`. (`NixStringContextEleme::DrvDeep` + * is not permitted). + * + * The string is parsed based on the context --- the context is the + * source of truth, and ultimately tells us what we want, and then + * we ensure the string corresponds to it. + */ + DerivedPath coerceToDerivedPath(const PosIdx pos, Value & v, std::string_view errorCtx); public: @@ -525,7 +551,7 @@ private: char * text, size_t length, Pos::Origin origin, - Path basePath, + const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv); public: @@ -573,6 +599,37 @@ public: void mkThunk_(Value & v, Expr * expr); void mkPos(Value & v, PosIdx pos); + /** + * Create a string representing a store path. + * + * The string is the printed store path with a context containing a single + * `NixStringContextElem::Opaque` element of that store path. + */ + void mkStorePathString(const StorePath & storePath, Value & v); + + /** + * Create a string representing a `DerivedPath::Built`. + * + * The string is the printed store path with a context containing a single + * `NixStringContextElem::Built` element of the drv path and output name. + * + * @param value Value we are settings + * + * @param drvPath Path the drv whose output we are making a string for + * + * @param outputName Name of the output + * + * @param optOutputPath Optional output path for that string. Must + * be passed if and only if output store object is input-addressed. + * Will be printed to form string if passed, otherwise a placeholder + * will be used (see `DownstreamPlaceholder`). + */ + void mkOutputString( + Value & value, + const StorePath & drvPath, + const std::string outputName, + std::optional<StorePath> optOutputPath); + void concatLists(Value & v, size_t nrLists, Value * * lists, const PosIdx pos, std::string_view errorCtx); /** @@ -584,7 +641,7 @@ public: * Realise the given context, and return a mapping from the placeholders * used to construct the associated value to their final store path */ - [[nodiscard]] StringMap realiseContext(const PathSet & context); + [[nodiscard]] StringMap realiseContext(const NixStringContext & context); private: @@ -650,7 +707,7 @@ std::string showType(const Value & v); /** * If `path` refers to a directory, then append "/default.nix". */ -Path resolveExprPath(Path path); +SourcePath resolveExprPath(const SourcePath & path); struct InvalidPathError : EvalError { @@ -688,7 +745,13 @@ struct EvalSettings : Config )"}; Setting<bool> pureEval{this, false, "pure-eval", - "Whether to restrict file system and network access to files specified by cryptographic hash."}; + R"( + Pure evaluation mode ensures that the result of Nix expressions is fully determined by explicitly declared inputs, and not influenced by external state: + + - Restrict file system and network access to files specified by cryptographic hash + - Disable [`bultins.currentSystem`](@docroot@/language/builtin-constants.md#builtins-currentSystem) and [`builtins.currentTime`](@docroot@/language/builtin-constants.md#builtins-currentTime) + )" + }; Setting<bool> enableImportFromDerivation{ this, true, "allow-import-from-derivation", diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index ac396236f..60bb6a71e 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -222,9 +222,9 @@ static Flake getFlake( throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); Value vInfo; - state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack + state.evalFile(CanonPath(flakeFile), vInfo, true); // FIXME: symlink attack - expectType(state, nAttrs, vInfo, state.positions.add({flakeFile}, 1, 1)); + expectType(state, nAttrs, vInfo, state.positions.add({CanonPath(flakeFile)}, 1, 1)); if (auto description = vInfo.attrs->get(state.sDescription)) { expectType(state, nString, *description->value, description->pos); @@ -265,7 +265,7 @@ static Flake getFlake( state.symbols[setting.name], std::string(state.forceStringNoCtx(*setting.value, setting.pos, ""))); else if (setting.value->type() == nPath) { - PathSet emptyContext = {}; + NixStringContext emptyContext = {}; flake.config.settings.emplace( state.symbols[setting.name], state.coerceToString(setting.pos, *setting.value, emptyContext, "", false, true, true) .toOwned()); @@ -745,7 +745,7 @@ void callFlake(EvalState & state, state.vCallFlake = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "call-flake.nix.gen.hh" - , "/"), **state.vCallFlake); + , CanonPath::root), **state.vCallFlake); } state.callFunction(**state.vCallFlake, *vLocks, *vTmp1, noPos); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 1602fbffb..506a63677 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -71,7 +71,7 @@ std::optional<StorePath> DrvInfo::queryDrvPath() const { if (!drvPath && attrs) { Bindings::iterator i = attrs->find(state->sDrvPath); - PathSet context; + NixStringContext context; if (i == attrs->end()) drvPath = {std::nullopt}; else @@ -93,7 +93,7 @@ StorePath DrvInfo::queryOutPath() const { if (!outPath && attrs) { Bindings::iterator i = attrs->find(state->sOutPath); - PathSet context; + NixStringContext context; if (i != attrs->end()) outPath = state->coerceToStorePath(i->pos, *i->value, context, "while evaluating the output path of a derivation"); } @@ -124,7 +124,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall /* And evaluate its ‘outPath’ attribute. */ Bindings::iterator outPath = out->value->attrs->find(state->sOutPath); if (outPath == out->value->attrs->end()) continue; // FIXME: throw error? - PathSet context; + NixStringContext context; outputs.emplace(output, state->coerceToStorePath(outPath->pos, *outPath->value, context, "while evaluating an output path of a derivation")); } else outputs.emplace(output, std::nullopt); diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index 1557cbbeb..4566a1388 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -32,9 +32,9 @@ struct PosAdapter : AbstractPos // Get rid of the null terminators added by the parser. return std::string(s.source->c_str()); }, - [](const Path & path) -> std::optional<std::string> { + [](const SourcePath & path) -> std::optional<std::string> { try { - return readFile(path); + return path.readFile(); } catch (Error &) { return std::nullopt; } @@ -48,7 +48,7 @@ struct PosAdapter : AbstractPos [&](const Pos::none_tag &) { out << "«none»"; }, [&](const Pos::Stdin &) { out << "«stdin»"; }, [&](const Pos::String & s) { out << "«string»"; }, - [&](const Path & path) { out << path; } + [&](const SourcePath & path) { out << path; } }, origin); } }; diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index c2f817c9a..5ca3d1fa6 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -34,7 +34,7 @@ struct Pos struct Stdin { ref<std::string> source; }; struct String { ref<std::string> source; }; - typedef std::variant<none_tag, Stdin, String, Path> Origin; + typedef std::variant<none_tag, Stdin, String, SourcePath> Origin; Origin origin; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 97e615c37..4d981712a 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -31,7 +31,7 @@ namespace nix { EvalState & state; SymbolTable & symbols; Expr * result; - Path basePath; + SourcePath basePath; PosTable::Origin origin; std::optional<ErrorInfo> error; }; @@ -509,7 +509,7 @@ string_parts_interpolated path_start : PATH { - Path path(absPath({$1.p, $1.l}, data->basePath)); + Path path(absPath({$1.p, $1.l}, data->basePath.path.abs())); /* add back in the trailing '/' to the first segment */ if ($1.p[$1.l-1] == '/' && $1.l > 1) path += "/"; @@ -651,7 +651,7 @@ Expr * EvalState::parse( char * text, size_t length, Pos::Origin origin, - Path basePath, + const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv) { yyscan_t scanner; @@ -675,48 +675,36 @@ Expr * EvalState::parse( } -Path resolveExprPath(Path path) +SourcePath resolveExprPath(const SourcePath & path) { - assert(path[0] == '/'); - - unsigned int followCount = 0, maxFollow = 1024; - /* If `path' is a symlink, follow it. This is so that relative path references work. */ - struct stat st; - while (true) { - // Basic cycle/depth limit to avoid infinite loops. - if (++followCount >= maxFollow) - throw Error("too many symbolic links encountered while traversing the path '%s'", path); - st = lstat(path); - if (!S_ISLNK(st.st_mode)) break; - path = absPath(readLink(path), dirOf(path)); - } + auto path2 = path.resolveSymlinks(); /* If `path' refers to a directory, append `/default.nix'. */ - if (S_ISDIR(st.st_mode)) - path = canonPath(path + "/default.nix"); + if (path2.lstat().type == InputAccessor::tDirectory) + return path2 + "default.nix"; - return path; + return path2; } -Expr * EvalState::parseExprFromFile(const Path & path) +Expr * EvalState::parseExprFromFile(const SourcePath & path) { return parseExprFromFile(path, staticBaseEnv); } -Expr * EvalState::parseExprFromFile(const Path & path, std::shared_ptr<StaticEnv> & staticEnv) +Expr * EvalState::parseExprFromFile(const SourcePath & path, std::shared_ptr<StaticEnv> & staticEnv) { - auto buffer = readFile(path); - // readFile should have left some extra space for terminators + auto buffer = path.readFile(); + // readFile hopefully have left some extra space for terminators buffer.append("\0\0", 2); - return parse(buffer.data(), buffer.size(), path, dirOf(path), staticEnv); + return parse(buffer.data(), buffer.size(), Pos::Origin(path), path.parent(), staticEnv); } -Expr * EvalState::parseExprFromString(std::string s_, const Path & basePath, std::shared_ptr<StaticEnv> & staticEnv) +Expr * EvalState::parseExprFromString(std::string s_, const SourcePath & basePath, std::shared_ptr<StaticEnv> & staticEnv) { auto s = make_ref<std::string>(std::move(s_)); s->append("\0\0", 2); @@ -724,7 +712,7 @@ Expr * EvalState::parseExprFromString(std::string s_, const Path & basePath, std } -Expr * EvalState::parseExprFromString(std::string s, const Path & basePath) +Expr * EvalState::parseExprFromString(std::string s, const SourcePath & basePath) { return parseExprFromString(std::move(s), basePath, staticBaseEnv); } @@ -737,7 +725,7 @@ Expr * EvalState::parseStdin() // drainFD should have left some extra space for terminators buffer.append("\0\0", 2); auto s = make_ref<std::string>(std::move(buffer)); - return parse(s->data(), s->size(), Pos::Stdin{.source = s}, absPath("."), staticBaseEnv); + return parse(s->data(), s->size(), Pos::Stdin{.source = s}, rootPath(CanonPath::fromCwd()), staticBaseEnv); } @@ -757,13 +745,13 @@ void EvalState::addToSearchPath(const std::string & s) } -Path EvalState::findFile(const std::string_view path) +SourcePath EvalState::findFile(const std::string_view path) { return findFile(searchPath, path); } -Path EvalState::findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos) +SourcePath EvalState::findFile(SearchPath & searchPath, const std::string_view path, const PosIdx pos) { for (auto & i : searchPath) { std::string suffix; @@ -779,11 +767,11 @@ Path EvalState::findFile(SearchPath & searchPath, const std::string_view path, c auto r = resolveSearchPathElem(i); if (!r.first) continue; Path res = r.second + suffix; - if (pathExists(res)) return canonPath(res); + if (pathExists(res)) return CanonPath(canonPath(res)); } if (hasPrefix(path, "nix/")) - return concatStrings(corepkgsPrefix, path.substr(4)); + return CanonPath(concatStrings(corepkgsPrefix, path.substr(4))); debugThrow(ThrownError({ .msg = hintfmt(evalSettings.pureEval diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc new file mode 100644 index 000000000..1d690b722 --- /dev/null +++ b/src/libexpr/paths.cc @@ -0,0 +1,10 @@ +#include "eval.hh" + +namespace nix { + +SourcePath EvalState::rootPath(CanonPath path) +{ + return std::move(path); +} + +} diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 510f674eb..42efca4e7 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -1,5 +1,6 @@ #include "archive.hh" #include "derivations.hh" +#include "downstream-placeholder.hh" #include "eval-inline.hh" #include "eval.hh" #include "globals.hh" @@ -38,17 +39,16 @@ namespace nix { InvalidPathError::InvalidPathError(const Path & path) : EvalError("path '%s' is not valid", path), path(path) {} -StringMap EvalState::realiseContext(const PathSet & context) +StringMap EvalState::realiseContext(const NixStringContext & context) { std::vector<DerivedPath::Built> drvs; StringMap res; - for (auto & c_ : context) { + for (auto & c : context) { auto ensureValid = [&](const StorePath & p) { if (!store->isValidPath(p)) debugThrowLastTrace(InvalidPathError(store->printStorePath(p))); }; - auto c = NixStringContextElem::parse(*store, c_); std::visit(overloaded { [&](const NixStringContextElem::Built & b) { drvs.push_back(DerivedPath::Built { @@ -88,7 +88,7 @@ StringMap EvalState::realiseContext(const PathSet & context) auto outputs = resolveDerivedPath(*store, drv); for (auto & [outputName, outputPath] : outputs) { res.insert_or_assign( - downstreamPlaceholder(*store, drv.drvPath, outputName), + DownstreamPlaceholder::unknownCaOutput(drv.drvPath, outputName).render(), store->printStorePath(outputPath) ); } @@ -110,16 +110,16 @@ struct RealisePathFlags { bool checkForPureEval = true; }; -static Path realisePath(EvalState & state, const PosIdx pos, Value & v, const RealisePathFlags flags = {}) +static SourcePath realisePath(EvalState & state, const PosIdx pos, Value & v, const RealisePathFlags flags = {}) { - PathSet context; + NixStringContext context; auto path = state.coerceToPath(noPos, v, context, "while realising the context of a path"); try { StringMap rewrites = state.realiseContext(context); - auto realPath = state.toRealPath(rewriteStrings(path, rewrites), context); + auto realPath = state.rootPath(CanonPath(state.toRealPath(rewriteStrings(path.path.abs(), rewrites), context))); return flags.checkForPureEval ? state.checkSourcePath(realPath) @@ -130,35 +130,31 @@ static Path realisePath(EvalState & state, const PosIdx pos, Value & v, const Re } } -/* Add and attribute to the given attribute map from the output name to - the output path, or a placeholder. - - Where possible the path is used, but for floating CA derivations we - may not know it. For sake of determinism we always assume we don't - and instead put in a place holder. In either case, however, the - string context will contain the drv path and output name, so - downstream derivations will have the proper dependency, and in - addition, before building, the placeholder will be rewritten to be - the actual path. - - The 'drv' and 'drvPath' outputs must correspond. */ +/** + * Add and attribute to the given attribute map from the output name to + * the output path, or a placeholder. + * + * Where possible the path is used, but for floating CA derivations we + * may not know it. For sake of determinism we always assume we don't + * and instead put in a place holder. In either case, however, the + * string context will contain the drv path and output name, so + * downstream derivations will have the proper dependency, and in + * addition, before building, the placeholder will be rewritten to be + * the actual path. + * + * The 'drv' and 'drvPath' outputs must correspond. + */ static void mkOutputString( EvalState & state, BindingsBuilder & attrs, const StorePath & drvPath, - const BasicDerivation & drv, const std::pair<std::string, DerivationOutput> & o) { - auto optOutputPath = o.second.path(*state.store, drv.name, o.first); - attrs.alloc(o.first).mkString( - optOutputPath - ? state.store->printStorePath(*optOutputPath) - /* Downstream we would substitute this for an actual path once - we build the floating CA derivation */ - /* FIXME: we need to depend on the basic derivation, not - derivation */ - : downstreamPlaceholder(*state.store, drvPath, o.first), - {"!" + o.first + "!" + state.store->printStorePath(drvPath)}); + state.mkOutputString( + attrs.alloc(o.first), + drvPath, + o.first, + o.second.path(*state.store, Derivation::nameFromPath(drvPath), o.first)); } /* Load and evaluate an expression from path specified by the @@ -166,28 +162,30 @@ static void mkOutputString( static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * vScope, Value & v) { auto path = realisePath(state, pos, vPath); + auto path2 = path.path.abs(); // FIXME auto isValidDerivationInStore = [&]() -> std::optional<StorePath> { - if (!state.store->isStorePath(path)) + if (!state.store->isStorePath(path2)) return std::nullopt; - auto storePath = state.store->parseStorePath(path); - if (!(state.store->isValidPath(storePath) && isDerivation(path))) + auto storePath = state.store->parseStorePath(path2); + if (!(state.store->isValidPath(storePath) && isDerivation(path2))) return std::nullopt; return storePath; }; - if (auto optStorePath = isValidDerivationInStore()) { - auto storePath = *optStorePath; - Derivation drv = state.store->readDerivation(storePath); + if (auto storePath = isValidDerivationInStore()) { + Derivation drv = state.store->readDerivation(*storePath); auto attrs = state.buildBindings(3 + drv.outputs.size()); - attrs.alloc(state.sDrvPath).mkString(path, {"=" + path}); + attrs.alloc(state.sDrvPath).mkString(path2, { + NixStringContextElem::DrvDeep { .drvPath = *storePath }, + }); attrs.alloc(state.sName).mkString(drv.env["name"]); auto & outputsVal = attrs.alloc(state.sOutputs); state.mkList(outputsVal, drv.outputs.size()); for (const auto & [i, o] : enumerate(drv.outputs)) { - mkOutputString(state, attrs, storePath, drv, o); + mkOutputString(state, attrs, *storePath, o); (outputsVal.listElems()[i] = state.allocValue())->mkString(o.first); } @@ -198,7 +196,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v state.vImportedDrvToDerivation = allocRootValue(state.allocValue()); state.eval(state.parseExprFromString( #include "imported-drv-to-derivation.nix.gen.hh" - , "/"), **state.vImportedDrvToDerivation); + , CanonPath::root), **state.vImportedDrvToDerivation); } state.forceFunction(**state.vImportedDrvToDerivation, pos, "while evaluating imported-drv-to-derivation.nix.gen.hh"); @@ -206,10 +204,10 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v state.forceAttrs(v, pos, "while calling imported-drv-to-derivation.nix.gen.hh"); } - else if (path == corepkgsPrefix + "fetchurl.nix") { + else if (path2 == corepkgsPrefix + "fetchurl.nix") { state.eval(state.parseExprFromString( #include "fetchurl.nix.gen.hh" - , "/"), v); + , CanonPath::root), v); } else { @@ -330,7 +328,7 @@ void prim_importNative(EvalState & state, const PosIdx pos, Value * * args, Valu std::string sym(state.forceStringNoCtx(*args[1], pos, "while evaluating the second argument passed to builtins.importNative")); - void *handle = dlopen(path.c_str(), RTLD_LAZY | RTLD_LOCAL); + void *handle = dlopen(path.path.c_str(), RTLD_LAZY | RTLD_LOCAL); if (!handle) state.debugThrowLastTrace(EvalError("could not open '%1%': %2%", path, dlerror())); @@ -358,7 +356,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto count = args[0]->listSize(); if (count == 0) state.error("at least one argument to 'exec' required").atPos(pos).debugThrow<EvalError>(); - PathSet context; + NixStringContext context; auto program = state.coerceToString(pos, *elems[0], context, "while evaluating the first element of the argument passed to builtins.exec", false, false).toOwned(); @@ -378,7 +376,7 @@ void prim_exec(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto output = runProgram(program, true, commandArgs); Expr * parsed; try { - parsed = state.parseExprFromString(std::move(output), "/"); + parsed = state.parseExprFromString(std::move(output), state.rootPath(CanonPath::root)); } catch (Error & e) { e.addTrace(state.positions[pos], "while parsing the output from '%1%'", program); throw; @@ -588,7 +586,7 @@ struct CompareValues case nString: return strcmp(v1->string.s, v2->string.s) < 0; case nPath: - return strcmp(v1->path, v2->path) < 0; + return strcmp(v1->_path, v2->_path) < 0; case nList: // Lexicographic comparison for (size_t i = 0;; i++) { @@ -700,12 +698,14 @@ static RegisterPrimOp primop_genericClosure(RegisterPrimOp::Info { .arity = 1, .doc = R"( Take an *attrset* with values named `startSet` and `operator` in order to - return a *list of attrsets* by starting with the `startSet`, recursively - applying the `operator` function to each element. The *attrsets* in the - `startSet` and produced by the `operator` must each contain value named - `key` which are comparable to each other. The result is produced by - repeatedly calling the operator for each element encountered with a - unique key, terminating when no new elements are produced. For example, + return a *list of attrsets* by starting with the `startSet` and recursively + applying the `operator` function to each `item`. The *attrsets* in the + `startSet` and the *attrsets* produced by `operator` must contain a value + named `key` which is comparable. The result is produced by calling `operator` + for each `item` with a value for `key` that has not been called yet including + newly produced `item`s. The function terminates when no new `item`s are + produced. The resulting *list of attrsets* contains only *attrsets* with a + unique key. For example, ``` builtins.genericClosure { @@ -768,7 +768,7 @@ static RegisterPrimOp primop_abort({ )", .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.abort").toOwned(); state.debugThrowLastTrace(Abort("evaluation aborted with the following error message: '%1%'", s)); @@ -787,7 +787,7 @@ static RegisterPrimOp primop_throw({ )", .fun = [](EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtin.throw").toOwned(); state.debugThrowLastTrace(ThrownError(s)); @@ -800,7 +800,7 @@ static void prim_addErrorContext(EvalState & state, const PosIdx pos, Value * * state.forceValue(*args[1], pos); v = *args[1]; } catch (Error & e) { - PathSet context; + NixStringContext context; auto message = state.coerceToString(pos, *args[0], context, "while evaluating the error message passed to builtins.addErrorContext", false, false).toOwned(); @@ -1086,13 +1086,13 @@ drvName, Bindings * attrs, Value & v) Derivation drv; drv.name = drvName; - PathSet context; + NixStringContext context; bool contentAddressed = false; bool isImpure = false; std::optional<std::string> outputHash; std::string outputHashAlgo; - std::optional<FileIngestionMethod> ingestionMethod; + std::optional<ContentAddressMethod> ingestionMethod; StringSet outputs; outputs.insert("out"); @@ -1105,7 +1105,10 @@ drvName, Bindings * attrs, Value & v) auto handleHashMode = [&](const std::string_view s) { if (s == "recursive") ingestionMethod = FileIngestionMethod::Recursive; else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat; - else + else if (s == "text") { + experimentalFeatureSettings.require(Xp::DynamicDerivations); + ingestionMethod = TextIngestionMethod {}; + } else state.debugThrowLastTrace(EvalError({ .msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s), .errPos = state.positions[noPos] @@ -1149,16 +1152,14 @@ drvName, Bindings * attrs, Value & v) if (i->value->type() == nNull) continue; } - if (i->name == state.sContentAddressed) { - contentAddressed = state.forceBool(*i->value, noPos, context_below); - if (contentAddressed) - experimentalFeatureSettings.require(Xp::CaDerivations); + if (i->name == state.sContentAddressed && state.forceBool(*i->value, noPos, context_below)) { + contentAddressed = true; + experimentalFeatureSettings.require(Xp::CaDerivations); } - else if (i->name == state.sImpure) { - isImpure = state.forceBool(*i->value, noPos, context_below); - if (isImpure) - experimentalFeatureSettings.require(Xp::ImpureDerivations); + else if (i->name == state.sImpure && state.forceBool(*i->value, noPos, context_below)) { + isImpure = true; + experimentalFeatureSettings.require(Xp::ImpureDerivations); } /* The `args' attribute is special: it supplies the @@ -1232,8 +1233,7 @@ drvName, Bindings * attrs, Value & v) /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ - for (auto & c_ : context) { - auto c = NixStringContextElem::parse(*state.store, c_); + for (auto & c : context) { std::visit(overloaded { /* Since this allows the builder to gain access to every path in the dependency graph of the derivation (including @@ -1273,11 +1273,16 @@ drvName, Bindings * attrs, Value & v) })); /* Check whether the derivation name is valid. */ - if (isDerivation(drvName)) + if (isDerivation(drvName) && + !(ingestionMethod == ContentAddressMethod { TextIngestionMethod { } } && + outputs.size() == 1 && + *(outputs.begin()) == "out")) + { state.debugThrowLastTrace(EvalError({ - .msg = hintfmt("derivation names are not allowed to end in '%s'", drvExtension), + .msg = hintfmt("derivation names are allowed to end in '%s' only if they produce a single derivation file", drvExtension), .errPos = state.positions[noPos] })); + } if (outputHash) { /* Handle fixed-output derivations. @@ -1293,21 +1298,15 @@ drvName, Bindings * attrs, Value & v) auto h = newHashAllowEmpty(*outputHash, parseHashTypeOpt(outputHashAlgo)); auto method = ingestionMethod.value_or(FileIngestionMethod::Flat); - auto outPath = state.store->makeFixedOutputPath(drvName, FixedOutputInfo { - .hash = { - .method = method, - .hash = h, - }, - .references = {}, - }); - drv.env["out"] = state.store->printStorePath(outPath); - drv.outputs.insert_or_assign("out", - DerivationOutput::CAFixed { - .hash = FixedOutputHash { - .method = method, - .hash = std::move(h), - }, - }); + + DerivationOutput::CAFixed dof { + .ca = ContentAddress::fromParts( + std::move(method), + std::move(h)), + }; + + drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out")); + drv.outputs.insert_or_assign("out", std::move(dof)); } else if (contentAddressed || isImpure) { @@ -1325,13 +1324,13 @@ drvName, Bindings * attrs, Value & v) if (isImpure) drv.outputs.insert_or_assign(i, DerivationOutput::Impure { - .method = method, + .method = method.raw, .hashType = ht, }); else drv.outputs.insert_or_assign(i, DerivationOutput::CAFloating { - .method = method, + .method = method.raw, .hashType = ht, }); } @@ -1392,9 +1391,11 @@ drvName, Bindings * attrs, Value & v) } auto result = state.buildBindings(1 + drv.outputs.size()); - result.alloc(state.sDrvPath).mkString(drvPathS, {"=" + drvPathS}); + result.alloc(state.sDrvPath).mkString(drvPathS, { + NixStringContextElem::DrvDeep { .drvPath = drvPath }, + }); for (auto & i : drv.outputs) - mkOutputString(state, result, drvPath, drv, i); + mkOutputString(state, result, drvPath, i); v.mkAttrs(result); } @@ -1437,9 +1438,9 @@ static RegisterPrimOp primop_placeholder({ /* Convert the argument to a path. !!! obsolete? */ static void prim_toPath(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.toPath"); - v.mkString(canonPath(path), context); + NixStringContext context; + auto path = state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.toPath"); + v.mkString(path.path.abs(), context); } static RegisterPrimOp primop_toPath({ @@ -1468,22 +1469,23 @@ static void prim_storePath(EvalState & state, const PosIdx pos, Value * * args, .errPos = state.positions[pos] })); - PathSet context; - Path path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")); + NixStringContext context; + auto path = state.checkSourcePath(state.coerceToPath(pos, *args[0], context, "while evaluating the first argument passed to builtins.storePath")).path; /* Resolve symlinks in ‘path’, unless ‘path’ itself is a symlink directly in the store. The latter condition is necessary so e.g. nix-push does the right thing. */ - if (!state.store->isStorePath(path)) path = canonPath(path, true); - if (!state.store->isInStore(path)) + if (!state.store->isStorePath(path.abs())) + path = CanonPath(canonPath(path.abs(), true)); + if (!state.store->isInStore(path.abs())) state.debugThrowLastTrace(EvalError({ .msg = hintfmt("path '%1%' is not in the Nix store", path), .errPos = state.positions[pos] })); - auto path2 = state.store->toStorePath(path).first; + auto path2 = state.store->toStorePath(path.abs()).first; if (!settings.readOnlyMode) state.store->ensurePath(path2); - context.insert(state.store->printStorePath(path2)); - v.mkString(path, context); + context.insert(NixStringContextElem::Opaque { .path = path2 }); + v.mkString(path.abs(), context); } static RegisterPrimOp primop_storePath({ @@ -1499,7 +1501,7 @@ static RegisterPrimOp primop_storePath({ causes the path to be *copied* again to the Nix store, resulting in a new path (e.g. `/nix/store/ld01dnzc…-source-source`). - This function is not available in pure evaluation mode. + Not available in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). )", .fun = prim_storePath, }); @@ -1514,7 +1516,7 @@ static void prim_pathExists(EvalState & state, const PosIdx pos, Value * * args, auto path = realisePath(state, pos, *args[0], { .checkForPureEval = false }); try { - v.mkBool(pathExists(state.checkSourcePath(path))); + v.mkBool(state.checkSourcePath(path).pathExists()); } catch (SysError & e) { /* Don't give away info from errors while canonicalising ‘path’ in restricted mode. */ @@ -1538,7 +1540,7 @@ static RegisterPrimOp primop_pathExists({ following the last slash. */ static void prim_baseNameOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; v.mkString(baseNameOf(*state.coerceToString(pos, *args[0], context, "while evaluating the first argument passed to builtins.baseNameOf", false, false)), context); @@ -1560,12 +1562,18 @@ static RegisterPrimOp primop_baseNameOf({ of the argument. */ static void prim_dirOf(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - auto path = state.coerceToString(pos, *args[0], context, - "while evaluating the first argument passed to builtins.dirOf", + state.forceValue(*args[0], pos); + if (args[0]->type() == nPath) { + auto path = args[0]->path(); + v.mkPath(path.path.isRoot() ? path : path.parent()); + } else { + NixStringContext context; + auto path = state.coerceToString(pos, *args[0], context, + "while evaluating the first argument passed to 'builtins.dirOf'", false, false); - auto dir = dirOf(*path); - if (args[0]->type() == nPath) v.mkPath(dir); else v.mkString(dir, context); + auto dir = dirOf(*path); + v.mkString(dir, context); + } } static RegisterPrimOp primop_dirOf({ @@ -1583,13 +1591,13 @@ static RegisterPrimOp primop_dirOf({ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { auto path = realisePath(state, pos, *args[0]); - auto s = readFile(path); + auto s = path.readFile(); if (s.find((char) 0) != std::string::npos) state.debugThrowLastTrace(Error("the contents of the file '%1%' cannot be represented as a Nix string", path)); StorePathSet refs; - if (state.store->isInStore(path)) { + if (state.store->isInStore(path.path.abs())) { try { - refs = state.store->queryPathInfo(state.store->toStorePath(path).first)->references; + refs = state.store->queryPathInfo(state.store->toStorePath(path.path.abs()).first)->references; } catch (Error &) { // FIXME: should be InvalidPathError } // Re-scan references to filter down to just the ones that actually occur in the file. @@ -1597,7 +1605,12 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value * * args, V refsSink << s; refs = refsSink.getResultPaths(); } - auto context = state.store->printStorePathSet(refs); + NixStringContext context; + for (auto && p : std::move(refs)) { + context.insert(NixStringContextElem::Opaque { + .path = std::move((StorePath &&)p), + }); + } v.mkString(s, context); } @@ -1628,7 +1641,7 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value * * args, V i = getAttr(state, state.sPath, v2->attrs, "in an element of the __nixPath"); - PathSet context; + NixStringContext context; auto path = state.coerceToString(pos, *i->value, context, "while evaluating the `path` attribute of an element of the list passed to builtins.findFile", false, false).toOwned(); @@ -1670,7 +1683,7 @@ static void prim_hashFile(EvalState & state, const PosIdx pos, Value * * args, V auto path = realisePath(state, pos, *args[1]); - v.mkString(hashFile(*ht, path).to_string(Base16, false)); + v.mkString(hashString(*ht, path.readFile()).to_string(Base16, false)); } static RegisterPrimOp primop_hashFile({ @@ -1684,26 +1697,20 @@ static RegisterPrimOp primop_hashFile({ .fun = prim_hashFile, }); - -/* Stringize a directory entry enum. Used by `readFileType' and `readDir'. */ -static const char * dirEntTypeToString(unsigned char dtType) +static std::string_view fileTypeToString(InputAccessor::Type type) { - /* Enum DT_(DIR|LNK|REG|UNKNOWN) */ - switch(dtType) { - case DT_REG: return "regular"; break; - case DT_DIR: return "directory"; break; - case DT_LNK: return "symlink"; break; - default: return "unknown"; break; - } - return "unknown"; /* Unreachable */ + return + type == InputAccessor::Type::tRegular ? "regular" : + type == InputAccessor::Type::tDirectory ? "directory" : + type == InputAccessor::Type::tSymlink ? "symlink" : + "unknown"; } - static void prim_readFileType(EvalState & state, const PosIdx pos, Value * * args, Value & v) { auto path = realisePath(state, pos, *args[0]); /* Retrieve the directory entry type and stringize it. */ - v.mkString(dirEntTypeToString(getFileType(path))); + v.mkString(fileTypeToString(path.lstat().type)); } static RegisterPrimOp primop_readFileType({ @@ -1724,8 +1731,7 @@ static void prim_readDir(EvalState & state, const PosIdx pos, Value * * args, Va // Retrieve directory entries for all nodes in a directory. // This is similar to `getFileType` but is optimized to reduce system calls // on many systems. - DirEntries entries = readDirectory(path); - + auto entries = path.readDirectory(); auto attrs = state.buildBindings(entries.size()); // If we hit unknown directory entry types we may need to fallback to @@ -1734,22 +1740,21 @@ static void prim_readDir(EvalState & state, const PosIdx pos, Value * * args, Va // `builtins.readFileType` application. Value * readFileType = nullptr; - for (auto & ent : entries) { - auto & attr = attrs.alloc(ent.name); - if (ent.type == DT_UNKNOWN) { + for (auto & [name, type] : entries) { + auto & attr = attrs.alloc(name); + if (!type) { // Some filesystems or operating systems may not be able to return // detailed node info quickly in this case we produce a thunk to // query the file type lazily. auto epath = state.allocValue(); - Path path2 = path + "/" + ent.name; - epath->mkString(path2); + epath->mkPath(path + name); if (!readFileType) readFileType = &state.getBuiltin("readFileType"); attr.mkApp(readFileType, epath); } else { // This branch of the conditional is much more likely. // Here we just stringize the directory entry type. - attr.mkString(dirEntTypeToString(ent.type)); + attr.mkString(fileTypeToString(*type)); } } @@ -1787,7 +1792,7 @@ static RegisterPrimOp primop_readDir({ static void prim_toXML(EvalState & state, const PosIdx pos, Value * * args, Value & v) { std::ostringstream out; - PathSet context; + NixStringContext context; printValueAsXML(state, true, false, *args[0], out, context, pos); v.mkString(out.str(), context); } @@ -1895,7 +1900,7 @@ static RegisterPrimOp primop_toXML({ static void prim_toJSON(EvalState & state, const PosIdx pos, Value * * args, Value & v) { std::ostringstream out; - PathSet context; + NixStringContext context; printValueAsJSON(state, true, *args[0], pos, out, context); v.mkString(out.str(), context); } @@ -1945,22 +1950,23 @@ static RegisterPrimOp primop_fromJSON({ as an input by derivations. */ static void prim_toFile(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; std::string name(state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.toFile")); std::string contents(state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.toFile")); StorePathSet refs; - for (auto path : context) { - if (path.at(0) != '/') + for (auto c : context) { + if (auto p = std::get_if<NixStringContextElem::Opaque>(&c)) + refs.insert(p->path); + else state.debugThrowLastTrace(EvalError({ .msg = hintfmt( "in 'toFile': the file named '%1%' must not contain a reference " "to a derivation but contains (%2%)", - name, path), + name, c.to_string()), .errPos = state.positions[pos] })); - refs.insert(state.store->parseStorePath(path)); } auto storePath = settings.readOnlyMode @@ -2055,13 +2061,13 @@ static RegisterPrimOp primop_toFile({ static void addPath( EvalState & state, const PosIdx pos, - const std::string & name, + std::string_view name, Path path, Value * filterFun, FileIngestionMethod method, const std::optional<Hash> expectedHash, Value & v, - const PathSet & context) + const NixStringContext & context) { try { // FIXME: handle CA derivation outputs (where path needs to @@ -2083,7 +2089,7 @@ static void addPath( path = evalSettings.pureEval && expectedHash ? path - : state.checkSourcePath(path); + : state.checkSourcePath(CanonPath(path)).path.abs(); PathFilter filter = filterFun ? ([&](const Path & path) { auto st = lstat(path); @@ -2135,10 +2141,11 @@ static void addPath( static void prim_filterSource(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; - Path path = state.coerceToPath(pos, *args[1], context, "while evaluating the second argument (the path to filter) passed to builtins.filterSource"); + NixStringContext context; + auto path = state.coerceToPath(pos, *args[1], context, + "while evaluating the second argument (the path to filter) passed to builtins.filterSource"); state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); - addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); + addPath(state, pos, path.baseName(), path.path.abs(), args[0], FileIngestionMethod::Recursive, std::nullopt, v, context); } static RegisterPrimOp primop_filterSource({ @@ -2198,18 +2205,19 @@ static RegisterPrimOp primop_filterSource({ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - state.forceAttrs(*args[0], pos, "while evaluating the argument passed to builtins.path"); - Path path; + std::optional<SourcePath> path; std::string name; Value * filterFun = nullptr; auto method = FileIngestionMethod::Recursive; std::optional<Hash> expectedHash; - PathSet context; + NixStringContext context; + + state.forceAttrs(*args[0], pos, "while evaluating the argument passed to 'builtins.path'"); for (auto & attr : *args[0]->attrs) { auto n = state.symbols[attr.name]; if (n == "path") - path = state.coerceToPath(attr.pos, *attr.value, context, "while evaluating the `path` attribute passed to builtins.path"); + path.emplace(state.coerceToPath(attr.pos, *attr.value, context, "while evaluating the 'path' attribute passed to 'builtins.path'")); else if (attr.name == state.sName) name = state.forceStringNoCtx(*attr.value, attr.pos, "while evaluating the `name` attribute passed to builtins.path"); else if (n == "filter") @@ -2224,15 +2232,15 @@ static void prim_path(EvalState & state, const PosIdx pos, Value * * args, Value .errPos = state.positions[attr.pos] })); } - if (path.empty()) + if (!path) state.debugThrowLastTrace(EvalError({ .msg = hintfmt("missing required 'path' attribute in the first argument to builtins.path"), .errPos = state.positions[pos] })); if (name.empty()) - name = baseNameOf(path); + name = path->baseName(); - addPath(state, pos, name, path, filterFun, method, expectedHash, v, context); + addPath(state, pos, name, path->path.abs(), filterFun, method, expectedHash, v, context); } static RegisterPrimOp primop_path({ @@ -3538,7 +3546,7 @@ static RegisterPrimOp primop_lessThan({ `"/nix/store/whatever..."'. */ static void prim_toString(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the first argument passed to builtins.toString", true, false); @@ -3577,7 +3585,7 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value * * args, { int start = state.forceInt(*args[0], pos, "while evaluating the first argument (the start offset) passed to builtins.substring"); int len = state.forceInt(*args[1], pos, "while evaluating the second argument (the substring length) passed to builtins.substring"); - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[2], context, "while evaluating the third argument (the string) passed to builtins.substring"); if (start < 0) @@ -3611,7 +3619,7 @@ static RegisterPrimOp primop_substring({ static void prim_stringLength(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.stringLength"); v.mkInt(s->size()); } @@ -3637,7 +3645,7 @@ static void prim_hashString(EvalState & state, const PosIdx pos, Value * * args, .errPos = state.positions[pos] })); - PathSet context; // discarded + NixStringContext context; // discarded auto s = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.hashString"); v.mkString(hashString(*ht, s).to_string(Base16, false)); @@ -3683,7 +3691,7 @@ void prim_match(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto regex = state.regexCache->get(re); - PathSet context; + NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.match"); std::cmatch match; @@ -3763,7 +3771,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value * * args, Value & v) auto regex = state.regexCache->get(re); - PathSet context; + NixStringContext context; const auto str = state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.split"); auto begin = std::cregex_iterator(str.begin(), str.end(), regex); @@ -3860,7 +3868,7 @@ static RegisterPrimOp primop_split({ static void prim_concatStringsSep(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto sep = state.forceString(*args[0], context, pos, "while evaluating the first argument (the separator string) passed to builtins.concatStringsSep"); state.forceList(*args[1], pos, "while evaluating the second argument (the list of strings to concat) passed to builtins.concatStringsSep"); @@ -3900,15 +3908,10 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a for (auto elem : args[0]->listItems()) from.emplace_back(state.forceString(*elem, pos, "while evaluating one of the strings to replace passed to builtins.replaceStrings")); - std::vector<std::pair<std::string, PathSet>> to; - to.reserve(args[1]->listSize()); - for (auto elem : args[1]->listItems()) { - PathSet ctx; - auto s = state.forceString(*elem, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings"); - to.emplace_back(s, std::move(ctx)); - } + std::unordered_map<size_t, std::string> cache; + auto to = args[1]->listItems(); - PathSet context; + NixStringContext context; auto s = state.forceString(*args[2], context, pos, "while evaluating the third argument passed to builtins.replaceStrings"); std::string res; @@ -3917,10 +3920,19 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a bool found = false; auto i = from.begin(); auto j = to.begin(); - for (; i != from.end(); ++i, ++j) + size_t j_index = 0; + for (; i != from.end(); ++i, ++j, ++j_index) if (s.compare(p, i->size(), *i) == 0) { found = true; - res += j->first; + auto v = cache.find(j_index); + if (v == cache.end()) { + NixStringContext ctx; + auto ts = state.forceString(**j, ctx, pos, "while evaluating one of the replacement strings passed to builtins.replaceStrings"); + v = (cache.emplace(j_index, ts)).first; + for (auto& path : ctx) + context.insert(path); + } + res += v->second; if (i->empty()) { if (p < s.size()) res += s[p]; @@ -3928,9 +3940,6 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value * * a } else { p += i->size(); } - for (auto& path : j->second) - context.insert(path); - j->second.clear(); break; } if (!found) { @@ -3948,7 +3957,11 @@ static RegisterPrimOp primop_replaceStrings({ .args = {"from", "to", "s"}, .doc = R"( Given string *s*, replace every occurrence of the strings in *from* - with the corresponding string in *to*. For example, + with the corresponding string in *to*. + + The argument *to* is lazy, that is, it is only evaluated when its corresponding pattern in *from* is matched in the string *s* + + Example: ```nix builtins.replaceStrings ["oo" "a"] ["a" "i"] "foobar" @@ -4150,7 +4163,6 @@ void EvalState::createBaseEnv() /* Add a wrapper around the derivation primop that computes the `drvPath' and `outPath' attributes lazily. */ - sDerivationNix = symbols.create(derivationNixPath); auto vDerivation = allocValue(); addConstant("derivation", vDerivation); @@ -4167,7 +4179,7 @@ void EvalState::createBaseEnv() // the parser needs two NUL bytes as terminators; one of them // is implied by being a C string. "\0"; - eval(parse(code, sizeof(code), derivationNixPath, "/", staticBaseEnv), *vDerivation); + eval(parse(code, sizeof(code), derivationInternal, {CanonPath::root}, staticBaseEnv), *vDerivation); } diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index db43e5771..07bf400cf 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -7,7 +7,7 @@ namespace nix { static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext"); v.mkString(*s); } @@ -17,7 +17,7 @@ static RegisterPrimOp primop_unsafeDiscardStringContext("__unsafeDiscardStringCo static void prim_hasContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.hasContext"); v.mkBool(!context.empty()); } @@ -33,17 +33,18 @@ static RegisterPrimOp primop_hasContext("__hasContext", 1, prim_hasContext); drv.inputDrvs. */ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto s = state.coerceToString(pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardOutputDependency"); - PathSet context2; - for (auto && p : context) { - auto c = NixStringContextElem::parse(*state.store, p); + NixStringContext context2; + for (auto && c : context) { if (auto * ptr = std::get_if<NixStringContextElem::DrvDeep>(&c)) { - context2.emplace(state.store->printStorePath(ptr->drvPath)); + context2.emplace(NixStringContextElem::Opaque { + .path = ptr->drvPath + }); } else { /* Can reuse original item */ - context2.emplace(std::move(p)); + context2.emplace(std::move(c)); } } @@ -79,22 +80,21 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value * * args, bool allOutputs = false; Strings outputs; }; - PathSet context; + NixStringContext context; state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.getContext"); auto contextInfos = std::map<StorePath, ContextInfo>(); - for (const auto & p : context) { - NixStringContextElem ctx = NixStringContextElem::parse(*state.store, p); + for (auto && i : context) { std::visit(overloaded { - [&](NixStringContextElem::DrvDeep & d) { - contextInfos[d.drvPath].allOutputs = true; + [&](NixStringContextElem::DrvDeep && d) { + contextInfos[std::move(d.drvPath)].allOutputs = true; }, - [&](NixStringContextElem::Built & b) { - contextInfos[b.drvPath].outputs.emplace_back(std::move(b.output)); + [&](NixStringContextElem::Built && b) { + contextInfos[std::move(b.drvPath)].outputs.emplace_back(std::move(b.output)); }, - [&](NixStringContextElem::Opaque & o) { - contextInfos[o.path].path = true; + [&](NixStringContextElem::Opaque && o) { + contextInfos[std::move(o.path)].path = true; }, - }, ctx.raw()); + }, ((NixStringContextElem &&) i).raw()); } auto attrs = state.buildBindings(contextInfos.size()); @@ -129,7 +129,7 @@ static RegisterPrimOp primop_getContext("__getContext", 1, prim_getContext); */ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * args, Value & v) { - PathSet context; + NixStringContext context; auto orig = state.forceString(*args[0], context, noPos, "while evaluating the first argument passed to builtins.appendContext"); state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.appendContext"); @@ -143,13 +143,16 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar .msg = hintfmt("context key '%s' is not a store path", name), .errPos = state.positions[i.pos] }); + auto namePath = state.store->parseStorePath(name); if (!settings.readOnlyMode) - state.store->ensurePath(state.store->parseStorePath(name)); + state.store->ensurePath(namePath); state.forceAttrs(*i.value, i.pos, "while evaluating the value of a string context"); auto iter = i.value->attrs->find(sPath); if (iter != i.value->attrs->end()) { if (state.forceBool(*iter->value, iter->pos, "while evaluating the `path` attribute of a string context")) - context.emplace(name); + context.emplace(NixStringContextElem::Opaque { + .path = namePath, + }); } iter = i.value->attrs->find(sAllOutputs); @@ -161,7 +164,9 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar .errPos = state.positions[i.pos] }); } - context.insert(concatStrings("=", name)); + context.emplace(NixStringContextElem::DrvDeep { + .drvPath = namePath, + }); } } @@ -176,7 +181,10 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value * * ar } for (auto elem : iter->value->listItems()) { auto outputName = state.forceStringNoCtx(*elem, iter->pos, "while evaluating an output name within a string context"); - context.insert(concatStrings("!", outputName, "!", name)); + context.emplace(NixStringContextElem::Built { + .drvPath = namePath, + .output = std::string { outputName }, + }); } } } diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 0dfa97fa3..4cf1f1e0b 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -18,7 +18,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg const auto & attrName = state.symbols[attr.name]; if (attrName == "fromPath") { - PathSet context; + NixStringContext context; fromPath = state.coerceToStorePath(attr.pos, *attr.value, context, "while evaluating the 'fromPath' attribute passed to builtins.fetchClosure"); } @@ -27,7 +27,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg state.forceValue(*attr.value, attr.pos); toCA = true; if (attr.value->type() != nString || attr.value->string.s != std::string("")) { - PathSet context; + NixStringContext context; toPath = state.coerceToStorePath(attr.pos, *attr.value, context, "while evaluating the 'toPath' attribute passed to builtins.fetchClosure"); } @@ -114,8 +114,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value * * arg }); } - auto toPathS = state.store->printStorePath(*toPath); - v.mkString(toPathS, {toPathS}); + state.mkStorePathString(*toPath, v); } static RegisterPrimOp primop_fetchClosure({ diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index c41bd60b6..2c0d98e74 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -13,7 +13,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a std::optional<Hash> rev; std::optional<std::string> ref; std::string_view name = "source"; - PathSet context; + NixStringContext context; state.forceValue(*args[0], pos); @@ -73,8 +73,7 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value * * a auto [tree, input2] = input.fetch(state.store); auto attrs2 = state.buildBindings(8); - auto storePath = state.store->printStorePath(tree.storePath); - attrs2.alloc(state.sOutPath).mkString(storePath, {storePath}); + state.mkStorePathString(tree.storePath, attrs2.alloc(state.sOutPath)); if (input2.getRef()) attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 2e150c9d0..fe880aaa8 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -24,9 +24,8 @@ void emitTreeAttrs( auto attrs = state.buildBindings(8); - auto storePath = state.store->printStorePath(tree.storePath); - attrs.alloc(state.sOutPath).mkString(storePath, {storePath}); + state.mkStorePathString(tree.storePath, attrs.alloc(state.sOutPath)); // FIXME: support arbitrary input attributes. @@ -107,7 +106,7 @@ static void fetchTree( const FetchTreeParams & params = FetchTreeParams{} ) { fetchers::Input input; - PathSet context; + NixStringContext context; state.forceValue(*args[0], pos); @@ -287,9 +286,9 @@ static RegisterPrimOp primop_fetchurl({ .name = "__fetchurl", .args = {"url"}, .doc = R"( - Download the specified URL and return the path of the downloaded - file. This function is not available if [restricted evaluation - mode](../command-ref/conf-file.md) is enabled. + Download the specified URL and return the path of the downloaded file. + + Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval). )", .fun = prim_fetchurl, }); @@ -339,8 +338,7 @@ static RegisterPrimOp primop_fetchTarball({ stdenv.mkDerivation { … } ``` - This function is not available if [restricted evaluation - mode](../command-ref/conf-file.md) is enabled. + Not available in [restricted evaluation mode](@docroot@/command-ref/conf-file.md#conf-restrict-eval). )", .fun = prim_fetchTarball, }); @@ -471,14 +469,9 @@ static RegisterPrimOp primop_fetchGit({ } ``` - > **Note** - > - > Nix will refetch the branch in accordance with - > the option `tarball-ttl`. + Nix will refetch the branch according to the [`tarball-ttl`](@docroot@/command-ref/conf-file.md#conf-tarball-ttl) setting. - > **Note** - > - > This behavior is disabled in *Pure evaluation mode*. + This behavior is disabled in [pure evaluation mode](@docroot@/command-ref/conf-file.md#conf-pure-eval). - To fetch the content of a checked-out work directory: diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 8a5231781..e2a8b3c3a 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -3,6 +3,8 @@ #include "../../toml11/toml.hpp" +#include <sstream> + namespace nix { static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, Value & val) @@ -58,8 +60,18 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value * * args, V case toml::value_t::offset_datetime: case toml::value_t::local_date: case toml::value_t::local_time: - // We fail since Nix doesn't have date and time types - throw std::runtime_error("Dates and times are not supported"); + { + if (experimentalFeatureSettings.isEnabled(Xp::ParseTomlTimestamps)) { + auto attrs = state.buildBindings(2); + attrs.alloc("_type").mkString("timestamp"); + std::ostringstream s; + s << t; + attrs.alloc("value").mkString(s.str()); + v.mkAttrs(attrs); + } else { + throw std::runtime_error("Dates and times are not supported"); + } + } break;; case toml::value_t::empty: v.mkNull(); diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index d08672cfc..53ba70bdd 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -1,4 +1,5 @@ #include "print.hh" +#include <unordered_set> namespace nix { @@ -25,11 +26,26 @@ printLiteralBool(std::ostream & str, bool boolean) return str; } +// Returns `true' is a string is a reserved keyword which requires quotation +// when printing attribute set field names. +// +// This list should generally be kept in sync with `./lexer.l'. +// You can test if a keyword needs to be added by running: +// $ nix eval --expr '{ <KEYWORD> = 1; }' +// For example `or' doesn't need to be quoted. +bool isReservedKeyword(const std::string_view str) +{ + static const std::unordered_set<std::string_view> reservedKeywords = { + "if", "then", "else", "assert", "with", "let", "in", "rec", "inherit" + }; + return reservedKeywords.contains(str); +} + std::ostream & printIdentifier(std::ostream & str, std::string_view s) { if (s.empty()) str << "\"\""; - else if (s == "if") // FIXME: handle other keywords + else if (isReservedKeyword(s)) str << '"' << s << '"'; else { char c = s[0]; @@ -50,10 +66,10 @@ printIdentifier(std::ostream & str, std::string_view s) { return str; } -// FIXME: keywords static bool isVarName(std::string_view s) { if (s.size() == 0) return false; + if (isReservedKeyword(s)) return false; char c = s[0]; if ((c >= '0' && c <= '9') || c == '-' || c == '\'') return false; for (auto & i : s) diff --git a/src/libexpr/print.hh b/src/libexpr/print.hh index f9cfc3964..3b72ae201 100644 --- a/src/libexpr/print.hh +++ b/src/libexpr/print.hh @@ -36,6 +36,12 @@ namespace nix { std::ostream & printAttributeName(std::ostream & o, std::string_view s); /** + * Returns `true' is a string is a reserved keyword which requires quotation + * when printing attribute set field names. + */ + bool isReservedKeyword(const std::string_view str); + + /** * Print a string as an identifier in the Nix expression language syntax. * * FIXME: "identifier" is ambiguous. Identifiers do not have a single diff --git a/src/libexpr/tests/derived-path.cc b/src/libexpr/tests/derived-path.cc new file mode 100644 index 000000000..8210efef2 --- /dev/null +++ b/src/libexpr/tests/derived-path.cc @@ -0,0 +1,65 @@ +#include <nlohmann/json.hpp> +#include <gtest/gtest.h> +#include <rapidcheck/gtest.h> + +#include "tests/derived-path.hh" +#include "tests/libexpr.hh" + +namespace nix { + +// Testing of trivial expressions +class DerivedPathExpressionTest : public LibExprTest {}; + +// FIXME: `RC_GTEST_FIXTURE_PROP` isn't calling `SetUpTestSuite` because it is +// no a real fixture. +// +// See https://github.com/emil-e/rapidcheck/blob/master/doc/gtest.md#rc_gtest_fixture_propfixture-name-args +TEST_F(DerivedPathExpressionTest, force_init) +{ +} + +RC_GTEST_FIXTURE_PROP( + DerivedPathExpressionTest, + prop_opaque_path_round_trip, + (const DerivedPath::Opaque & o)) +{ + auto * v = state.allocValue(); + state.mkStorePathString(o.path, *v); + auto d = state.coerceToDerivedPath(noPos, *v, ""); + RC_ASSERT(DerivedPath { o } == d); +} + +// TODO use DerivedPath::Built for parameter once it supports a single output +// path only. + +RC_GTEST_FIXTURE_PROP( + DerivedPathExpressionTest, + prop_built_path_placeholder_round_trip, + (const StorePath & drvPath, const StorePathName & outputName)) +{ + auto * v = state.allocValue(); + state.mkOutputString(*v, drvPath, outputName.name, std::nullopt); + auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); + DerivedPath::Built b { + .drvPath = drvPath, + .outputs = OutputsSpec::Names { outputName.name }, + }; + RC_ASSERT(DerivedPath { b } == d); +} + +RC_GTEST_FIXTURE_PROP( + DerivedPathExpressionTest, + prop_built_path_out_path_round_trip, + (const StorePath & drvPath, const StorePathName & outputName, const StorePath & outPath)) +{ + auto * v = state.allocValue(); + state.mkOutputString(*v, drvPath, outputName.name, outPath); + auto [d, _] = state.coerceToDerivedPathUnchecked(noPos, *v, ""); + DerivedPath::Built b { + .drvPath = drvPath, + .outputs = OutputsSpec::Names { outputName.name }, + }; + RC_ASSERT(DerivedPath { b } == d); +} + +} /* namespace nix */ diff --git a/src/libexpr/tests/error_traces.cc b/src/libexpr/tests/error_traces.cc index 24e95ac39..285651256 100644 --- a/src/libexpr/tests/error_traces.cc +++ b/src/libexpr/tests/error_traces.cc @@ -171,7 +171,7 @@ namespace nix { hintfmt("value is %s while a string was expected", "an integer"), hintfmt("while evaluating one of the strings to replace passed to builtins.replaceStrings")); - ASSERT_TRACE2("replaceStrings [ \"old\" ] [ true ] {}", + ASSERT_TRACE2("replaceStrings [ \"oo\" ] [ true ] \"foo\"", TypeError, hintfmt("value is %s while a string was expected", "a Boolean"), hintfmt("while evaluating one of the replacement strings passed to builtins.replaceStrings")); diff --git a/src/libexpr/tests/json.cc b/src/libexpr/tests/json.cc index 411bc0ac3..7586bdd9b 100644 --- a/src/libexpr/tests/json.cc +++ b/src/libexpr/tests/json.cc @@ -8,7 +8,7 @@ namespace nix { protected: std::string getJSONValue(Value& value) { std::stringstream ss; - PathSet ps; + NixStringContext ps; printValueAsJSON(state, true, value, noPos, ss, ps); return ss.str(); } diff --git a/src/libexpr/tests/libexpr.hh b/src/libexpr/tests/libexpr.hh index 69c932f05..b8e65aafe 100644 --- a/src/libexpr/tests/libexpr.hh +++ b/src/libexpr/tests/libexpr.hh @@ -28,7 +28,7 @@ namespace nix { } Value eval(std::string input, bool forceValue = true) { Value v; - Expr * e = state.parseExprFromString(input, ""); + Expr * e = state.parseExprFromString(input, state.rootPath(CanonPath::root)); assert(e); state.eval(e, v); if (forceValue) diff --git a/src/libexpr/tests/local.mk b/src/libexpr/tests/local.mk index 3e5504f71..331a5ead6 100644 --- a/src/libexpr/tests/local.mk +++ b/src/libexpr/tests/local.mk @@ -12,7 +12,7 @@ libexpr-tests_SOURCES := \ $(wildcard $(d)/*.cc) \ $(wildcard $(d)/value/*.cc) -libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests +libexpr-tests_CXXFLAGS += -I src/libexpr -I src/libutil -I src/libstore -I src/libexpr/tests -I src/libfetchers libexpr-tests_LIBS = libstore-tests libutils-tests libexpr libutil libstore libfetchers diff --git a/src/libexpr/tests/value/context.cc b/src/libexpr/tests/value/context.cc index 083359b7a..0d9381577 100644 --- a/src/libexpr/tests/value/context.cc +++ b/src/libexpr/tests/value/context.cc @@ -8,69 +8,62 @@ namespace nix { -// Testing of trivial expressions -struct NixStringContextElemTest : public LibExprTest { - const Store & store() const { - return *LibExprTest::store; - } -}; - -TEST_F(NixStringContextElemTest, empty_invalid) { +TEST(NixStringContextElemTest, empty_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), ""), + NixStringContextElem::parse(""), BadNixStringContextElem); } -TEST_F(NixStringContextElemTest, single_bang_invalid) { +TEST(NixStringContextElemTest, single_bang_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), "!"), + NixStringContextElem::parse("!"), BadNixStringContextElem); } -TEST_F(NixStringContextElemTest, double_bang_invalid) { +TEST(NixStringContextElemTest, double_bang_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), "!!/"), + NixStringContextElem::parse("!!/"), BadStorePath); } -TEST_F(NixStringContextElemTest, eq_slash_invalid) { +TEST(NixStringContextElemTest, eq_slash_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), "=/"), + NixStringContextElem::parse("=/"), BadStorePath); } -TEST_F(NixStringContextElemTest, slash_invalid) { +TEST(NixStringContextElemTest, slash_invalid) { EXPECT_THROW( - NixStringContextElem::parse(store(), "/"), + NixStringContextElem::parse("/"), BadStorePath); } -TEST_F(NixStringContextElemTest, opaque) { - std::string_view opaque = "/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; - auto elem = NixStringContextElem::parse(store(), opaque); +TEST(NixStringContextElemTest, opaque) { + std::string_view opaque = "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x"; + auto elem = NixStringContextElem::parse(opaque); auto * p = std::get_if<NixStringContextElem::Opaque>(&elem); ASSERT_TRUE(p); - ASSERT_EQ(p->path, store().parseStorePath(opaque)); - ASSERT_EQ(elem.to_string(store()), opaque); + ASSERT_EQ(p->path, StorePath { opaque }); + ASSERT_EQ(elem.to_string(), opaque); } -TEST_F(NixStringContextElemTest, drvDeep) { - std::string_view drvDeep = "=/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; - auto elem = NixStringContextElem::parse(store(), drvDeep); +TEST(NixStringContextElemTest, drvDeep) { + std::string_view drvDeep = "=g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(drvDeep); auto * p = std::get_if<NixStringContextElem::DrvDeep>(&elem); ASSERT_TRUE(p); - ASSERT_EQ(p->drvPath, store().parseStorePath(drvDeep.substr(1))); - ASSERT_EQ(elem.to_string(store()), drvDeep); + ASSERT_EQ(p->drvPath, StorePath { drvDeep.substr(1) }); + ASSERT_EQ(elem.to_string(), drvDeep); } -TEST_F(NixStringContextElemTest, built) { - std::string_view built = "!foo!/nix/store/g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; - auto elem = NixStringContextElem::parse(store(), built); +TEST(NixStringContextElemTest, built) { + std::string_view built = "!foo!g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-x.drv"; + auto elem = NixStringContextElem::parse(built); auto * p = std::get_if<NixStringContextElem::Built>(&elem); ASSERT_TRUE(p); ASSERT_EQ(p->output, "foo"); - ASSERT_EQ(p->drvPath, store().parseStorePath(built.substr(5))); - ASSERT_EQ(elem.to_string(store()), built); + ASSERT_EQ(p->drvPath, StorePath { built.substr(5) }); + ASSERT_EQ(elem.to_string(), built); } } @@ -102,13 +95,15 @@ Gen<NixStringContextElem::Built> Arbitrary<NixStringContextElem::Built>::arbitra Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary() { - switch (*gen::inRange<uint8_t>(0, 2)) { + switch (*gen::inRange<uint8_t>(0, std::variant_size_v<NixStringContextElem::Raw>)) { case 0: return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Opaque>()); case 1: return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::DrvDeep>()); - default: + case 2: return gen::just<NixStringContextElem>(*gen::arbitrary<NixStringContextElem::Built>()); + default: + assert(false); } } @@ -116,12 +111,12 @@ Gen<NixStringContextElem> Arbitrary<NixStringContextElem>::arbitrary() namespace nix { -RC_GTEST_FIXTURE_PROP( +RC_GTEST_PROP( NixStringContextElemTest, prop_round_rip, (const NixStringContextElem & o)) { - RC_ASSERT(o == NixStringContextElem::parse(store(), o.to_string(store()))); + RC_ASSERT(o == NixStringContextElem::parse(o.to_string())); } } diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index c35c876e3..4996a5bde 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -11,7 +11,7 @@ namespace nix { using json = nlohmann::json; json printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, PathSet & context, bool copyToStore) + Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore) { checkInterrupt(); @@ -36,9 +36,10 @@ json printValueAsJSON(EvalState & state, bool strict, case nPath: if (copyToStore) - out = state.store->printStorePath(state.copyPathToStore(context, v.path)); + out = state.store->printStorePath( + state.copyPathToStore(context, v.path())); else - out = v.path; + out = v.path().path.abs(); break; case nNull: @@ -94,13 +95,13 @@ json printValueAsJSON(EvalState & state, bool strict, } void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore) + Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore) { str << printValueAsJSON(state, strict, v, pos, context, copyToStore); } json ExternalValueBase::printValueAsJSON(EvalState & state, bool strict, - PathSet & context, bool copyToStore) const + NixStringContext & context, bool copyToStore) const { state.debugThrowLastTrace(TypeError("cannot convert %1% to JSON", showType())); } diff --git a/src/libexpr/value-to-json.hh b/src/libexpr/value-to-json.hh index 713356c7f..47ac90313 100644 --- a/src/libexpr/value-to-json.hh +++ b/src/libexpr/value-to-json.hh @@ -11,9 +11,9 @@ namespace nix { nlohmann::json printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, PathSet & context, bool copyToStore = true); + Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore = true); void printValueAsJSON(EvalState & state, bool strict, - Value & v, const PosIdx pos, std::ostream & str, PathSet & context, bool copyToStore = true); + Value & v, const PosIdx pos, std::ostream & str, NixStringContext & context, bool copyToStore = true); } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 341c8922f..2539ad1c1 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -18,21 +18,21 @@ static XMLAttrs singletonAttrs(const std::string & name, const std::string & val static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + Value & v, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos); static void posToXML(EvalState & state, XMLAttrs & xmlAttrs, const Pos & pos) { - if (auto path = std::get_if<Path>(&pos.origin)) - xmlAttrs["path"] = *path; + if (auto path = std::get_if<SourcePath>(&pos.origin)) + xmlAttrs["path"] = path->path.abs(); xmlAttrs["line"] = fmt("%1%", pos.line); xmlAttrs["column"] = fmt("%1%", pos.column); } static void showAttrs(EvalState & state, bool strict, bool location, - Bindings & attrs, XMLWriter & doc, PathSet & context, PathSet & drvsSeen) + Bindings & attrs, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen) { StringSet names; @@ -54,7 +54,7 @@ static void showAttrs(EvalState & state, bool strict, bool location, static void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + Value & v, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos) { checkInterrupt(); @@ -78,7 +78,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, break; case nPath: - doc.writeEmptyElement("path", singletonAttrs("value", v.path)); + doc.writeEmptyElement("path", singletonAttrs("value", v.path().to_string())); break; case nNull: @@ -166,7 +166,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, - bool location, XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + bool location, XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos) const { doc.writeEmptyElement("unevaluated"); @@ -174,7 +174,7 @@ void ExternalValueBase::printValueAsXML(EvalState & state, bool strict, void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context, const PosIdx pos) + Value & v, std::ostream & out, NixStringContext & context, const PosIdx pos) { XMLWriter doc(true, out); XMLOpenElement root(doc, "expr"); diff --git a/src/libexpr/value-to-xml.hh b/src/libexpr/value-to-xml.hh index ace7ead0f..6d702c0f2 100644 --- a/src/libexpr/value-to-xml.hh +++ b/src/libexpr/value-to-xml.hh @@ -10,6 +10,6 @@ namespace nix { void printValueAsXML(EvalState & state, bool strict, bool location, - Value & v, std::ostream & out, PathSet & context, const PosIdx pos); + Value & v, std::ostream & out, NixStringContext & context, const PosIdx pos); } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 7739f99df..89c0c36fd 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -5,6 +5,7 @@ #include "symbol-table.hh" #include "value/context.hh" +#include "input-accessor.hh" #if HAVE_BOEHMGC #include <gc/gc_allocator.h> @@ -100,7 +101,7 @@ class ExternalValueBase * Coerce the value to a string. Defaults to uncoercable, i.e. throws an * error. */ - virtual std::string coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const; + virtual std::string coerceToString(const Pos & pos, NixStringContext & context, bool copyMore, bool copyToStore) const; /** * Compare to another value of the same type. Defaults to uncomparable, @@ -112,13 +113,13 @@ class ExternalValueBase * Print the value as JSON. Defaults to unconvertable, i.e. throws an error */ virtual nlohmann::json printValueAsJSON(EvalState & state, bool strict, - PathSet & context, bool copyToStore = true) const; + NixStringContext & context, bool copyToStore = true) const; /** * Print the value as XML. Defaults to unevaluated */ virtual void printValueAsXML(EvalState & state, bool strict, bool location, - XMLWriter & doc, PathSet & context, PathSet & drvsSeen, + XMLWriter & doc, NixStringContext & context, PathSet & drvsSeen, const PosIdx pos) const; virtual ~ExternalValueBase() @@ -188,7 +189,7 @@ public: const char * * context; // must be in sorted order } string; - const char * path; + const char * _path; Bindings * attrs; struct { size_t size; @@ -268,19 +269,24 @@ public: void mkString(std::string_view s); - void mkString(std::string_view s, const PathSet & context); + void mkString(std::string_view s, const NixStringContext & context); - void mkStringMove(const char * s, const PathSet & context); + void mkStringMove(const char * s, const NixStringContext & context); - inline void mkPath(const char * s) + inline void mkString(const Symbol & s) + { + mkString(((const std::string &) s).c_str()); + } + + void mkPath(const SourcePath & path); + + inline void mkPath(const char * path) { clearValue(); internalType = tPath; - path = s; + _path = path; } - void mkPath(std::string_view s); - inline void mkNull() { clearValue(); @@ -394,8 +400,6 @@ public: */ bool isTrivial() const; - NixStringContext getContext(const Store &); - auto listItems() { struct ListIterable @@ -423,6 +427,18 @@ public: auto begin = listElems(); return ConstListIterable { begin, begin + listSize() }; } + + SourcePath path() const + { + assert(internalType == tPath); + return SourcePath{CanonPath(_path)}; + } + + std::string_view str() const + { + assert(internalType == tString); + return std::string_view(string.s); + } }; diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index 61d9c53df..f76fc76e4 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -1,11 +1,10 @@ #include "value/context.hh" -#include "store-api.hh" #include <optional> namespace nix { -NixStringContextElem NixStringContextElem::parse(const Store & store, std::string_view s0) +NixStringContextElem NixStringContextElem::parse(std::string_view s0) { std::string_view s = s0; @@ -25,41 +24,41 @@ NixStringContextElem NixStringContextElem::parse(const Store & store, std::strin "String content element beginning with '!' should have a second '!'"); } return NixStringContextElem::Built { - .drvPath = store.parseStorePath(s.substr(index + 1)), + .drvPath = StorePath { s.substr(index + 1) }, .output = std::string(s.substr(0, index)), }; } case '=': { return NixStringContextElem::DrvDeep { - .drvPath = store.parseStorePath(s.substr(1)), + .drvPath = StorePath { s.substr(1) }, }; } default: { return NixStringContextElem::Opaque { - .path = store.parseStorePath(s), + .path = StorePath { s }, }; } } } -std::string NixStringContextElem::to_string(const Store & store) const { +std::string NixStringContextElem::to_string() const { return std::visit(overloaded { [&](const NixStringContextElem::Built & b) { std::string res; res += '!'; res += b.output; res += '!'; - res += store.printStorePath(b.drvPath); + res += b.drvPath.to_string(); return res; }, [&](const NixStringContextElem::DrvDeep & d) { std::string res; res += '='; - res += store.printStorePath(d.drvPath); + res += d.drvPath.to_string(); return res; }, [&](const NixStringContextElem::Opaque & o) { - return store.printStorePath(o.path); + return std::string { o.path.to_string() }; }, }, raw()); } diff --git a/src/libexpr/value/context.hh b/src/libexpr/value/context.hh index 8719602d8..287ae08a9 100644 --- a/src/libexpr/value/context.hh +++ b/src/libexpr/value/context.hh @@ -26,8 +26,6 @@ public: } }; -class Store; - /** * Plain opaque path to some store object. * @@ -80,12 +78,15 @@ struct NixStringContextElem : _NixStringContextElem_Raw { using DrvDeep = NixStringContextElem_DrvDeep; using Built = NixStringContextElem_Built; - inline const Raw & raw() const { + inline const Raw & raw() const & { return static_cast<const Raw &>(*this); } - inline Raw & raw() { + inline Raw & raw() & { return static_cast<Raw &>(*this); } + inline Raw && raw() && { + return static_cast<Raw &&>(*this); + } /** * Decode a context string, one of: @@ -93,10 +94,10 @@ struct NixStringContextElem : _NixStringContextElem_Raw { * - ‘=<path>’ * - ‘!<name>!<path>’ */ - static NixStringContextElem parse(const Store & store, std::string_view s); - std::string to_string(const Store & store) const; + static NixStringContextElem parse(std::string_view s); + std::string to_string() const; }; -typedef std::vector<NixStringContextElem> NixStringContext; +typedef std::set<NixStringContextElem> NixStringContext; } |