aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorJohn Ericson <John.Ericson@Obsidian.Systems>2022-03-25 19:40:52 +0000
committerJohn Ericson <John.Ericson@Obsidian.Systems>2022-03-25 19:40:52 +0000
commitff2a8ccfe176fa3e075ed8925a371eeb17e627e6 (patch)
tree7c20ce8a8ae5370f6fc078fa26888c7417f47c7a /src
parent938650700fafe76e3755982d670855fed3db35c6 (diff)
parent0dc2974930df57cac6673c02e9bc6eb6fd16ba48 (diff)
Merge branch 'path-info' into ca-drv-exotic
Diffstat (limited to 'src')
-rw-r--r--src/build-remote/build-remote.cc2
-rw-r--r--src/libcmd/command.cc3
-rw-r--r--src/libcmd/installables.cc9
-rw-r--r--src/libexpr/eval-cache.cc21
-rw-r--r--src/libexpr/eval-cache.hh2
-rw-r--r--src/libexpr/eval-inline.hh91
-rw-r--r--src/libexpr/eval.cc163
-rw-r--r--src/libexpr/eval.hh19
-rw-r--r--src/libexpr/flake/flake.cc27
-rw-r--r--src/libexpr/get-drvs.cc31
-rw-r--r--src/libexpr/get-drvs.hh7
-rw-r--r--src/libexpr/lexer.l23
-rw-r--r--src/libexpr/nixexpr.hh13
-rw-r--r--src/libexpr/primops.cc120
-rw-r--r--src/libexpr/primops.hh2
-rw-r--r--src/libexpr/primops/context.cc5
-rw-r--r--src/libexpr/primops/fetchClosure.cc154
-rw-r--r--src/libexpr/primops/fetchTree.cc4
-rw-r--r--src/libexpr/value.hh11
-rw-r--r--src/libfetchers/git.cc54
-rw-r--r--src/libfetchers/github.cc10
-rw-r--r--src/libfetchers/path.cc14
-rw-r--r--src/libstore/binary-cache-store.hh5
-rw-r--r--src/libstore/build/derivation-goal.cc45
-rw-r--r--src/libstore/build/derivation-goal.hh8
-rw-r--r--src/libstore/build/goal.cc2
-rw-r--r--src/libstore/build/goal.hh13
-rw-r--r--src/libstore/build/local-derivation-goal.cc35
-rw-r--r--src/libstore/builtins/buildenv.cc10
-rw-r--r--src/libstore/daemon.cc19
-rw-r--r--src/libstore/derivations.cc299
-rw-r--r--src/libstore/derivations.hh131
-rw-r--r--src/libstore/derived-path.cc32
-rw-r--r--src/libstore/derived-path.hh2
-rw-r--r--src/libstore/gc-store.cc13
-rw-r--r--src/libstore/gc-store.hh4
-rw-r--r--src/libstore/gc.cc6
-rw-r--r--src/libstore/local-fs-store.hh6
-rw-r--r--src/libstore/local-store.cc15
-rw-r--r--src/libstore/log-store.hh21
-rw-r--r--src/libstore/make-content-addressed.cc83
-rw-r--r--src/libstore/make-content-addressed.hh12
-rw-r--r--src/libstore/misc.cc2
-rw-r--r--src/libstore/parsed-derivations.cc2
-rw-r--r--src/libstore/parsed-derivations.hh1
-rw-r--r--src/libstore/remote-store.hh6
-rw-r--r--src/libstore/repair-flag.hh7
-rw-r--r--src/libstore/ssh-store.cc4
-rw-r--r--src/libstore/store-api.cc1
-rw-r--r--src/libstore/store-api.hh12
-rw-r--r--src/libstore/store-cast.hh16
-rw-r--r--src/libutil/archive.cc19
-rw-r--r--src/libutil/archive.hh4
-rw-r--r--src/libutil/error.hh1
-rw-r--r--src/libutil/experimental-features.cc1
-rw-r--r--src/libutil/experimental-features.hh3
-rw-r--r--src/libutil/finally.hh7
-rw-r--r--src/libutil/tarfile.cc26
-rw-r--r--src/libutil/tarfile.hh3
-rw-r--r--src/libutil/util.cc40
-rwxr-xr-xsrc/nix-build/nix-build.cc31
-rw-r--r--src/nix-collect-garbage/nix-collect-garbage.cc3
-rw-r--r--src/nix-env/nix-env.cc144
-rw-r--r--src/nix-env/user-env.cc14
-rw-r--r--src/nix-store/nix-store.cc16
-rw-r--r--src/nix/app.cc3
-rw-r--r--src/nix/build.cc17
-rw-r--r--src/nix/develop.cc13
-rw-r--r--src/nix/edit.md6
-rw-r--r--src/nix/log.cc14
-rw-r--r--src/nix/main.cc3
-rw-r--r--src/nix/make-content-addressable.cc102
-rw-r--r--src/nix/make-content-addressed.cc55
-rw-r--r--src/nix/make-content-addressed.md (renamed from src/nix/make-content-addressable.md)4
-rw-r--r--src/nix/profile.cc5
-rw-r--r--src/nix/profile.md6
-rw-r--r--src/nix/repl.cc13
-rw-r--r--src/nix/show-derivation.cc10
-rw-r--r--src/nix/store-copy-log.cc9
-rw-r--r--src/nix/store-delete.cc3
-rw-r--r--src/nix/store-gc.cc3
81 files changed, 1422 insertions, 753 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 8c9133c17..ff8ba2724 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -300,7 +300,7 @@ connected:
std::set<Realisation> missingRealisations;
StorePathSet missingPaths;
- if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !derivationHasKnownOutputPaths(drv.type())) {
+ if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) && !drv.type().hasKnownOutputPaths()) {
for (auto & outputName : wantedOutputs) {
auto thisOutputHash = outputHashes.at(outputName);
auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc
index dc8fa9e5a..a53b029b7 100644
--- a/src/libcmd/command.cc
+++ b/src/libcmd/command.cc
@@ -204,7 +204,8 @@ Strings editorFor(const Pos & pos)
if (pos.line > 0 && (
editor.find("emacs") != std::string::npos ||
editor.find("nano") != std::string::npos ||
- editor.find("vim") != std::string::npos))
+ editor.find("vim") != std::string::npos ||
+ editor.find("kak") != std::string::npos))
args.push_back(fmt("+%d", pos.line));
args.push_back(pos.file);
return args;
diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc
index b7623d4ba..784117569 100644
--- a/src/libcmd/installables.cc
+++ b/src/libcmd/installables.cc
@@ -134,7 +134,9 @@ SourceExprCommand::SourceExprCommand()
addFlag({
.longName = "file",
.shortName = 'f',
- .description = "Interpret installables as attribute paths relative to the Nix expression stored in *file*.",
+ .description =
+ "Interpret installables as attribute paths relative to the Nix expression stored in *file*. "
+ "If *file* is the character -, then a Nix expression will be read from standard input.",
.category = installablesCategory,
.labels = {"file"},
.handler = {&file},
@@ -695,7 +697,10 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
auto state = getEvalState();
auto vFile = state->allocValue();
- if (file)
+ if (file == "-") {
+ auto e = state->parseStdin();
+ state->eval(e, *vFile);
+ } else if (file)
state->evalFile(lookupFileArg(*state, *file), *vFile);
else {
auto e = state->parseExprFromString(*expr, absPath("."));
diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
index 188223957..54fa9b741 100644
--- a/src/libexpr/eval-cache.cc
+++ b/src/libexpr/eval-cache.cc
@@ -21,6 +21,8 @@ struct AttrDb
{
std::atomic_bool failed{false};
+ const Store & cfg;
+
struct State
{
SQLite db;
@@ -33,8 +35,9 @@ struct AttrDb
std::unique_ptr<Sync<State>> _state;
- AttrDb(const Hash & fingerprint)
- : _state(std::make_unique<Sync<State>>())
+ AttrDb(const Store & cfg, const Hash & fingerprint)
+ : cfg(cfg)
+ , _state(std::make_unique<Sync<State>>())
{
auto state(_state->lock());
@@ -254,10 +257,10 @@ struct AttrDb
return {{rowId, attrs}};
}
case AttrType::String: {
- std::vector<std::pair<Path, std::string>> context;
+ NixStringContext context;
if (!queryAttribute.isNull(3))
for (auto & s : tokenizeString<std::vector<std::string>>(queryAttribute.getStr(3), ";"))
- context.push_back(decodeContext(s));
+ context.push_back(decodeContext(cfg, s));
return {{rowId, string_t{queryAttribute.getStr(2), context}}};
}
case AttrType::Bool:
@@ -274,10 +277,10 @@ struct AttrDb
}
};
-static std::shared_ptr<AttrDb> makeAttrDb(const Hash & fingerprint)
+static std::shared_ptr<AttrDb> makeAttrDb(const Store & cfg, const Hash & fingerprint)
{
try {
- return std::make_shared<AttrDb>(fingerprint);
+ return std::make_shared<AttrDb>(cfg, fingerprint);
} catch (SQLiteError &) {
ignoreException();
return nullptr;
@@ -288,7 +291,7 @@ EvalCache::EvalCache(
std::optional<std::reference_wrapper<const Hash>> useCache,
EvalState & state,
RootLoader rootLoader)
- : db(useCache ? makeAttrDb(*useCache) : nullptr)
+ : db(useCache ? makeAttrDb(*state.store, *useCache) : nullptr)
, state(state)
, rootLoader(rootLoader)
{
@@ -546,7 +549,7 @@ string_t AttrCursor::getStringWithContext()
if (auto s = std::get_if<string_t>(&cachedValue->second)) {
bool valid = true;
for (auto & c : s->second) {
- if (!root->state.store->isValidPath(root->state.store->parseStorePath(c.first))) {
+ if (!root->state.store->isValidPath(c.first)) {
valid = false;
break;
}
@@ -563,7 +566,7 @@ string_t AttrCursor::getStringWithContext()
auto & v = forceValue();
if (v.type() == nString)
- return {v.string.s, v.getContext()};
+ return {v.string.s, v.getContext(*root->state.store)};
else if (v.type() == nPath)
return {v.path, {}};
else
diff --git a/src/libexpr/eval-cache.hh b/src/libexpr/eval-cache.hh
index 40f1d4ffc..c9a9bf471 100644
--- a/src/libexpr/eval-cache.hh
+++ b/src/libexpr/eval-cache.hh
@@ -52,7 +52,7 @@ struct misc_t {};
struct failed_t {};
typedef uint64_t AttrId;
typedef std::pair<AttrId, Symbol> AttrKey;
-typedef std::pair<std::string, std::vector<std::pair<Path, std::string>>> string_t;
+typedef std::pair<std::string, NixStringContext> string_t;
typedef std::variant<
std::vector<Symbol>,
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index aef1f6351..08a419923 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -24,6 +24,81 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
}
+/* Note: Various places expect the allocated memory to be zeroed. */
+[[gnu::always_inline]]
+inline void * allocBytes(size_t n)
+{
+ void * p;
+#if HAVE_BOEHMGC
+ p = GC_MALLOC(n);
+#else
+ p = calloc(n, 1);
+#endif
+ if (!p) throw std::bad_alloc();
+ return p;
+}
+
+
+[[gnu::always_inline]]
+Value * EvalState::allocValue()
+{
+#if HAVE_BOEHMGC
+ /* We use the boehm batch allocator to speed up allocations of Values (of which there are many).
+ GC_malloc_many returns a linked list of objects of the given size, where the first word
+ of each object is also the pointer to the next object in the list. This also means that we
+ have to explicitly clear the first word of every object we take. */
+ if (!*valueAllocCache) {
+ *valueAllocCache = GC_malloc_many(sizeof(Value));
+ if (!*valueAllocCache) throw std::bad_alloc();
+ }
+
+ /* GC_NEXT is a convenience macro for accessing the first word of an object.
+ Take the first list item, advance the list to the next item, and clear the next pointer. */
+ void * p = *valueAllocCache;
+ *valueAllocCache = GC_NEXT(p);
+ GC_NEXT(p) = nullptr;
+#else
+ void * p = allocBytes(sizeof(Value));
+#endif
+
+ nrValues++;
+ return (Value *) p;
+}
+
+
+[[gnu::always_inline]]
+Env & EvalState::allocEnv(size_t size)
+{
+ nrEnvs++;
+ nrValuesInEnvs += size;
+
+ Env * env;
+
+#if HAVE_BOEHMGC
+ if (size == 1) {
+ /* see allocValue for explanations. */
+ if (!*env1AllocCache) {
+ *env1AllocCache = GC_malloc_many(sizeof(Env) + sizeof(Value *));
+ if (!*env1AllocCache) throw std::bad_alloc();
+ }
+
+ void * p = *env1AllocCache;
+ *env1AllocCache = GC_NEXT(p);
+ GC_NEXT(p) = nullptr;
+ env = (Env *) p;
+ } else
+#endif
+ env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
+
+ env->type = Env::Plain;
+
+ /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
+
+ return *env;
+}
+
+
+[[gnu::always_inline]]
void EvalState::forceValue(Value & v, const Pos & pos)
{
forceValue(v, [&]() { return pos; });
@@ -52,6 +127,7 @@ void EvalState::forceValue(Value & v, Callable getPos)
}
+[[gnu::always_inline]]
inline void EvalState::forceAttrs(Value & v, const Pos & pos)
{
forceAttrs(v, [&]() { return pos; });
@@ -59,6 +135,7 @@ inline void EvalState::forceAttrs(Value & v, const Pos & pos)
template <typename Callable>
+[[gnu::always_inline]]
inline void EvalState::forceAttrs(Value & v, Callable getPos)
{
forceValue(v, getPos);
@@ -67,6 +144,7 @@ inline void EvalState::forceAttrs(Value & v, Callable getPos)
}
+[[gnu::always_inline]]
inline void EvalState::forceList(Value & v, const Pos & pos)
{
forceValue(v, pos);
@@ -74,18 +152,5 @@ inline void EvalState::forceList(Value & v, const Pos & pos)
throwTypeError(pos, "value is %1% while a list was expected", v);
}
-/* Note: Various places expect the allocated memory to be zeroed. */
-inline void * allocBytes(size_t n)
-{
- void * p;
-#if HAVE_BOEHMGC
- p = GC_MALLOC(n);
-#else
- p = calloc(n, 1);
-#endif
- if (!p) throw std::bad_alloc();
- return p;
-}
-
}
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 5bf161cc0..437c7fc53 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -96,20 +96,20 @@ RootValue allocRootValue(Value * v)
}
-void printValue(std::ostream & str, std::set<const void *> & seen, const Value & v)
+void Value::print(std::ostream & str, std::set<const void *> * seen) const
{
checkInterrupt();
- switch (v.internalType) {
+ switch (internalType) {
case tInt:
- str << v.integer;
+ str << integer;
break;
case tBool:
- str << (v.boolean ? "true" : "false");
+ str << (boolean ? "true" : "false");
break;
case tString:
str << "\"";
- for (const char * i = v.string.s; *i; i++)
+ for (const char * i = string.s; *i; i++)
if (*i == '\"' || *i == '\\') str << "\\" << *i;
else if (*i == '\n') str << "\\n";
else if (*i == '\r') str << "\\r";
@@ -119,19 +119,19 @@ void printValue(std::ostream & str, std::set<const void *> & seen, const Value &
str << "\"";
break;
case tPath:
- str << v.path; // !!! escaping?
+ str << path; // !!! escaping?
break;
case tNull:
str << "null";
break;
case tAttrs: {
- if (!v.attrs->empty() && !seen.insert(v.attrs).second)
- str << "<REPEAT>";
+ if (seen && !attrs->empty() && !seen->insert(attrs).second)
+ str << "«repeated»";
else {
str << "{ ";
- for (auto & i : v.attrs->lexicographicOrder()) {
+ for (auto & i : attrs->lexicographicOrder()) {
str << i->name << " = ";
- printValue(str, seen, *i->value);
+ i->value->print(str, seen);
str << "; ";
}
str << "}";
@@ -141,12 +141,12 @@ void printValue(std::ostream & str, std::set<const void *> & seen, const Value &
case tList1:
case tList2:
case tListN:
- if (v.listSize() && !seen.insert(v.listElems()).second)
- str << "<REPEAT>";
+ if (seen && listSize() && !seen->insert(listElems()).second)
+ str << "«repeated»";
else {
str << "[ ";
- for (auto v2 : v.listItems()) {
- printValue(str, seen, *v2);
+ for (auto v2 : listItems()) {
+ v2->print(str, seen);
str << " ";
}
str << "]";
@@ -166,10 +166,10 @@ void printValue(std::ostream & str, std::set<const void *> & seen, const Value &
str << "<PRIMOP-APP>";
break;
case tExternal:
- str << *v.external;
+ str << *external;
break;
case tFloat:
- str << v.fpoint;
+ str << fpoint;
break;
default:
abort();
@@ -177,10 +177,16 @@ void printValue(std::ostream & str, std::set<const void *> & seen, const Value &
}
-std::ostream & operator << (std::ostream & str, const Value & v)
+void Value::print(std::ostream & str, bool showRepeated) const
{
std::set<const void *> seen;
- printValue(str, seen, v);
+ print(str, showRepeated ? nullptr : &seen);
+}
+
+
+std::ostream & operator << (std::ostream & str, const Value & v)
+{
+ v.print(str, false);
return str;
}
@@ -449,8 +455,10 @@ EvalState::EvalState(
, regexCache(makeRegexCache())
#if HAVE_BOEHMGC
, valueAllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
+ , env1AllocCache(std::allocate_shared<void *>(traceable_allocator<void *>(), nullptr))
#else
, valueAllocCache(std::make_shared<void *>(nullptr))
+ , env1AllocCache(std::make_shared<void *>(nullptr))
#endif
, baseEnv(allocEnv(128))
, staticBaseEnv(false, 0)
@@ -499,23 +507,6 @@ EvalState::~EvalState()
}
-void EvalState::requireExperimentalFeatureOnEvaluation(
- const ExperimentalFeature & feature,
- const std::string_view fName,
- const Pos & pos)
-{
- if (!settings.isExperimentalFeatureEnabled(feature)) {
- throw EvalError({
- .msg = hintfmt(
- "Cannot call '%2%' because experimental Nix feature '%1%' is disabled. You can enable it via '--extra-experimental-features %1%'.",
- feature,
- fName
- ),
- .errPos = pos
- });
- }
-}
-
void EvalState::allowPath(const Path & path)
{
if (allowedPaths)
@@ -727,9 +718,18 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const std::string & s2
throw EvalError(s, s2);
}
+LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const Suggestions & suggestions, const char * s, const std::string & s2))
+{
+ throw EvalError(ErrorInfo {
+ .msg = hintfmt(s, s2),
+ .errPos = pos,
+ .suggestions = suggestions,
+ });
+}
+
LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const std::string & s2))
{
- throw EvalError({
+ throw EvalError(ErrorInfo {
.msg = hintfmt(s, s2),
.errPos = pos
});
@@ -773,6 +773,16 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
});
}
+LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const Suggestions & suggestions, const char * s, const ExprLambda & fun, const Symbol & s2))
+{
+ throw TypeError(ErrorInfo {
+ .msg = hintfmt(s, fun.showNamePos(), s2),
+ .errPos = pos,
+ .suggestions = suggestions,
+ });
+}
+
+
LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
{
throw TypeError(s, showType(v));
@@ -876,42 +886,6 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval)
}
-Value * EvalState::allocValue()
-{
- /* We use the boehm batch allocator to speed up allocations of Values (of which there are many).
- GC_malloc_many returns a linked list of objects of the given size, where the first word
- of each object is also the pointer to the next object in the list. This also means that we
- have to explicitly clear the first word of every object we take. */
- if (!*valueAllocCache) {
- *valueAllocCache = GC_malloc_many(sizeof(Value));
- if (!*valueAllocCache) throw std::bad_alloc();
- }
-
- /* GC_NEXT is a convenience macro for accessing the first word of an object.
- Take the first list item, advance the list to the next item, and clear the next pointer. */
- void * p = *valueAllocCache;
- GC_PTR_STORE_AND_DIRTY(&*valueAllocCache, GC_NEXT(p));
- GC_NEXT(p) = nullptr;
-
- nrValues++;
- auto v = (Value *) p;
- return v;
-}
-
-
-Env & EvalState::allocEnv(size_t size)
-{
- nrEnvs++;
- nrValuesInEnvs += size;
- Env * env = (Env *) allocBytes(sizeof(Env) + size * sizeof(Value *));
- env->type = Env::Plain;
-
- /* We assume that env->values has been cleared by the allocator; maybeThunk() and lookupVar fromWith expect this. */
-
- return *env;
-}
-
-
void EvalState::mkList(Value & v, size_t size)
{
v.mkList(size);
@@ -1281,8 +1255,15 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
}
} else {
state.forceAttrs(*vAttrs, pos);
- if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
- throwEvalError(pos, "attribute '%1%' missing", name);
+ if ((j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) {
+ std::set<std::string> allAttrNames;
+ for (auto & attr : *vAttrs->attrs)
+ allAttrNames.insert(attr.name);
+ throwEvalError(
+ pos,
+ Suggestions::bestMatches(allAttrNames, name),
+ "attribute '%1%' missing", name);
+ }
}
vAttrs = j->value;
pos2 = j->pos;
@@ -1398,8 +1379,17 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value &
/* Nope, so show the first unexpected argument to the
user. */
for (auto & i : *args[0]->attrs)
- if (!lambda.formals->has(i.name))
- throwTypeError(pos, "%1% called with unexpected argument '%2%'", lambda, i.name);
+ if (!lambda.formals->has(i.name)) {
+ std::set<std::string> formalNames;
+ for (auto & formal : lambda.formals->formals)
+ formalNames.insert(formal.name);
+ throwTypeError(
+ pos,
+ Suggestions::bestMatches(formalNames, i.name),
+ "%1% called with unexpected argument '%2%'",
+ lambda,
+ i.name);
+ }
abort(); // can't happen
}
}
@@ -1902,13 +1892,22 @@ std::string_view EvalState::forceString(Value & v, const Pos & pos)
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
-std::pair<std::string, std::string> decodeContext(std::string_view s)
+NixStringContextElem decodeContext(const Store & store, std::string_view s)
{
if (s.at(0) == '!') {
size_t index = s.find("!", 1);
- return {std::string(s.substr(index + 1)), std::string(s.substr(1, index - 1))};
+ return {
+ store.parseStorePath(s.substr(index + 1)),
+ std::string(s.substr(1, index - 1)),
+ };
} else
- return {s.at(0) == '/' ? std::string(s) : std::string(s.substr(1)), ""};
+ return {
+ store.parseStorePath(
+ s.at(0) == '/'
+ ? s
+ : s.substr(1)),
+ "",
+ };
}
@@ -1920,13 +1919,13 @@ void copyContext(const Value & v, PathSet & context)
}
-std::vector<std::pair<Path, std::string>> Value::getContext()
+NixStringContext Value::getContext(const Store & store)
{
- std::vector<std::pair<Path, std::string>> res;
+ NixStringContext res;
assert(internalType == tString);
if (string.context)
for (const char * * p = string.context; *p; ++p)
- res.push_back(decodeContext(*p));
+ res.push_back(decodeContext(store, *p));
return res;
}
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 800b00eef..e7915dd99 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -133,9 +133,14 @@ private:
/* Cache used by prim_match(). */
std::shared_ptr<RegexCache> regexCache;
+#if HAVE_BOEHMGC
/* Allocation cache for GC'd Value objects. */
std::shared_ptr<void *> valueAllocCache;
+ /* Allocation cache for size-1 Env objects. */
+ std::shared_ptr<void *> env1AllocCache;
+#endif
+
public:
EvalState(
@@ -144,12 +149,6 @@ public:
std::shared_ptr<Store> buildStore = nullptr);
~EvalState();
- void requireExperimentalFeatureOnEvaluation(
- const ExperimentalFeature &,
- const std::string_view fName,
- const Pos & pos
- );
-
void addToSearchPath(const std::string & s);
SearchPath getSearchPath() { return searchPath; }
@@ -347,8 +346,8 @@ public:
void autoCallFunction(Bindings & args, Value & fun, Value & res);
/* Allocation primitives. */
- Value * allocValue();
- Env & allocEnv(size_t size);
+ inline Value * allocValue();
+ inline Env & allocEnv(size_t size);
Value * allocAttr(Value & vAttrs, const Symbol & name);
Value * allocAttr(Value & vAttrs, std::string_view name);
@@ -425,7 +424,7 @@ std::string showType(const Value & v);
/* Decode a context string ‘!<name>!<path>’ into a pair <path,
name>. */
-std::pair<std::string, std::string> decodeContext(std::string_view s);
+NixStringContextElem decodeContext(const Store & store, std::string_view s);
/* If `path' refers to a directory, then append "/default.nix". */
Path resolveExprPath(Path path);
@@ -509,3 +508,5 @@ extern EvalSettings evalSettings;
static const std::string corepkgsPrefix{"/__corepkgs__/"};
}
+
+#include "eval-inline.hh"
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 6a1aca40d..22257c6b3 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -706,8 +706,6 @@ void callFlake(EvalState & state,
static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
- state.requireExperimentalFeatureOnEvaluation(Xp::Flakes, "builtins.getFlake", pos);
-
std::string flakeRefS(state.forceStringNoCtx(*args[0], pos));
auto flakeRef = parseFlakeRef(flakeRefS, {}, true);
if (evalSettings.pureEval && !flakeRef.input.isLocked())
@@ -723,7 +721,30 @@ static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Va
v);
}
-static RegisterPrimOp r2("__getFlake", 1, prim_getFlake);
+static RegisterPrimOp r2({
+ .name = "__getFlake",
+ .args = {"args"},
+ .doc = R"(
+ Fetch a flake from a flake reference, and return its output attributes and some metadata. For example:
+
+ ```nix
+ (builtins.getFlake "nix/55bc52401966fbffa525c574c14f67b00bc4fb3a").packages.x86_64-linux.nix
+ ```
+
+ Unless impure evaluation is allowed (`--impure`), the flake reference
+ must be "locked", e.g. contain a Git revision or content hash. An
+ example of an unlocked usage is:
+
+ ```nix
+ (builtins.getFlake "github:edolstra/dwarffs").rev
+ ```
+
+ This function is only available if you enable the experimental feature
+ `flakes`.
+ )",
+ .fun = prim_getFlake,
+ .experimentalFeature = Xp::Flakes,
+});
}
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index 7630c5ff4..bb7e77b61 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -1,6 +1,7 @@
#include "get-drvs.hh"
#include "util.hh"
#include "eval-inline.hh"
+#include "derivations.hh"
#include "store-api.hh"
#include "path-with-outputs.hh"
@@ -102,7 +103,7 @@ StorePath DrvInfo::queryOutPath() const
}
-DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
+DrvInfo::Outputs DrvInfo::queryOutputs(bool withPaths, bool onlyOutputsToInstall)
{
if (outputs.empty()) {
/* Get the ‘outputs’ list. */
@@ -112,20 +113,24 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
/* For each output... */
for (auto elem : i->value->listItems()) {
- /* Evaluate the corresponding set. */
- std::string name(state->forceStringNoCtx(*elem, *i->pos));
- Bindings::iterator out = attrs->find(state->symbols.create(name));
- if (out == attrs->end()) continue; // FIXME: throw error?
- state->forceAttrs(*out->value, *i->pos);
-
- /* And evaluate its ‘outPath’ attribute. */
- Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
- if (outPath == out->value->attrs->end()) continue; // FIXME: throw error?
- PathSet context;
- outputs.emplace(name, state->coerceToStorePath(*outPath->pos, *outPath->value, context));
+ std::string output(state->forceStringNoCtx(*elem, *i->pos));
+
+ if (withPaths) {
+ /* Evaluate the corresponding set. */
+ Bindings::iterator out = attrs->find(state->symbols.create(output));
+ if (out == attrs->end()) continue; // FIXME: throw error?
+ state->forceAttrs(*out->value, *i->pos);
+
+ /* And evaluate its ‘outPath’ attribute. */
+ Bindings::iterator outPath = out->value->attrs->find(state->sOutPath);
+ if (outPath == out->value->attrs->end()) continue; // FIXME: throw error?
+ PathSet context;
+ outputs.emplace(output, state->coerceToStorePath(*outPath->pos, *outPath->value, context));
+ } else
+ outputs.emplace(output, std::nullopt);
}
} else
- outputs.emplace("out", queryOutPath());
+ outputs.emplace("out", withPaths ? std::optional{queryOutPath()} : std::nullopt);
}
if (!onlyOutputsToInstall || !attrs)
return outputs;
diff --git a/src/libexpr/get-drvs.hh b/src/libexpr/get-drvs.hh
index 3ca6f1fca..7cc1abef2 100644
--- a/src/libexpr/get-drvs.hh
+++ b/src/libexpr/get-drvs.hh
@@ -13,7 +13,7 @@ namespace nix {
struct DrvInfo
{
public:
- typedef std::map<std::string, StorePath> Outputs;
+ typedef std::map<std::string, std::optional<StorePath>> Outputs;
private:
EvalState * state;
@@ -46,8 +46,9 @@ public:
StorePath requireDrvPath() const;
StorePath queryOutPath() const;
std::string queryOutputName() const;
- /** Return the list of outputs. The "outputs to install" are determined by `meta.outputsToInstall`. */
- Outputs queryOutputs(bool onlyOutputsToInstall = false);
+ /** Return the unordered map of output names to (optional) output paths.
+ * The "outputs to install" are determined by `meta.outputsToInstall`. */
+ Outputs queryOutputs(bool withPaths = true, bool onlyOutputsToInstall = false);
StringSet queryMetaNames();
Value * queryMeta(const std::string & name);
diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index e276b0467..d574121b0 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -28,6 +28,13 @@ using namespace nix;
namespace nix {
+static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
+{
+ return Pos(data->origin, data->file, loc.first_line, loc.first_column);
+}
+
+#define CUR_POS makeCurPos(*yylloc, data)
+
// backup to recover from yyless(0)
YYLTYPE prev_yylloc;
@@ -37,7 +44,6 @@ static void initLoc(YYLTYPE * loc)
loc->first_column = loc->last_column = 1;
}
-
static void adjustLoc(YYLTYPE * loc, const char * s, size_t len)
{
prev_yylloc = *loc;
@@ -147,14 +153,20 @@ or { return OR_KW; }
try {
yylval->n = boost::lexical_cast<int64_t>(yytext);
} catch (const boost::bad_lexical_cast &) {
- throw ParseError("invalid integer '%1%'", yytext);
+ throw ParseError({
+ .msg = hintfmt("invalid integer '%1%'", yytext),
+ .errPos = CUR_POS,
+ });
}
return INT;
}
{FLOAT} { errno = 0;
yylval->nf = strtod(yytext, 0);
if (errno != 0)
- throw ParseError("invalid float '%1%'", yytext);
+ throw ParseError({
+ .msg = hintfmt("invalid float '%1%'", yytext),
+ .errPos = CUR_POS,
+ });
return FLOAT;
}
@@ -280,7 +292,10 @@ or { return OR_KW; }
<INPATH_SLASH>{ANY} |
<INPATH_SLASH><<EOF>> {
- throw ParseError("path has a trailing slash");
+ throw ParseError({
+ .msg = hintfmt("path has a trailing slash"),
+ .errPos = CUR_POS,
+ });
}
{SPATH} { yylval->path = {yytext, (size_t) yyleng}; return SPATH; }
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 12b54b8eb..4dbe31510 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -23,14 +23,13 @@ MakeError(RestrictedPathError, Error);
struct Pos
{
- FileOrigin origin;
Symbol file;
- unsigned int line, column;
-
- Pos() : origin(foString), line(0), column(0) { }
- Pos(FileOrigin origin, const Symbol & file, unsigned int line, unsigned int column)
- : origin(origin), file(file), line(line), column(column) { }
-
+ uint32_t line;
+ FileOrigin origin:2;
+ uint32_t column:30;
+ Pos() : line(0), origin(foString), column(0) { };
+ Pos(FileOrigin origin, const Symbol & file, uint32_t line, uint32_t column)
+ : file(file), line(line), origin(origin), column(column) { };
operator bool() const
{
return line != 0;
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 027030879..7d1d78d2e 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -43,8 +43,8 @@ StringMap EvalState::realiseContext(const PathSet & context)
StringMap res;
for (auto & i : context) {
- auto [ctxS, outputName] = decodeContext(i);
- auto ctx = store->parseStorePath(ctxS);
+ auto [ctx, outputName] = decodeContext(*store, i);
+ auto ctxS = store->printStorePath(ctx);
if (!store->isValidPath(ctx))
throw InvalidPathError(store->printStorePath(ctx));
if (!outputName.empty() && ctx.isDerivation()) {
@@ -694,7 +694,32 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
static RegisterPrimOp primop_genericClosure(RegisterPrimOp::Info {
.name = "__genericClosure",
+ .args = {"attrset"},
.arity = 1,
+ .doc = R"(
+ Take an *attrset* with values named `startSet` and `operator` in order to
+ return a *list of attrsets* by starting with the `startSet`, recursively
+ applying the `operator` function to each element. The *attrsets* in the
+ `startSet` and produced by the `operator` must each contain value named
+ `key` which are comparable to each other. The result is produced by
+ repeatedly calling the operator for each element encountered with a
+ unique key, terminating when no new elements are produced. For example,
+
+ ```
+ builtins.genericClosure {
+ startSet = [ {key = 5;} ];
+ operator = item: [{
+ key = if (item.key / 2 ) * 2 == item.key
+ then item.key / 2
+ else 3 * item.key + 1;
+ }];
+ }
+ ```
+ evaluates to
+ ```
+ [ { key = 5; } { key = 16; } { key = 8; } { key = 4; } { key = 2; } { key = 1; } ]
+ ```
+ )",
.fun = prim_genericClosure,
});
@@ -1114,7 +1139,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
drv.inputSrcs.insert(j);
if (j.isDerivation()) {
Derivation jDrv = state.store->readDerivation(j);
- if(jDrv.type() != DerivationType::CAFloating)
+ if(jDrv.type().hasKnownOutputPaths())
drv.inputDrvs[j] = jDrv.outputNames();
}
}
@@ -1122,8 +1147,8 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
/* Handle derivation outputs of the form ‘!<name>!<path>’. */
else if (path.at(0) == '!') {
- auto ctx = decodeContext(path);
- drv.inputDrvs[state.store->parseStorePath(ctx.first)].insert(ctx.second);
+ auto ctx = decodeContext(*state.store, path);
+ drv.inputDrvs[ctx.first].insert(ctx.second);
}
/* Otherwise it's a source file. */
@@ -1171,22 +1196,21 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
std::move(h),
{});
- DerivationOutputCAFixed dof { .ca = ca };
+ DerivationOutput::CAFixed dof { .ca = ca };
drv.env["out"] = state.store->printStorePath(dof.path(*state.store, drvName, "out"));
- drv.outputs.insert_or_assign("out", DerivationOutput { .output = dof });
+ drv.outputs.insert_or_assign("out", dof);
}
else if (contentAddressed) {
HashType ht = parseHashType(outputHashAlgo);
for (auto & i : outputs) {
drv.env[i] = hashPlaceholder(i);
- drv.outputs.insert_or_assign(i, DerivationOutput {
- .output = DerivationOutputCAFloating {
+ drv.outputs.insert_or_assign(i,
+ DerivationOutput::CAFloating {
.method = ingestionMethod,
.hashType = ht,
- },
- });
+ });
}
}
@@ -1200,43 +1224,36 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
for (auto & i : outputs) {
drv.env[i] = "";
drv.outputs.insert_or_assign(i,
- DerivationOutput {
- .output = DerivationOutputInputAddressed {
- .path = StorePath::dummy,
- },
- });
+ DerivationOutput::Deferred { });
}
// Regular, non-CA derivation should always return a single hash and not
// hash per output.
- auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
+ auto hashModulo = hashDerivationModulo(*state.store, drv, true);
std::visit(overloaded {
- [&](Hash & h) {
- for (auto & i : outputs) {
- auto outPath = state.store->makeOutputPath(i, h, drvName);
- drv.env[i] = state.store->printStorePath(outPath);
- drv.outputs.insert_or_assign(i,
- DerivationOutput {
- .output = DerivationOutputInputAddressed {
- .path = std::move(outPath),
- },
- });
+ [&](const DrvHash & drvHash) {
+ auto & h = drvHash.hash;
+ switch (drvHash.kind) {
+ case DrvHash::Kind::Deferred:
+ /* Outputs already deferred, nothing to do */
+ break;
+ case DrvHash::Kind::Regular:
+ for (auto & [outputName, output] : drv.outputs) {
+ auto outPath = state.store->makeOutputPath(outputName, h, drvName);
+ drv.env[outputName] = state.store->printStorePath(outPath);
+ output = DerivationOutput::InputAddressed {
+ .path = std::move(outPath),
+ };
+ }
+ break;
}
},
- [&](CaOutputHashes &) {
+ [&](const CaOutputHashes &) {
// Shouldn't happen as the toplevel derivation is not CA.
assert(false);
},
- [&](DeferredHash &) {
- for (auto & i : outputs) {
- drv.outputs.insert_or_assign(i,
- DerivationOutput {
- .output = DerivationOutputDeferred{},
- });
- }
- },
},
- hashModulo);
+ hashModulo.raw());
}
@@ -1248,12 +1265,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
/* Optimisation, but required in read-only mode! because in that
case we don't actually write store derivations, so we can't
- read them later.
-
- However, we don't bother doing this for floating CA derivations because
- their "hash modulo" is indeterminate until built. */
- if (drv.type() != DerivationType::CAFloating) {
- auto h = hashDerivationModulo(*state.store, Derivation(drv), false);
+ read them later. */
+ {
+ auto h = hashDerivationModulo(*state.store, drv, false);
drvHashes.lock()->insert_or_assign(drvPath, h);
}
@@ -3811,7 +3825,7 @@ RegisterPrimOp::RegisterPrimOp(std::string name, size_t arity, PrimOpFun fun)
.name = name,
.args = {},
.arity = arity,
- .fun = fun
+ .fun = fun,
});
}
@@ -3883,13 +3897,17 @@ void EvalState::createBaseEnv()
if (RegisterPrimOp::primOps)
for (auto & primOp : *RegisterPrimOp::primOps)
- addPrimOp({
- .fun = primOp.fun,
- .arity = std::max(primOp.args.size(), primOp.arity),
- .name = symbols.create(primOp.name),
- .args = primOp.args,
- .doc = primOp.doc,
- });
+ if (!primOp.experimentalFeature
+ || settings.isExperimentalFeatureEnabled(*primOp.experimentalFeature))
+ {
+ addPrimOp({
+ .fun = primOp.fun,
+ .arity = std::max(primOp.args.size(), primOp.arity),
+ .name = symbols.create(primOp.name),
+ .args = primOp.args,
+ .doc = primOp.doc,
+ });
+ }
/* Add a wrapper around the derivation primop that computes the
`drvPath' and `outPath' attributes lazily. */
diff --git a/src/libexpr/primops.hh b/src/libexpr/primops.hh
index 5b16e075f..905bd0366 100644
--- a/src/libexpr/primops.hh
+++ b/src/libexpr/primops.hh
@@ -16,6 +16,7 @@ struct RegisterPrimOp
size_t arity = 0;
const char * doc;
PrimOpFun fun;
+ std::optional<ExperimentalFeature> experimentalFeature;
};
typedef std::vector<Info> PrimOps;
@@ -35,6 +36,7 @@ struct RegisterPrimOp
/* These primops are disabled without enableNativeCode, but plugins
may wish to use them in limited contexts without globally enabling
them. */
+
/* Load a ValueInitializer from a DSO and return whatever it initializes */
void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value & v);
diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc
index 3701bd442..cc74c7f58 100644
--- a/src/libexpr/primops/context.cc
+++ b/src/libexpr/primops/context.cc
@@ -1,5 +1,6 @@
#include "primops.hh"
#include "eval-inline.hh"
+#include "derivations.hh"
#include "store-api.hh"
namespace nix {
@@ -82,8 +83,8 @@ static void prim_getContext(EvalState & state, const Pos & pos, Value * * args,
drv = std::string(p, 1);
path = &drv;
} else if (p.at(0) == '!') {
- std::pair<std::string, std::string> ctx = decodeContext(p);
- drv = ctx.first;
+ NixStringContextElem ctx = decodeContext(*state.store, p);
+ drv = state.store->printStorePath(ctx.first);
output = ctx.second;
path = &drv;
}
diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc
new file mode 100644
index 000000000..efeb93daf
--- /dev/null
+++ b/src/libexpr/primops/fetchClosure.cc
@@ -0,0 +1,154 @@
+#include "primops.hh"
+#include "store-api.hh"
+#include "make-content-addressed.hh"
+#include "url.hh"
+
+namespace nix {
+
+static void prim_fetchClosure(EvalState & state, const Pos & pos, Value * * args, Value & v)
+{
+ state.forceAttrs(*args[0], pos);
+
+ std::optional<std::string> fromStoreUrl;
+ std::optional<StorePath> fromPath;
+ bool toCA = false;
+ std::optional<StorePath> toPath;
+
+ for (auto & attr : *args[0]->attrs) {
+ if (attr.name == "fromPath") {
+ PathSet context;
+ fromPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
+ }
+
+ else if (attr.name == "toPath") {
+ state.forceValue(*attr.value, *attr.pos);
+ toCA = true;
+ if (attr.value->type() != nString || attr.value->string.s != std::string("")) {
+ PathSet context;
+ toPath = state.coerceToStorePath(*attr.pos, *attr.value, context);
+ }
+ }
+
+ else if (attr.name == "fromStore")
+ fromStoreUrl = state.forceStringNoCtx(*attr.value, *attr.pos);
+
+ else
+ throw Error({
+ .msg = hintfmt("attribute '%s' isn't supported in call to 'fetchClosure'", attr.name),
+ .errPos = pos
+ });
+ }
+
+ if (!fromPath)
+ throw Error({
+ .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromPath"),
+ .errPos = pos
+ });
+
+ if (!fromStoreUrl)
+ throw Error({
+ .msg = hintfmt("attribute '%s' is missing in call to 'fetchClosure'", "fromStore"),
+ .errPos = pos
+ });
+
+ auto parsedURL = parseURL(*fromStoreUrl);
+
+ if (parsedURL.scheme != "http" &&
+ parsedURL.scheme != "https" &&
+ !(getEnv("_NIX_IN_TEST").has_value() && parsedURL.scheme == "file"))
+ throw Error({
+ .msg = hintfmt("'fetchClosure' only supports http:// and https:// stores"),
+ .errPos = pos
+ });
+
+ auto fromStore = openStore(parsedURL.to_string());
+
+ if (toCA) {
+ if (!toPath || !state.store->isValidPath(*toPath)) {
+ auto remappings = makeContentAddressed(*fromStore, *state.store, { *fromPath });
+ auto i = remappings.find(*fromPath);
+ assert(i != remappings.end());
+ if (toPath && *toPath != i->second)
+ throw Error({
+ .msg = hintfmt("rewriting '%s' to content-addressed form yielded '%s', while '%s' was expected",
+ state.store->printStorePath(*fromPath),
+ state.store->printStorePath(i->second),
+ state.store->printStorePath(*toPath)),
+ .errPos = pos
+ });
+ if (!toPath)
+ throw Error({
+ .msg = hintfmt(
+ "rewriting '%s' to content-addressed form yielded '%s'; "
+ "please set this in the 'toPath' attribute passed to 'fetchClosure'",
+ state.store->printStorePath(*fromPath),
+ state.store->printStorePath(i->second)),
+ .errPos = pos
+ });
+ }
+ } else {
+ copyClosure(*fromStore, *state.store, RealisedPath::Set { *fromPath });
+ toPath = fromPath;
+ }
+
+ /* In pure mode, require a CA path. */
+ if (evalSettings.pureEval) {
+ auto info = state.store->queryPathInfo(*toPath);
+ if (!info->isContentAddressed(*state.store))
+ throw Error({
+ .msg = hintfmt("in pure mode, 'fetchClosure' requires a content-addressed path, which '%s' isn't",
+ state.store->printStorePath(*toPath)),
+ .errPos = pos
+ });
+ }
+
+ auto toPathS = state.store->printStorePath(*toPath);
+ v.mkString(toPathS, {toPathS});
+}
+
+static RegisterPrimOp primop_fetchClosure({
+ .name = "__fetchClosure",
+ .args = {"args"},
+ .doc = R"(
+ Fetch a Nix store closure from a binary cache, rewriting it into
+ content-addressed form. For example,
+
+ ```nix
+ builtins.fetchClosure {
+ fromStore = "https://cache.nixos.org";
+ fromPath = /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1;
+ toPath = /nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1;
+ }
+ ```
+
+ fetches `/nix/store/r2jd...` from the specified binary cache,
+ and rewrites it into the content-addressed store path
+ `/nix/store/ldbh...`.
+
+ If `fromPath` is already content-addressed, or if you are
+ allowing impure evaluation (`--impure`), then `toPath` may be
+ omitted.
+
+ To find out the correct value for `toPath` given a `fromPath`,
+ you can use `nix store make-content-addressed`:
+
+ ```console
+ # nix store make-content-addressed --from https://cache.nixos.org /nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1
+ rewrote '/nix/store/r2jd6ygnmirm2g803mksqqjm4y39yi6i-git-2.33.1' to '/nix/store/ldbhlwhh39wha58rm61bkiiwm6j7211j-git-2.33.1'
+ ```
+
+ This function is similar to `builtins.storePath` in that it
+ allows you to use a previously built store path in a Nix
+ expression. However, it is more reproducible because it requires
+ specifying a binary cache from which the path can be fetched.
+ Also, requiring a content-addressed final store path avoids the
+ need for users to configure binary cache public keys.
+
+ This function is only available if you enable the experimental
+ feature `fetch-closure`.
+ )",
+ .fun = prim_fetchClosure,
+ .experimentalFeature = Xp::FetchClosure,
+});
+
+}
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 281f9bc2c..c12872084 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -145,7 +145,7 @@ static void fetchTree(
if (!params.allowNameArgument)
if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
throw Error({
- .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"),
+ .msg = hintfmt("attribute 'name' isn't supported in call to 'fetchTree'"),
.errPos = pos
});
@@ -334,7 +334,7 @@ static RegisterPrimOp primop_fetchTarball({
.fun = prim_fetchTarball,
});
-static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v)
+static void prim_fetchGit(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true });
}
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index d0fa93e92..3d07c3198 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -57,6 +57,8 @@ struct ExprLambda;
struct PrimOp;
class Symbol;
struct Pos;
+class StorePath;
+class Store;
class EvalState;
class XMLWriter;
class JSONPlaceholder;
@@ -64,6 +66,8 @@ class JSONPlaceholder;
typedef int64_t NixInt;
typedef double NixFloat;
+typedef std::pair<StorePath, std::string> NixStringContextElem;
+typedef std::vector<NixStringContextElem> NixStringContext;
/* External values must descend from ExternalValueBase, so that
* type-agnostic nix functions (e.g. showType) can be implemented
@@ -115,10 +119,13 @@ private:
InternalType internalType;
friend std::string showType(const Value & v);
- friend void printValue(std::ostream & str, std::set<const void *> & seen, const Value & v);
+
+ void print(std::ostream & str, std::set<const void *> * seen) const;
public:
+ void print(std::ostream & str, bool showRepeated = false) const;
+
// Functions needed to distinguish the type
// These should be removed eventually, by putting the functionality that's
// needed by callers into methods of this type
@@ -368,7 +375,7 @@ public:
non-trivial. */
bool isTrivial() const;
- std::vector<std::pair<Path, std::string>> getContext();
+ NixStringContext getContext(const Store &);
auto listItems()
{
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index c0beca2f2..d75c5d3ae 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -222,22 +222,46 @@ struct GitInputScheme : InputScheme
if (!input.getRef() && !input.getRev() && isLocal) {
bool clean = false;
- /* Check whether this repo has any commits. There are
- probably better ways to do this. */
- auto gitDir = actualUrl + "/.git";
- auto commonGitDir = chomp(runProgram(
- "git",
- true,
- { "-C", actualUrl, "rev-parse", "--git-common-dir" }
- ));
- if (commonGitDir != ".git")
- gitDir = commonGitDir;
-
- bool haveCommits = !readDirectory(gitDir + "/refs/heads").empty();
+ auto env = getEnv();
+ // Set LC_ALL to C: because we rely on the error messages from git rev-parse to determine what went wrong
+ // that way unknown errors can lead to a failure instead of continuing through the wrong code path
+ env["LC_ALL"] = "C";
+
+ /* Check whether HEAD points to something that looks like a commit,
+ since that is the refrence we want to use later on. */
+ auto result = runProgram(RunOptions {
+ .program = "git",
+ .args = { "-C", actualUrl, "--git-dir=.git", "rev-parse", "--verify", "--no-revs", "HEAD^{commit}" },
+ .environment = env,
+ .mergeStderrToStdout = true
+ });
+ auto exitCode = WEXITSTATUS(result.first);
+ auto errorMessage = result.second;
+
+ if (errorMessage.find("fatal: not a git repository") != std::string::npos) {
+ throw Error("'%s' is not a Git repository", actualUrl);
+ } else if (errorMessage.find("fatal: Needed a single revision") != std::string::npos) {
+ // indicates that the repo does not have any commits
+ // we want to proceed and will consider it dirty later
+ } else if (exitCode != 0) {
+ // any other errors should lead to a failure
+ throw Error("getting the HEAD of the Git tree '%s' failed with exit code %d:\n%s", actualUrl, exitCode, errorMessage);
+ }
+ bool hasHead = exitCode == 0;
try {
- if (haveCommits) {
- runProgram("git", true, { "-C", actualUrl, "diff-index", "--quiet", "HEAD", "--" });
+ if (hasHead) {
+ // Using git diff is preferrable over lower-level operations here,
+ // because its conceptually simpler and we only need the exit code anyways.
+ auto gitDiffOpts = Strings({ "-C", actualUrl, "diff", "HEAD", "--quiet"});
+ if (!submodules) {
+ // Changes in submodules should only make the tree dirty
+ // when those submodules will be copied as well.
+ gitDiffOpts.emplace_back("--ignore-submodules");
+ }
+ gitDiffOpts.emplace_back("--");
+ runProgram("git", true, gitDiffOpts);
+
clean = true;
}
} catch (ExecError & e) {
@@ -282,7 +306,7 @@ struct GitInputScheme : InputScheme
// modified dirty file?
input.attrs.insert_or_assign(
"lastModified",
- haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
+ hasHead ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
return {std::move(storePath), input};
}
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index a1430f087..58b6e7c04 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -390,7 +390,7 @@ struct SourceHutInputScheme : GitArchiveInputScheme
ref_uri = line.substr(ref_index+5, line.length()-1);
} else
- ref_uri = fmt("refs/heads/%s", ref);
+ ref_uri = fmt("refs/(heads|tags)/%s", ref);
auto file = store->toRealPath(
downloadFile(store, fmt("%s/info/refs", base_url), "source", false, headers).storePath);
@@ -399,9 +399,11 @@ struct SourceHutInputScheme : GitArchiveInputScheme
std::string line;
std::string id;
while(getline(is, line)) {
- auto index = line.find(ref_uri);
- if (index != std::string::npos) {
- id = line.substr(0, index-1);
+ // Append $ to avoid partial name matches
+ std::regex pattern(fmt("%s$", ref_uri));
+
+ if (std::regex_search(line, pattern)) {
+ id = line.substr(0, line.find('\t'));
break;
}
}
diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
index 59e228e97..f0ef97da5 100644
--- a/src/libfetchers/path.cc
+++ b/src/libfetchers/path.cc
@@ -1,5 +1,6 @@
#include "fetchers.hh"
#include "store-api.hh"
+#include "archive.hh"
namespace nix::fetchers {
@@ -80,8 +81,9 @@ struct PathInputScheme : InputScheme
// nothing to do
}
- std::pair<StorePath, Input> fetch(ref<Store> store, const Input & input) override
+ std::pair<StorePath, Input> fetch(ref<Store> store, const Input & _input) override
{
+ Input input(_input);
std::string absPath;
auto path = getStrAttr(input.attrs, "path");
@@ -111,9 +113,15 @@ struct PathInputScheme : InputScheme
if (storePath)
store->addTempRoot(*storePath);
- if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath))
+ time_t mtime = 0;
+ if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) {
// FIXME: try to substitute storePath.
- storePath = store->addToStore("source", absPath);
+ auto src = sinkToSource([&](Sink & sink) {
+ mtime = dumpPathAndGetMtime(absPath, sink, defaultPathFilter);
+ });
+ storePath = store->addToStoreFromDump(*src, "source");
+ }
+ input.attrs.insert_or_assign("lastModified", uint64_t(mtime));
return {std::move(*storePath), input};
}
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 9603a8caa..ca538b3cb 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -2,6 +2,7 @@
#include "crypto.hh"
#include "store-api.hh"
+#include "log-store.hh"
#include "pool.hh"
@@ -28,7 +29,9 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
"other than -1 which we reserve to indicate Nix defaults should be used"};
};
-class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store
+class BinaryCacheStore : public virtual BinaryCacheStoreConfig,
+ public virtual Store,
+ public virtual LogStore
{
private:
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index afed9bf16..3d1c4fbc1 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -204,11 +204,9 @@ void DerivationGoal::haveDerivation()
{
trace("have derivation");
- if (drv->type() == DerivationType::CAFloating)
+ if (!drv->type().hasKnownOutputPaths())
settings.requireExperimentalFeature(Xp::CaDerivations);
- retrySubstitution = false;
-
for (auto & i : drv->outputsAndOptPaths(worker.store))
if (i.second.second)
worker.store.addTempRoot(*i.second.second);
@@ -311,14 +309,11 @@ void DerivationGoal::outputsSubstitutionTried()
gaveUpOnSubstitution();
}
+
/* At least one of the output paths could not be
produced using a substitute. So we have to build instead. */
void DerivationGoal::gaveUpOnSubstitution()
{
- /* Make sure checkPathValidity() from now on checks all
- outputs. */
- wantedOutputs.clear();
-
/* The inputs must be built before we can build this goal. */
if (useDerivation)
for (auto & i : dynamic_cast<Derivation *>(drv.get())->inputDrvs)
@@ -426,7 +421,8 @@ void DerivationGoal::inputsRealised()
return;
}
- if (retrySubstitution) {
+ if (retrySubstitution && !retriedSubstitution) {
+ retriedSubstitution = true;
haveDerivation();
return;
}
@@ -440,9 +436,28 @@ void DerivationGoal::inputsRealised()
if (useDerivation) {
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
- if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations) &&
- ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
- || fullDrv.type() == DerivationType::DeferredInputAddressed)) {
+ auto drvType = fullDrv.type();
+ bool resolveDrv = std::visit(overloaded {
+ [&](const DerivationType::InputAddressed & ia) {
+ /* must resolve if deferred. */
+ return ia.deferred;
+ },
+ [&](const DerivationType::ContentAddressed & ca) {
+ return !fullDrv.inputDrvs.empty() && (
+ ca.fixed
+ /* Can optionally resolve if fixed, which is good
+ for avoiding unnecessary rebuilds. */
+ ? settings.isExperimentalFeatureEnabled(Xp::CaDerivations)
+ /* Must resolve if floating and there are any inputs
+ drvs. */
+ : true);
+ },
+ }, drvType.raw());
+
+ if (resolveDrv)
+ {
+ settings.requireExperimentalFeature(Xp::CaDerivations);
+
/* We are be able to resolve this derivation based on the
now-known results of dependencies. If so, we become a stub goal
aliasing that resolved derivation goal */
@@ -501,7 +516,7 @@ void DerivationGoal::inputsRealised()
/* Don't repeat fixed-output derivations since they're already
verified by their output hash.*/
- nrRounds = derivationIsFixed(derivationType) ? 1 : settings.buildRepeat + 1;
+ nrRounds = derivationType.isFixed() ? 1 : settings.buildRepeat + 1;
/* Okay, try to build. Note that here we don't wait for a build
slot to become available, since we don't need one if there is a
@@ -908,7 +923,7 @@ void DerivationGoal::buildDone()
st =
dynamic_cast<NotDeterministic*>(&e) ? BuildResult::NotDeterministic :
statusOk(status) ? BuildResult::OutputRejected :
- derivationIsImpure(derivationType) || diskFull ? BuildResult::TransientFailure :
+ derivationType.isImpure() || diskFull ? BuildResult::TransientFailure :
BuildResult::PermanentFailure;
}
@@ -1221,7 +1236,7 @@ void DerivationGoal::flushLine()
std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDerivationOutputMap()
{
- if (!useDerivation || drv->type() != DerivationType::CAFloating) {
+ if (!useDerivation || drv->type().hasKnownOutputPaths()) {
std::map<std::string, std::optional<StorePath>> res;
for (auto & [name, output] : drv->outputs)
res.insert_or_assign(name, output.path(worker.store, drv->name, name));
@@ -1233,7 +1248,7 @@ std::map<std::string, std::optional<StorePath>> DerivationGoal::queryPartialDeri
OutputPathMap DerivationGoal::queryDerivationOutputMap()
{
- if (!useDerivation || drv->type() != DerivationType::CAFloating) {
+ if (!useDerivation || drv->type().hasKnownOutputPaths()) {
OutputPathMap res;
for (auto & [name, output] : drv->outputsAndOptPaths(worker.store))
res.insert_or_assign(name, *output.second);
diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh
index ea2db89b2..f556b6f25 100644
--- a/src/libstore/build/derivation-goal.hh
+++ b/src/libstore/build/derivation-goal.hh
@@ -61,8 +61,12 @@ struct DerivationGoal : public Goal
bool needRestart = false;
/* Whether to retry substituting the outputs after building the
- inputs. */
- bool retrySubstitution;
+ inputs. This is done in case of an incomplete closure. */
+ bool retrySubstitution = false;
+
+ /* Whether we've retried substitution, in which case we won't try
+ again. */
+ bool retriedSubstitution = false;
/* The derivation stored at drvPath. */
std::unique_ptr<Derivation> drv;
diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc
index d2420b107..58e805f55 100644
--- a/src/libstore/build/goal.cc
+++ b/src/libstore/build/goal.cc
@@ -28,7 +28,7 @@ void Goal::addWaitee(GoalPtr waitee)
void Goal::waiteeDone(GoalPtr waitee, ExitCode result)
{
- assert(waitees.find(waitee) != waitees.end());
+ assert(waitees.count(waitee));
waitees.erase(waitee);
trace(fmt("waitee '%s' done; %d left", waitee->name, waitees.size()));
diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh
index 07c752bb9..35121c5d9 100644
--- a/src/libstore/build/goal.hh
+++ b/src/libstore/build/goal.hh
@@ -40,21 +40,21 @@ struct Goal : public std::enable_shared_from_this<Goal>
WeakGoals waiters;
/* Number of goals we are/were waiting for that have failed. */
- unsigned int nrFailed;
+ size_t nrFailed = 0;
/* Number of substitution goals we are/were waiting for that
failed because there are no substituters. */
- unsigned int nrNoSubstituters;
+ size_t nrNoSubstituters = 0;
/* Number of substitution goals we are/were waiting for that
failed because they had unsubstitutable references. */
- unsigned int nrIncompleteClosure;
+ size_t nrIncompleteClosure = 0;
/* Name of this goal for debugging purposes. */
std::string name;
/* Whether the goal is finished. */
- ExitCode exitCode;
+ ExitCode exitCode = ecBusy;
/* Build result. */
BuildResult buildResult;
@@ -65,10 +65,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
Goal(Worker & worker, DerivedPath path)
: worker(worker)
, buildResult { .path = std::move(path) }
- {
- nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
- exitCode = ecBusy;
- }
+ { }
virtual ~Goal()
{
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index 14b1582f1..412c3df29 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -395,7 +395,7 @@ void LocalDerivationGoal::startBuilder()
else if (settings.sandboxMode == smDisabled)
useChroot = false;
else if (settings.sandboxMode == smRelaxed)
- useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
+ useChroot = !(derivationType.isImpure()) && !noChroot;
}
auto & localStore = getLocalStore();
@@ -608,7 +608,7 @@ void LocalDerivationGoal::startBuilder()
"nogroup:x:65534:\n", sandboxGid()));
/* Create /etc/hosts with localhost entry. */
- if (!(derivationIsImpure(derivationType)))
+ if (!(derivationType.isImpure()))
writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
/* Make the closure of the inputs available in the chroot,
@@ -796,7 +796,7 @@ void LocalDerivationGoal::startBuilder()
us.
*/
- if (!(derivationIsImpure(derivationType)))
+ if (!(derivationType.isImpure()))
privateNetwork = true;
userNamespaceSync.create();
@@ -1049,7 +1049,7 @@ void LocalDerivationGoal::initEnv()
derivation, tell the builder, so that for instance `fetchurl'
can skip checking the output. On older Nixes, this environment
variable won't be set, so `fetchurl' will do the check. */
- if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
+ if (derivationType.isFixed()) env["NIX_OUTPUT_CHECKED"] = "1";
/* *Only* if this is a fixed-output derivation, propagate the
values of the environment variables specified in the
@@ -1060,7 +1060,7 @@ void LocalDerivationGoal::initEnv()
to the builder is generally impure, but the output of
fixed-output derivations is by definition pure (since we
already know the cryptographic hash of the output). */
- if (derivationIsImpure(derivationType)) {
+ if (derivationType.isImpure()) {
for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
env[i] = getEnv(i).value_or("");
}
@@ -1340,6 +1340,12 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo
next->queryMissing(allowed, willBuild, willSubstitute,
unknown, downloadSize, narSize);
}
+
+ virtual std::optional<std::string> getBuildLog(const StorePath & path) override
+ { return std::nullopt; }
+
+ virtual void addBuildLog(const StorePath & path, std::string_view log) override
+ { unsupported("addBuildLog"); }
};
@@ -1668,7 +1674,7 @@ void LocalDerivationGoal::runChild()
/* Fixed-output derivations typically need to access the
network, so give them access to /etc/resolv.conf and so
on. */
- if (derivationIsImpure(derivationType)) {
+ if (derivationType.isImpure()) {
// Only use nss functions to resolve hosts and
// services. Don’t use it for anything else that may
// be configured for this system. This limits the
@@ -1912,7 +1918,7 @@ void LocalDerivationGoal::runChild()
sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
- if (derivationIsImpure(derivationType))
+ if (derivationType.isImpure())
sandboxProfile += "(import \"sandbox-network.sb\")\n";
/* Add the output paths we'll use at build-time to the chroot */
@@ -2272,7 +2278,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
return res;
};
- auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
+ auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo {
auto & st = outputStats.at(outputName);
if (outputHash.method == ContentAddressMethod { FileIngestionMethod::Flat } ||
outputHash.method == ContentAddressMethod { TextHashMethod {} })
@@ -2340,7 +2346,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
ValidPathInfo newInfo = std::visit(overloaded {
- [&](const DerivationOutputInputAddressed & output) {
+ [&](const DerivationOutput::InputAddressed & output) {
/* input-addressed case */
auto requiredFinalPath = output.path;
/* Preemptively add rewrite rule for final hash, as that is
@@ -2357,7 +2363,7 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
return newInfo0;
},
- [&](const DerivationOutputCAFixed & dof) {
+ [&](const DerivationOutput::CAFixed & dof) {
auto wanted = getContentAddressHash(dof.ca);
auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
@@ -2386,17 +2392,17 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
return newInfo0;
},
- [&](DerivationOutputCAFloating & dof) {
+ [&](const DerivationOutput::CAFloating & dof) {
return newInfoFromCA(dof);
},
- [&](DerivationOutputDeferred) -> ValidPathInfo {
+ [&](const DerivationOutput::Deferred &) -> ValidPathInfo {
// No derivation should reach that point without having been
// rewritten first
assert(false);
},
- }, output.output);
+ }, output.raw());
/* FIXME: set proper permissions in restorePath() so
we don't have to do another traversal. */
@@ -2610,7 +2616,8 @@ DrvOutputs LocalDerivationGoal::registerOutputs()
signRealisation(thisRealisation);
worker.store.registerDrvOutput(thisRealisation);
}
- builtOutputs.emplace(thisRealisation.id, thisRealisation);
+ if (wantOutput(outputName, wantedOutputs))
+ builtOutputs.emplace(thisRealisation.id, thisRealisation);
}
return builtOutputs;
diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc
index 25d015cb9..6f6ad57cb 100644
--- a/src/libstore/builtins/buildenv.cc
+++ b/src/libstore/builtins/buildenv.cc
@@ -47,9 +47,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
throw;
}
- /* The files below are special-cased to that they don't show up
- * in user profiles, either because they are useless, or
- * because they would cauase pointless collisions (e.g., each
+ /* The files below are special-cased to that they don't show
+ * up in user profiles, either because they are useless, or
+ * because they would cause pointless collisions (e.g., each
* Python package brings its own
* `$out/lib/pythonX.Y/site-packages/easy-install.pth'.)
*/
@@ -57,7 +57,9 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
hasSuffix(srcFile, "/nix-support") ||
hasSuffix(srcFile, "/perllocal.pod") ||
hasSuffix(srcFile, "/info/dir") ||
- hasSuffix(srcFile, "/log"))
+ hasSuffix(srcFile, "/log") ||
+ hasSuffix(srcFile, "/manifest.nix") ||
+ hasSuffix(srcFile, "/manifest.json"))
continue;
else if (S_ISDIR(srcSt.st_mode)) {
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 4b5a57ff1..8271bc8d6 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -3,7 +3,9 @@
#include "worker-protocol.hh"
#include "build-result.hh"
#include "store-api.hh"
+#include "store-cast.hh"
#include "gc-store.hh"
+#include "log-store.hh"
#include "path-with-outputs.hh"
#include "finally.hh"
#include "archive.hh"
@@ -562,6 +564,8 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
BuildMode buildMode = (BuildMode) readInt(from);
logger->startWork();
+ auto drvType = drv.type();
+
/* Content-addressed derivations are trustless because their output paths
are verified by their content alone, so any derivation is free to
try to produce such a path.
@@ -594,12 +598,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
derivations, we throw out the precomputed output paths and just
store the hashes, so there aren't two competing sources of truth an
attacker could exploit. */
- if (drv.type() == DerivationType::InputAddressed && !trusted)
+ if (!(drvType.isCA() || trusted))
throw Error("you are not privileged to build input-addressed derivations");
/* Make sure that the non-input-addressed derivations that got this far
are in fact content-addressed if we don't trust them. */
- assert(derivationIsCA(drv.type()) || trusted);
+ assert(drvType.isCA() || trusted);
/* Recompute the derivation path when we cannot trust the original. */
if (!trusted) {
@@ -608,7 +612,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
original not-necessarily-resolved derivation to verify the drv
derivation as adequate claim to the input-addressed output
paths. */
- assert(derivationIsCA(drv.type()));
+ assert(drvType.isCA());
Derivation drv2;
static_cast<BasicDerivation &>(drv2) = drv;
@@ -649,7 +653,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
Path path = absPath(readString(from));
logger->startWork();
- auto & gcStore = requireGcStore(*store);
+ auto & gcStore = require<GcStore>(*store);
gcStore.addIndirectRoot(path);
logger->stopWork();
@@ -667,7 +671,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopFindRoots: {
logger->startWork();
- auto & gcStore = requireGcStore(*store);
+ auto & gcStore = require<GcStore>(*store);
Roots roots = gcStore.findRoots(!trusted);
logger->stopWork();
@@ -699,7 +703,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
if (options.ignoreLiveness)
throw Error("you are not allowed to ignore liveness");
- auto & gcStore = requireGcStore(*store);
+ auto & gcStore = require<GcStore>(*store);
gcStore.collectGarbage(options, results);
logger->stopWork();
@@ -957,11 +961,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
logger->startWork();
if (!trusted)
throw Error("you are not privileged to add logs");
+ auto & logStore = require<LogStore>(*store);
{
FramedSource source(from);
StringSink sink;
source.drainInto(sink);
- store->addBuildLog(path, sink.s);
+ logStore.addBuildLog(path, sink.s);
}
logger->stopWork();
to << 1;
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index f6f41361b..00757abc8 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -12,25 +12,25 @@ namespace nix {
std::optional<StorePath> DerivationOutput::path(const Store & store, std::string_view drvName, std::string_view outputName) const
{
return std::visit(overloaded {
- [](const DerivationOutputInputAddressed & doi) -> std::optional<StorePath> {
+ [](const DerivationOutput::InputAddressed & doi) -> std::optional<StorePath> {
return { doi.path };
},
- [&](const DerivationOutputCAFixed & dof) -> std::optional<StorePath> {
+ [&](const DerivationOutput::CAFixed & dof) -> std::optional<StorePath> {
return {
dof.path(store, drvName, outputName)
};
},
- [](const DerivationOutputCAFloating & dof) -> std::optional<StorePath> {
+ [](const DerivationOutput::CAFloating & dof) -> std::optional<StorePath> {
return std::nullopt;
},
- [](const DerivationOutputDeferred &) -> std::optional<StorePath> {
+ [](const DerivationOutput::Deferred &) -> std::optional<StorePath> {
return std::nullopt;
},
- }, output);
+ }, raw());
}
-StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
+StorePath DerivationOutput::CAFixed::path(const Store & store, std::string_view drvName, std::string_view outputName) const {
return store.makeFixedOutputPathFromCA(StorePathDescriptor {
.name = outputPathName(drvName, outputName),
.info = ca,
@@ -38,47 +38,46 @@ StorePath DerivationOutputCAFixed::path(const Store & store, std::string_view dr
}
-bool derivationIsCA(DerivationType dt) {
- switch (dt) {
- case DerivationType::InputAddressed: return false;
- case DerivationType::CAFixed: return true;
- case DerivationType::CAFloating: return true;
- case DerivationType::DeferredInputAddressed: return false;
- };
- // Since enums can have non-variant values, but making a `default:` would
- // disable exhaustiveness warnings.
- assert(false);
+bool DerivationType::isCA() const {
+ /* Normally we do the full `std::visit` to make sure we have
+ exhaustively handled all variants, but so long as there is a
+ variant called `ContentAddressed`, it must be the only one for
+ which `isCA` is true for this to make sense!. */
+ return std::holds_alternative<ContentAddressed>(raw());
}
-bool derivationIsFixed(DerivationType dt) {
- switch (dt) {
- case DerivationType::InputAddressed: return false;
- case DerivationType::CAFixed: return true;
- case DerivationType::CAFloating: return false;
- case DerivationType::DeferredInputAddressed: return false;
- };
- assert(false);
+bool DerivationType::isFixed() const {
+ return std::visit(overloaded {
+ [](const InputAddressed & ia) {
+ return false;
+ },
+ [](const ContentAddressed & ca) {
+ return ca.fixed;
+ },
+ }, raw());
}
-bool derivationHasKnownOutputPaths(DerivationType dt) {
- switch (dt) {
- case DerivationType::InputAddressed: return true;
- case DerivationType::CAFixed: return true;
- case DerivationType::CAFloating: return false;
- case DerivationType::DeferredInputAddressed: return false;
- };
- assert(false);
+bool DerivationType::hasKnownOutputPaths() const {
+ return std::visit(overloaded {
+ [](const InputAddressed & ia) {
+ return !ia.deferred;
+ },
+ [](const ContentAddressed & ca) {
+ return ca.fixed;
+ },
+ }, raw());
}
-bool derivationIsImpure(DerivationType dt) {
- switch (dt) {
- case DerivationType::InputAddressed: return false;
- case DerivationType::CAFixed: return true;
- case DerivationType::CAFloating: return false;
- case DerivationType::DeferredInputAddressed: return false;
- };
- assert(false);
+bool DerivationType::isImpure() const {
+ return std::visit(overloaded {
+ [](const InputAddressed & ia) {
+ return false;
+ },
+ [](const ContentAddressed & ca) {
+ return !ca.pure;
+ },
+ }, raw());
}
@@ -178,34 +177,26 @@ static DerivationOutput parseDerivationOutput(const Store & store,
if (hashS != "") {
validatePath(pathS);
auto hash = Hash::parseNonSRIUnprefixed(hashS, hashType);
- return DerivationOutput {
- .output = DerivationOutputCAFixed {
- // FIXME non-trivial fixed refs set
- .ca = contentAddressFromMethodHashAndRefs(
- method, std::move(hash), {}),
- },
+ return DerivationOutput::CAFixed {
+ // FIXME non-trivial fixed refs set
+ .ca = contentAddressFromMethodHashAndRefs(
+ method, std::move(hash), {}),
};
} else {
settings.requireExperimentalFeature(Xp::CaDerivations);
assert(pathS == "");
- return DerivationOutput {
- .output = DerivationOutputCAFloating {
- .method = std::move(method),
- .hashType = std::move(hashType),
- },
+ return DerivationOutput::CAFloating {
+ .method = std::move(method),
+ .hashType = std::move(hashType),
};
}
} else {
if (pathS == "") {
- return DerivationOutput {
- .output = DerivationOutputDeferred { }
- };
+ return DerivationOutput::Deferred { };
}
validatePath(pathS);
- return DerivationOutput {
- .output = DerivationOutputInputAddressed {
- .path = store.parseStorePath(pathS),
- }
+ return DerivationOutput::InputAddressed {
+ .path = store.parseStorePath(pathS),
};
}
}
@@ -333,27 +324,27 @@ std::string Derivation::unparse(const Store & store, bool maskOutputs,
if (first) first = false; else s += ',';
s += '('; printUnquotedString(s, i.first);
std::visit(overloaded {
- [&](const DerivationOutputInputAddressed & doi) {
+ [&](const DerivationOutput::InputAddressed & doi) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(doi.path));
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
},
- [&](const DerivationOutputCAFixed & dof) {
+ [&](const DerivationOutput::CAFixed & dof) {
s += ','; printUnquotedString(s, maskOutputs ? "" : store.printStorePath(dof.path(store, name, i.first)));
s += ','; printUnquotedString(s, printMethodAlgo(dof.ca));
s += ','; printUnquotedString(s, getContentAddressHash(dof.ca).to_string(Base16, false));
},
- [&](const DerivationOutputCAFloating & dof) {
+ [&](const DerivationOutput::CAFloating & dof) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, makeContentAddressingPrefix(dof.method) + printHashType(dof.hashType));
s += ','; printUnquotedString(s, "");
},
- [&](const DerivationOutputDeferred &) {
+ [&](const DerivationOutput::Deferred &) {
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
s += ','; printUnquotedString(s, "");
}
- }, i.second.output);
+ }, i.second.raw());
s += ')';
}
@@ -421,13 +412,13 @@ DerivationType BasicDerivation::type() const
std::optional<HashType> floatingHashType;
for (auto & i : outputs) {
std::visit(overloaded {
- [&](const DerivationOutputInputAddressed &) {
+ [&](const DerivationOutput::InputAddressed &) {
inputAddressedOutputs.insert(i.first);
},
- [&](const DerivationOutputCAFixed &) {
+ [&](const DerivationOutput::CAFixed &) {
fixedCAOutputs.insert(i.first);
},
- [&](const DerivationOutputCAFloating & dof) {
+ [&](const DerivationOutput::CAFloating & dof) {
floatingCAOutputs.insert(i.first);
if (!floatingHashType) {
floatingHashType = dof.hashType;
@@ -436,27 +427,37 @@ DerivationType BasicDerivation::type() const
throw Error("All floating outputs must use the same hash type");
}
},
- [&](const DerivationOutputDeferred &) {
+ [&](const DerivationOutput::Deferred &) {
deferredIAOutputs.insert(i.first);
},
- }, i.second.output);
+ }, i.second.raw());
}
if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
throw Error("Must have at least one output");
} else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
- return DerivationType::InputAddressed;
+ return DerivationType::InputAddressed {
+ .deferred = false,
+ };
} else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
if (fixedCAOutputs.size() > 1)
// FIXME: Experimental feature?
throw Error("Only one fixed output is allowed for now");
if (*fixedCAOutputs.begin() != "out")
throw Error("Single fixed output must be named \"out\"");
- return DerivationType::CAFixed;
+ return DerivationType::ContentAddressed {
+ .pure = false,
+ .fixed = true,
+ };
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
- return DerivationType::CAFloating;
+ return DerivationType::ContentAddressed {
+ .pure = true,
+ .fixed = false,
+ };
} else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) {
- return DerivationType::DeferredInputAddressed;
+ return DerivationType::InputAddressed {
+ .deferred = true,
+ };
} else {
throw Error("Can't mix derivation output types");
}
@@ -508,13 +509,13 @@ static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath &
*/
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
{
- bool isDeferred = false;
+ auto type = drv.type();
+
/* Return a fixed hash for fixed-output derivations. */
- switch (drv.type()) {
- case DerivationType::CAFixed: {
+ if (type.isFixed()) {
std::map<std::string, Hash> outputHashes;
for (const auto & i : drv.outputs) {
- auto & dof = std::get<DerivationOutputCAFixed>(i.second.output);
+ auto & dof = std::get<DerivationOutput::CAFixed>(i.second.raw());
auto hash = hashString(htSHA256, "fixed:out:"
+ printMethodAlgo(dof.ca) + ":"
+ getContentAddressHash(dof.ca).to_string(Base16, false) + ":"
@@ -523,33 +524,37 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
}
return outputHashes;
}
- case DerivationType::CAFloating:
- isDeferred = true;
- break;
- case DerivationType::InputAddressed:
- break;
- case DerivationType::DeferredInputAddressed:
- break;
- }
+
+ auto kind = std::visit(overloaded {
+ [](const DerivationType::InputAddressed & ia) {
+ /* This might be a "pesimistically" deferred output, so we don't
+ "taint" the kind yet. */
+ return DrvHash::Kind::Regular;
+ },
+ [](const DerivationType::ContentAddressed & ca) {
+ return ca.fixed
+ ? DrvHash::Kind::Regular
+ : DrvHash::Kind::Deferred;
+ },
+ }, drv.type().raw());
/* For other derivations, replace the inputs paths with recursive
calls to this function. */
std::map<std::string, StringSet> inputs2;
- for (auto & i : drv.inputDrvs) {
- const auto & res = pathDerivationModulo(store, i.first);
+ for (auto & [drvPath, inputOutputs0] : drv.inputDrvs) {
+ // Avoid lambda capture restriction with standard / Clang
+ auto & inputOutputs = inputOutputs0;
+ const auto & res = pathDerivationModulo(store, drvPath);
std::visit(overloaded {
// Regular non-CA derivation, replace derivation
- [&](const Hash & drvHash) {
- inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
- },
- [&](const DeferredHash & deferredHash) {
- isDeferred = true;
- inputs2.insert_or_assign(deferredHash.hash.to_string(Base16, false), i.second);
+ [&](const DrvHash & drvHash) {
+ kind |= drvHash.kind;
+ inputs2.insert_or_assign(drvHash.hash.to_string(Base16, false), inputOutputs);
},
// CA derivation's output hashes
[&](const CaOutputHashes & outputHashes) {
std::set<std::string> justOut = { "out" };
- for (auto & output : i.second) {
+ for (auto & output : inputOutputs) {
/* Put each one in with a single "out" output.. */
const auto h = outputHashes.at(output);
inputs2.insert_or_assign(
@@ -557,15 +562,24 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
justOut);
}
},
- }, res);
+ }, res.raw());
}
auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
- if (isDeferred)
- return DeferredHash { hash };
- else
- return hash;
+ return DrvHash { .hash = hash, .kind = kind };
+}
+
+
+void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept
+{
+ switch (other) {
+ case DrvHash::Kind::Regular:
+ break;
+ case DrvHash::Kind::Deferred:
+ self = other;
+ break;
+ }
}
@@ -573,20 +587,15 @@ std::map<std::string, Hash> staticOutputHashes(Store & store, const Derivation &
{
std::map<std::string, Hash> res;
std::visit(overloaded {
- [&](const Hash & drvHash) {
- for (auto & outputName : drv.outputNames()) {
- res.insert({outputName, drvHash});
- }
- },
- [&](const DeferredHash & deferredHash) {
+ [&](const DrvHash & drvHash) {
for (auto & outputName : drv.outputNames()) {
- res.insert({outputName, deferredHash.hash});
+ res.insert({outputName, drvHash.hash});
}
},
[&](const CaOutputHashes & outputHashes) {
res = outputHashes;
},
- }, hashDerivationModulo(store, drv, true));
+ }, hashDerivationModulo(store, drv, true).raw());
return res;
}
@@ -667,27 +676,27 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
for (auto & i : drv.outputs) {
out << i.first;
std::visit(overloaded {
- [&](const DerivationOutputInputAddressed & doi) {
+ [&](const DerivationOutput::InputAddressed & doi) {
out << store.printStorePath(doi.path)
<< ""
<< "";
},
- [&](const DerivationOutputCAFixed & dof) {
+ [&](const DerivationOutput::CAFixed & dof) {
out << store.printStorePath(dof.path(store, drv.name, i.first))
<< printMethodAlgo(dof.ca)
<< getContentAddressHash(dof.ca).to_string(Base16, false);
},
- [&](const DerivationOutputCAFloating & dof) {
+ [&](const DerivationOutput::CAFloating & dof) {
out << ""
<< (makeContentAddressingPrefix(dof.method) + printHashType(dof.hashType))
<< "";
},
- [&](const DerivationOutputDeferred &) {
+ [&](const DerivationOutput::Deferred &) {
out << ""
<< ""
<< "";
},
- }, i.second.output);
+ }, i.second.raw());
}
worker_proto::write(store, out, drv.inputSrcs);
out << drv.platform << drv.builder << drv.args;
@@ -735,45 +744,63 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
for (auto & [outputName, output] : drv.outputs) {
- if (std::holds_alternative<DerivationOutputDeferred>(output.output)) {
- Hash h = std::get<Hash>(hashModulo);
+ if (std::holds_alternative<DerivationOutput::Deferred>(output.raw())) {
+ auto & h = hashModulo.requireNoFixedNonDeferred();
auto outPath = store.makeOutputPath(outputName, h, drv.name);
drv.env[outputName] = store.printStorePath(outPath);
- output = DerivationOutput {
- .output = DerivationOutputInputAddressed {
- .path = std::move(outPath),
- },
+ output = DerivationOutput::InputAddressed {
+ .path = std::move(outPath),
};
}
}
}
+const Hash & DrvHashModulo::requireNoFixedNonDeferred() const {
+ auto * drvHashOpt = std::get_if<DrvHash>(&raw());
+ assert(drvHashOpt);
+ assert(drvHashOpt->kind == DrvHash::Kind::Regular);
+ return drvHashOpt->hash;
+}
+
+static bool tryResolveInput(
+ Store & store, StorePathSet & inputSrcs, StringMap & inputRewrites,
+ const StorePath & inputDrv, const StringSet & inputOutputs)
+{
+ auto inputDrvOutputs = store.queryPartialDerivationOutputMap(inputDrv);
+
+ auto getOutput = [&](const std::string & outputName) {
+ auto & actualPathOpt = inputDrvOutputs.at(outputName);
+ if (!actualPathOpt)
+ warn("output %s of input %s missing, aborting the resolving",
+ outputName,
+ store.printStorePath(inputDrv)
+ );
+ return actualPathOpt;
+ };
+
+ for (auto & outputName : inputOutputs) {
+ auto actualPathOpt = getOutput(outputName);
+ if (!actualPathOpt) return false;
+ auto actualPath = *actualPathOpt;
+ inputRewrites.emplace(
+ downstreamPlaceholder(store, inputDrv, outputName),
+ store.printStorePath(actualPath));
+ inputSrcs.insert(std::move(actualPath));
+ }
+
+ return true;
+}
+
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
BasicDerivation resolved { *this };
// Input paths that we'll want to rewrite in the derivation
StringMap inputRewrites;
- for (auto & input : inputDrvs) {
- auto inputDrvOutputs = store.queryPartialDerivationOutputMap(input.first);
- StringSet newOutputNames;
- for (auto & outputName : input.second) {
- auto actualPathOpt = inputDrvOutputs.at(outputName);
- if (!actualPathOpt) {
- warn("output %s of input %s missing, aborting the resolving",
- outputName,
- store.printStorePath(input.first)
- );
- return std::nullopt;
- }
- auto actualPath = *actualPathOpt;
- inputRewrites.emplace(
- downstreamPlaceholder(store, input.first, outputName),
- store.printStorePath(actualPath));
- resolved.inputSrcs.insert(std::move(actualPath));
- }
- }
+ for (auto & [inputDrv, inputOutputs] : inputDrvs)
+ if (!tryResolveInput(store, resolved.inputSrcs, inputRewrites, inputDrv, inputOutputs))
+ return std::nullopt;
rewriteDerivation(store, resolved, inputRewrites);
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index f59887465..f41a9d93b 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -4,6 +4,7 @@
#include "types.hh"
#include "hash.hh"
#include "content-address.hh"
+#include "repair-flag.hh"
#include "sync.hh"
#include <map>
@@ -44,19 +45,31 @@ struct DerivationOutputCAFloating
*/
struct DerivationOutputDeferred {};
-struct DerivationOutput
+typedef std::variant<
+ DerivationOutputInputAddressed,
+ DerivationOutputCAFixed,
+ DerivationOutputCAFloating,
+ DerivationOutputDeferred
+> _DerivationOutputRaw;
+
+struct DerivationOutput : _DerivationOutputRaw
{
- std::variant<
- DerivationOutputInputAddressed,
- DerivationOutputCAFixed,
- DerivationOutputCAFloating,
- DerivationOutputDeferred
- > output;
+ using Raw = _DerivationOutputRaw;
+ using Raw::Raw;
+
+ using InputAddressed = DerivationOutputInputAddressed;
+ using CAFixed = DerivationOutputCAFixed;
+ using CAFloating = DerivationOutputCAFloating;
+ using Deferred = DerivationOutputDeferred;
/* Note, when you use this function you should make sure that you're passing
the right derivation name. When in doubt, you should use the safer
interface provided by BasicDerivation::outputsAndOptPaths */
std::optional<StorePath> path(const Store & store, std::string_view drvName, std::string_view outputName) const;
+
+ inline const Raw & raw() const {
+ return static_cast<const Raw &>(*this);
+ }
};
typedef std::map<std::string, DerivationOutput> DerivationOutputs;
@@ -72,30 +85,50 @@ typedef std::map<std::string, std::pair<DerivationOutput, std::optional<StorePat
output IDs we are interested in. */
typedef std::map<StorePath, StringSet> DerivationInputs;
-enum struct DerivationType : uint8_t {
- InputAddressed,
- DeferredInputAddressed,
- CAFixed,
- CAFloating,
+struct DerivationType_InputAddressed {
+ bool deferred;
};
-/* Do the outputs of the derivation have paths calculated from their content,
- or from the derivation itself? */
-bool derivationIsCA(DerivationType);
-
-/* Is the content of the outputs fixed a-priori via a hash? Never true for
- non-CA derivations. */
-bool derivationIsFixed(DerivationType);
-
-/* Is the derivation impure and needs to access non-deterministic resources, or
- pure and can be sandboxed? Note that whether or not we actually sandbox the
- derivation is controlled separately. Never true for non-CA derivations. */
-bool derivationIsImpure(DerivationType);
+struct DerivationType_ContentAddressed {
+ bool pure;
+ bool fixed;
+};
-/* Does the derivation knows its own output paths?
- * Only true when there's no floating-ca derivation involved in the closure.
- */
-bool derivationHasKnownOutputPaths(DerivationType);
+typedef std::variant<
+ DerivationType_InputAddressed,
+ DerivationType_ContentAddressed
+> _DerivationTypeRaw;
+
+struct DerivationType : _DerivationTypeRaw {
+ using Raw = _DerivationTypeRaw;
+ using Raw::Raw;
+ using InputAddressed = DerivationType_InputAddressed;
+ using ContentAddressed = DerivationType_ContentAddressed;
+
+
+ /* Do the outputs of the derivation have paths calculated from their content,
+ or from the derivation itself? */
+ bool isCA() const;
+
+ /* Is the content of the outputs fixed a-priori via a hash? Never true for
+ non-CA derivations. */
+ bool isFixed() const;
+
+ /* Is the derivation impure and needs to access non-deterministic resources, or
+ pure and can be sandboxed? Note that whether or not we actually sandbox the
+ derivation is controlled separately. Never true for non-CA derivations. */
+ bool isImpure() const;
+
+ /* Does the derivation knows its own output paths?
+ Only true when there's no floating-ca derivation involved in the
+ closure, or if fixed output.
+ */
+ bool hasKnownOutputPaths() const;
+
+ inline const Raw & raw() const {
+ return static_cast<const Raw &>(*this);
+ }
+};
struct BasicDerivation
{
@@ -150,8 +183,6 @@ struct Derivation : BasicDerivation
class Store;
-enum RepairFlag : bool { NoRepair = false, Repair = true };
-
/* Write a derivation to the Nix store, and return its path. */
StorePath writeDerivation(Store & store,
const Derivation & drv,
@@ -175,13 +206,43 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
// whose output hashes are always known since they are fixed up-front.
typedef std::map<std::string, Hash> CaOutputHashes;
-struct DeferredHash { Hash hash; };
+struct DrvHash {
+ Hash hash;
+
+ enum struct Kind: bool {
+ // Statically determined derivations.
+ // This hash will be directly used to compute the output paths
+ Regular,
+ // Floating-output derivations (and their reverse dependencies).
+ Deferred,
+ };
+
+ Kind kind;
+};
+
+void operator |= (DrvHash::Kind & self, const DrvHash::Kind & other) noexcept;
typedef std::variant<
- Hash, // regular DRV normalized hash
- CaOutputHashes, // Fixed-output derivation hashes
- DeferredHash // Deferred hashes for floating outputs drvs and their dependencies
-> DrvHashModulo;
+ // Regular normalized derivation hash, and whether it was deferred (because
+ // an ancestor derivation is a floating content addressed derivation).
+ DrvHash,
+ // Fixed-output derivation hashes
+ CaOutputHashes
+> _DrvHashModuloRaw;
+
+struct DrvHashModulo : _DrvHashModuloRaw {
+ using Raw = _DrvHashModuloRaw;
+ using Raw::Raw;
+
+ /* Get hash, throwing if it is per-output CA hashes or a
+ deferred Drv hash.
+ */
+ const Hash & requireNoFixedNonDeferred() const;
+
+ inline const Raw & raw() const {
+ return static_cast<const Raw &>(*this);
+ }
+};
/* Returns hashes with the details of fixed-output subderivations
expunged.
diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc
index 194489580..319b1c790 100644
--- a/src/libstore/derived-path.cc
+++ b/src/libstore/derived-path.cc
@@ -1,4 +1,5 @@
#include "derived-path.hh"
+#include "derivations.hh"
#include "store-api.hh"
#include <nlohmann/json.hpp>
@@ -11,6 +12,21 @@ nlohmann::json DerivedPath::Opaque::toJSON(ref<Store> store) const {
return res;
}
+nlohmann::json DerivedPath::Built::toJSON(ref<Store> store) const {
+ nlohmann::json res;
+ res["drvPath"] = store->printStorePath(drvPath);
+ // Fallback for the input-addressed derivation case: We expect to always be
+ // able to print the output paths, so let’s do it
+ auto knownOutputs = store->queryPartialDerivationOutputMap(drvPath);
+ for (const auto& output : outputs) {
+ if (knownOutputs.at(output))
+ res["outputs"][output] = store->printStorePath(knownOutputs.at(output).value());
+ else
+ res["outputs"][output] = nullptr;
+ }
+ return res;
+}
+
nlohmann::json BuiltPath::Built::toJSON(ref<Store> store) const {
nlohmann::json res;
res["drvPath"] = store->printStorePath(drvPath);
@@ -35,16 +51,22 @@ StorePathSet BuiltPath::outPaths() const
);
}
-nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref<Store> store) {
+template<typename T>
+nlohmann::json stuffToJSON(const std::vector<T> & ts, ref<Store> store) {
auto res = nlohmann::json::array();
- for (const BuiltPath & buildable : buildables) {
- std::visit([&res, store](const auto & buildable) {
- res.push_back(buildable.toJSON(store));
- }, buildable.raw());
+ for (const T & t : ts) {
+ std::visit([&res, store](const auto & t) {
+ res.push_back(t.toJSON(store));
+ }, t.raw());
}
return res;
}
+nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref<Store> store)
+{ return stuffToJSON<BuiltPath>(buildables, store); }
+nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, ref<Store> store)
+{ return stuffToJSON<DerivedPath>(paths, store); }
+
std::string DerivedPath::Opaque::to_string(const Store & store) const {
return store.printStorePath(path);
diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh
index ef7c5326c..5a38203ae 100644
--- a/src/libstore/derived-path.hh
+++ b/src/libstore/derived-path.hh
@@ -46,6 +46,7 @@ struct DerivedPathBuilt {
std::string to_string(const Store & store) const;
static DerivedPathBuilt parse(const Store & store, std::string_view);
+ nlohmann::json toJSON(ref<Store> store) const;
};
using _DerivedPathRaw = std::variant<
@@ -120,5 +121,6 @@ typedef std::vector<DerivedPath> DerivedPaths;
typedef std::vector<BuiltPath> BuiltPaths;
nlohmann::json derivedPathsWithHintsToJSON(const BuiltPaths & buildables, ref<Store> store);
+nlohmann::json derivedPathsToJSON(const DerivedPaths & , ref<Store> store);
}
diff --git a/src/libstore/gc-store.cc b/src/libstore/gc-store.cc
deleted file mode 100644
index 3dbdec53b..000000000
--- a/src/libstore/gc-store.cc
+++ /dev/null
@@ -1,13 +0,0 @@
-#include "gc-store.hh"
-
-namespace nix {
-
-GcStore & requireGcStore(Store & store)
-{
- auto * gcStore = dynamic_cast<GcStore *>(&store);
- if (!gcStore)
- throw UsageError("Garbage collection not supported by this store");
- return *gcStore;
-}
-
-}
diff --git a/src/libstore/gc-store.hh b/src/libstore/gc-store.hh
index 829f70dc4..b3cbbad74 100644
--- a/src/libstore/gc-store.hh
+++ b/src/libstore/gc-store.hh
@@ -61,6 +61,8 @@ struct GCResults
struct GcStore : public virtual Store
{
+ inline static std::string operationName = "Garbage collection";
+
/* Add an indirect root, which is merely a symlink to `path' from
/nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed
to be a symlink to a store path. The garbage collector will
@@ -79,6 +81,4 @@ struct GcStore : public virtual Store
virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0;
};
-GcStore & requireGcStore(Store & store);
-
}
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 024da66c1..f65fb1b2e 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -678,7 +678,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
alive.insert(start);
try {
StorePathSet closure;
- computeFSClosure(*path, closure);
+ computeFSClosure(*path, closure,
+ /* flipDirection */ false, gcKeepOutputs, gcKeepDerivations);
for (auto & p : closure)
alive.insert(p);
} catch (InvalidPath &) { }
@@ -841,7 +842,8 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results)
if (unlink(path.c_str()) == -1)
throw SysError("deleting '%1%'", path);
- results.bytesFreed += st.st_size;
+ /* Do not accound for deleted file here. Rely on deletePath()
+ accounting. */
}
struct stat st;
diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh
index fbd49dc2c..e6fb3201a 100644
--- a/src/libstore/local-fs-store.hh
+++ b/src/libstore/local-fs-store.hh
@@ -2,6 +2,7 @@
#include "store-api.hh"
#include "gc-store.hh"
+#include "log-store.hh"
namespace nix {
@@ -24,7 +25,10 @@ struct LocalFSStoreConfig : virtual StoreConfig
"physical path to the Nix store"};
};
-class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store, virtual GcStore
+class LocalFSStore : public virtual LocalFSStoreConfig,
+ public virtual Store,
+ public virtual GcStore,
+ public virtual LogStore
{
public:
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 52ca5a34a..9c0d482f7 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -698,11 +698,11 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
std::optional<Hash> h;
for (auto & i : drv.outputs) {
std::visit(overloaded {
- [&](const DerivationOutputInputAddressed & doia) {
+ [&](const DerivationOutput::InputAddressed & doia) {
if (!h) {
// somewhat expensive so we do lazily
- auto temp = hashDerivationModulo(*this, drv, true);
- h = std::get<Hash>(temp);
+ auto h0 = hashDerivationModulo(*this, drv, true);
+ h = h0.requireNoFixedNonDeferred();
}
StorePath recomputed = makeOutputPath(i.first, *h, drvName);
if (doia.path != recomputed)
@@ -710,16 +710,17 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
printStorePath(drvPath), printStorePath(doia.path), printStorePath(recomputed));
envHasRightPath(doia.path, i.first);
},
- [&](const DerivationOutputCAFixed & dof) {
+ [&](const DerivationOutput::CAFixed & dof) {
auto path = dof.path(*this, drvName, i.first);
envHasRightPath(path, i.first);
},
- [&](const DerivationOutputCAFloating &) {
+ [&](const DerivationOutput::CAFloating &) {
/* Nothing to check */
},
- [&](const DerivationOutputDeferred &) {
+ [&](const DerivationOutput::Deferred &) {
+ /* Nothing to check */
},
- }, i.second.output);
+ }, i.second.raw());
}
}
diff --git a/src/libstore/log-store.hh b/src/libstore/log-store.hh
new file mode 100644
index 000000000..ff1b92e17
--- /dev/null
+++ b/src/libstore/log-store.hh
@@ -0,0 +1,21 @@
+#pragma once
+
+#include "store-api.hh"
+
+
+namespace nix {
+
+struct LogStore : public virtual Store
+{
+ inline static std::string operationName = "Build log storage and retrieval";
+
+ /* Return the build log of the specified store path, if available,
+ or null otherwise. */
+ virtual std::optional<std::string> getBuildLog(const StorePath & path) = 0;
+
+ virtual void addBuildLog(const StorePath & path, std::string_view log) = 0;
+
+ static LogStore & require(Store & store);
+};
+
+}
diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc
new file mode 100644
index 000000000..9655a0555
--- /dev/null
+++ b/src/libstore/make-content-addressed.cc
@@ -0,0 +1,83 @@
+#include "make-content-addressed.hh"
+#include "references.hh"
+
+namespace nix {
+
+std::map<StorePath, StorePath> makeContentAddressed(
+ Store & srcStore,
+ Store & dstStore,
+ const StorePathSet & storePaths)
+{
+ StorePathSet closure;
+ srcStore.computeFSClosure(storePaths, closure);
+
+ auto paths = srcStore.topoSortPaths(closure);
+
+ std::reverse(paths.begin(), paths.end());
+
+ std::map<StorePath, StorePath> remappings;
+
+ for (auto & path : paths) {
+ auto pathS = srcStore.printStorePath(path);
+ auto oldInfo = srcStore.queryPathInfo(path);
+ std::string oldHashPart(path.hashPart());
+
+ StringSink sink;
+ srcStore.narFromPath(path, sink);
+
+ StringMap rewrites;
+
+ PathReferences<StorePath> refs;
+ refs.hasSelfReference = oldInfo->hasSelfReference;
+ for (auto & ref : oldInfo->references) {
+ auto i = remappings.find(ref);
+ auto replacement = i != remappings.end() ? i->second : ref;
+ // FIXME: warn about unremapped paths?
+ if (replacement != ref) {
+ rewrites.insert_or_assign(srcStore.printStorePath(ref), srcStore.printStorePath(replacement));
+ refs.references.insert(std::move(replacement));
+ }
+ }
+
+ sink.s = rewriteStrings(sink.s, rewrites);
+
+ HashModuloSink hashModuloSink(htSHA256, oldHashPart);
+ hashModuloSink(sink.s);
+
+ auto narModuloHash = hashModuloSink.finish().first;
+
+ ValidPathInfo info {
+ dstStore,
+ StorePathDescriptor {
+ .name = std::string { path.name() },
+ .info = FixedOutputInfo {
+ {
+ .method = FileIngestionMethod::Recursive,
+ .hash = narModuloHash,
+ },
+ std::move(refs),
+ },
+ },
+ Hash::dummy,
+ };
+
+ printInfo("rewriting '%s' to '%s'", pathS, dstStore.printStorePath(info.path));
+
+ StringSink sink2;
+ RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), sink2);
+ rsink2(sink.s);
+ rsink2.flush();
+
+ info.narHash = hashString(htSHA256, sink2.s);
+ info.narSize = sink.s.size();
+
+ StringSource source(sink2.s);
+ dstStore.addToStore(info, source);
+
+ remappings.insert_or_assign(std::move(path), std::move(info.path));
+ }
+
+ return remappings;
+}
+
+}
diff --git a/src/libstore/make-content-addressed.hh b/src/libstore/make-content-addressed.hh
new file mode 100644
index 000000000..c4a66ed41
--- /dev/null
+++ b/src/libstore/make-content-addressed.hh
@@ -0,0 +1,12 @@
+#pragma once
+
+#include "store-api.hh"
+
+namespace nix {
+
+std::map<StorePath, StorePath> makeContentAddressed(
+ Store & srcStore,
+ Store & dstStore,
+ const StorePathSet & storePaths);
+
+}
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index e411cbe04..42965d66e 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -87,7 +87,7 @@ std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
auto out = drv.outputs.find("out");
if (out == drv.outputs.end())
return std::nullopt;
- if (auto dof = std::get_if<DerivationOutputCAFixed>(&out->second.output)) {
+ if (auto dof = std::get_if<DerivationOutput::CAFixed>(&out->second)) {
return std::visit(overloaded {
[&](const TextInfo & ti) -> std::optional<ContentAddress> {
if (!ti.references.empty())
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index 8c65053e4..f2288a04e 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -93,7 +93,7 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const
StringSet res;
for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings()))
res.insert(i);
- if (!derivationHasKnownOutputPaths(drv.type()))
+ if (!drv.type().hasKnownOutputPaths())
res.insert("ca-derivations");
return res;
}
diff --git a/src/libstore/parsed-derivations.hh b/src/libstore/parsed-derivations.hh
index effcf099d..95bec21e8 100644
--- a/src/libstore/parsed-derivations.hh
+++ b/src/libstore/parsed-derivations.hh
@@ -1,5 +1,6 @@
#pragma once
+#include "derivations.hh"
#include "store-api.hh"
#include <nlohmann/json_fwd.hpp>
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index eb74d88c9..961d1bc08 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -5,6 +5,7 @@
#include "store-api.hh"
#include "gc-store.hh"
+#include "log-store.hh"
namespace nix {
@@ -30,7 +31,10 @@ struct RemoteStoreConfig : virtual StoreConfig
/* FIXME: RemoteStore is a misnomer - should be something like
DaemonStore. */
-class RemoteStore : public virtual RemoteStoreConfig, public virtual Store, public virtual GcStore
+class RemoteStore : public virtual RemoteStoreConfig,
+ public virtual Store,
+ public virtual GcStore,
+ public virtual LogStore
{
public:
diff --git a/src/libstore/repair-flag.hh b/src/libstore/repair-flag.hh
new file mode 100644
index 000000000..a13cda312
--- /dev/null
+++ b/src/libstore/repair-flag.hh
@@ -0,0 +1,7 @@
+#pragma once
+
+namespace nix {
+
+enum RepairFlag : bool { NoRepair = false, Repair = true };
+
+}
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index bb03daef4..62daa838c 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -52,6 +52,10 @@ public:
bool sameMachine() override
{ return false; }
+ // FIXME extend daemon protocol, move implementation to RemoteStore
+ std::optional<std::string> getBuildLog(const StorePath & path) override
+ { unsupported("getBuildLog"); }
+
private:
struct Connection : RemoteStore::Connection
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 6f4b6b98c..4362177f9 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -1,6 +1,7 @@
#include "crypto.hh"
#include "fs-accessor.hh"
#include "globals.hh"
+#include "derivations.hh"
#include "store-api.hh"
#include "util.hh"
#include "nar-info-disk-cache.hh"
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 1a320bcb1..5ea606886 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -10,8 +10,8 @@
#include "sync.hh"
#include "globals.hh"
#include "config.hh"
-#include "derivations.hh"
#include "path-info.hh"
+#include "repair-flag.hh"
#include <atomic>
#include <limits>
@@ -62,6 +62,8 @@ MakeError(BadStorePath, Error);
MakeError(InvalidStoreURI, Error);
+struct BasicDerivation;
+struct Derivation;
class FSAccessor;
class NarInfoDiskCache;
class Store;
@@ -601,14 +603,6 @@ public:
*/
StorePathSet exportReferences(const StorePathSet & storePaths, const StorePathSet & inputPaths);
- /* Return the build log of the specified store path, if available,
- or null otherwise. */
- virtual std::optional<std::string> getBuildLog(const StorePath & path)
- { return std::nullopt; }
-
- virtual void addBuildLog(const StorePath & path, std::string_view log)
- { unsupported("addBuildLog"); }
-
/* Hack to allow long-running processes like hydra-queue-runner to
occasionally flush their path info cache. */
void clearPathInfoCache()
diff --git a/src/libstore/store-cast.hh b/src/libstore/store-cast.hh
new file mode 100644
index 000000000..ff62fc359
--- /dev/null
+++ b/src/libstore/store-cast.hh
@@ -0,0 +1,16 @@
+#pragma once
+
+#include "store-api.hh"
+
+namespace nix {
+
+template<typename T>
+T & require(Store & store)
+{
+ auto * castedStore = dynamic_cast<T *>(&store);
+ if (!castedStore)
+ throw UsageError("%s not supported by store '%s'", T::operationName, store.getUri());
+ return *castedStore;
+}
+
+}
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index eda004756..30b471af5 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -64,11 +64,12 @@ static void dumpContents(const Path & path, off_t size,
}
-static void dump(const Path & path, Sink & sink, PathFilter & filter)
+static time_t dump(const Path & path, Sink & sink, PathFilter & filter)
{
checkInterrupt();
auto st = lstat(path);
+ time_t result = st.st_mtime;
sink << "(";
@@ -103,7 +104,10 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
for (auto & i : unhacked)
if (filter(path + "/" + i.first)) {
sink << "entry" << "(" << "name" << i.first << "node";
- dump(path + "/" + i.second, sink, filter);
+ auto tmp_mtime = dump(path + "/" + i.second, sink, filter);
+ if (tmp_mtime > result) {
+ result = tmp_mtime;
+ }
sink << ")";
}
}
@@ -114,13 +118,20 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
else throw Error("file '%1%' has an unsupported type", path);
sink << ")";
+
+ return result;
}
-void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
+time_t dumpPathAndGetMtime(const Path & path, Sink & sink, PathFilter & filter)
{
sink << narVersionMagic1;
- dump(path, sink, filter);
+ return dump(path, sink, filter);
+}
+
+void dumpPath(const Path & path, Sink & sink, PathFilter & filter)
+{
+ dumpPathAndGetMtime(path, sink, filter);
}
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index fca351605..79ce08df0 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -48,6 +48,10 @@ namespace nix {
void dumpPath(const Path & path, Sink & sink,
PathFilter & filter = defaultPathFilter);
+/* Same as `void dumpPath()`, but returns the last modified date of the path */
+time_t dumpPathAndGetMtime(const Path & path, Sink & sink,
+ PathFilter & filter = defaultPathFilter);
+
void dumpString(std::string_view s, Sink & sink);
/* FIXME: fix this API, it sucks. */
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index cb6532587..a7cd08d74 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -54,6 +54,7 @@ typedef enum {
lvlVomit
} Verbosity;
+/* adjust Pos::origin bit width when adding stuff here */
typedef enum {
foFile,
foStdin,
diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc
index b49f47e1d..01f318fa3 100644
--- a/src/libutil/experimental-features.cc
+++ b/src/libutil/experimental-features.cc
@@ -11,6 +11,7 @@ std::map<ExperimentalFeature, std::string> stringifiedXpFeatures = {
{ Xp::NixCommand, "nix-command" },
{ Xp::RecursiveNix, "recursive-nix" },
{ Xp::NoUrlLiterals, "no-url-literals" },
+ { Xp::FetchClosure, "fetch-closure" },
};
const std::optional<ExperimentalFeature> parseExperimentalFeature(const std::string_view & name)
diff --git a/src/libutil/experimental-features.hh b/src/libutil/experimental-features.hh
index 291a58e32..b5140dcfe 100644
--- a/src/libutil/experimental-features.hh
+++ b/src/libutil/experimental-features.hh
@@ -19,7 +19,8 @@ enum struct ExperimentalFeature
Flakes,
NixCommand,
RecursiveNix,
- NoUrlLiterals
+ NoUrlLiterals,
+ FetchClosure,
};
/**
diff --git a/src/libutil/finally.hh b/src/libutil/finally.hh
index 7760cfe9a..dee2e8d2f 100644
--- a/src/libutil/finally.hh
+++ b/src/libutil/finally.hh
@@ -1,14 +1,13 @@
#pragma once
-#include <functional>
-
/* A trivial class to run a function at the end of a scope. */
+template<typename Fn>
class Finally
{
private:
- std::function<void()> fun;
+ Fn fun;
public:
- Finally(std::function<void()> fun) : fun(fun) { }
+ Finally(Fn fun) : fun(std::move(fun)) { }
~Finally() { fun(); }
};
diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index 790bc943a..a7db58559 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -39,28 +39,30 @@ void TarArchive::check(int err, const std::string & reason)
throw Error(reason, archive_error_string(this->archive));
}
-TarArchive::TarArchive(Source & source, bool raw)
- : source(&source), buffer(4096)
+TarArchive::TarArchive(Source & source, bool raw) : buffer(4096)
{
- init();
- if (!raw)
+ this->archive = archive_read_new();
+ this->source = &source;
+
+ if (!raw) {
+ archive_read_support_filter_all(archive);
archive_read_support_format_all(archive);
- else
+ } else {
+ archive_read_support_filter_all(archive);
archive_read_support_format_raw(archive);
+ archive_read_support_format_empty(archive);
+ }
check(archive_read_open(archive, (void *)this, callback_open, callback_read, callback_close), "Failed to open archive (%s)");
}
+
TarArchive::TarArchive(const Path & path)
{
- init();
- archive_read_support_format_all(archive);
- check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
-}
+ this->archive = archive_read_new();
-void TarArchive::init()
-{
- archive = archive_read_new();
archive_read_support_filter_all(archive);
+ archive_read_support_format_all(archive);
+ check(archive_read_open_filename(archive, path.c_str(), 16384), "failed to open archive: %s");
}
void TarArchive::close()
diff --git a/src/libutil/tarfile.hh b/src/libutil/tarfile.hh
index f107a7e2e..4d9141fd4 100644
--- a/src/libutil/tarfile.hh
+++ b/src/libutil/tarfile.hh
@@ -17,13 +17,10 @@ struct TarArchive {
// disable copy constructor
TarArchive(const TarArchive &) = delete;
- void init();
-
void close();
~TarArchive();
};
-
void unpackTarfile(Source & source, const Path & destDir);
void unpackTarfile(const Path & tarFile, const Path & destDir);
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index b833038a9..70eaf4f9c 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -406,8 +406,29 @@ static void _deletePath(int parentfd, const Path & path, uint64_t & bytesFreed)
throw SysError("getting status of '%1%'", path);
}
- if (!S_ISDIR(st.st_mode) && st.st_nlink == 1)
- bytesFreed += st.st_size;
+ if (!S_ISDIR(st.st_mode)) {
+ /* We are about to delete a file. Will it likely free space? */
+
+ switch (st.st_nlink) {
+ /* Yes: last link. */
+ case 1:
+ bytesFreed += st.st_size;
+ break;
+ /* Maybe: yes, if 'auto-optimise-store' or manual optimisation
+ was performed. Instead of checking for real let's assume
+ it's an optimised file and space will be freed.
+
+ In worst case we will double count on freed space for files
+ with exactly two hardlinks for unoptimised packages.
+ */
+ case 2:
+ bytesFreed += st.st_size;
+ break;
+ /* No: 3+ links. */
+ default:
+ break;
+ }
+ }
if (S_ISDIR(st.st_mode)) {
/* Make the directory accessible. */
@@ -682,7 +703,14 @@ std::string drainFD(int fd, bool block, const size_t reserveSize)
void drainFD(int fd, Sink & sink, bool block)
{
- int saved;
+ // silence GCC maybe-uninitialized warning in finally
+ int saved = 0;
+
+ if (!block) {
+ saved = fcntl(fd, F_GETFL);
+ if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1)
+ throw SysError("making file descriptor non-blocking");
+ }
Finally finally([&]() {
if (!block) {
@@ -691,12 +719,6 @@ void drainFD(int fd, Sink & sink, bool block)
}
});
- if (!block) {
- saved = fcntl(fd, F_GETFL);
- if (fcntl(fd, F_SETFL, saved | O_NONBLOCK) == -1)
- throw SysError("making file descriptor non-blocking");
- }
-
std::vector<unsigned char> buf(64 * 1024);
while (1) {
checkInterrupt();
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index 795a4f7bd..faa8c078f 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -325,8 +325,7 @@ static void main_nix_build(int argc, char * * argv)
state->printStats();
- auto buildPaths = [&](const std::vector<StorePathWithOutputs> & paths0) {
- auto paths = toDerivedPaths(paths0);
+ auto buildPaths = [&](const std::vector<DerivedPath> & paths) {
/* Note: we do this even when !printMissing to efficiently
fetch binary cache data. */
uint64_t downloadSize, narSize;
@@ -348,7 +347,7 @@ static void main_nix_build(int argc, char * * argv)
auto & drvInfo = drvs.front();
auto drv = evalStore->derivationFromPath(drvInfo.requireDrvPath());
- std::vector<StorePathWithOutputs> pathsToBuild;
+ std::vector<DerivedPath> pathsToBuild;
RealisedPath::Set pathsToCopy;
/* Figure out what bash shell to use. If $NIX_BUILD_SHELL
@@ -370,7 +369,10 @@ static void main_nix_build(int argc, char * * argv)
throw Error("the 'bashInteractive' attribute in <nixpkgs> did not evaluate to a derivation");
auto bashDrv = drv->requireDrvPath();
- pathsToBuild.push_back({bashDrv});
+ pathsToBuild.push_back(DerivedPath::Built {
+ .drvPath = bashDrv,
+ .outputs = {},
+ });
pathsToCopy.insert(bashDrv);
shellDrv = bashDrv;
@@ -382,17 +384,24 @@ static void main_nix_build(int argc, char * * argv)
}
// Build or fetch all dependencies of the derivation.
- for (const auto & input : drv.inputDrvs)
+ for (const auto & [inputDrv0, inputOutputs] : drv.inputDrvs) {
+ // To get around lambda capturing restrictions in the
+ // standard.
+ const auto & inputDrv = inputDrv0;
if (std::all_of(envExclude.cbegin(), envExclude.cend(),
[&](const std::string & exclude) {
- return !std::regex_search(store->printStorePath(input.first), std::regex(exclude));
+ return !std::regex_search(store->printStorePath(inputDrv), std::regex(exclude));
}))
{
- pathsToBuild.push_back({input.first, input.second});
- pathsToCopy.insert(input.first);
+ pathsToBuild.push_back(DerivedPath::Built {
+ .drvPath = inputDrv,
+ .outputs = inputOutputs
+ });
+ pathsToCopy.insert(inputDrv);
}
+ }
for (const auto & src : drv.inputSrcs) {
- pathsToBuild.push_back({src});
+ pathsToBuild.push_back(DerivedPath::Opaque{src});
pathsToCopy.insert(src);
}
@@ -543,7 +552,7 @@ static void main_nix_build(int argc, char * * argv)
else {
- std::vector<StorePathWithOutputs> pathsToBuild;
+ std::vector<DerivedPath> pathsToBuild;
std::vector<std::pair<StorePath, std::string>> pathsToBuildOrdered;
RealisedPath::Set drvsToCopy;
@@ -556,7 +565,7 @@ static void main_nix_build(int argc, char * * argv)
if (outputName == "")
throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath));
- pathsToBuild.push_back({drvPath, {outputName}});
+ pathsToBuild.push_back(DerivedPath::Built{drvPath, {outputName}});
pathsToBuildOrdered.push_back({drvPath, {outputName}});
drvsToCopy.insert(drvPath);
diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc
index 4b28ea6a4..af6f1c88c 100644
--- a/src/nix-collect-garbage/nix-collect-garbage.cc
+++ b/src/nix-collect-garbage/nix-collect-garbage.cc
@@ -1,4 +1,5 @@
#include "store-api.hh"
+#include "store-cast.hh"
#include "gc-store.hh"
#include "profiles.hh"
#include "shared.hh"
@@ -81,7 +82,7 @@ static int main_nix_collect_garbage(int argc, char * * argv)
// Run the actual garbage collector.
if (!dryRun) {
auto store = openStore();
- auto & gcStore = requireGcStore(*store);
+ auto & gcStore = require<GcStore>(*store);
options.action = GCOptions::gcDeleteDead;
GCResults results;
PrintFreed freed(true, results);
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index 40c3c5d65..9a68899cd 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -128,7 +128,12 @@ static void getAllExprs(EvalState & state,
if (hasSuffix(attrName, ".nix"))
attrName = std::string(attrName, 0, attrName.size() - 4);
if (!seen.insert(attrName).second) {
- printError("warning: name collision in input Nix expressions, skipping '%1%'", path2);
+ std::string suggestionMessage = "";
+ if (path2.find("channels") != std::string::npos && path.find("channels") != std::string::npos) {
+ suggestionMessage = fmt("\nsuggestion: remove '%s' from either the root channels or the user channels", attrName);
+ }
+ printError("warning: name collision in input Nix expressions, skipping '%1%'"
+ "%2%", path2, suggestionMessage);
continue;
}
/* Load the expression on demand. */
@@ -918,12 +923,17 @@ static void queryJSON(Globals & globals, std::vector<DrvInfo> & elems, bool prin
pkgObj.attr("pname", drvName.name);
pkgObj.attr("version", drvName.version);
pkgObj.attr("system", i.querySystem());
+ pkgObj.attr("outputName", i.queryOutputName());
- if (printOutPath) {
- DrvInfo::Outputs outputs = i.queryOutputs();
+ {
+ DrvInfo::Outputs outputs = i.queryOutputs(printOutPath);
JSONObject outputObj = pkgObj.object("outputs");
- for (auto & j : outputs)
- outputObj.attr(j.first, globals.state->store->printStorePath(j.second));
+ for (auto & j : outputs) {
+ if (j.second)
+ outputObj.attr(j.first, globals.state->store->printStorePath(*j.second));
+ else
+ outputObj.attr(j.first, nullptr);
+ }
}
if (printMeta) {
@@ -1052,6 +1062,7 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
/* Print the desired columns, or XML output. */
if (jsonOutput) {
queryJSON(globals, elems, printOutPath, printMeta);
+ cout << '\n';
return;
}
@@ -1154,13 +1165,16 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
columns.push_back(drvPath ? store.printStorePath(*drvPath) : "-");
}
+ if (xmlOutput)
+ attrs["outputName"] = i.queryOutputName();
+
if (printOutPath && !xmlOutput) {
DrvInfo::Outputs outputs = i.queryOutputs();
std::string s;
for (auto & j : outputs) {
if (!s.empty()) s += ';';
if (j.first != "out") { s += j.first; s += "="; }
- s += store.printStorePath(j.second);
+ s += store.printStorePath(*j.second);
}
columns.push_back(s);
}
@@ -1174,71 +1188,67 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
}
if (xmlOutput) {
- if (printOutPath || printMeta) {
- XMLOpenElement item(xml, "item", attrs);
- if (printOutPath) {
- DrvInfo::Outputs outputs = i.queryOutputs();
- for (auto & j : outputs) {
- XMLAttrs attrs2;
- attrs2["name"] = j.first;
- attrs2["path"] = store.printStorePath(j.second);
- xml.writeEmptyElement("output", attrs2);
- }
- }
- if (printMeta) {
- StringSet metaNames = i.queryMetaNames();
- for (auto & j : metaNames) {
- XMLAttrs attrs2;
- attrs2["name"] = j;
- Value * v = i.queryMeta(j);
- if (!v)
- printError(
- "derivation '%s' has invalid meta attribute '%s'",
- i.queryName(), j);
- else {
- if (v->type() == nString) {
- attrs2["type"] = "string";
- attrs2["value"] = v->string.s;
- xml.writeEmptyElement("meta", attrs2);
- } else if (v->type() == nInt) {
- attrs2["type"] = "int";
- attrs2["value"] = (format("%1%") % v->integer).str();
- xml.writeEmptyElement("meta", attrs2);
- } else if (v->type() == nFloat) {
- attrs2["type"] = "float";
- attrs2["value"] = (format("%1%") % v->fpoint).str();
- xml.writeEmptyElement("meta", attrs2);
- } else if (v->type() == nBool) {
- attrs2["type"] = "bool";
- attrs2["value"] = v->boolean ? "true" : "false";
- xml.writeEmptyElement("meta", attrs2);
- } else if (v->type() == nList) {
- attrs2["type"] = "strings";
- XMLOpenElement m(xml, "meta", attrs2);
- for (auto elem : v->listItems()) {
- if (elem->type() != nString) continue;
- XMLAttrs attrs3;
- attrs3["value"] = elem->string.s;
- xml.writeEmptyElement("string", attrs3);
- }
- } else if (v->type() == nAttrs) {
- attrs2["type"] = "strings";
- XMLOpenElement m(xml, "meta", attrs2);
- Bindings & attrs = *v->attrs;
- for (auto &i : attrs) {
- Attr & a(*attrs.find(i.name));
- if(a.value->type() != nString) continue;
- XMLAttrs attrs3;
- attrs3["type"] = i.name;
- attrs3["value"] = a.value->string.s;
- xml.writeEmptyElement("string", attrs3);
+ XMLOpenElement item(xml, "item", attrs);
+ DrvInfo::Outputs outputs = i.queryOutputs(printOutPath);
+ for (auto & j : outputs) {
+ XMLAttrs attrs2;
+ attrs2["name"] = j.first;
+ if (j.second)
+ attrs2["path"] = store.printStorePath(*j.second);
+ xml.writeEmptyElement("output", attrs2);
+ }
+ if (printMeta) {
+ StringSet metaNames = i.queryMetaNames();
+ for (auto & j : metaNames) {
+ XMLAttrs attrs2;
+ attrs2["name"] = j;
+ Value * v = i.queryMeta(j);
+ if (!v)
+ printError(
+ "derivation '%s' has invalid meta attribute '%s'",
+ i.queryName(), j);
+ else {
+ if (v->type() == nString) {
+ attrs2["type"] = "string";
+ attrs2["value"] = v->string.s;
+ xml.writeEmptyElement("meta", attrs2);
+ } else if (v->type() == nInt) {
+ attrs2["type"] = "int";
+ attrs2["value"] = (format("%1%") % v->integer).str();
+ xml.writeEmptyElement("meta", attrs2);
+ } else if (v->type() == nFloat) {
+ attrs2["type"] = "float";
+ attrs2["value"] = (format("%1%") % v->fpoint).str();
+ xml.writeEmptyElement("meta", attrs2);
+ } else if (v->type() == nBool) {
+ attrs2["type"] = "bool";
+ attrs2["value"] = v->boolean ? "true" : "false";
+ xml.writeEmptyElement("meta", attrs2);
+ } else if (v->type() == nList) {
+ attrs2["type"] = "strings";
+ XMLOpenElement m(xml, "meta", attrs2);
+ for (auto elem : v->listItems()) {
+ if (elem->type() != nString) continue;
+ XMLAttrs attrs3;
+ attrs3["value"] = elem->string.s;
+ xml.writeEmptyElement("string", attrs3);
}
- }
+ } else if (v->type() == nAttrs) {
+ attrs2["type"] = "strings";
+ XMLOpenElement m(xml, "meta", attrs2);
+ Bindings & attrs = *v->attrs;
+ for (auto &i : attrs) {
+ Attr & a(*attrs.find(i.name));
+ if(a.value->type() != nString) continue;
+ XMLAttrs attrs3;
+ attrs3["type"] = i.name;
+ attrs3["value"] = a.value->string.s;
+ xml.writeEmptyElement("string", attrs3);
+ }
}
}
}
- } else
- xml.writeEmptyElement("item", attrs);
+ }
} else
table.push_back(columns);
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index af4f350ff..78692b9c6 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -56,7 +56,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
output paths, and optionally the derivation path, as well
as the meta attributes. */
std::optional<StorePath> drvPath = keepDerivations ? i.queryDrvPath() : std::nullopt;
- DrvInfo::Outputs outputs = i.queryOutputs(true);
+ DrvInfo::Outputs outputs = i.queryOutputs(true, true);
StringSet metaNames = i.queryMetaNames();
auto attrs = state.buildBindings(7 + outputs.size());
@@ -76,15 +76,15 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
for (const auto & [m, j] : enumerate(outputs)) {
(vOutputs.listElems()[m] = state.allocValue())->mkString(j.first);
auto outputAttrs = state.buildBindings(2);
- outputAttrs.alloc(state.sOutPath).mkString(state.store->printStorePath(j.second));
+ outputAttrs.alloc(state.sOutPath).mkString(state.store->printStorePath(*j.second));
attrs.alloc(j.first).mkAttrs(outputAttrs);
/* This is only necessary when installing store paths, e.g.,
`nix-env -i /nix/store/abcd...-foo'. */
- state.store->addTempRoot(j.second);
- state.store->ensurePath(j.second);
+ state.store->addTempRoot(*j.second);
+ state.store->ensurePath(*j.second);
- references.insert(j.second);
+ references.insert(*j.second);
}
// Copy the meta attributes.
@@ -105,8 +105,10 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
/* Also write a copy of the list of user environment elements to
the store; we need it for future modifications of the
environment. */
+ std::ostringstream str;
+ manifest.print(str, true);
auto manifestFile = state.store->addTextToStore("env-manifest.nix",
- fmt("%s", manifest), references);
+ str.str(), references);
/* Get the environment builder expression. */
Value envBuilder;
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index ad14ba9db..a97a82379 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -3,7 +3,9 @@
#include "dotgraph.hh"
#include "globals.hh"
#include "build-result.hh"
+#include "store-cast.hh"
#include "gc-store.hh"
+#include "log-store.hh"
#include "local-store.hh"
#include "monitor-fd.hh"
#include "serve-protocol.hh"
@@ -435,7 +437,7 @@ static void opQuery(Strings opFlags, Strings opArgs)
store->computeFSClosure(
args, referrers, true, settings.gcKeepOutputs, settings.gcKeepDerivations);
- auto & gcStore = requireGcStore(*store);
+ auto & gcStore = require<GcStore>(*store);
Roots roots = gcStore.findRoots(false);
for (auto & [target, links] : roots)
if (referrers.find(target) != referrers.end())
@@ -480,13 +482,15 @@ static void opReadLog(Strings opFlags, Strings opArgs)
{
if (!opFlags.empty()) throw UsageError("unknown flag");
+ auto & logStore = require<LogStore>(*store);
+
RunPager pager;
for (auto & i : opArgs) {
- auto path = store->followLinksToStorePath(i);
- auto log = store->getBuildLog(path);
+ auto path = logStore.followLinksToStorePath(i);
+ auto log = logStore.getBuildLog(path);
if (!log)
- throw Error("build log of derivation '%s' is not available", store->printStorePath(path));
+ throw Error("build log of derivation '%s' is not available", logStore.printStorePath(path));
std::cout << *log;
}
}
@@ -596,7 +600,7 @@ static void opGC(Strings opFlags, Strings opArgs)
if (!opArgs.empty()) throw UsageError("no arguments expected");
- auto & gcStore = requireGcStore(*store);
+ auto & gcStore = require<GcStore>(*store);
if (printRoots) {
Roots roots = gcStore.findRoots(false);
@@ -635,7 +639,7 @@ static void opDelete(Strings opFlags, Strings opArgs)
for (auto & i : opArgs)
options.pathsToDelete.insert(store->followLinksToStorePath(i));
- auto & gcStore = requireGcStore(*store);
+ auto & gcStore = require<GcStore>(*store);
GCResults results;
PrintFreed freed(true, results);
diff --git a/src/nix/app.cc b/src/nix/app.cc
index 2563180fb..803d028f0 100644
--- a/src/nix/app.cc
+++ b/src/nix/app.cc
@@ -4,6 +4,7 @@
#include "eval-cache.hh"
#include "names.hh"
#include "command.hh"
+#include "derivations.hh"
namespace nix {
@@ -70,7 +71,7 @@ UnresolvedApp Installable::toApp(EvalState & state)
std::vector<StorePathWithOutputs> context2;
for (auto & [path, name] : context)
- context2.push_back({state.store->parseStorePath(path), {name}});
+ context2.push_back({path, {name}});
return UnresolvedApp{App {
.context = std::move(context2),
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 680db1c60..840c7ca38 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -52,15 +52,26 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
void run(ref<Store> store) override
{
+ if (dryRun) {
+ std::vector<DerivedPath> pathsToBuild;
+
+ for (auto & i : installables) {
+ auto b = i->toDerivedPaths();
+ pathsToBuild.insert(pathsToBuild.end(), b.begin(), b.end());
+ }
+ printMissing(store, pathsToBuild, lvlError);
+ if (json)
+ logger->cout("%s", derivedPathsToJSON(pathsToBuild, store).dump());
+ return;
+ }
+
auto buildables = Installable::build(
getEvalStore(), store,
- dryRun ? Realise::Derivation : Realise::Outputs,
+ Realise::Outputs,
installables, buildMode);
if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump());
- if (dryRun) return;
-
if (outLink != "")
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
for (const auto & [_i, buildable] : enumerate(buildables)) {
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 8af5da9d0..d2f9b5a6a 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -196,21 +196,22 @@ static StorePath getDerivationEnvironment(ref<Store> store, ref<Store> evalStore
drv.inputSrcs.insert(std::move(getEnvShPath));
if (settings.isExperimentalFeatureEnabled(Xp::CaDerivations)) {
for (auto & output : drv.outputs) {
- output.second = {
- .output = DerivationOutputDeferred{},
- };
+ output.second = DerivationOutput::Deferred {},
drv.env[output.first] = hashPlaceholder(output.first);
}
} else {
for (auto & output : drv.outputs) {
- output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } };
+ output.second = DerivationOutput::Deferred { };
drv.env[output.first] = "";
}
- Hash h = std::get<0>(hashDerivationModulo(*evalStore, drv, true));
+ auto h0 = hashDerivationModulo(*evalStore, drv, true);
+ const Hash & h = h0.requireNoFixedNonDeferred();
for (auto & output : drv.outputs) {
auto outPath = store->makeOutputPath(output.first, h, drv.name);
- output.second = { .output = DerivationOutputInputAddressed { .path = outPath } };
+ output.second = DerivationOutput::InputAddressed {
+ .path = outPath,
+ };
drv.env[output.first] = store->printStorePath(outPath);
}
}
diff --git a/src/nix/edit.md b/src/nix/edit.md
index 80563d06b..89bd09abf 100644
--- a/src/nix/edit.md
+++ b/src/nix/edit.md
@@ -24,8 +24,8 @@ this attribute to the location of the definition of the
`meta.description`, `version` or `name` derivation attributes.
The editor to invoke is specified by the `EDITOR` environment
-variable. It defaults to `cat`. If the editor is `emacs`, `nano` or
-`vim`, it is passed the line number of the derivation using the
-argument `+<lineno>`.
+variable. It defaults to `cat`. If the editor is `emacs`, `nano`,
+`vim` or `kak`, it is passed the line number of the derivation using
+the argument `+<lineno>`.
)""
diff --git a/src/nix/log.cc b/src/nix/log.cc
index fd3c1d787..72d02ef11 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -2,6 +2,7 @@
#include "common-args.hh"
#include "shared.hh"
#include "store-api.hh"
+#include "log-store.hh"
#include "progress-bar.hh"
using namespace nix;
@@ -34,17 +35,24 @@ struct CmdLog : InstallableCommand
RunPager pager;
for (auto & sub : subs) {
+ auto * logSubP = dynamic_cast<LogStore *>(&*sub);
+ if (!logSubP) {
+ printInfo("Skipped '%s' which does not support retrieving build logs", sub->getUri());
+ continue;
+ }
+ auto & logSub = *logSubP;
+
auto log = std::visit(overloaded {
[&](const DerivedPath::Opaque & bo) {
- return sub->getBuildLog(bo.path);
+ return logSub.getBuildLog(bo.path);
},
[&](const DerivedPath::Built & bfd) {
- return sub->getBuildLog(bfd.drvPath);
+ return logSub.getBuildLog(bfd.drvPath);
},
}, b.raw());
if (!log) continue;
stopProgressBar();
- printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri());
+ printInfo("got build log for '%s' from '%s'", installable->what(), logSub.getUri());
std::cout << *log;
return;
}
diff --git a/src/nix/main.cc b/src/nix/main.cc
index b923f2535..6198681e7 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -117,7 +117,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
{"hash-path", {"hash", "path"}},
{"ls-nar", {"nar", "ls"}},
{"ls-store", {"store", "ls"}},
- {"make-content-addressable", {"store", "make-content-addressable"}},
+ {"make-content-addressable", {"store", "make-content-addressed"}},
{"optimise-store", {"store", "optimise"}},
{"ping-store", {"store", "ping"}},
{"sign-paths", {"store", "sign"}},
@@ -289,6 +289,7 @@ void mainWrapped(int argc, char * * argv)
}
if (argc == 2 && std::string(argv[1]) == "__dump-builtins") {
+ settings.experimentalFeatures = {Xp::Flakes, Xp::FetchClosure};
evalSettings.pureEval = false;
EvalState state({}, openStore("dummy://"));
auto res = nlohmann::json::object();
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
deleted file mode 100644
index cceaacac9..000000000
--- a/src/nix/make-content-addressable.cc
+++ /dev/null
@@ -1,102 +0,0 @@
-#include "command.hh"
-#include "store-api.hh"
-#include "references.hh"
-#include "common-args.hh"
-#include "json.hh"
-
-using namespace nix;
-
-struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
-{
- CmdMakeContentAddressable()
- {
- realiseMode = Realise::Outputs;
- }
-
- std::string description() override
- {
- return "rewrite a path or closure to content-addressed form";
- }
-
- std::string doc() override
- {
- return
- #include "make-content-addressable.md"
- ;
- }
-
- void run(ref<Store> store, StorePaths && storePaths) override
- {
- auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end()));
-
- std::reverse(paths.begin(), paths.end());
-
- std::map<StorePath, StorePath> remappings;
-
- auto jsonRoot = json ? std::make_unique<JSONObject>(std::cout) : nullptr;
- auto jsonRewrites = json ? std::make_unique<JSONObject>(jsonRoot->object("rewrites")) : nullptr;
-
- for (auto & path : paths) {
- auto pathS = store->printStorePath(path);
- auto oldInfo = store->queryPathInfo(path);
- std::string oldHashPart(path.hashPart());
-
- StringSink sink;
- store->narFromPath(path, sink);
-
- StringMap rewrites;
-
- PathReferences<StorePath> refs;
- refs.hasSelfReference = oldInfo->hasSelfReference;
- for (auto & ref : oldInfo->references) {
- auto i = remappings.find(ref);
- auto replacement = i != remappings.end() ? i->second : ref;
- // FIXME: warn about unremapped paths?
- if (replacement != ref)
- rewrites.insert_or_assign(store->printStorePath(ref), store->printStorePath(replacement));
- refs.references.insert(std::move(replacement));
- }
-
- sink.s = rewriteStrings(sink.s, rewrites);
-
- HashModuloSink hashModuloSink(htSHA256, oldHashPart);
- hashModuloSink(sink.s);
-
- auto narHash = hashModuloSink.finish().first;
-
- ValidPathInfo info {
- *store,
- StorePathDescriptor {
- .name = std::string { path.name() },
- .info = FixedOutputInfo {
- {
- .method = FileIngestionMethod::Recursive,
- .hash = narHash,
- },
- std::move(refs),
- },
- },
- narHash,
- };
- info.narSize = sink.s.size();
-
- if (!json)
- notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
-
- auto source = sinkToSource([&](Sink & nextSink) {
- RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);
- rsink2(sink.s);
- rsink2.flush();
- });
-
- store->addToStore(info, *source);
-
- if (json)
- jsonRewrites->attr(store->printStorePath(path), store->printStorePath(info.path));
-
- remappings.insert_or_assign(std::move(path), std::move(info.path));
- }
- }
-};
-
-static auto rCmdMakeContentAddressable = registerCommand2<CmdMakeContentAddressable>({"store", "make-content-addressable"});
diff --git a/src/nix/make-content-addressed.cc b/src/nix/make-content-addressed.cc
new file mode 100644
index 000000000..34860c38f
--- /dev/null
+++ b/src/nix/make-content-addressed.cc
@@ -0,0 +1,55 @@
+#include "command.hh"
+#include "store-api.hh"
+#include "make-content-addressed.hh"
+#include "common-args.hh"
+#include "json.hh"
+
+using namespace nix;
+
+struct CmdMakeContentAddressed : virtual CopyCommand, virtual StorePathsCommand, MixJSON
+{
+ CmdMakeContentAddressed()
+ {
+ realiseMode = Realise::Outputs;
+ }
+
+ std::string description() override
+ {
+ return "rewrite a path or closure to content-addressed form";
+ }
+
+ std::string doc() override
+ {
+ return
+ #include "make-content-addressed.md"
+ ;
+ }
+
+ void run(ref<Store> srcStore, StorePaths && storePaths) override
+ {
+ auto dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
+
+ auto remappings = makeContentAddressed(*srcStore, *dstStore,
+ StorePathSet(storePaths.begin(), storePaths.end()));
+
+ if (json) {
+ JSONObject jsonRoot(std::cout);
+ JSONObject jsonRewrites(jsonRoot.object("rewrites"));
+ for (auto & path : storePaths) {
+ auto i = remappings.find(path);
+ assert(i != remappings.end());
+ jsonRewrites.attr(srcStore->printStorePath(path), srcStore->printStorePath(i->second));
+ }
+ } else {
+ for (auto & path : storePaths) {
+ auto i = remappings.find(path);
+ assert(i != remappings.end());
+ notice("rewrote '%s' to '%s'",
+ srcStore->printStorePath(path),
+ srcStore->printStorePath(i->second));
+ }
+ }
+ }
+};
+
+static auto rCmdMakeContentAddressed = registerCommand2<CmdMakeContentAddressed>({"store", "make-content-addressed"});
diff --git a/src/nix/make-content-addressable.md b/src/nix/make-content-addressed.md
index 3dd847edc..215683e6d 100644
--- a/src/nix/make-content-addressable.md
+++ b/src/nix/make-content-addressed.md
@@ -5,7 +5,7 @@ R""(
* Create a content-addressed representation of the closure of GNU Hello:
```console
- # nix store make-content-addressable -r nixpkgs#hello
+ # nix store make-content-addressed nixpkgs#hello
rewrote '/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10' to '/nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10'
```
@@ -29,7 +29,7 @@ R""(
system closure:
```console
- # nix store make-content-addressable -r /run/current-system
+ # nix store make-content-addressed /run/current-system
```
# Description
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 6260b0c10..1d9da087c 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -107,8 +107,9 @@ struct ProfileManifest
element.storePaths.insert(state.store->parseStorePath((std::string) p));
element.active = e["active"];
if (e.value("uri", "") != "") {
+ auto originalUrl = e.value("originalUrl", e["originalUri"]);
element.source = ProfileElementSource{
- parseFlakeRef(e["originalUri"]),
+ parseFlakeRef(originalUrl),
parseFlakeRef(e["uri"]),
e["attrPath"]
};
@@ -143,7 +144,7 @@ struct ProfileManifest
obj["storePaths"] = paths;
obj["active"] = element.active;
if (element.source) {
- obj["originalUri"] = element.source->originalRef.to_string();
+ obj["originalUrl"] = element.source->originalRef.to_string();
obj["uri"] = element.source->resolvedRef.to_string();
obj["attrPath"] = element.source->attrPath;
}
diff --git a/src/nix/profile.md b/src/nix/profile.md
index 0a4ff2fa9..8dade051d 100644
--- a/src/nix/profile.md
+++ b/src/nix/profile.md
@@ -70,7 +70,7 @@ are installed in this version of the profile. It looks like this:
{
"active": true,
"attrPath": "legacyPackages.x86_64-linux.zoom-us",
- "originalUri": "flake:nixpkgs",
+ "originalUrl": "flake:nixpkgs",
"storePaths": [
"/nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927"
],
@@ -84,11 +84,11 @@ are installed in this version of the profile. It looks like this:
Each object in the array `elements` denotes an installed package and
has the following fields:
-* `originalUri`: The [flake reference](./nix3-flake.md) specified by
+* `originalUrl`: The [flake reference](./nix3-flake.md) specified by
the user at the time of installation (e.g. `nixpkgs`). This is also
the flake reference that will be used by `nix profile upgrade`.
-* `uri`: The immutable flake reference to which `originalUri`
+* `uri`: The immutable flake reference to which `originalUrl`
resolved.
* `attrPath`: The flake output attribute that provided this
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 3a51a13e6..1f9d4fb4e 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -25,6 +25,7 @@ extern "C" {
#include "eval-inline.hh"
#include "attr-path.hh"
#include "store-api.hh"
+#include "log-store.hh"
#include "common-eval-args.hh"
#include "get-drvs.hh"
#include "derivations.hh"
@@ -395,6 +396,7 @@ StorePath NixRepl::getDerivationPath(Value & v) {
bool NixRepl::processLine(std::string line)
{
+ line = trim(line);
if (line == "") return true;
_isInterrupted = false;
@@ -526,9 +528,16 @@ bool NixRepl::processLine(std::string line)
bool foundLog = false;
RunPager pager;
for (auto & sub : subs) {
- auto log = sub->getBuildLog(drvPath);
+ auto * logSubP = dynamic_cast<LogStore *>(&*sub);
+ if (!logSubP) {
+ printInfo("Skipped '%s' which does not support retrieving build logs", sub->getUri());
+ continue;
+ }
+ auto & logSub = *logSubP;
+
+ auto log = logSub.getBuildLog(drvPath);
if (log) {
- printInfo("got build log for '%s' from '%s'", drvPathRaw, sub->getUri());
+ printInfo("got build log for '%s' from '%s'", drvPathRaw, logSub.getUri());
logger->writeToStdout(*log);
foundLog = true;
break;
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index 66d172441..3dfc4547b 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -65,20 +65,20 @@ struct CmdShowDerivation : InstallablesCommand
auto & outputName = _outputName; // work around clang bug
auto outputObj { outputsObj.object(outputName) };
std::visit(overloaded {
- [&](const DerivationOutputInputAddressed & doi) {
+ [&](const DerivationOutput::InputAddressed & doi) {
outputObj.attr("path", store->printStorePath(doi.path));
},
- [&](const DerivationOutputCAFixed & dof) {
+ [&](const DerivationOutput::CAFixed & dof) {
outputObj.attr("path", store->printStorePath(dof.path(*store, drv.name, outputName)));
outputObj.attr("hashAlgo", printMethodAlgo(dof.ca));
outputObj.attr("hash", getContentAddressHash(dof.ca).to_string(Base16, false));
// FIXME print refs?
},
- [&](const DerivationOutputCAFloating & dof) {
+ [&](const DerivationOutput::CAFloating & dof) {
outputObj.attr("hashAlgo", makeContentAddressingPrefix(dof.method) + printHashType(dof.hashType));
},
- [&](const DerivationOutputDeferred &) {},
- }, output.output);
+ [&](const DerivationOutput::Deferred &) {},
+ }, output.raw());
}
}
diff --git a/src/nix/store-copy-log.cc b/src/nix/store-copy-log.cc
index 079cd6b3e..2e288f743 100644
--- a/src/nix/store-copy-log.cc
+++ b/src/nix/store-copy-log.cc
@@ -1,6 +1,8 @@
#include "command.hh"
#include "shared.hh"
#include "store-api.hh"
+#include "store-cast.hh"
+#include "log-store.hh"
#include "sync.hh"
#include "thread-pool.hh"
@@ -26,7 +28,10 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand
void run(ref<Store> srcStore) override
{
+ auto & srcLogStore = require<LogStore>(*srcStore);
+
auto dstStore = getDstStore();
+ auto & dstLogStore = require<LogStore>(*dstStore);
StorePathSet drvPaths;
@@ -35,8 +40,8 @@ struct CmdCopyLog : virtual CopyCommand, virtual InstallablesCommand
drvPaths.insert(drvPath);
for (auto & drvPath : drvPaths) {
- if (auto log = srcStore->getBuildLog(drvPath))
- dstStore->addBuildLog(drvPath, *log);
+ if (auto log = srcLogStore.getBuildLog(drvPath))
+ dstLogStore.addBuildLog(drvPath, *log);
else
throw Error("build log for '%s' is not available", srcStore->printStorePath(drvPath));
}
diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc
index aa7a8b12f..ca43f1530 100644
--- a/src/nix/store-delete.cc
+++ b/src/nix/store-delete.cc
@@ -2,6 +2,7 @@
#include "common-args.hh"
#include "shared.hh"
#include "store-api.hh"
+#include "store-cast.hh"
#include "gc-store.hh"
using namespace nix;
@@ -33,7 +34,7 @@ struct CmdStoreDelete : StorePathsCommand
void run(ref<Store> store, std::vector<StorePath> && storePaths) override
{
- auto & gcStore = requireGcStore(*store);
+ auto & gcStore = require<GcStore>(*store);
for (auto & path : storePaths)
options.pathsToDelete.insert(path);
diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc
index 21718dc0c..8b9b5d164 100644
--- a/src/nix/store-gc.cc
+++ b/src/nix/store-gc.cc
@@ -2,6 +2,7 @@
#include "common-args.hh"
#include "shared.hh"
#include "store-api.hh"
+#include "store-cast.hh"
#include "gc-store.hh"
using namespace nix;
@@ -34,7 +35,7 @@ struct CmdStoreGC : StoreCommand, MixDryRun
void run(ref<Store> store) override
{
- auto & gcStore = requireGcStore(*store);
+ auto & gcStore = require<GcStore>(*store);
options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead;
GCResults results;