aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/libexpr/eval.cc25
-rw-r--r--src/libexpr/eval.hh9
-rw-r--r--src/libexpr/flake/flake.cc5
-rw-r--r--src/libexpr/nixexpr.cc4
-rw-r--r--src/libexpr/nixexpr.hh6
-rw-r--r--src/libexpr/parser.y8
-rw-r--r--src/libexpr/primops.cc160
-rw-r--r--src/libexpr/primops/fetchMercurial.cc3
-rw-r--r--src/libexpr/primops/fetchTree.cc72
-rw-r--r--src/libexpr/value-to-xml.cc2
-rw-r--r--src/libstore/build/local-derivation-goal.cc14
-rw-r--r--src/libstore/build/worker.cc2
-rw-r--r--src/libstore/references.cc93
-rw-r--r--src/libstore/references.hh24
-rw-r--r--src/libstore/tests/local.mk15
-rw-r--r--src/libstore/tests/references.cc45
-rw-r--r--src/libstore/uds-remote-store.cc11
-rw-r--r--src/libutil/archive.cc4
-rw-r--r--src/libutil/util.cc81
-rw-r--r--src/libutil/util.hh10
-rw-r--r--src/nix/daemon.cc6
-rw-r--r--src/nix/flake.cc6
-rw-r--r--src/nix/flake.md2
-rw-r--r--src/nix/main.cc7
-rw-r--r--src/nix/profile.cc6
25 files changed, 389 insertions, 231 deletions
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 800839a8d..3ae05a8d8 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -445,12 +445,12 @@ EvalState::EvalState(
StorePathSet closure;
store->computeFSClosure(store->toStorePath(r.second).first, closure);
for (auto & path : closure)
- allowedPaths->insert(store->printStorePath(path));
+ allowPath(path);
} catch (InvalidPath &) {
- allowedPaths->insert(r.second);
+ allowPath(r.second);
}
} else
- allowedPaths->insert(r.second);
+ allowPath(r.second);
}
}
@@ -482,6 +482,18 @@ void EvalState::requireExperimentalFeatureOnEvaluation(
}
}
+void EvalState::allowPath(const Path & path)
+{
+ if (allowedPaths)
+ allowedPaths->insert(path);
+}
+
+void EvalState::allowPath(const StorePath & storePath)
+{
+ if (allowedPaths)
+ allowedPaths->insert(store->toRealPath(storePath));
+}
+
Path EvalState::checkSourcePath(const Path & path_)
{
if (!allowedPaths) return path_;
@@ -1316,13 +1328,13 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
auto size =
(lambda.arg.empty() ? 0 : 1) +
- (lambda.matchAttrs ? lambda.formals->formals.size() : 0);
+ (lambda.hasFormals() ? lambda.formals->formals.size() : 0);
Env & env2(allocEnv(size));
env2.up = fun.lambda.env;
size_t displ = 0;
- if (!lambda.matchAttrs)
+ if (!lambda.hasFormals())
env2.values[displ++] = &arg;
else {
@@ -1402,7 +1414,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
}
}
- if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) {
+ if (!fun.isLambda() || !fun.lambda.fun->hasFormals()) {
res = fun;
return;
}
@@ -1889,6 +1901,7 @@ string EvalState::copyPathToStore(PathSet & context, const Path & path)
? store->computeStorePathForPath(std::string(baseNameOf(path)), checkSourcePath(path)).first
: store->addToStore(std::string(baseNameOf(path)), checkSourcePath(path), FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, repair);
dstPath = store->printStorePath(p);
+ allowPath(p);
srcToStore.insert_or_assign(path, std::move(p));
printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, dstPath);
}
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 9df6150c6..7cc16ef0a 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -150,6 +150,15 @@ public:
SearchPath getSearchPath() { return searchPath; }
+ /* Allow access to a path. */
+ void allowPath(const Path & path);
+
+ /* Allow access to a store path. Note that this gets remapped to
+ the real store path if `store` is a chroot store. */
+ void allowPath(const StorePath & storePath);
+
+ /* Check whether access to a path is allowed and throw an error if
+ not. Otherwise return the canonicalised path. */
Path checkSourcePath(const Path & path);
void checkURI(const std::string & uri);
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index 1a1fa6938..43bfc3644 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -64,8 +64,7 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
debug("got tree '%s' from '%s'",
state.store->printStorePath(tree.storePath), lockedRef);
- if (state.allowedPaths)
- state.allowedPaths->insert(tree.actualPath);
+ state.allowPath(tree.storePath);
assert(!originalRef.input.getNarHash() || tree.storePath == originalRef.input.computeStorePath(*state.store));
@@ -231,7 +230,7 @@ static Flake getFlake(
if (auto outputs = vInfo.attrs->get(sOutputs)) {
expectType(state, nFunction, *outputs->value, *outputs->pos);
- if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) {
+ if (outputs->value->isLambda() && outputs->value->lambda.fun->hasFormals()) {
for (auto & formal : outputs->value->lambda.fun->formals->formals) {
if (formal.name != state.sSelf)
flake.inputs.emplace(formal.name, FlakeInput {
diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc
index 492b819e7..0d0f3e469 100644
--- a/src/libexpr/nixexpr.cc
+++ b/src/libexpr/nixexpr.cc
@@ -124,7 +124,7 @@ void ExprList::show(std::ostream & str) const
void ExprLambda::show(std::ostream & str) const
{
str << "(";
- if (matchAttrs) {
+ if (hasFormals()) {
str << "{ ";
bool first = true;
for (auto & i : formals->formals) {
@@ -348,7 +348,7 @@ void ExprLambda::bindVars(const StaticEnv & env)
if (!arg.empty()) newEnv.vars[arg] = displ++;
- if (matchAttrs) {
+ if (hasFormals()) {
for (auto & i : formals->formals)
newEnv.vars[i.name] = displ++;
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index 51a14cd59..851e875bd 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -233,11 +233,10 @@ struct ExprLambda : Expr
Pos pos;
Symbol name;
Symbol arg;
- bool matchAttrs;
Formals * formals;
Expr * body;
- ExprLambda(const Pos & pos, const Symbol & arg, bool matchAttrs, Formals * formals, Expr * body)
- : pos(pos), arg(arg), matchAttrs(matchAttrs), formals(formals), body(body)
+ ExprLambda(const Pos & pos, const Symbol & arg, Formals * formals, Expr * body)
+ : pos(pos), arg(arg), formals(formals), body(body)
{
if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
throw ParseError({
@@ -247,6 +246,7 @@ struct ExprLambda : Expr
};
void setName(Symbol & name);
string showNamePos() const;
+ inline bool hasFormals() const { return formals != nullptr; }
COMMON_METHODS
};
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index e3749783a..8a0a79c96 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -324,13 +324,13 @@ expr: expr_function;
expr_function
: ID ':' expr_function
- { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), false, 0, $3); }
+ { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), 0, $3); }
| '{' formals '}' ':' expr_function
- { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), true, $2, $5); }
+ { $$ = new ExprLambda(CUR_POS, data->symbols.create(""), $2, $5); }
| '{' formals '}' '@' ID ':' expr_function
- { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), true, $2, $7); }
+ { $$ = new ExprLambda(CUR_POS, data->symbols.create($5), $2, $7); }
| ID '@' '{' formals '}' ':' expr_function
- { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), true, $4, $7); }
+ { $$ = new ExprLambda(CUR_POS, data->symbols.create($1), $4, $7); }
| ASSERT expr ';' expr_function
{ $$ = new ExprAssert(CUR_POS, $2, $4); }
| WITH expr ';' expr_function
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 25b9c32b2..8f6e937ec 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -56,13 +56,9 @@ void EvalState::realiseContext(const PathSet & context)
"cannot build '%1%' during evaluation because the option 'allow-import-from-derivation' is disabled",
store->printStorePath(drvs.begin()->drvPath));
- /* For performance, prefetch all substitute info. */
- StorePathSet willBuild, willSubstitute, unknown;
- uint64_t downloadSize, narSize;
+ /* Build/substitute the context. */
std::vector<DerivedPath> buildReqs;
for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
- store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize);
-
store->buildPaths(buildReqs);
/* Add the output of this derivations to the allowed
@@ -414,7 +410,7 @@ static RegisterPrimOp primop_isNull({
Return `true` if *e* evaluates to `null`, and `false` otherwise.
> **Warning**
- >
+ >
> This function is *deprecated*; just write `e == null` instead.
)",
.fun = prim_isNull,
@@ -1851,56 +1847,87 @@ static RegisterPrimOp primop_toFile({
.fun = prim_toFile,
});
-static void addPath(EvalState & state, const Pos & pos, const string & name, const Path & path_,
- Value * filterFun, FileIngestionMethod method, const std::optional<Hash> expectedHash, Value & v)
+static void addPath(
+ EvalState & state,
+ const Pos & pos,
+ const string & name,
+ Path path,
+ Value * filterFun,
+ FileIngestionMethod method,
+ const std::optional<Hash> expectedHash,
+ Value & v,
+ const PathSet & context)
{
- const auto path = evalSettings.pureEval && expectedHash ?
- path_ :
- state.checkSourcePath(path_);
- PathFilter filter = filterFun ? ([&](const Path & path) {
- auto st = lstat(path);
+ try {
+ // FIXME: handle CA derivation outputs (where path needs to
+ // be rewritten to the actual output).
+ state.realiseContext(context);
- /* Call the filter function. The first argument is the path,
- the second is a string indicating the type of the file. */
- Value arg1;
- mkString(arg1, path);
+ if (state.store->isInStore(path)) {
+ auto [storePath, subPath] = state.store->toStorePath(path);
+ auto info = state.store->queryPathInfo(storePath);
+ if (!info->references.empty())
+ throw EvalError("store path '%s' is not allowed to have references",
+ state.store->printStorePath(storePath));
+ path = state.store->toRealPath(storePath) + subPath;
+ }
- Value fun2;
- state.callFunction(*filterFun, arg1, fun2, noPos);
+ path = evalSettings.pureEval && expectedHash
+ ? path
+ : state.checkSourcePath(path);
- Value arg2;
- mkString(arg2,
- S_ISREG(st.st_mode) ? "regular" :
- S_ISDIR(st.st_mode) ? "directory" :
- S_ISLNK(st.st_mode) ? "symlink" :
- "unknown" /* not supported, will fail! */);
+ PathFilter filter = filterFun ? ([&](const Path & path) {
+ auto st = lstat(path);
- Value res;
- state.callFunction(fun2, arg2, res, noPos);
+ /* Call the filter function. The first argument is the path,
+ the second is a string indicating the type of the file. */
+ Value arg1;
+ mkString(arg1, path);
- return state.forceBool(res, pos);
- }) : defaultPathFilter;
+ Value fun2;
+ state.callFunction(*filterFun, arg1, fun2, noPos);
- std::optional<StorePath> expectedStorePath;
- if (expectedHash)
- expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
- {
- .method = method,
- .hash = *expectedHash,
- },
- {},
- });
- Path dstPath;
- if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
- dstPath = state.store->printStorePath(settings.readOnlyMode
- ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
- : state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
- if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
- throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
- } else
- dstPath = state.store->printStorePath(*expectedStorePath);
+ Value arg2;
+ mkString(arg2,
+ S_ISREG(st.st_mode) ? "regular" :
+ S_ISDIR(st.st_mode) ? "directory" :
+ S_ISLNK(st.st_mode) ? "symlink" :
+ "unknown" /* not supported, will fail! */);
+
+ Value res;
+ state.callFunction(fun2, arg2, res, noPos);
+
+ return state.forceBool(res, pos);
+ }) : defaultPathFilter;
+
+ std::optional<StorePath> expectedStorePath;
+ if (expectedHash)
+ expectedStorePath = state.store->makeFixedOutputPath(name, FixedOutputInfo {
+ {
+ .method = method,
+ .hash = *expectedHash,
+ },
+ {},
+ });
+
+ Path dstPath;
+ if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) {
+ dstPath = state.store->printStorePath(settings.readOnlyMode
+ ? state.store->computeStorePathForPath(name, path, method, htSHA256, filter).first
+ : state.store->addToStore(name, path, method, htSHA256, filter, state.repair));
+ if (expectedHash && expectedStorePath != state.store->parseStorePath(dstPath))
+ throw Error("store path mismatch in (possibly filtered) path added from '%s'", path);
+ } else
+ dstPath = state.store->printStorePath(*expectedStorePath);
- mkString(v, dstPath, {dstPath});
+ mkString(v, dstPath, {dstPath});
+
+ state.allowPath(dstPath);
+
+ } catch (Error & e) {
+ e.addTrace(pos, "while adding path '%s'", path);
+ throw;
+ }
}
@@ -1908,11 +1935,6 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
{
PathSet context;
Path path = state.coerceToPath(pos, *args[1], context);
- if (!context.empty())
- throw EvalError({
- .msg = hintfmt("string '%1%' cannot refer to other paths", path),
- .errPos = pos
- });
state.forceValue(*args[0], pos);
if (args[0]->type() != nFunction)
@@ -1923,13 +1945,26 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
.errPos = pos
});
- addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v);
+ addPath(state, pos, std::string(baseNameOf(path)), path, args[0], FileIngestionMethod::Recursive, std::nullopt, v, context);
}
static RegisterPrimOp primop_filterSource({
.name = "__filterSource",
.args = {"e1", "e2"},
.doc = R"(
+ > **Warning**
+ >
+ > `filterSource` should not be used to filter store paths. Since
+ > `filterSource` uses the name of the input directory while naming
+ > the output directory, doing so will produce a directory name in
+ > the form of `<hash2>-<hash>-<name>`, where `<hash>-<name>` is
+ > the name of the input directory. Since `<hash>` depends on the
+ > unfiltered directory, the name of the output directory will
+ > indirectly depend on files that are filtered out by the
+ > function. This will trigger a rebuild even when a filtered out
+ > file is changed. Use `builtins.path` instead, which allows
+ > specifying the name of the output directory.
+
This function allows you to copy sources into the Nix store while
filtering certain files. For instance, suppose that you want to use
the directory `source-dir` as an input to a Nix expression, e.g.
@@ -1976,18 +2011,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
Value * filterFun = nullptr;
auto method = FileIngestionMethod::Recursive;
std::optional<Hash> expectedHash;
+ PathSet context;
for (auto & attr : *args[0]->attrs) {
const string & n(attr.name);
- if (n == "path") {
- PathSet context;
+ if (n == "path")
path = state.coerceToPath(*attr.pos, *attr.value, context);
- if (!context.empty())
- throw EvalError({
- .msg = hintfmt("string '%1%' cannot refer to other paths", path),
- .errPos = *attr.pos
- });
- } else if (attr.name == state.sName)
+ else if (attr.name == state.sName)
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else if (n == "filter") {
state.forceValue(*attr.value, pos);
@@ -2010,7 +2040,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
if (name.empty())
name = baseNameOf(path);
- addPath(state, pos, name, path, filterFun, method, expectedHash, v);
+ addPath(state, pos, name, path, filterFun, method, expectedHash, v, context);
}
static RegisterPrimOp primop_path({
@@ -2375,7 +2405,7 @@ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args
.errPos = pos
});
- if (!args[0]->lambda.fun->matchAttrs) {
+ if (!args[0]->lambda.fun->hasFormals()) {
state.mkAttrs(v, 0);
return;
}
@@ -2530,7 +2560,7 @@ static RegisterPrimOp primop_tail({
the argument isn’t a list or is an empty list.
> **Warning**
- >
+ >
> This function should generally be avoided since it's inefficient:
> unlike Haskell's `tail`, it takes O(n) time, so recursing over a
> list by repeatedly calling `tail` takes O(n^2) time.
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index 3f88ccb91..1cd481243 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -84,8 +84,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
v.attrs->sort();
- if (state.allowedPaths)
- state.allowedPaths->insert(tree.actualPath);
+ state.allowPath(tree.storePath);
}
static RegisterPrimOp r_fetchMercurial("fetchMercurial", 1, prim_fetchMercurial);
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 06bdec003..f570f19ae 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -66,7 +66,7 @@ void emitTreeAttrs(
v.attrs->sort();
}
-std::string fixURI(std::string uri, EvalState &state, const std::string & defaultScheme = "file")
+std::string fixURI(std::string uri, EvalState & state, const std::string & defaultScheme = "file")
{
state.checkURI(uri);
return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri;
@@ -81,23 +81,17 @@ std::string fixURIForGit(std::string uri, EvalState & state)
return fixURI(uri, state);
}
-void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v)
-{
- string n(name);
- attrs.emplace(name, n == "url" ? fixURI(v, state) : v);
-}
-
struct FetchTreeParams {
bool emptyRevFallback = false;
bool allowNameArgument = false;
};
static void fetchTree(
- EvalState &state,
- const Pos &pos,
- Value **args,
- Value &v,
- const std::optional<std::string> type,
+ EvalState & state,
+ const Pos & pos,
+ Value * * args,
+ Value & v,
+ std::optional<std::string> type,
const FetchTreeParams & params = FetchTreeParams{}
) {
fetchers::Input input;
@@ -110,17 +104,33 @@ static void fetchTree(
fetchers::Attrs attrs;
+ if (auto aType = args[0]->attrs->get(state.sType)) {
+ if (type)
+ throw Error({
+ .msg = hintfmt("unexpected attribute 'type'"),
+ .errPos = pos
+ });
+ type = state.forceStringNoCtx(*aType->value, *aType->pos);
+ } else if (!type)
+ throw Error({
+ .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
+ .errPos = pos
+ });
+
+ attrs.emplace("type", type.value());
+
for (auto & attr : *args[0]->attrs) {
+ if (attr.name == state.sType) continue;
state.forceValue(*attr.value);
- if (attr.value->type() == nPath || attr.value->type() == nString)
- addURI(
- state,
- attrs,
- attr.name,
- state.coerceToString(*attr.pos, *attr.value, context, false, false)
- );
- else if (attr.value->type() == nString)
- addURI(state, attrs, attr.name, attr.value->string.s);
+ if (attr.value->type() == nPath || attr.value->type() == nString) {
+ auto s = state.coerceToString(*attr.pos, *attr.value, context, false, false);
+ attrs.emplace(attr.name,
+ attr.name == "url"
+ ? type == "git"
+ ? fixURIForGit(s, state)
+ : fixURI(s, state)
+ : s);
+ }
else if (attr.value->type() == nBool)
attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
else if (attr.value->type() == nInt)
@@ -130,15 +140,6 @@ static void fetchTree(
attr.name, showType(*attr.value));
}
- if (type)
- attrs.emplace("type", type.value());
-
- if (!attrs.count("type"))
- throw Error({
- .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
- .errPos = pos
- });
-
if (!params.allowNameArgument)
if (auto nameIter = attrs.find("name"); nameIter != attrs.end())
throw Error({
@@ -146,7 +147,6 @@ static void fetchTree(
.errPos = pos
});
-
input = fetchers::Input::fromAttrs(std::move(attrs));
} else {
auto url = state.coerceToString(pos, *args[0], context, false, false);
@@ -169,8 +169,7 @@ static void fetchTree(
auto [tree, input2] = input.fetch(state.store);
- if (state.allowedPaths)
- state.allowedPaths->insert(tree.actualPath);
+ state.allowPath(tree.storePath);
emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback, false);
}
@@ -234,19 +233,16 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
? fetchers::downloadTarball(state.store, *url, name, (bool) expectedHash).first.storePath
: fetchers::downloadFile(state.store, *url, name, (bool) expectedHash).storePath;
- auto realPath = state.store->toRealPath(storePath);
-
if (expectedHash) {
auto hash = unpack
? state.store->queryPathInfo(storePath)->narHash
- : hashFile(htSHA256, realPath);
+ : hashFile(htSHA256, state.store->toRealPath(storePath));
if (hash != *expectedHash)
throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s",
*url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
}
- if (state.allowedPaths)
- state.allowedPaths->insert(realPath);
+ state.allowPath(storePath);
auto path = state.store->printStorePath(storePath);
mkString(v, path, PathSet({path}));
diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc
index 2ddc5f751..b44455f5f 100644
--- a/src/libexpr/value-to-xml.cc
+++ b/src/libexpr/value-to-xml.cc
@@ -135,7 +135,7 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
if (location) posToXML(xmlAttrs, v.lambda.fun->pos);
XMLOpenElement _(doc, "function", xmlAttrs);
- if (v.lambda.fun->matchAttrs) {
+ if (v.lambda.fun->hasFormals()) {
XMLAttrs attrs;
if (!v.lambda.fun->arg.empty()) attrs["name"] = v.lambda.fun->arg;
if (v.lambda.fun->formals->ellipsis) attrs["ellipsis"] = "1";
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
index f574e64cf..e739f4d9d 100644
--- a/src/libstore/build/local-derivation-goal.cc
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -711,6 +711,7 @@ void LocalDerivationGoal::startBuilder()
if (!builderOut.readSide)
throw SysError("opening pseudoterminal master");
+ // FIXME: not thread-safe, use ptsname_r
std::string slaveName(ptsname(builderOut.readSide.get()));
if (buildUser) {
@@ -754,7 +755,6 @@ void LocalDerivationGoal::startBuilder()
result.startTime = time(0);
/* Fork a child to build the package. */
- ProcessOptions options;
#if __linux__
if (useChroot) {
@@ -797,8 +797,6 @@ void LocalDerivationGoal::startBuilder()
userNamespaceSync.create();
- options.allowVfork = false;
-
Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
static bool userNamespacesEnabled =
pathExists(maxUserNamespaces)
@@ -856,7 +854,7 @@ void LocalDerivationGoal::startBuilder()
writeFull(builderOut.writeSide.get(),
fmt("%d %d\n", usingUserNamespace, child));
_exit(0);
- }, options);
+ });
int res = helper.wait();
if (res != 0 && settings.sandboxFallback) {
@@ -920,10 +918,9 @@ void LocalDerivationGoal::startBuilder()
#endif
{
fallback:
- options.allowVfork = !buildUser && !drv->isBuiltin();
pid = startProcess([&]() {
runChild();
- }, options);
+ });
}
/* parent */
@@ -2140,8 +2137,7 @@ void LocalDerivationGoal::registerOutputs()
/* Pass blank Sink as we are not ready to hash data at this stage. */
NullSink blank;
- auto references = worker.store.parseStorePathSet(
- scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
+ auto references = scanForReferences(blank, actualPath, referenceablePaths);
outputReferencesIfUnregistered.insert_or_assign(
outputName,
@@ -2208,7 +2204,7 @@ void LocalDerivationGoal::registerOutputs()
auto rewriteOutput = [&]() {
/* Apply hash rewriting if necessary. */
if (!outputRewrites.empty()) {
- warn("rewriting hashes in '%1%'; cross fingers", actualPath);
+ debug("rewriting hashes in '%1%'; cross fingers", actualPath);
/* FIXME: this is in-memory. */
StringSink sink;
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index a7a6b92a6..55afb5cca 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -239,7 +239,7 @@ void Worker::run(const Goals & _topGoals)
}
}
- /* Call queryMissing() efficiently query substitutes. */
+ /* Call queryMissing() to efficiently query substitutes. */
StorePathSet willBuild, willSubstitute, unknown;
uint64_t downloadSize, narSize;
store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize);
diff --git a/src/libstore/references.cc b/src/libstore/references.cc
index 3a07c1411..c369b14ac 100644
--- a/src/libstore/references.cc
+++ b/src/libstore/references.cc
@@ -11,11 +11,13 @@
namespace nix {
-static unsigned int refLength = 32; /* characters */
+static size_t refLength = 32; /* characters */
-static void search(const unsigned char * s, size_t len,
- StringSet & hashes, StringSet & seen)
+static void search(
+ std::string_view s,
+ StringSet & hashes,
+ StringSet & seen)
{
static std::once_flag initialised;
static bool isBase32[256];
@@ -25,7 +27,7 @@ static void search(const unsigned char * s, size_t len,
isBase32[(unsigned char) base32Chars[i]] = true;
});
- for (size_t i = 0; i + refLength <= len; ) {
+ for (size_t i = 0; i + refLength <= s.size(); ) {
int j;
bool match = true;
for (j = refLength - 1; j >= 0; --j)
@@ -35,7 +37,7 @@ static void search(const unsigned char * s, size_t len,
break;
}
if (!match) continue;
- string ref((const char *) s + i, refLength);
+ std::string ref(s.substr(i, refLength));
if (hashes.erase(ref)) {
debug(format("found reference to '%1%' at offset '%2%'")
% ref % i);
@@ -46,69 +48,60 @@ static void search(const unsigned char * s, size_t len,
}
-struct RefScanSink : Sink
+void RefScanSink::operator () (std::string_view data)
{
- StringSet hashes;
- StringSet seen;
-
- string tail;
-
- RefScanSink() { }
-
- void operator () (std::string_view data) override
- {
- /* It's possible that a reference spans the previous and current
- fragment, so search in the concatenation of the tail of the
- previous fragment and the start of the current fragment. */
- string s = tail + std::string(data, 0, refLength);
- search((const unsigned char *) s.data(), s.size(), hashes, seen);
-
- search((const unsigned char *) data.data(), data.size(), hashes, seen);
-
- size_t tailLen = data.size() <= refLength ? data.size() : refLength;
- tail = std::string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen));
- tail.append({data.data() + data.size() - tailLen, tailLen});
- }
-};
+ /* It's possible that a reference spans the previous and current
+ fragment, so search in the concatenation of the tail of the
+ previous fragment and the start of the current fragment. */
+ auto s = tail;
+ s.append(data.data(), refLength);
+ search(s, hashes, seen);
+
+ search(data, hashes, seen);
+
+ auto tailLen = std::min(data.size(), refLength);
+ auto rest = refLength - tailLen;
+ if (rest < tail.size())
+ tail = tail.substr(tail.size() - rest);
+ tail.append(data.data() + data.size() - tailLen, tailLen);
+}
-std::pair<PathSet, HashResult> scanForReferences(const string & path,
- const PathSet & refs)
+std::pair<StorePathSet, HashResult> scanForReferences(
+ const string & path,
+ const StorePathSet & refs)
{
HashSink hashSink { htSHA256 };
auto found = scanForReferences(hashSink, path, refs);
auto hash = hashSink.finish();
- return std::pair<PathSet, HashResult>(found, hash);
+ return std::pair<StorePathSet, HashResult>(found, hash);
}
-PathSet scanForReferences(Sink & toTee,
- const string & path, const PathSet & refs)
+StorePathSet scanForReferences(
+ Sink & toTee,
+ const Path & path,
+ const StorePathSet & refs)
{
- RefScanSink refsSink;
- TeeSink sink { refsSink, toTee };
- std::map<string, Path> backMap;
+ StringSet hashes;
+ std::map<std::string, StorePath> backMap;
for (auto & i : refs) {
- auto baseName = std::string(baseNameOf(i));
- string::size_type pos = baseName.find('-');
- if (pos == string::npos)
- throw Error("bad reference '%1%'", i);
- string s = string(baseName, 0, pos);
- assert(s.size() == refLength);
- assert(backMap.find(s) == backMap.end());
- // parseHash(htSHA256, s);
- refsSink.hashes.insert(s);
- backMap[s] = i;
+ std::string hashPart(i.hashPart());
+ auto inserted = backMap.emplace(hashPart, i).second;
+ assert(inserted);
+ hashes.insert(hashPart);
}
/* Look for the hashes in the NAR dump of the path. */
+ RefScanSink refsSink(std::move(hashes));
+ TeeSink sink { refsSink, toTee };
dumpPath(path, sink);
/* Map the hashes found back to their store paths. */
- PathSet found;
- for (auto & i : refsSink.seen) {
- std::map<string, Path>::iterator j;
- if ((j = backMap.find(i)) == backMap.end()) abort();
+ StorePathSet found;
+ for (auto & i : refsSink.getResult()) {
+ auto j = backMap.find(i);
+ assert(j != backMap.end());
found.insert(j->second);
}
diff --git a/src/libstore/references.hh b/src/libstore/references.hh
index 4f12e6b21..a6119c861 100644
--- a/src/libstore/references.hh
+++ b/src/libstore/references.hh
@@ -1,13 +1,31 @@
#pragma once
-#include "types.hh"
#include "hash.hh"
+#include "path.hh"
namespace nix {
-std::pair<PathSet, HashResult> scanForReferences(const Path & path, const PathSet & refs);
+std::pair<StorePathSet, HashResult> scanForReferences(const Path & path, const StorePathSet & refs);
-PathSet scanForReferences(Sink & toTee, const Path & path, const PathSet & refs);
+StorePathSet scanForReferences(Sink & toTee, const Path & path, const StorePathSet & refs);
+
+class RefScanSink : public Sink
+{
+ StringSet hashes;
+ StringSet seen;
+
+ std::string tail;
+
+public:
+
+ RefScanSink(StringSet && hashes) : hashes(hashes)
+ { }
+
+ StringSet & getResult()
+ { return seen; }
+
+ void operator () (std::string_view data) override;
+};
struct RewritingSink : Sink
{
diff --git a/src/libstore/tests/local.mk b/src/libstore/tests/local.mk
new file mode 100644
index 000000000..f74295d97
--- /dev/null
+++ b/src/libstore/tests/local.mk
@@ -0,0 +1,15 @@
+check: libstore-tests_RUN
+
+programs += libstore-tests
+
+libstore-tests_DIR := $(d)
+
+libstore-tests_INSTALL_DIR :=
+
+libstore-tests_SOURCES := $(wildcard $(d)/*.cc)
+
+libstore-tests_CXXFLAGS += -I src/libstore -I src/libutil
+
+libstore-tests_LIBS = libstore libutil
+
+libstore-tests_LDFLAGS := $(GTEST_LIBS)
diff --git a/src/libstore/tests/references.cc b/src/libstore/tests/references.cc
new file mode 100644
index 000000000..d91d1cedd
--- /dev/null
+++ b/src/libstore/tests/references.cc
@@ -0,0 +1,45 @@
+#include "references.hh"
+
+#include <gtest/gtest.h>
+
+namespace nix {
+
+TEST(references, scan)
+{
+ std::string hash1 = "dc04vv14dak1c1r48qa0m23vr9jy8sm0";
+ std::string hash2 = "zc842j0rz61mjsp3h3wp5ly71ak6qgdn";
+
+ {
+ RefScanSink scanner(StringSet{hash1});
+ auto s = "foobar";
+ scanner(s);
+ ASSERT_EQ(scanner.getResult(), StringSet{});
+ }
+
+ {
+ RefScanSink scanner(StringSet{hash1});
+ auto s = "foobar" + hash1 + "xyzzy";
+ scanner(s);
+ ASSERT_EQ(scanner.getResult(), StringSet{hash1});
+ }
+
+ {
+ RefScanSink scanner(StringSet{hash1, hash2});
+ auto s = "foobar" + hash1 + "xyzzy" + hash2;
+ scanner(((std::string_view) s).substr(0, 10));
+ scanner(((std::string_view) s).substr(10, 5));
+ scanner(((std::string_view) s).substr(15, 5));
+ scanner(((std::string_view) s).substr(20));
+ ASSERT_EQ(scanner.getResult(), StringSet({hash1, hash2}));
+ }
+
+ {
+ RefScanSink scanner(StringSet{hash1, hash2});
+ auto s = "foobar" + hash1 + "xyzzy" + hash2;
+ for (auto & i : s)
+ scanner(std::string(1, i));
+ ASSERT_EQ(scanner.getResult(), StringSet({hash1, hash2}));
+ }
+}
+
+}
diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc
index cfadccf68..02e81b022 100644
--- a/src/libstore/uds-remote-store.cc
+++ b/src/libstore/uds-remote-store.cc
@@ -65,16 +65,7 @@ ref<RemoteStore::Connection> UDSRemoteStore::openConnection()
throw SysError("cannot create Unix domain socket");
closeOnExec(conn->fd.get());
- string socketPath = path ? *path : settings.nixDaemonSocketFile;
-
- struct sockaddr_un addr;
- addr.sun_family = AF_UNIX;
- if (socketPath.size() + 1 >= sizeof(addr.sun_path))
- throw Error("socket path '%1%' is too long", socketPath);
- strcpy(addr.sun_path, socketPath.c_str());
-
- if (::connect(conn->fd.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
- throw SysError("cannot connect to daemon at '%1%'", socketPath);
+ nix::connect(conn->fd.get(), path ? *path : settings.nixDaemonSocketFile);
conn->from.fd = conn->fd.get();
conn->to.fd = conn->fd.get();
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index ed0eb2fb5..d78ec2b93 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -42,7 +42,7 @@ static string caseHackSuffix = "~nix~case~hack~";
PathFilter defaultPathFilter = [](const Path &) { return true; };
-static void dumpContents(const Path & path, size_t size,
+static void dumpContents(const Path & path, off_t size,
Sink & sink)
{
sink << "contents" << size;
@@ -76,7 +76,7 @@ static void dump(const Path & path, Sink & sink, PathFilter & filter)
sink << "type" << "regular";
if (st.st_mode & S_IXUSR)
sink << "executable" << "";
- dumpContents(path, (size_t) st.st_size, sink);
+ dumpContents(path, st.st_size, sink);
}
else if (S_ISDIR(st.st_mode)) {
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index bc841f425..563a72c12 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -903,7 +903,7 @@ int Pid::wait()
return status;
}
if (errno != EINTR)
- throw SysError("cannot get child exit status");
+ throw SysError("cannot get exit status of PID %d", pid);
checkInterrupt();
}
}
@@ -939,9 +939,6 @@ void killUser(uid_t uid)
users to which the current process can send signals. So we
fork a process, switch to uid, and send a mass kill. */
- ProcessOptions options;
- options.allowVfork = false;
-
Pid pid = startProcess([&]() {
if (setuid(uid) == -1)
@@ -964,7 +961,7 @@ void killUser(uid_t uid)
}
_exit(0);
- }, options);
+ });
int status = pid.wait();
if (status != 0)
@@ -1085,8 +1082,7 @@ void runProgram2(const RunOptions & options)
// vfork implies that the environment of the main process and the fork will
// be shared (technically this is undefined, but in practice that's the
// case), so we can't use it if we alter the environment
- if (options.environment)
- processOptions.allowVfork = false;
+ processOptions.allowVfork = !options.environment;
/* Fork. */
Pid pid = startProcess([&]() {
@@ -1686,16 +1682,7 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
closeOnExec(fdSocket.get());
- struct sockaddr_un addr;
- addr.sun_family = AF_UNIX;
- if (path.size() + 1 >= sizeof(addr.sun_path))
- throw Error("socket path '%1%' is too long", path);
- strcpy(addr.sun_path, path.c_str());
-
- unlink(path.c_str());
-
- if (bind(fdSocket.get(), (struct sockaddr *) &addr, sizeof(addr)) == -1)
- throw SysError("cannot bind to socket '%1%'", path);
+ bind(fdSocket.get(), path);
if (chmod(path.c_str(), mode) == -1)
throw SysError("changing permissions on '%1%'", path);
@@ -1707,6 +1694,66 @@ AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode)
}
+void bind(int fd, const std::string & path)
+{
+ unlink(path.c_str());
+
+ struct sockaddr_un addr;
+ addr.sun_family = AF_UNIX;
+
+ if (path.size() + 1 >= sizeof(addr.sun_path)) {
+ Pid pid = startProcess([&]() {
+ auto dir = dirOf(path);
+ if (chdir(dir.c_str()) == -1)
+ throw SysError("chdir to '%s' failed", dir);
+ std::string base(baseNameOf(path));
+ if (base.size() + 1 >= sizeof(addr.sun_path))
+ throw Error("socket path '%s' is too long", base);
+ strcpy(addr.sun_path, base.c_str());
+ if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+ throw SysError("cannot bind to socket '%s'", path);
+ _exit(0);
+ });
+ int status = pid.wait();
+ if (status != 0)
+ throw Error("cannot bind to socket '%s'", path);
+ } else {
+ strcpy(addr.sun_path, path.c_str());
+ if (bind(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+ throw SysError("cannot bind to socket '%s'", path);
+ }
+}
+
+
+void connect(int fd, const std::string & path)
+{
+ struct sockaddr_un addr;
+ addr.sun_family = AF_UNIX;
+
+ if (path.size() + 1 >= sizeof(addr.sun_path)) {
+ Pid pid = startProcess([&]() {
+ auto dir = dirOf(path);
+ if (chdir(dir.c_str()) == -1)
+ throw SysError("chdir to '%s' failed", dir);
+ std::string base(baseNameOf(path));
+ if (base.size() + 1 >= sizeof(addr.sun_path))
+ throw Error("socket path '%s' is too long", base);
+ strcpy(addr.sun_path, base.c_str());
+ if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+ throw SysError("cannot connect to socket at '%s'", path);
+ _exit(0);
+ });
+ int status = pid.wait();
+ if (status != 0)
+ throw Error("cannot connect to socket at '%s'", path);
+ } else {
+ strcpy(addr.sun_path, path.c_str());
+ if (connect(fd, (struct sockaddr *) &addr, sizeof(addr)) == -1)
+ throw SysError("cannot connect to socket at '%s'", path);
+ }
+}
+
+
string showBytes(uint64_t bytes)
{
return fmt("%.2f MiB", bytes / (1024.0 * 1024.0));
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index bee77b53f..29232453f 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -259,10 +259,10 @@ void killUser(uid_t uid);
pid to the caller. */
struct ProcessOptions
{
- string errorPrefix = "error: ";
+ string errorPrefix = "";
bool dieWithParent = true;
bool runExitHandlers = false;
- bool allowVfork = true;
+ bool allowVfork = false;
};
pid_t startProcess(std::function<void()> fun, const ProcessOptions & options = ProcessOptions());
@@ -574,6 +574,12 @@ void commonChildInit(Pipe & logPipe);
/* Create a Unix domain socket in listen mode. */
AutoCloseFD createUnixDomainSocket(const Path & path, mode_t mode);
+/* Bind a Unix domain socket to a path. */
+void bind(int fd, const std::string & path);
+
+/* Connect to a Unix domain socket. */
+void connect(int fd, const std::string & path);
+
// A Rust/Python-like enumerate() iterator adapter.
// Borrowed from http://reedbeta.com/blog/python-like-enumerate-in-cpp17.
diff --git a/src/nix/daemon.cc b/src/nix/daemon.cc
index 2cf2a04c9..6a40a0bd3 100644
--- a/src/nix/daemon.cc
+++ b/src/nix/daemon.cc
@@ -156,9 +156,6 @@ static void daemonLoop()
if (chdir("/") == -1)
throw SysError("cannot change current directory");
- // Get rid of children automatically; don't let them become zombies.
- setSigChldAction(true);
-
AutoCloseFD fdSocket;
// Handle socket-based activation by systemd.
@@ -176,6 +173,9 @@ static void daemonLoop()
fdSocket = createUnixDomainSocket(settings.nixDaemonSocketFile, 0666);
}
+ // Get rid of children automatically; don't let them become zombies.
+ setSigChldAction(true);
+
// Loop accepting connections.
while (1) {
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index 7d7ada707..7e4d23f6e 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -346,10 +346,10 @@ struct CmdFlakeCheck : FlakeCommand
auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) {
try {
state->forceValue(v, pos);
- if (!v.isLambda() || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final")
+ if (!v.isLambda() || v.lambda.fun->hasFormals() || std::string(v.lambda.fun->arg) != "final")
throw Error("overlay does not take an argument named 'final'");
auto body = dynamic_cast<ExprLambda *>(v.lambda.fun->body);
- if (!body || body->matchAttrs || std::string(body->arg) != "prev")
+ if (!body || body->hasFormals() || std::string(body->arg) != "prev")
throw Error("overlay does not take an argument named 'prev'");
// FIXME: if we have a 'nixpkgs' input, use it to
// evaluate the overlay.
@@ -363,7 +363,7 @@ struct CmdFlakeCheck : FlakeCommand
try {
state->forceValue(v, pos);
if (v.isLambda()) {
- if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis)
+ if (!v.lambda.fun->hasFormals() || !v.lambda.fun->formals->ellipsis)
throw Error("module must match an open attribute set ('{ config, ... }')");
} else if (v.type() == nAttrs) {
for (auto & attr : *v.attrs)
diff --git a/src/nix/flake.md b/src/nix/flake.md
index 3d273100b..3b5812a0a 100644
--- a/src/nix/flake.md
+++ b/src/nix/flake.md
@@ -225,7 +225,7 @@ Currently the `type` attribute can be one of the following:
[flake:]<flake-id>(/<rev-or-ref>(/rev)?)?
```
- These perform a lookup of `<flake-id>` in the flake registry. or
+ These perform a lookup of `<flake-id>` in the flake registry. For
example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake
references. The specified `rev` and/or `ref` are merged with the
entry in the registry; see [nix registry](./nix3-registry.md) for
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 8aaf08813..2c3976689 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -187,11 +187,14 @@ static void showHelp(std::vector<std::string> subcommand, MultiCommand & topleve
, "/"),
*vUtils);
- auto vJson = state.allocValue();
+ auto vArgs = state.allocValue();
+ state.mkAttrs(*vArgs, 16);
+ auto vJson = state.allocAttr(*vArgs, state.symbols.create("command"));
mkString(*vJson, toplevel.toJSON().dump());
+ vArgs->attrs->sort();
auto vRes = state.allocValue();
- state.callFunction(*vGenerateManpage, *vJson, *vRes, noPos);
+ state.callFunction(*vGenerateManpage, *vArgs, *vRes, noPos);
auto attr = vRes->attrs->get(state.symbols.create(mdName + ".md"));
if (!attr)
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 916966997..b49be256e 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -98,10 +98,8 @@ struct ProfileManifest
else if (pathExists(profile + "/manifest.nix")) {
// FIXME: needed because of pure mode; ugly.
- if (state.allowedPaths) {
- state.allowedPaths->insert(state.store->followLinksToStore(profile));
- state.allowedPaths->insert(state.store->followLinksToStore(profile + "/manifest.nix"));
- }
+ state.allowPath(state.store->followLinksToStore(profile));
+ state.allowPath(state.store->followLinksToStore(profile + "/manifest.nix"));
auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile));