diff options
Diffstat (limited to 'src')
176 files changed, 5435 insertions, 1666 deletions
diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 83854df49..9dd557205 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -52,9 +52,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr for (auto & attr : tokens) { /* Is i an index (integer) or a normal attribute name? */ - enum { apAttr, apIndex } apType = apAttr; - unsigned int attrIndex; - if (string2Int(attr, attrIndex)) apType = apIndex; + auto attrIndex = string2Int<unsigned int>(attr); /* Evaluate the expression. */ Value * vNew = state.allocValue(); @@ -65,9 +63,9 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr /* It should evaluate to either a set or an expression, according to what is specified in the attrPath. */ - if (apType == apAttr) { + if (!attrIndex) { - if (v->type != tAttrs) + if (v->type() != nAttrs) throw TypeError( "the expression selected by the selection path '%1%' should be a set but is %2%", attrPath, @@ -82,17 +80,17 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr pos = *a->pos; } - else if (apType == apIndex) { + else { if (!v->isList()) throw TypeError( "the expression selected by the selection path '%1%' should be a list but is %2%", attrPath, showType(*v)); - if (attrIndex >= v->listSize()) - throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath); + if (*attrIndex >= v->listSize()) + throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", *attrIndex, attrPath); - v = v->listElems()[attrIndex]; + v = v->listElems()[*attrIndex]; pos = noPos; } diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index b1d61a285..b6091c955 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -24,9 +24,7 @@ void EvalState::mkAttrs(Value & v, size_t capacity) v = vEmptySet; return; } - clearValue(v); - v.type = tAttrs; - v.attrs = allocBindings(capacity); + v.mkAttrs(allocBindings(capacity)); nrAttrsets++; nrAttrsInAttrsets += capacity; } diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 10c1a6975..ffe782454 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -14,14 +14,14 @@ MixEvalArgs::MixEvalArgs() { addFlag({ .longName = "arg", - .description = "argument to be passed to Nix functions", + .description = "Pass the value *expr* as the argument *name* to Nix functions.", .labels = {"name", "expr"}, .handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }} }); addFlag({ .longName = "argstr", - .description = "string-valued argument to be passed to Nix functions", + .description = "Pass the string *string* as the argument *name* to Nix functions.", .labels = {"name", "string"}, .handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }}, }); @@ -29,14 +29,14 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "include", .shortName = 'I', - .description = "add a path to the list of locations used to look up `<...>` file names", + .description = "Add *path* to the list of locations used to look up `<...>` file names.", .labels = {"path"}, .handler = {[&](std::string s) { searchPath.push_back(s); }} }); addFlag({ .longName = "impure", - .description = "allow access to mutable paths and repositories", + .description = "Allow access to mutable paths and repositories.", .handler = {[&]() { evalSettings.pureEval = false; }}, @@ -44,7 +44,7 @@ MixEvalArgs::MixEvalArgs() addFlag({ .longName = "override-flake", - .description = "override a flake registry value", + .description = "Override the flake registries, redirecting *original-ref* to *resolved-ref*.", .labels = {"original-ref", "resolved-ref"}, .handler = {[&](std::string _from, std::string _to) { auto from = parseFlakeRef(_from, absPath(".")); diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 7b025be23..98d91c905 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -390,14 +390,14 @@ Value & AttrCursor::forceValue() } if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) { - if (v.type == tString) + if (v.type() == nString) cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context), string_t{v.string.s, {}}}; - else if (v.type == tPath) - cachedValue = {root->db->setString(getKey(), v.path), v.path}; - else if (v.type == tBool) + else if (v.type() == nPath) + cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}}; + else if (v.type() == nBool) cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean}; - else if (v.type == tAttrs) + else if (v.type() == nAttrs) ; // FIXME: do something? else cachedValue = {root->db->setMisc(getKey()), misc_t()}; @@ -442,7 +442,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro auto & v = forceValue(); - if (v.type != tAttrs) + if (v.type() != nAttrs) return nullptr; //throw TypeError("'%s' is not an attribute set", getAttrPathStr()); @@ -512,10 +512,10 @@ std::string AttrCursor::getString() auto & v = forceValue(); - if (v.type != tString && v.type != tPath) - throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type)); + if (v.type() != nString && v.type() != nPath) + throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type())); - return v.type == tString ? v.string.s : v.path; + return v.type() == nString ? v.string.s : v.path; } string_t AttrCursor::getStringWithContext() @@ -543,12 +543,12 @@ string_t AttrCursor::getStringWithContext() auto & v = forceValue(); - if (v.type == tString) + if (v.type() == nString) return {v.string.s, v.getContext()}; - else if (v.type == tPath) + else if (v.type() == nPath) return {v.path, {}}; else - throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type)); + throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type())); } bool AttrCursor::getBool() @@ -567,7 +567,7 @@ bool AttrCursor::getBool() auto & v = forceValue(); - if (v.type != tBool) + if (v.type() != nBool) throw TypeError("'%s' is not a Boolean", getAttrPathStr()); return v.boolean; @@ -589,7 +589,7 @@ std::vector<Symbol> AttrCursor::getAttrs() auto & v = forceValue(); - if (v.type != tAttrs) + if (v.type() != nAttrs) throw TypeError("'%s' is not an attribute set", getAttrPathStr()); std::vector<Symbol> attrs; diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh index 30f6ec7db..f6dead6b0 100644 --- a/src/libexpr/eval-inline.hh +++ b/src/libexpr/eval-inline.hh @@ -32,23 +32,21 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const void EvalState::forceValue(Value & v, const Pos & pos) { - if (v.type == tThunk) { + if (v.isThunk()) { Env * env = v.thunk.env; Expr * expr = v.thunk.expr; try { - v.type = tBlackhole; + v.mkBlackhole(); //checkInterrupt(); expr->eval(*this, *env, v); } catch (...) { - v.type = tThunk; - v.thunk.env = env; - v.thunk.expr = expr; + v.mkThunk(env, expr); throw; } } - else if (v.type == tApp) + else if (v.isApp()) callFunction(*v.app.left, *v.app.right, v, noPos); - else if (v.type == tBlackhole) + else if (v.isBlackhole()) throwEvalError(pos, "infinite recursion encountered"); } @@ -56,7 +54,7 @@ void EvalState::forceValue(Value & v, const Pos & pos) inline void EvalState::forceAttrs(Value & v) { forceValue(v); - if (v.type != tAttrs) + if (v.type() != nAttrs) throwTypeError("value is %1% while a set was expected", v); } @@ -64,7 +62,7 @@ inline void EvalState::forceAttrs(Value & v) inline void EvalState::forceAttrs(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tAttrs) + if (v.type() != nAttrs) throwTypeError(pos, "value is %1% while a set was expected", v); } diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index c6f4d1716..f3471aac7 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -68,7 +68,7 @@ RootValue allocRootValue(Value * v) } -static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v) +void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v) { checkInterrupt(); @@ -77,7 +77,7 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con return; } - switch (v.type) { + switch (v.internalType) { case tInt: str << v.integer; break; @@ -158,32 +158,27 @@ std::ostream & operator << (std::ostream & str, const Value & v) const Value *getPrimOp(const Value &v) { const Value * primOp = &v; - while (primOp->type == tPrimOpApp) { + while (primOp->isPrimOpApp()) { primOp = primOp->primOpApp.left; } - assert(primOp->type == tPrimOp); + assert(primOp->isPrimOp()); return primOp; } - string showType(ValueType type) { switch (type) { - case tInt: return "an integer"; - case tBool: return "a Boolean"; - case tString: return "a string"; - case tPath: return "a path"; - case tNull: return "null"; - case tAttrs: return "a set"; - case tList1: case tList2: case tListN: return "a list"; - case tThunk: return "a thunk"; - case tApp: return "a function application"; - case tLambda: return "a function"; - case tBlackhole: return "a black hole"; - case tPrimOp: return "a built-in function"; - case tPrimOpApp: return "a partially applied built-in function"; - case tExternal: return "an external value"; - case tFloat: return "a float"; + case nInt: return "an integer"; + case nBool: return "a Boolean"; + case nString: return "a string"; + case nPath: return "a path"; + case nNull: return "null"; + case nAttrs: return "a set"; + case nList: return "a list"; + case nFunction: return "a function"; + case nExternal: return "an external value"; + case nFloat: return "a float"; + case nThunk: return "a thunk"; } abort(); } @@ -191,15 +186,18 @@ string showType(ValueType type) string showType(const Value & v) { - switch (v.type) { + switch (v.internalType) { case tString: return v.string.context ? "a string with context" : "a string"; case tPrimOp: return fmt("the built-in function '%s'", string(v.primOp->name)); case tPrimOpApp: return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); case tExternal: return v.external->showType(); + case tThunk: return "a thunk"; + case tApp: return "a function application"; + case tBlackhole: return "a black hole"; default: - return showType(v.type); + return showType(v.type()); } } @@ -207,9 +205,9 @@ string showType(const Value & v) bool Value::isTrivial() const { return - type != tApp - && type != tPrimOpApp - && (type != tThunk + internalType != tApp + && internalType != tPrimOpApp + && (internalType != tThunk || (dynamic_cast<ExprAttrs *>(thunk.expr) && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty()) || dynamic_cast<ExprLambda *>(thunk.expr) @@ -404,11 +402,6 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i); } - try { - addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true)); - } catch (Error &) { - } - if (evalSettings.restrictEval || evalSettings.pureEval) { allowedPaths = PathSet(); @@ -432,9 +425,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) } } - clearValue(vEmptySet); - vEmptySet.type = tAttrs; - vEmptySet.attrs = allocBindings(0); + vEmptySet.mkAttrs(allocBindings(0)); createBaseEnv(); } @@ -461,6 +452,8 @@ Path EvalState::checkSourcePath(const Path & path_) */ Path abspath = canonPath(path_); + if (hasPrefix(abspath, corepkgsPrefix)) return abspath; + for (auto & i : *allowedPaths) { if (isDirOrInDir(abspath, i)) { found = true; @@ -550,16 +543,14 @@ Value * EvalState::addPrimOp(const string & name, the primop to a dummy value. */ if (arity == 0) { auto vPrimOp = allocValue(); - vPrimOp->type = tPrimOp; - vPrimOp->primOp = new PrimOp { .fun = primOp, .arity = 1, .name = sym }; + vPrimOp->mkPrimOp(new PrimOp { .fun = primOp, .arity = 1, .name = sym }); Value v; mkApp(v, *vPrimOp, *vPrimOp); return addConstant(name, v); } Value * v = allocValue(); - v->type = tPrimOp; - v->primOp = new PrimOp { .fun = primOp, .arity = arity, .name = sym }; + v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = sym }); staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(sym, v)); @@ -574,8 +565,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) if (primOp.arity == 0) { primOp.arity = 1; auto vPrimOp = allocValue(); - vPrimOp->type = tPrimOp; - vPrimOp->primOp = new PrimOp(std::move(primOp)); + vPrimOp->mkPrimOp(new PrimOp(std::move(primOp))); Value v; mkApp(v, *vPrimOp, *vPrimOp); return addConstant(primOp.name, v); @@ -586,8 +576,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp) primOp.name = symbols.create(std::string(primOp.name, 2)); Value * v = allocValue(); - v->type = tPrimOp; - v->primOp = new PrimOp(std::move(primOp)); + v->mkPrimOp(new PrimOp(std::move(primOp))); staticBaseEnv.vars[envName] = baseEnvDispl; baseEnv.values[baseEnvDispl++] = v; baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v)); @@ -603,9 +592,9 @@ Value & EvalState::getBuiltin(const string & name) std::optional<EvalState::Doc> EvalState::getDoc(Value & v) { - if (v.type == tPrimOp || v.type == tPrimOpApp) { + if (v.isPrimOp() || v.isPrimOpApp()) { auto v2 = &v; - while (v2->type == tPrimOpApp) + while (v2->isPrimOpApp()) v2 = v2->primOpApp.left; if (v2->primOp->doc) return Doc { @@ -668,11 +657,6 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s)) }); } -LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1)) -{ - throw TypeError(s, s1); -} - LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2)) { throw TypeError({ @@ -697,6 +681,14 @@ LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * }); } +LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char * s, const string & s1)) +{ + throw MissingArgumentError({ + .hint = hintfmt(s, s1), + .errPos = pos + }); +} + LocalNoInline(void addErrorTrace(Error & e, const char * s, const string & s2)) { e.addTrace(std::nullopt, s, s2); @@ -710,15 +702,13 @@ LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, con void mkString(Value & v, const char * s) { - mkStringNoCopy(v, dupString(s)); + v.mkString(dupString(s)); } Value & mkString(Value & v, std::string_view s, const PathSet & context) { - v.type = tString; - v.string.s = dupStringWithLen(s.data(), s.size()); - v.string.context = 0; + v.mkString(dupStringWithLen(s.data(), s.size())); if (!context.empty()) { size_t n = 0; v.string.context = (const char * *) @@ -733,7 +723,7 @@ Value & mkString(Value & v, std::string_view s, const PathSet & context) void mkPath(Value & v, const char * s) { - mkPathNoCopy(v, dupString(s)); + v.mkPath(dupString(s)); } @@ -794,16 +784,9 @@ Env & EvalState::allocEnv(size_t size) void EvalState::mkList(Value & v, size_t size) { - clearValue(v); - if (size == 1) - v.type = tList1; - else if (size == 2) - v.type = tList2; - else { - v.type = tListN; - v.bigList.size = size; - v.bigList.elems = size ? (Value * *) allocBytes(size * sizeof(Value *)) : 0; - } + v.mkList(size); + if (size > 2) + v.bigList.elems = (Value * *) allocBytes(size * sizeof(Value *)); nrListElems += size; } @@ -812,9 +795,7 @@ unsigned long nrThunks = 0; static inline void mkThunk(Value & v, Env & env, Expr * expr) { - v.type = tThunk; - v.thunk.env = &env; - v.thunk.expr = expr; + v.mkThunk(&env, expr); nrThunks++; } @@ -949,7 +930,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e) { Value v; e->eval(*this, env, v); - if (v.type != tBool) + if (v.type() != nBool) throwTypeError("value is %1% while a Boolean was expected", v); return v.boolean; } @@ -959,7 +940,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) { Value v; e->eval(*this, env, v); - if (v.type != tBool) + if (v.type() != nBool) throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -968,7 +949,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos) inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v) { e->eval(*this, env, v); - if (v.type != tAttrs) + if (v.type() != nAttrs) throwTypeError("value is %1% while a set was expected", v); } @@ -1068,7 +1049,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) Value nameVal; i.nameExpr->eval(state, *dynamicEnv, nameVal); state.forceValue(nameVal, i.pos); - if (nameVal.type == tNull) + if (nameVal.type() == nNull) continue; state.forceStringNoCtx(nameVal); Symbol nameSym = state.symbols.create(nameVal.string.s); @@ -1153,7 +1134,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) Symbol name = getName(i, state, env); if (def) { state.forceValue(*vAttrs, pos); - if (vAttrs->type != tAttrs || + if (vAttrs->type() != nAttrs || (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { def->eval(state, env, v); @@ -1193,7 +1174,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) state.forceValue(*vAttrs); Bindings::iterator j; Symbol name = getName(i, state, env); - if (vAttrs->type != tAttrs || + if (vAttrs->type() != nAttrs || (j = vAttrs->attrs->find(name)) == vAttrs->attrs->end()) { mkBool(v, false); @@ -1209,9 +1190,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v) void ExprLambda::eval(EvalState & state, Env & env, Value & v) { - v.type = tLambda; - v.lambda.env = &env; - v.lambda.fun = this; + v.mkLambda(&env, this); } @@ -1229,11 +1208,11 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) /* Figure out the number of arguments still needed. */ size_t argsDone = 0; Value * primOp = &fun; - while (primOp->type == tPrimOpApp) { + while (primOp->isPrimOpApp()) { argsDone++; primOp = primOp->primOpApp.left; } - assert(primOp->type == tPrimOp); + assert(primOp->isPrimOp()); auto arity = primOp->primOp->arity; auto argsLeft = arity - argsDone; @@ -1244,7 +1223,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) Value * vArgs[arity]; auto n = arity - 1; vArgs[n--] = &arg; - for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left) + for (Value * arg = &fun; arg->isPrimOpApp(); arg = arg->primOpApp.left) vArgs[n--] = arg->primOpApp.right; /* And call the primop. */ @@ -1254,9 +1233,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos) } else { Value * fun2 = allocValue(); *fun2 = fun; - v.type = tPrimOpApp; - v.primOpApp.left = fun2; - v.primOpApp.right = &arg; + v.mkPrimOpApp(fun2, &arg); } } @@ -1266,12 +1243,12 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po forceValue(fun, pos); - if (fun.type == tPrimOp || fun.type == tPrimOpApp) { + if (fun.isPrimOp() || fun.isPrimOpApp()) { callPrimOp(fun, arg, v, pos); return; } - if (fun.type == tAttrs) { + if (fun.type() == nAttrs) { auto found = fun.attrs->find(sFunctor); if (found != fun.attrs->end()) { /* fun may be allocated on the stack of the calling function, @@ -1287,7 +1264,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po } } - if (fun.type != tLambda) + if (!fun.isLambda()) throwTypeError(pos, "attempt to call something which is not a function but %1%", fun); ExprLambda & lambda(*fun.lambda.fun); @@ -1370,7 +1347,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) { forceValue(fun); - if (fun.type == tAttrs) { + if (fun.type() == nAttrs) { auto found = fun.attrs->find(sFunctor); if (found != fun.attrs->end()) { Value * v = allocValue(); @@ -1380,7 +1357,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) } } - if (fun.type != tLambda || !fun.lambda.fun->matchAttrs) { + if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) { res = fun; return; } @@ -1402,7 +1379,13 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res) if (j != args.end()) { actualArgs->attrs->push_back(*j); } else if (!i.def) { - throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name); + throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%') + +nix attempted to evaluate a function as a top level expression; in this case it must have its +arguments supplied either by default values, or passed explicitly with --arg or --argstr. + +https://nixos.org/manual/nix/stable/#ss-functions)", i.name); + } } } @@ -1564,7 +1547,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) NixFloat nf = 0; bool first = !forceString; - ValueType firstType = tString; + ValueType firstType = nString; for (auto & i : *es) { Value vTmp; @@ -1575,36 +1558,36 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) since paths are copied when they are used in a derivation), and none of the strings are allowed to have contexts. */ if (first) { - firstType = vTmp.type; + firstType = vTmp.type(); first = false; } - if (firstType == tInt) { - if (vTmp.type == tInt) { + if (firstType == nInt) { + if (vTmp.type() == nInt) { n += vTmp.integer; - } else if (vTmp.type == tFloat) { + } else if (vTmp.type() == nFloat) { // Upgrade the type from int to float; - firstType = tFloat; + firstType = nFloat; nf = n; nf += vTmp.fpoint; } else throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp)); - } else if (firstType == tFloat) { - if (vTmp.type == tInt) { + } else if (firstType == nFloat) { + if (vTmp.type() == nInt) { nf += vTmp.integer; - } else if (vTmp.type == tFloat) { + } else if (vTmp.type() == nFloat) { nf += vTmp.fpoint; } else throwEvalError(pos, "cannot add %1% to a float", showType(vTmp)); } else - s << state.coerceToString(pos, vTmp, context, false, firstType == tString); + s << state.coerceToString(pos, vTmp, context, false, firstType == nString); } - if (firstType == tInt) + if (firstType == nInt) mkInt(v, n); - else if (firstType == tFloat) + else if (firstType == nFloat) mkFloat(v, nf); - else if (firstType == tPath) { + else if (firstType == nPath) { if (!context.empty()) throwEvalError(pos, "a string that refers to a store path cannot be appended to a path"); auto path = canonPath(s.str()); @@ -1631,7 +1614,7 @@ void EvalState::forceValueDeep(Value & v) forceValue(v); - if (v.type == tAttrs) { + if (v.type() == nAttrs) { for (auto & i : *v.attrs) try { recurse(*i.value); @@ -1654,7 +1637,7 @@ void EvalState::forceValueDeep(Value & v) NixInt EvalState::forceInt(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tInt) + if (v.type() != nInt) throwTypeError(pos, "value is %1% while an integer was expected", v); return v.integer; } @@ -1663,9 +1646,9 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos) NixFloat EvalState::forceFloat(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type == tInt) + if (v.type() == nInt) return v.integer; - else if (v.type != tFloat) + else if (v.type() != nFloat) throwTypeError(pos, "value is %1% while a float was expected", v); return v.fpoint; } @@ -1674,7 +1657,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos) bool EvalState::forceBool(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tBool) + if (v.type() != nBool) throwTypeError(pos, "value is %1% while a Boolean was expected", v); return v.boolean; } @@ -1682,14 +1665,14 @@ bool EvalState::forceBool(Value & v, const Pos & pos) bool EvalState::isFunctor(Value & fun) { - return fun.type == tAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); + return fun.type() == nAttrs && fun.attrs->find(sFunctor) != fun.attrs->end(); } void EvalState::forceFunction(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v)) + if (v.type() != nFunction && !isFunctor(v)) throwTypeError(pos, "value is %1% while a function was expected", v); } @@ -1697,7 +1680,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos) string EvalState::forceString(Value & v, const Pos & pos) { forceValue(v, pos); - if (v.type != tString) { + if (v.type() != nString) { if (pos) throwTypeError(pos, "value is %1% while a string was expected", v); else @@ -1730,7 +1713,7 @@ void copyContext(const Value & v, PathSet & context) std::vector<std::pair<Path, std::string>> Value::getContext() { std::vector<std::pair<Path, std::string>> res; - assert(type == tString); + assert(internalType == tString); if (string.context) for (const char * * p = string.context; *p; ++p) res.push_back(decodeContext(*p)); @@ -1763,11 +1746,11 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos) bool EvalState::isDerivation(Value & v) { - if (v.type != tAttrs) return false; + if (v.type() != nAttrs) return false; Bindings::iterator i = v.attrs->find(sType); if (i == v.attrs->end()) return false; forceValue(*i->value); - if (i->value->type != tString) return false; + if (i->value->type() != nString) return false; return strcmp(i->value->string.s, "derivation") == 0; } @@ -1792,17 +1775,17 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, string s; - if (v.type == tString) { + if (v.type() == nString) { copyContext(v, context); return v.string.s; } - if (v.type == tPath) { + if (v.type() == nPath) { Path path(canonPath(v.path)); return copyToStore ? copyPathToStore(context, path) : path; } - if (v.type == tAttrs) { + if (v.type() == nAttrs) { auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore); if (maybeString) { return *maybeString; @@ -1812,18 +1795,18 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context, return coerceToString(pos, *i->value, context, coerceMore, copyToStore); } - if (v.type == tExternal) + if (v.type() == nExternal) return v.external->coerceToString(pos, context, coerceMore, copyToStore); if (coerceMore) { /* Note that `false' is represented as an empty string for shell scripting convenience, just like `null'. */ - if (v.type == tBool && v.boolean) return "1"; - if (v.type == tBool && !v.boolean) return ""; - if (v.type == tInt) return std::to_string(v.integer); - if (v.type == tFloat) return std::to_string(v.fpoint); - if (v.type == tNull) return ""; + if (v.type() == nBool && v.boolean) return "1"; + if (v.type() == nBool && !v.boolean) return ""; + if (v.type() == nInt) return std::to_string(v.integer); + if (v.type() == nFloat) return std::to_string(v.fpoint); + if (v.type() == nNull) return ""; if (v.isList()) { string result; @@ -1886,40 +1869,38 @@ bool EvalState::eqValues(Value & v1, Value & v2) if (&v1 == &v2) return true; // Special case type-compatibility between float and int - if (v1.type == tInt && v2.type == tFloat) + if (v1.type() == nInt && v2.type() == nFloat) return v1.integer == v2.fpoint; - if (v1.type == tFloat && v2.type == tInt) + if (v1.type() == nFloat && v2.type() == nInt) return v1.fpoint == v2.integer; // All other types are not compatible with each other. - if (v1.type != v2.type) return false; + if (v1.type() != v2.type()) return false; - switch (v1.type) { + switch (v1.type()) { - case tInt: + case nInt: return v1.integer == v2.integer; - case tBool: + case nBool: return v1.boolean == v2.boolean; - case tString: + case nString: return strcmp(v1.string.s, v2.string.s) == 0; - case tPath: + case nPath: return strcmp(v1.path, v2.path) == 0; - case tNull: + case nNull: return true; - case tList1: - case tList2: - case tListN: + case nList: if (v1.listSize() != v2.listSize()) return false; for (size_t n = 0; n < v1.listSize(); ++n) if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false; return true; - case tAttrs: { + case nAttrs: { /* If both sets denote a derivation (type = "derivation"), then compare their outPaths. */ if (isDerivation(v1) && isDerivation(v2)) { @@ -1941,15 +1922,13 @@ bool EvalState::eqValues(Value & v1, Value & v2) } /* Functions are incomparable. */ - case tLambda: - case tPrimOp: - case tPrimOpApp: + case nFunction: return false; - case tExternal: + case nExternal: return *v1.external == *v2.external; - case tFloat: + case nFloat: return v1.fpoint == v2.fpoint; default: diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 0e1f61baa..e3eaed6d3 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -432,4 +432,6 @@ struct EvalSettings : Config extern EvalSettings evalSettings; +static const std::string corepkgsPrefix{"/__corepkgs__/"}; + } diff --git a/src/libexpr/fetchurl.nix b/src/libexpr/fetchurl.nix new file mode 100644 index 000000000..02531103b --- /dev/null +++ b/src/libexpr/fetchurl.nix @@ -0,0 +1,41 @@ +{ system ? "" # obsolete +, url +, hash ? "" # an SRI hash + +# Legacy hash specification +, md5 ? "", sha1 ? "", sha256 ? "", sha512 ? "" +, outputHash ? + if hash != "" then hash else if sha512 != "" then sha512 else if sha1 != "" then sha1 else if md5 != "" then md5 else sha256 +, outputHashAlgo ? + if hash != "" then "" else if sha512 != "" then "sha512" else if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256" + +, executable ? false +, unpack ? false +, name ? baseNameOf (toString url) +}: + +derivation { + builder = "builtin:fetchurl"; + + # New-style output content requirements. + inherit outputHashAlgo outputHash; + outputHashMode = if unpack || executable then "recursive" else "flat"; + + inherit name url executable unpack; + + system = "builtin"; + + # No need to double the amount of network traffic + preferLocalBuild = true; + + impureEnvVars = [ + # We borrow these environment variables from the caller to allow + # easy proxy configuration. This is impure, but a fixed-output + # derivation like fetchurl is allowed to do so since its result is + # by definition pure. + "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy" + ]; + + # To make "nix-prefetch-url" work. + urls = [ url ]; +} diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 3e866e1f9..0786fef3d 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -73,7 +73,7 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree( static void forceTrivialValue(EvalState & state, Value & value, const Pos & pos) { - if (value.type == tThunk && value.isTrivial()) + if (value.isThunk() && value.isTrivial()) state.forceValue(value, pos); } @@ -82,9 +82,9 @@ static void expectType(EvalState & state, ValueType type, Value & value, const Pos & pos) { forceTrivialValue(state, value, pos); - if (value.type != type) + if (value.type() != type) throw Error("expected %s but got %s at %s", - showType(type), showType(value.type), pos); + showType(type), showType(value.type()), pos); } static std::map<FlakeId, FlakeInput> parseFlakeInputs( @@ -93,7 +93,7 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs( static FlakeInput parseFlakeInput(EvalState & state, const std::string & inputName, Value * value, const Pos & pos) { - expectType(state, tAttrs, *value, pos); + expectType(state, nAttrs, *value, pos); FlakeInput input; @@ -108,23 +108,32 @@ static FlakeInput parseFlakeInput(EvalState & state, for (nix::Attr attr : *(value->attrs)) { try { if (attr.name == sUrl) { - expectType(state, tString, *attr.value, *attr.pos); + expectType(state, nString, *attr.value, *attr.pos); url = attr.value->string.s; attrs.emplace("url", *url); } else if (attr.name == sFlake) { - expectType(state, tBool, *attr.value, *attr.pos); + expectType(state, nBool, *attr.value, *attr.pos); input.isFlake = attr.value->boolean; } else if (attr.name == sInputs) { input.overrides = parseFlakeInputs(state, attr.value, *attr.pos); } else if (attr.name == sFollows) { - expectType(state, tString, *attr.value, *attr.pos); + expectType(state, nString, *attr.value, *attr.pos); input.follows = parseInputPath(attr.value->string.s); } else { - if (attr.value->type == tString) - attrs.emplace(attr.name, attr.value->string.s); - else - throw TypeError("flake input attribute '%s' is %s while a string is expected", - attr.name, showType(*attr.value)); + switch (attr.value->type()) { + case nString: + attrs.emplace(attr.name, attr.value->string.s); + break; + case nBool: + attrs.emplace(attr.name, Explicit<bool> { attr.value->boolean }); + break; + case nInt: + attrs.emplace(attr.name, (long unsigned int)attr.value->integer); + break; + default: + throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected", + attr.name, showType(*attr.value)); + } } } catch (Error & e) { e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name)); @@ -158,7 +167,7 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs( { std::map<FlakeId, FlakeInput> inputs; - expectType(state, tAttrs, *value, pos); + expectType(state, nAttrs, *value, pos); for (nix::Attr & inputAttr : *(*value).attrs) { inputs.emplace(inputAttr.name, @@ -199,10 +208,10 @@ static Flake getFlake( Value vInfo; state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack - expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0)); + expectType(state, nAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0)); if (auto description = vInfo.attrs->get(state.sDescription)) { - expectType(state, tString, *description->value, *description->pos); + expectType(state, nString, *description->value, *description->pos); flake.description = description->value->string.s; } @@ -214,9 +223,9 @@ static Flake getFlake( auto sOutputs = state.symbols.create("outputs"); if (auto outputs = vInfo.attrs->get(sOutputs)) { - expectType(state, tLambda, *outputs->value, *outputs->pos); + expectType(state, nFunction, *outputs->value, *outputs->pos); - if (outputs->value->lambda.fun->matchAttrs) { + if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) { for (auto & formal : outputs->value->lambda.fun->formals->formals) { if (formal.name != state.sSelf) flake.inputs.emplace(formal.name, FlakeInput { @@ -231,21 +240,21 @@ static Flake getFlake( auto sNixConfig = state.symbols.create("nixConfig"); if (auto nixConfig = vInfo.attrs->get(sNixConfig)) { - expectType(state, tAttrs, *nixConfig->value, *nixConfig->pos); + expectType(state, nAttrs, *nixConfig->value, *nixConfig->pos); for (auto & setting : *nixConfig->value->attrs) { forceTrivialValue(state, *setting.value, *setting.pos); - if (setting.value->type == tString) + if (setting.value->type() == nString) flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)}); - else if (setting.value->type == tInt) + else if (setting.value->type() == nInt) flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)}); - else if (setting.value->type == tBool) + else if (setting.value->type() == nBool) flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)}); - else if (setting.value->isList()) { + else if (setting.value->type() == nList) { std::vector<std::string> ss; for (unsigned int n = 0; n < setting.value->listSize(); ++n) { auto elem = setting.value->listElems()[n]; - if (elem->type != tString) + if (elem->type() != nString) throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected", setting.name, showType(*setting.value)); ss.push_back(state.forceStringNoCtx(*elem, *setting.pos)); diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 91916e8bf..1a3990ea1 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -128,7 +128,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall) if (!outTI->isList()) throw errMsg; Outputs result; for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) { - if ((*i)->type != tString) throw errMsg; + if ((*i)->type() != nString) throw errMsg; auto out = outputs.find((*i)->string.s); if (out == outputs.end()) throw errMsg; result.insert(*out); @@ -172,20 +172,20 @@ StringSet DrvInfo::queryMetaNames() bool DrvInfo::checkMeta(Value & v) { state->forceValue(v); - if (v.isList()) { + if (v.type() == nList) { for (unsigned int n = 0; n < v.listSize(); ++n) if (!checkMeta(*v.listElems()[n])) return false; return true; } - else if (v.type == tAttrs) { + else if (v.type() == nAttrs) { Bindings::iterator i = v.attrs->find(state->sOutPath); if (i != v.attrs->end()) return false; for (auto & i : *v.attrs) if (!checkMeta(*i.value)) return false; return true; } - else return v.type == tInt || v.type == tBool || v.type == tString || - v.type == tFloat; + else return v.type() == nInt || v.type() == nBool || v.type() == nString || + v.type() == nFloat; } @@ -201,7 +201,7 @@ Value * DrvInfo::queryMeta(const string & name) string DrvInfo::queryMetaString(const string & name) { Value * v = queryMeta(name); - if (!v || v->type != tString) return ""; + if (!v || v->type() != nString) return ""; return v->string.s; } @@ -210,12 +210,12 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type == tInt) return v->integer; - if (v->type == tString) { + if (v->type() == nInt) return v->integer; + if (v->type() == nString) { /* Backwards compatibility with before we had support for integer meta fields. */ - NixInt n; - if (string2Int(v->string.s, n)) return n; + if (auto n = string2Int<NixInt>(v->string.s)) + return *n; } return def; } @@ -224,12 +224,12 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type == tFloat) return v->fpoint; - if (v->type == tString) { + if (v->type() == nFloat) return v->fpoint; + if (v->type() == nString) { /* Backwards compatibility with before we had support for float meta fields. */ - NixFloat n; - if (string2Float(v->string.s, n)) return n; + if (auto n = string2Float<NixFloat>(v->string.s)) + return *n; } return def; } @@ -239,8 +239,8 @@ bool DrvInfo::queryMetaBool(const string & name, bool def) { Value * v = queryMeta(name); if (!v) return def; - if (v->type == tBool) return v->boolean; - if (v->type == tString) { + if (v->type() == nBool) return v->boolean; + if (v->type() == nString) { /* Backwards compatibility with before we had support for Boolean meta fields. */ if (strcmp(v->string.s, "true") == 0) return true; @@ -331,7 +331,7 @@ static void getDerivations(EvalState & state, Value & vIn, /* Process the expression. */ if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ; - else if (v.type == tAttrs) { + else if (v.type() == nAttrs) { /* !!! undocumented hackery to support combining channels in nix-env.cc. */ @@ -353,7 +353,7 @@ static void getDerivations(EvalState & state, Value & vIn, /* If the value of this attribute is itself a set, should we recurse into it? => Only if it has a `recurseForDerivations = true' attribute. */ - if (i->value->type == tAttrs) { + if (i->value->type() == nAttrs) { Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations); if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos)) getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures); @@ -362,7 +362,7 @@ static void getDerivations(EvalState & state, Value & vIn, } } - else if (v.isList()) { + else if (v.type() == nList) { for (unsigned int n = 0; n < v.listSize(); ++n) { string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str()); if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures)) diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 519da33f7..26c53d301 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -40,6 +40,6 @@ $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644)) $(foreach i, $(wildcard src/libexpr/flake/*.hh), \ $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) -$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh +$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh $(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh index e4cbc660f..530202ff6 100644 --- a/src/libexpr/nixexpr.hh +++ b/src/libexpr/nixexpr.hh @@ -17,6 +17,7 @@ MakeError(ThrownError, AssertionError); MakeError(Abort, EvalError); MakeError(TypeError, EvalError); MakeError(UndefinedVarError, Error); +MakeError(MissingArgumentError, Error); MakeError(RestrictedPathError, Error); @@ -129,7 +130,7 @@ struct ExprPath : Expr { string s; Value v; - ExprPath(const string & s) : s(s) { mkPathNoCopy(v, this->s.c_str()); }; + ExprPath(const string & s) : s(s) { v.mkPath(this->s.c_str()); }; COMMON_METHODS Value * maybeThunk(EvalState & state, Env & env); }; diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index a4c84c526..85eb05d61 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -698,6 +698,10 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos Path res = r.second + suffix; if (pathExists(res)) return canonPath(res); } + + if (hasPrefix(path, "nix/")) + return corepkgsPrefix + path.substr(4); + throw ThrownError({ .hint = hintfmt(evalSettings.pureEval ? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)" diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index e9c1f3b43..c73a94f4e 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -164,7 +164,15 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS state.forceFunction(**fun, pos); mkApp(v, **fun, w); state.forceAttrs(v, pos); - } else { + } + + else if (path == corepkgsPrefix + "fetchurl.nix") { + state.eval(state.parseExprFromString( + #include "fetchurl.nix.gen.hh" + , "/"), v); + } + + else { if (!vScope) state.evalFile(realPath, v); else { @@ -356,24 +364,20 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu { state.forceValue(*args[0], pos); string t; - switch (args[0]->type) { - case tInt: t = "int"; break; - case tBool: t = "bool"; break; - case tString: t = "string"; break; - case tPath: t = "path"; break; - case tNull: t = "null"; break; - case tAttrs: t = "set"; break; - case tList1: case tList2: case tListN: t = "list"; break; - case tLambda: - case tPrimOp: - case tPrimOpApp: - t = "lambda"; - break; - case tExternal: + switch (args[0]->type()) { + case nInt: t = "int"; break; + case nBool: t = "bool"; break; + case nString: t = "string"; break; + case nPath: t = "path"; break; + case nNull: t = "null"; break; + case nAttrs: t = "set"; break; + case nList: t = "list"; break; + case nFunction: t = "lambda"; break; + case nExternal: t = args[0]->external->typeOf(); break; - case tFloat: t = "float"; break; - default: abort(); + case nFloat: t = "float"; break; + case nThunk: abort(); } mkString(v, state.symbols.create(t)); } @@ -393,7 +397,7 @@ static RegisterPrimOp primop_typeOf({ static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tNull); + mkBool(v, args[0]->type() == nNull); } static RegisterPrimOp primop_isNull({ @@ -413,18 +417,7 @@ static RegisterPrimOp primop_isNull({ static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - bool res; - switch (args[0]->type) { - case tLambda: - case tPrimOp: - case tPrimOpApp: - res = true; - break; - default: - res = false; - break; - } - mkBool(v, res); + mkBool(v, args[0]->type() == nFunction); } static RegisterPrimOp primop_isFunction({ @@ -440,7 +433,7 @@ static RegisterPrimOp primop_isFunction({ static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tInt); + mkBool(v, args[0]->type() == nInt); } static RegisterPrimOp primop_isInt({ @@ -456,7 +449,7 @@ static RegisterPrimOp primop_isInt({ static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tFloat); + mkBool(v, args[0]->type() == nFloat); } static RegisterPrimOp primop_isFloat({ @@ -472,7 +465,7 @@ static RegisterPrimOp primop_isFloat({ static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tString); + mkBool(v, args[0]->type() == nString); } static RegisterPrimOp primop_isString({ @@ -488,7 +481,7 @@ static RegisterPrimOp primop_isString({ static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tBool); + mkBool(v, args[0]->type() == nBool); } static RegisterPrimOp primop_isBool({ @@ -504,7 +497,7 @@ static RegisterPrimOp primop_isBool({ static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tPath); + mkBool(v, args[0]->type() == nPath); } static RegisterPrimOp primop_isPath({ @@ -520,20 +513,20 @@ struct CompareValues { bool operator () (const Value * v1, const Value * v2) const { - if (v1->type == tFloat && v2->type == tInt) + if (v1->type() == nFloat && v2->type() == nInt) return v1->fpoint < v2->integer; - if (v1->type == tInt && v2->type == tFloat) + if (v1->type() == nInt && v2->type() == nFloat) return v1->integer < v2->fpoint; - if (v1->type != v2->type) + if (v1->type() != v2->type()) throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); - switch (v1->type) { - case tInt: + switch (v1->type()) { + case nInt: return v1->integer < v2->integer; - case tFloat: + case nFloat: return v1->fpoint < v2->fpoint; - case tString: + case nString: return strcmp(v1->string.s, v2->string.s) < 0; - case tPath: + case nPath: return strcmp(v1->path, v2->path) < 0; default: throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2)); @@ -777,7 +770,7 @@ static RegisterPrimOp primop_deepSeq({ static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - if (args[0]->type == tString) + if (args[0]->type() == nString) printError("trace: %1%", args[0]->string.s); else printError("trace: %1%", *args[0]); @@ -902,7 +895,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * if (ignoreNulls) { state.forceValue(*i->value, pos); - if (i->value->type == tNull) continue; + if (i->value->type() == nNull) continue; } if (i->name == state.sContentAddressed) { @@ -1308,7 +1301,7 @@ static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value { PathSet context; Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false)); - if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); + if (args[0]->type() == nPath) mkPath(v, dir.c_str()); else mkString(v, dir, context); } static RegisterPrimOp primop_dirOf({ @@ -1449,7 +1442,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val Value * ent_val = state.allocAttr(v, state.symbols.create(ent.name)); if (ent.type == DT_UNKNOWN) ent.type = getFileType(path + "/" + ent.name); - mkStringNoCopy(*ent_val, + ent_val->mkString( ent.type == DT_REG ? "regular" : ent.type == DT_DIR ? "directory" : ent.type == DT_LNK ? "symlink" : @@ -1813,7 +1806,7 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args }); state.forceValue(*args[0], pos); - if (args[0]->type != tLambda) + if (args[0]->type() != nFunction) throw TypeError({ .hint = hintfmt( "first argument in call to 'filterSource' is not a function but %1%", @@ -2079,7 +2072,7 @@ static RegisterPrimOp primop_hasAttr({ static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->type == tAttrs); + mkBool(v, args[0]->type() == nAttrs); } static RegisterPrimOp primop_isAttrs({ @@ -2259,11 +2252,11 @@ static RegisterPrimOp primop_catAttrs({ static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - if (args[0]->type == tPrimOpApp || args[0]->type == tPrimOp) { + if (args[0]->isPrimOpApp() || args[0]->isPrimOp()) { state.mkAttrs(v, 0); return; } - if (args[0]->type != tLambda) + if (!args[0]->isLambda()) throw TypeError({ .hint = hintfmt("'functionArgs' requires a function"), .errPos = pos @@ -2342,7 +2335,7 @@ static RegisterPrimOp primop_mapAttrs({ static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v) { state.forceValue(*args[0], pos); - mkBool(v, args[0]->isList()); + mkBool(v, args[0]->type() == nList); } static RegisterPrimOp primop_isList({ @@ -2694,7 +2687,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value auto comparator = [&](Value * a, Value * b) { /* Optimization: if the comparator is lessThan, bypass callFunction. */ - if (args[0]->type == tPrimOp && args[0]->primOp->fun == prim_lessThan) + if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan) return CompareValues()(a, b); Value vTmp1, vTmp2; @@ -2836,7 +2829,7 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value & { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->type == tFloat || args[1]->type == tFloat) + if (args[0]->type() == nFloat || args[1]->type() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos)); @@ -2855,7 +2848,7 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value & { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->type == tFloat || args[1]->type == tFloat) + if (args[0]->type() == nFloat || args[1]->type() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos)); @@ -2874,7 +2867,7 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value & { state.forceValue(*args[0], pos); state.forceValue(*args[1], pos); - if (args[0]->type == tFloat || args[1]->type == tFloat) + if (args[0]->type() == nFloat || args[1]->type() == nFloat) mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos)); else mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos)); @@ -2901,7 +2894,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value & .errPos = pos }); - if (args[0]->type == tFloat || args[1]->type == tFloat) { + if (args[0]->type() == nFloat || args[1]->type() == nFloat) { mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos)); } else { NixInt i1 = state.forceInt(*args[0], pos); diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index a77035c16..845a1ed1b 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -17,7 +17,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar state.forceValue(*args[0]); - if (args[0]->type == tAttrs) { + if (args[0]->type() == nAttrs) { state.forceAttrs(*args[0], pos); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index d094edf92..ab80be2d3 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -85,26 +85,26 @@ static void fetchTree( state.forceValue(*args[0]); - if (args[0]->type == tAttrs) { + if (args[0]->type() == nAttrs) { state.forceAttrs(*args[0], pos); fetchers::Attrs attrs; for (auto & attr : *args[0]->attrs) { state.forceValue(*attr.value); - if (attr.value->type == tPath || attr.value->type == tString) + if (attr.value->type() == nPath || attr.value->type() == nString) addURI( state, attrs, attr.name, state.coerceToString(*attr.pos, *attr.value, context, false, false) ); - else if (attr.value->type == tString) + else if (attr.value->type() == nString) addURI(state, attrs, attr.name, attr.value->string.s); - else if (attr.value->type == tBool) + else if (attr.value->type() == nBool) attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean}); - else if (attr.value->type == tInt) - attrs.emplace(attr.name, attr.value->integer); + else if (attr.value->type() == nInt) + attrs.emplace(attr.name, uint64_t(attr.value->integer)); else throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", attr.name, showType(*attr.value)); @@ -163,7 +163,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v, state.forceValue(*args[0]); - if (args[0]->type == tAttrs) { + if (args[0]->type() == nAttrs) { state.forceAttrs(*args[0], pos); @@ -324,6 +324,11 @@ static RegisterPrimOp primop_fetchGit({ A Boolean parameter that specifies whether submodules should be checked out. Defaults to `false`. + - allRefs + Whether to fetch all refs of the repository. With this argument being + true, it's possible to load a `rev` from *any* `ref` (by default only + `rev`s from the specified `ref` are supported). + Here are some examples of how to use `fetchGit`. - To fetch a private repository over SSH: diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 6ec8315ba..bfea24d40 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -16,30 +16,30 @@ void printValueAsJSON(EvalState & state, bool strict, if (strict) state.forceValue(v); - switch (v.type) { + switch (v.type()) { - case tInt: + case nInt: out.write(v.integer); break; - case tBool: + case nBool: out.write(v.boolean); break; - case tString: + case nString: copyContext(v, context); out.write(v.string.s); break; - case tPath: + case nPath: out.write(state.copyPathToStore(context, v.path)); break; - case tNull: + case nNull: out.write(nullptr); break; - case tAttrs: { + case nAttrs: { auto maybeString = state.tryAttrsToString(noPos, v, context, false, false); if (maybeString) { out.write(*maybeString); @@ -61,7 +61,7 @@ void printValueAsJSON(EvalState & state, bool strict, break; } - case tList1: case tList2: case tListN: { + case nList: { auto list(out.list()); for (unsigned int n = 0; n < v.listSize(); ++n) { auto placeholder(list.placeholder()); @@ -70,15 +70,18 @@ void printValueAsJSON(EvalState & state, bool strict, break; } - case tExternal: + case nExternal: v.external->printValueAsJSON(state, strict, out, context); break; - case tFloat: + case nFloat: out.write(v.fpoint); break; - default: + case nThunk: + throw TypeError("cannot convert %1% to JSON", showType(v)); + + case nFunction: throw TypeError("cannot convert %1% to JSON", showType(v)); } } diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index 1f0b1541d..7464455d8 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -58,31 +58,31 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, if (strict) state.forceValue(v); - switch (v.type) { + switch (v.type()) { - case tInt: + case nInt: doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str())); break; - case tBool: + case nBool: doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false")); break; - case tString: + case nString: /* !!! show the context? */ copyContext(v, context); doc.writeEmptyElement("string", singletonAttrs("value", v.string.s)); break; - case tPath: + case nPath: doc.writeEmptyElement("path", singletonAttrs("value", v.path)); break; - case tNull: + case nNull: doc.writeEmptyElement("null"); break; - case tAttrs: + case nAttrs: if (state.isDerivation(v)) { XMLAttrs xmlAttrs; @@ -92,14 +92,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, a = v.attrs->find(state.sDrvPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value); - if (a->value->type == tString) + if (a->value->type() == nString) xmlAttrs["drvPath"] = drvPath = a->value->string.s; } a = v.attrs->find(state.sOutPath); if (a != v.attrs->end()) { if (strict) state.forceValue(*a->value); - if (a->value->type == tString) + if (a->value->type() == nString) xmlAttrs["outPath"] = a->value->string.s; } @@ -118,14 +118,19 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, break; - case tList1: case tList2: case tListN: { + case nList: { XMLOpenElement _(doc, "list"); for (unsigned int n = 0; n < v.listSize(); ++n) printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen); break; } - case tLambda: { + case nFunction: { + if (!v.isLambda()) { + // FIXME: Serialize primops and primopapps + doc.writeEmptyElement("unevaluated"); + break; + } XMLAttrs xmlAttrs; if (location) posToXML(xmlAttrs, v.lambda.fun->pos); XMLOpenElement _(doc, "function", xmlAttrs); @@ -143,15 +148,15 @@ static void printValueAsXML(EvalState & state, bool strict, bool location, break; } - case tExternal: + case nExternal: v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen); break; - case tFloat: + case nFloat: doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str())); break; - default: + case nThunk: doc.writeEmptyElement("unevaluated"); } } diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index fe11bb2ed..b317c1898 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -27,8 +27,24 @@ typedef enum { tPrimOpApp, tExternal, tFloat -} ValueType; +} InternalType; +// This type abstracts over all actual value types in the language, +// grouping together implementation details like tList*, different function +// types, and types in non-normal form (so thunks and co.) +typedef enum { + nThunk, + nInt, + nFloat, + nBool, + nString, + nPath, + nNull, + nAttrs, + nList, + nFunction, + nExternal +} ValueType; class Bindings; struct Env; @@ -90,7 +106,28 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v); struct Value { - ValueType type; +private: + InternalType internalType; + +friend std::string showType(const Value & v); +friend void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v); + +public: + + // Functions needed to distinguish the type + // These should be removed eventually, by putting the functionality that's + // needed by callers into methods of this type + + // type() == nThunk + inline bool isThunk() const { return internalType == tThunk; }; + inline bool isApp() const { return internalType == tApp; }; + inline bool isBlackhole() const { return internalType == tBlackhole; }; + + // type() == nFunction + inline bool isLambda() const { return internalType == tLambda; }; + inline bool isPrimOp() const { return internalType == tPrimOp; }; + inline bool isPrimOpApp() const { return internalType == tPrimOpApp; }; + union { NixInt integer; @@ -147,24 +184,161 @@ struct Value NixFloat fpoint; }; + // Returns the normal type of a Value. This only returns nThunk if the + // Value hasn't been forceValue'd + inline ValueType type() const + { + switch (internalType) { + case tInt: return nInt; + case tBool: return nBool; + case tString: return nString; + case tPath: return nPath; + case tNull: return nNull; + case tAttrs: return nAttrs; + case tList1: case tList2: case tListN: return nList; + case tLambda: case tPrimOp: case tPrimOpApp: return nFunction; + case tExternal: return nExternal; + case tFloat: return nFloat; + case tThunk: case tApp: case tBlackhole: return nThunk; + } + abort(); + } + + /* After overwriting an app node, be sure to clear pointers in the + Value to ensure that the target isn't kept alive unnecessarily. */ + inline void clearValue() + { + app.left = app.right = 0; + } + + inline void mkInt(NixInt n) + { + clearValue(); + internalType = tInt; + integer = n; + } + + inline void mkBool(bool b) + { + clearValue(); + internalType = tBool; + boolean = b; + } + + inline void mkString(const char * s, const char * * context = 0) + { + internalType = tString; + string.s = s; + string.context = context; + } + + inline void mkPath(const char * s) + { + clearValue(); + internalType = tPath; + path = s; + } + + inline void mkNull() + { + clearValue(); + internalType = tNull; + } + + inline void mkAttrs(Bindings * a) + { + clearValue(); + internalType = tAttrs; + attrs = a; + } + + inline void mkList(size_t size) + { + clearValue(); + if (size == 1) + internalType = tList1; + else if (size == 2) + internalType = tList2; + else { + internalType = tListN; + bigList.size = size; + } + } + + inline void mkThunk(Env * e, Expr * ex) + { + internalType = tThunk; + thunk.env = e; + thunk.expr = ex; + } + + inline void mkApp(Value * l, Value * r) + { + internalType = tApp; + app.left = l; + app.right = r; + } + + inline void mkLambda(Env * e, ExprLambda * f) + { + internalType = tLambda; + lambda.env = e; + lambda.fun = f; + } + + inline void mkBlackhole() + { + internalType = tBlackhole; + // Value will be overridden anyways + } + + inline void mkPrimOp(PrimOp * p) + { + clearValue(); + internalType = tPrimOp; + primOp = p; + } + + + inline void mkPrimOpApp(Value * l, Value * r) + { + internalType = tPrimOpApp; + app.left = l; + app.right = r; + } + + inline void mkExternal(ExternalValueBase * e) + { + clearValue(); + internalType = tExternal; + external = e; + } + + inline void mkFloat(NixFloat n) + { + clearValue(); + internalType = tFloat; + fpoint = n; + } + bool isList() const { - return type == tList1 || type == tList2 || type == tListN; + return internalType == tList1 || internalType == tList2 || internalType == tListN; } Value * * listElems() { - return type == tList1 || type == tList2 ? smallList : bigList.elems; + return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } const Value * const * listElems() const { - return type == tList1 || type == tList2 ? smallList : bigList.elems; + return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems; } size_t listSize() const { - return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size; + return internalType == tList1 ? 1 : internalType == tList2 ? 2 : bigList.size; } /* Check whether forcing this value requires a trivial amount of @@ -176,86 +350,42 @@ struct Value }; -/* After overwriting an app node, be sure to clear pointers in the - Value to ensure that the target isn't kept alive unnecessarily. */ -static inline void clearValue(Value & v) -{ - v.app.left = v.app.right = 0; -} - +// TODO: Remove these static functions, replace call sites with v.mk* instead static inline void mkInt(Value & v, NixInt n) { - clearValue(v); - v.type = tInt; - v.integer = n; + v.mkInt(n); } - static inline void mkFloat(Value & v, NixFloat n) { - clearValue(v); - v.type = tFloat; - v.fpoint = n; + v.mkFloat(n); } - static inline void mkBool(Value & v, bool b) { - clearValue(v); - v.type = tBool; - v.boolean = b; + v.mkBool(b); } - static inline void mkNull(Value & v) { - clearValue(v); - v.type = tNull; + v.mkNull(); } - static inline void mkApp(Value & v, Value & left, Value & right) { - v.type = tApp; - v.app.left = &left; - v.app.right = &right; -} - - -static inline void mkPrimOpApp(Value & v, Value & left, Value & right) -{ - v.type = tPrimOpApp; - v.app.left = &left; - v.app.right = &right; -} - - -static inline void mkStringNoCopy(Value & v, const char * s) -{ - v.type = tString; - v.string.s = s; - v.string.context = 0; + v.mkApp(&left, &right); } - static inline void mkString(Value & v, const Symbol & s) { - mkStringNoCopy(v, ((const string &) s).c_str()); + v.mkString(((const string &) s).c_str()); } void mkString(Value & v, const char * s); -static inline void mkPathNoCopy(Value & v, const char * s) -{ - clearValue(v); - v.type = tPath; - v.path = s; -} - - void mkPath(Value & v, const char * s); diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc index 720b19fcd..a565d19d4 100644 --- a/src/libfetchers/attrs.cc +++ b/src/libfetchers/attrs.cc @@ -11,11 +11,11 @@ Attrs jsonToAttrs(const nlohmann::json & json) for (auto & i : json.items()) { if (i.value().is_number()) - attrs.emplace(i.key(), i.value().get<int64_t>()); + attrs.emplace(i.key(), i.value().get<uint64_t>()); else if (i.value().is_string()) attrs.emplace(i.key(), i.value().get<std::string>()); else if (i.value().is_boolean()) - attrs.emplace(i.key(), i.value().get<bool>()); + attrs.emplace(i.key(), Explicit<bool> { i.value().get<bool>() }); else throw Error("unsupported input attribute type in lock file"); } diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index e7712c5fd..81c647f89 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -59,12 +59,13 @@ struct GitInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "git") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash") + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs") throw Error("unsupported Git input attribute '%s'", name); parseURL(getStrAttr(attrs, "url")); maybeGetBoolAttr(attrs, "shallow"); maybeGetBoolAttr(attrs, "submodules"); + maybeGetBoolAttr(attrs, "allRefs"); if (auto ref = maybeGetStrAttr(attrs, "ref")) { if (std::regex_search(*ref, badGitRefRegex)) @@ -169,10 +170,12 @@ struct GitInputScheme : InputScheme bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); + bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); std::string cacheType = "git"; if (shallow) cacheType += "-shallow"; if (submodules) cacheType += "-submodules"; + if (allRefs) cacheType += "-all-refs"; auto getImmutableAttrs = [&]() { @@ -338,11 +341,15 @@ struct GitInputScheme : InputScheme } } } else { - /* If the local ref is older than ‘tarball-ttl’ seconds, do a - git fetch to update the local ref to the remote ref. */ - struct stat st; - doFetch = stat(localRefFile.c_str(), &st) != 0 || - (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; + if (allRefs) { + doFetch = true; + } else { + /* If the local ref is older than ‘tarball-ttl’ seconds, do a + git fetch to update the local ref to the remote ref. */ + struct stat st; + doFetch = stat(localRefFile.c_str(), &st) != 0 || + (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now; + } } if (doFetch) { @@ -352,9 +359,11 @@ struct GitInputScheme : InputScheme // we're using --quiet for now. Should process its stderr. try { auto ref = input.getRef(); - auto fetchRef = ref->compare(0, 5, "refs/") == 0 - ? *ref - : "refs/heads/" + *ref; + auto fetchRef = allRefs + ? "refs/*" + : ref->compare(0, 5, "refs/") == 0 + ? *ref + : "refs/heads/" + *ref; runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) }); } catch (Error & e) { if (!pathExists(localRefFile)) throw; @@ -392,6 +401,28 @@ struct GitInputScheme : InputScheme AutoDelete delTmpDir(tmpDir, true); PathFilter filter = defaultPathFilter; + RunOptions checkCommitOpts( + "git", + { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() } + ); + checkCommitOpts.searchPath = true; + checkCommitOpts.mergeStderrToStdout = true; + + auto result = runProgram(checkCommitOpts); + if (WEXITSTATUS(result.first) == 128 + && result.second.find("bad file") != std::string::npos + ) { + throw Error( + "Cannot find Git revision '%s' in ref '%s' of repository '%s'! " + "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the " + ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD + "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".", + input.getRev()->gitRev(), + *input.getRef(), + actualUrl + ); + } + if (submodules) { Path tmpGitDir = createTempDir(); AutoDelete delTmpGitDir(tmpGitDir, true); diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 92ff224f7..8352ef02d 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -37,15 +37,29 @@ struct GitArchiveInputScheme : InputScheme std::optional<std::string> ref; std::optional<std::string> host_url; - if (path.size() == 2) { - } else if (path.size() == 3) { + auto size = path.size(); + if (size == 3) { if (std::regex_match(path[2], revRegex)) rev = Hash::parseAny(path[2], htSHA1); else if (std::regex_match(path[2], refRegex)) ref = path[2]; else throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]); - } else + } else if (size > 3) { + std::string rs; + for (auto i = std::next(path.begin(), 2); i != path.end(); i++) { + rs += *i; + if (std::next(i) != path.end()) { + rs += "/"; + } + } + + if (std::regex_match(rs, refRegex)) { + ref = rs; + } else { + throw BadURL("in URL '%s', '%s' is not a branch/tag name", url.url, rs); + } + } else if (size < 2) throw BadURL("URL '%s' is invalid", url.url); for (auto &[name, value] : url.query) { @@ -195,14 +209,14 @@ struct GitArchiveInputScheme : InputScheme auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers); - input.attrs.insert_or_assign("lastModified", lastModified); + input.attrs.insert_or_assign("lastModified", uint64_t(lastModified)); getCache()->add( store, immutableAttrs, { {"rev", rev->gitRev()}, - {"lastModified", lastModified} + {"lastModified", uint64_t(lastModified)} }, tree.storePath, true); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 07a51059d..0eb401e10 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -301,7 +301,7 @@ struct MercurialInputScheme : InputScheme Attrs infoAttrs({ {"rev", input.getRev()->gitRev()}, - {"revCount", (int64_t) revCount}, + {"revCount", (uint64_t) revCount}, }); if (!_input.getRev()) diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index bcb904c0d..d1003de57 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -20,10 +20,10 @@ struct PathInputScheme : InputScheme if (name == "rev" || name == "narHash") input.attrs.insert_or_assign(name, value); else if (name == "revCount" || name == "lastModified") { - uint64_t n; - if (!string2Int(value, n)) + if (auto n = string2Int<uint64_t>(value)) + input.attrs.insert_or_assign(name, *n); + else throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name); - input.attrs.insert_or_assign(name, n); } else throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 8c0f20475..56c014a8c 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -152,7 +152,7 @@ std::pair<Tree, time_t> downloadTarball( } Attrs infoAttrs({ - {"lastModified", lastModified}, + {"lastModified", uint64_t(lastModified)}, {"etag", res.etag}, }); diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc index 3e4e475e5..bd5573e5d 100644 --- a/src/libmain/common-args.cc +++ b/src/libmain/common-args.cc @@ -10,25 +10,25 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "verbose", .shortName = 'v', - .description = "increase verbosity level", + .description = "Increase the logging verbosity level.", .handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }}, }); addFlag({ .longName = "quiet", - .description = "decrease verbosity level", + .description = "Decrease the logging verbosity level.", .handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }}, }); addFlag({ .longName = "debug", - .description = "enable debug output", + .description = "Set the logging verbosity level to 'debug'.", .handler = {[]() { verbosity = lvlDebug; }}, }); addFlag({ .longName = "option", - .description = "set a Nix configuration option (overriding `nix.conf`)", + .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).", .labels = {"name", "value"}, .handler = {[](std::string name, std::string value) { try { @@ -51,8 +51,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "log-format", - .description = "format of log output; `raw`, `internal-json`, `bar` " - "or `bar-with-logs`", + .description = "Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`.", .labels = {"format"}, .handler = {[](std::string format) { setLogFormat(format); }}, }); @@ -60,7 +59,7 @@ MixCommonArgs::MixCommonArgs(const string & programName) addFlag({ .longName = "max-jobs", .shortName = 'j', - .description = "maximum number of parallel builds", + .description = "The maximum number of parallel builds.", .labels = Strings{"jobs"}, .handler = {[=](std::string s) { settings.set("max-jobs", s); diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh index a4de3dccf..47f341619 100644 --- a/src/libmain/common-args.hh +++ b/src/libmain/common-args.hh @@ -16,7 +16,7 @@ struct MixDryRun : virtual Args MixDryRun() { - mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun); + mkFlag(0, "dry-run", "Show what this command would do without doing it.", &dryRun); } }; @@ -26,7 +26,7 @@ struct MixJSON : virtual Args MixJSON() { - mkFlag(0, "json", "produce JSON output", &json); + mkFlag(0, "json", "Produce output in JSON format, suitable for consumption by another program.", &json); } }; diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 2247aeca4..7e27e95c2 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -18,6 +18,8 @@ #include <openssl/crypto.h> +#include <sodium.h> + namespace nix { @@ -126,6 +128,9 @@ void initNix() CRYPTO_set_locking_callback(opensslLockCallback); #endif + if (sodium_init() == -1) + throw Error("could not initialise libsodium"); + loadConfFile(); startSignalHandlerThread(); @@ -181,50 +186,58 @@ LegacyArgs::LegacyArgs(const std::string & programName, addFlag({ .longName = "no-build-output", .shortName = 'Q', - .description = "do not show build output", + .description = "Do not show build output.", .handler = {[&]() {setLogFormat(LogFormat::raw); }}, }); addFlag({ .longName = "keep-failed", .shortName ='K', - .description = "keep temporary directories of failed builds", + .description = "Keep temporary directories of failed builds.", .handler = {&(bool&) settings.keepFailed, true}, }); addFlag({ .longName = "keep-going", .shortName ='k', - .description = "keep going after a build fails", + .description = "Keep going after a build fails.", .handler = {&(bool&) settings.keepGoing, true}, }); addFlag({ .longName = "fallback", - .description = "build from source if substitution fails", + .description = "Build from source if substitution fails.", .handler = {&(bool&) settings.tryFallback, true}, }); auto intSettingAlias = [&](char shortName, const std::string & longName, - const std::string & description, const std::string & dest) { - mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) { - settings.set(dest, std::to_string(n)); + const std::string & description, const std::string & dest) + { + addFlag({ + .longName = longName, + .shortName = shortName, + .description = description, + .labels = {"n"}, + .handler = {[=](std::string s) { + auto n = string2IntWithUnitPrefix<uint64_t>(s); + settings.set(dest, std::to_string(n)); + }} }); }; - intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores"); - intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time"); - intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout"); + intSettingAlias(0, "cores", "Maximum number of CPU cores to use inside a build.", "cores"); + intSettingAlias(0, "max-silent-time", "Number of seconds of silence before a build is killed.", "max-silent-time"); + intSettingAlias(0, "timeout", "Number of seconds before a build is killed.", "timeout"); - mkFlag(0, "readonly-mode", "do not write to the Nix store", + mkFlag(0, "readonly-mode", "Do not write to the Nix store.", &settings.readOnlyMode); - mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'", + mkFlag(0, "no-gc-warning", "Disable warnings about not using `--add-root`.", &gcWarning, false); addFlag({ .longName = "store", - .description = "URI of the Nix store to use", + .description = "The URL of the Nix store to use.", .labels = {"store-uri"}, .handler = {&(std::string&) settings.storeUri}, }); @@ -274,9 +287,7 @@ void printVersion(const string & programName) #if HAVE_BOEHMGC cfg.push_back("gc"); #endif -#if HAVE_SODIUM cfg.push_back("signed-caches"); -#endif std::cout << "System type: " << settings.thisSystem << "\n"; std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n"; std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n"; diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index ffae5d796..edc7b5efa 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -57,23 +57,7 @@ template<class N> N getIntArg(const string & opt, { ++i; if (i == end) throw UsageError("'%1%' requires an argument", opt); - string s = *i; - N multiplier = 1; - if (allowUnit && !s.empty()) { - char u = std::toupper(*s.rbegin()); - if (std::isalpha(u)) { - if (u == 'K') multiplier = 1ULL << 10; - else if (u == 'M') multiplier = 1ULL << 20; - else if (u == 'G') multiplier = 1ULL << 30; - else if (u == 'T') multiplier = 1ULL << 40; - else throw UsageError("invalid unit specifier '%1%'", u); - s.resize(s.size() - 1); - } - } - N n; - if (!string2Int(s, n)) - throw UsageError("'%1%' requires an integer argument", opt); - return n * multiplier; + return string2IntWithUnitPrefix<N>(*i); } diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index 07a8b2beb..443a53cac 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -24,7 +24,7 @@ struct BinaryCacheStoreConfig : virtual StoreConfig "enable multi-threading compression, available for xz only currently"}; }; -class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig +class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store { private: diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index f494545fb..2e74cfd6c 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -50,6 +50,11 @@ #define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old)) #endif +#if __APPLE__ +#include <spawn.h> +#include <sys/sysctl.h> +#endif + #include <pwd.h> #include <grp.h> @@ -675,13 +680,9 @@ void DerivationGoal::tryToBuild() } void DerivationGoal::tryLocalBuild() { - bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store); - - /* Make sure that we are allowed to start a build. If this - derivation prefers to be done locally, do it even if - maxBuildJobs is 0. */ + /* Make sure that we are allowed to start a build. */ unsigned int curBuilds = worker.getNrLocalBuilds(); - if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) { + if (curBuilds >= settings.maxBuildJobs) { worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); return; @@ -1695,12 +1696,10 @@ void DerivationGoal::startBuilder() userNamespaceSync.writeSide = -1; }); - pid_t tmp; auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get())); assert(ss.size() == 2); usingUserNamespace = ss[0] == "1"; - if (!string2Int<pid_t>(ss[1], tmp)) abort(); - pid = tmp; + pid = string2Int<pid_t>(ss[1]).value(); if (usingUserNamespace) { /* Set the UID/GID mapping of the builder's user namespace @@ -1985,7 +1984,7 @@ void DerivationGoal::writeStructuredAttrs() chownToBuilder(tmpDir + "/.attrs.sh"); } -struct RestrictedStoreConfig : LocalFSStoreConfig +struct RestrictedStoreConfig : virtual LocalFSStoreConfig { using LocalFSStoreConfig::LocalFSStoreConfig; const std::string name() { return "Restricted Store"; } @@ -1994,14 +1993,19 @@ struct RestrictedStoreConfig : LocalFSStoreConfig /* A wrapper around LocalStore that only allows building/querying of paths that are in the input closures of the build or were added via recursive Nix calls. */ -struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig +struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore { ref<LocalStore> next; DerivationGoal & goal; RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal) - : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal) + : StoreConfig(params) + , LocalFSStoreConfig(params) + , RestrictedStoreConfig(params) + , Store(params) + , LocalFSStore(params) + , next(next), goal(goal) { } Path getRealStoreDir() override @@ -2852,7 +2856,31 @@ void DerivationGoal::runChild() } } +#if __APPLE__ + posix_spawnattr_t attrp; + + if (posix_spawnattr_init(&attrp)) + throw SysError("failed to initialize builder"); + + if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC)) + throw SysError("failed to initialize builder"); + + if (drv->platform == "aarch64-darwin") { + // Unset kern.curproc_arch_affinity so we can escape Rosetta + int affinity = 0; + sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity)); + + cpu_type_t cpu = CPU_TYPE_ARM64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } else if (drv->platform == "x86_64-darwin") { + cpu_type_t cpu = CPU_TYPE_X86_64; + posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL); + } + + posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#else execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data()); +#endif throw SysError("executing '%1%'", drv->builder); diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc index 9ec8abd22..1027469c9 100644 --- a/src/libstore/crypto.cc +++ b/src/libstore/crypto.cc @@ -2,21 +2,19 @@ #include "util.hh" #include "globals.hh" -#if HAVE_SODIUM #include <sodium.h> -#endif namespace nix { -static std::pair<std::string, std::string> split(const string & s) +static std::pair<std::string_view, std::string_view> split(std::string_view s) { size_t colon = s.find(':'); if (colon == std::string::npos || colon == 0) return {"", ""}; - return {std::string(s, 0, colon), std::string(s, colon + 1)}; + return {s.substr(0, colon), s.substr(colon + 1)}; } -Key::Key(const string & s) +Key::Key(std::string_view s) { auto ss = split(s); @@ -29,62 +27,57 @@ Key::Key(const string & s) key = base64Decode(key); } -SecretKey::SecretKey(const string & s) - : Key(s) +std::string Key::to_string() const { -#if HAVE_SODIUM - if (key.size() != crypto_sign_SECRETKEYBYTES) - throw Error("secret key is not valid"); -#endif + return name + ":" + base64Encode(key); } -#if !HAVE_SODIUM -[[noreturn]] static void noSodium() +SecretKey::SecretKey(std::string_view s) + : Key(s) { - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); + if (key.size() != crypto_sign_SECRETKEYBYTES) + throw Error("secret key is not valid"); } -#endif -std::string SecretKey::signDetached(const std::string & data) const +std::string SecretKey::signDetached(std::string_view data) const { -#if HAVE_SODIUM unsigned char sig[crypto_sign_BYTES]; unsigned long long sigLen; crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(), (unsigned char *) key.data()); return name + ":" + base64Encode(std::string((char *) sig, sigLen)); -#else - noSodium(); -#endif } PublicKey SecretKey::toPublicKey() const { -#if HAVE_SODIUM unsigned char pk[crypto_sign_PUBLICKEYBYTES]; crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data()); return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES)); -#else - noSodium(); -#endif } -PublicKey::PublicKey(const string & s) +SecretKey SecretKey::generate(std::string_view name) +{ + unsigned char pk[crypto_sign_PUBLICKEYBYTES]; + unsigned char sk[crypto_sign_SECRETKEYBYTES]; + if (crypto_sign_keypair(pk, sk) != 0) + throw Error("key generation failed"); + + return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES)); +} + +PublicKey::PublicKey(std::string_view s) : Key(s) { -#if HAVE_SODIUM if (key.size() != crypto_sign_PUBLICKEYBYTES) throw Error("public key is not valid"); -#endif } bool verifyDetached(const std::string & data, const std::string & sig, const PublicKeys & publicKeys) { -#if HAVE_SODIUM auto ss = split(sig); - auto key = publicKeys.find(ss.first); + auto key = publicKeys.find(std::string(ss.first)); if (key == publicKeys.end()) return false; auto sig2 = base64Decode(ss.second); @@ -94,9 +87,6 @@ bool verifyDetached(const std::string & data, const std::string & sig, return crypto_sign_verify_detached((unsigned char *) sig2.data(), (unsigned char *) data.data(), data.size(), (unsigned char *) key->second.key.data()) == 0; -#else - noSodium(); -#endif } PublicKeys getDefaultPublicKeys() diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh index 9110af3aa..03f85c103 100644 --- a/src/libstore/crypto.hh +++ b/src/libstore/crypto.hh @@ -13,32 +13,40 @@ struct Key /* Construct Key from a string in the format ‘<name>:<key-in-base64>’. */ - Key(const std::string & s); + Key(std::string_view s); + + std::string to_string() const; protected: - Key(const std::string & name, const std::string & key) - : name(name), key(key) { } + Key(std::string_view name, std::string && key) + : name(name), key(std::move(key)) { } }; struct PublicKey; struct SecretKey : Key { - SecretKey(const std::string & s); + SecretKey(std::string_view s); /* Return a detached signature of the given string. */ - std::string signDetached(const std::string & s) const; + std::string signDetached(std::string_view s) const; PublicKey toPublicKey() const; + + static SecretKey generate(std::string_view name); + +private: + SecretKey(std::string_view name, std::string && key) + : Key(name, std::move(key)) { } }; struct PublicKey : Key { - PublicKey(const std::string & data); + PublicKey(std::string_view data); private: - PublicKey(const std::string & name, const std::string & key) - : Key(name, key) { } + PublicKey(std::string_view name, std::string && key) + : Key(name, std::move(key)) { } friend struct SecretKey; }; diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 91fc178db..3c7caf8f2 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -9,7 +9,7 @@ struct DummyStoreConfig : virtual StoreConfig { const std::string name() override { return "Dummy Store"; } }; -struct DummyStore : public Store, public virtual DummyStoreConfig +struct DummyStore : public virtual DummyStoreConfig, public virtual Store { DummyStore(const std::string scheme, const std::string uri, const Params & params) : DummyStore(params) @@ -17,6 +17,7 @@ struct DummyStore : public Store, public virtual DummyStoreConfig DummyStore(const Params & params) : StoreConfig(params) + , DummyStoreConfig(params) , Store(params) { } diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh index afc7e7aa6..45d9ccf89 100644 --- a/src/libstore/filetransfer.hh +++ b/src/libstore/filetransfer.hh @@ -63,7 +63,7 @@ struct FileTransferRequest std::string mimeType; std::function<void(std::string_view data)> dataCallback; - FileTransferRequest(const std::string & uri) + FileTransferRequest(std::string_view uri) : uri(uri), parentAct(getCurActivity()) { } std::string verb() diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh index 64780a6da..c825e84f2 100644 --- a/src/libstore/fs-accessor.hh +++ b/src/libstore/fs-accessor.hh @@ -25,7 +25,14 @@ public: virtual StringSet readDirectory(const Path & path) = 0; - virtual std::string readFile(const Path & path) = 0; + /** + * Read a file inside the store. + * + * If `requireValidPath` is set to `true` (the default), the path must be + * inside a valid store path, otherwise it just needs to be physically + * present (but not necessarily properly registered) + */ + virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0; virtual std::string readLink(const Path & path) = 0; }; diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index f38601d6d..0531aad9f 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -131,6 +131,28 @@ StringSet Settings::getDefaultSystemFeatures() return features; } +StringSet Settings::getDefaultExtraPlatforms() +{ + if (std::string{SYSTEM} == "x86_64-linux" && !isWSL1()) + return StringSet{"i686-linux"}; +#if __APPLE__ + // Rosetta 2 emulation layer can run x86_64 binaries on aarch64 + // machines. Note that we can’t force processes from executing + // x86_64 in aarch64 environments or vice versa since they can + // always exec with their own binary preferences. + else if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) { + if (std::string{SYSTEM} == "x86_64-darwin") + return StringSet{"aarch64-darwin"}; + else if (std::string{SYSTEM} == "aarch64-darwin") + return StringSet{"x86_64-darwin"}; + else + return StringSet{}; + } +#endif + else + return StringSet{}; +} + bool Settings::isExperimentalFeatureEnabled(const std::string & name) { auto & f = experimentalFeatures.get(); @@ -206,8 +228,12 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s void MaxBuildJobsSetting::set(const std::string & str, bool append) { if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency()); - else if (!string2Int(str, value)) - throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); + else { + if (auto n = string2Int<decltype(value)>(str)) + value = *n; + else + throw UsageError("configuration setting '%s' should be 'auto' or an integer", name); + } } diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index 6b4775683..1d968ef3e 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -34,6 +34,8 @@ class Settings : public Config { StringSet getDefaultSystemFeatures(); + StringSet getDefaultExtraPlatforms(); + bool isWSL1(); public: @@ -545,7 +547,7 @@ public: Setting<StringSet> extraPlatforms{ this, - std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{}, + getDefaultExtraPlatforms(), "extra-platforms", R"( Platforms other than the native one which this machine is capable of diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 9d2a89f96..0a3afcd51 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -15,7 +15,7 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig const std::string name() override { return "Http Binary Cache Store"; } }; -class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig +class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore { private: @@ -36,6 +36,9 @@ public: const Path & _cacheUri, const Params & params) : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , HttpBinaryCacheStoreConfig(params) + , Store(params) , BinaryCacheStore(params) , cacheUri(scheme + "://" + _cacheUri) { diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index ad1779aea..253c0033e 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -22,7 +22,7 @@ struct LegacySSHStoreConfig : virtual StoreConfig const std::string name() override { return "Legacy SSH Store"; } }; -struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig +struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store { // Hack for getting remote build log output. // Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in @@ -48,6 +48,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig LegacySSHStore(const string & scheme, const string & host, const Params & params) : StoreConfig(params) + , LegacySSHStoreConfig(params) , Store(params) , host(host) , connections(make_ref<Pool<Connection>>( diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc index bb7464989..a58b7733f 100644 --- a/src/libstore/local-binary-cache-store.cc +++ b/src/libstore/local-binary-cache-store.cc @@ -11,7 +11,7 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig const std::string name() override { return "Local Binary Cache Store"; } }; -class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig +class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore { private: @@ -24,6 +24,9 @@ public: const Path & binaryCacheDir, const Params & params) : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , LocalBinaryCacheStoreConfig(params) + , Store(params) , BinaryCacheStore(params) , binaryCacheDir(binaryCacheDir) { diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index e7c3dae92..6de13c73a 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -19,10 +19,10 @@ struct LocalStoreAccessor : public FSAccessor LocalStoreAccessor(ref<LocalFSStore> store) : store(store) { } - Path toRealPath(const Path & path) + Path toRealPath(const Path & path, bool requireValidPath = true) { auto storePath = store->toStorePath(path).first; - if (!store->isValidPath(storePath)) + if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); return store->getRealStoreDir() + std::string(path, store->storeDir.size()); } @@ -61,9 +61,9 @@ struct LocalStoreAccessor : public FSAccessor return res; } - std::string readFile(const Path & path) override + std::string readFile(const Path & path, bool requireValidPath = true) override { - return nix::readFile(toRealPath(path)); + return nix::readFile(toRealPath(path, requireValidPath)); } std::string readLink(const Path & path) override diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh index 8eccd8236..55941b771 100644 --- a/src/libstore/local-fs-store.hh +++ b/src/libstore/local-fs-store.hh @@ -20,7 +20,7 @@ struct LocalFSStoreConfig : virtual StoreConfig "log", "directory where Nix will store state"}; }; -class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig +class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store { public: diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index e9f9bde4d..ab78f1435 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -66,8 +66,10 @@ int getSchema(Path schemaPath) int curSchema = 0; if (pathExists(schemaPath)) { string s = readFile(schemaPath); - if (!string2Int(s, curSchema)) + auto n = string2Int<int>(s); + if (!n) throw Error("'%1%' is corrupt", schemaPath); + curSchema = *n; } return curSchema; } @@ -100,6 +102,8 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) LocalStore::LocalStore(const Params & params) : StoreConfig(params) + , LocalFSStoreConfig(params) + , LocalStoreConfig(params) , Store(params) , LocalFSStore(params) , realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real", @@ -734,54 +738,59 @@ void LocalStore::queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept { try { - callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() { + callback(retrySQLite<std::shared_ptr<const ValidPathInfo>>([&]() { auto state(_state.lock()); + return queryPathInfoInternal(*state, path); + })); - /* Get the path info. */ - auto useQueryPathInfo(state->stmts->QueryPathInfo.use()(printStorePath(path))); + } catch (...) { callback.rethrow(); } +} - if (!useQueryPathInfo.next()) - return std::shared_ptr<ValidPathInfo>(); - auto id = useQueryPathInfo.getInt(0); +std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoInternal(State & state, const StorePath & path) +{ + /* Get the path info. */ + auto useQueryPathInfo(state.stmts->QueryPathInfo.use()(printStorePath(path))); - auto narHash = Hash::dummy; - try { - narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); - } catch (BadHash & e) { - throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); - } + if (!useQueryPathInfo.next()) + return std::shared_ptr<ValidPathInfo>(); - auto info = std::make_shared<ValidPathInfo>(path, narHash); + auto id = useQueryPathInfo.getInt(0); - info->id = id; + auto narHash = Hash::dummy; + try { + narHash = Hash::parseAnyPrefixed(useQueryPathInfo.getStr(1)); + } catch (BadHash & e) { + throw Error("invalid-path entry for '%s': %s", printStorePath(path), e.what()); + } - info->registrationTime = useQueryPathInfo.getInt(2); + auto info = std::make_shared<ValidPathInfo>(path, narHash); - auto s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 3); - if (s) info->deriver = parseStorePath(s); + info->id = id; - /* Note that narSize = NULL yields 0. */ - info->narSize = useQueryPathInfo.getInt(4); + info->registrationTime = useQueryPathInfo.getInt(2); - info->ultimate = useQueryPathInfo.getInt(5) == 1; + auto s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 3); + if (s) info->deriver = parseStorePath(s); - s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 6); - if (s) info->sigs = tokenizeString<StringSet>(s, " "); + /* Note that narSize = NULL yields 0. */ + info->narSize = useQueryPathInfo.getInt(4); - s = (const char *) sqlite3_column_text(state->stmts->QueryPathInfo, 7); - if (s) info->ca = parseContentAddressOpt(s); + info->ultimate = useQueryPathInfo.getInt(5) == 1; - /* Get the references. */ - auto useQueryReferences(state->stmts->QueryReferences.use()(info->id)); + s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 6); + if (s) info->sigs = tokenizeString<StringSet>(s, " "); - while (useQueryReferences.next()) - info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7); + if (s) info->ca = parseContentAddressOpt(s); - return info; - })); + /* Get the references. */ + auto useQueryReferences(state.stmts->QueryReferences.use()(info->id)); - } catch (...) { callback.rethrow(); } + while (useQueryReferences.next()) + info->references.insert(parseStorePath(useQueryReferences.getStr(0))); + + return info; } @@ -905,7 +914,7 @@ LocalStore::queryDerivationOutputMapNoResolve(const StorePath& path_) if (realisation) outputs.insert_or_assign(outputName, realisation->outPath); else - outputs.insert_or_assign(outputName, std::nullopt); + outputs.insert({outputName, std::nullopt}); } return outputs; @@ -1606,7 +1615,7 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si SQLiteTxn txn(state->db); - auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(storePath))); + auto info = std::const_pointer_cast<ValidPathInfo>(queryPathInfoInternal(*state, storePath)); info->sigs.insert(sigs.begin(), sigs.end()); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 877dba742..6c7ebac1e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -43,7 +43,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig }; -class LocalStore : public LocalFSStore, public virtual LocalStoreConfig +class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore { private: @@ -198,9 +198,7 @@ public: void vacuumDB(); - /* Repair the contents of the given path by redownloading it using - a substituter (if available). */ - void repairPath(const StorePath & path); + void repairPath(const StorePath & path) override; void addSignatures(const StorePath & storePath, const StringSet & sigs) override; @@ -235,6 +233,8 @@ private: void verifyPath(const Path & path, const StringSet & store, PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors); + std::shared_ptr<const ValidPathInfo> queryPathInfoInternal(State & state, const StorePath & path); + void updatePathInfo(State & state, const ValidPathInfo & info); void upgradeStore6(); diff --git a/src/libstore/names.cc b/src/libstore/names.cc index 41e28dc99..ce808accc 100644 --- a/src/libstore/names.cc +++ b/src/libstore/names.cc @@ -80,16 +80,16 @@ string nextComponent(string::const_iterator & p, static bool componentsLT(const string & c1, const string & c2) { - int n1, n2; - bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2); + auto n1 = string2Int<int>(c1); + auto n2 = string2Int<int>(c2); - if (c1Num && c2Num) return n1 < n2; - else if (c1 == "" && c2Num) return true; + if (n1 && n2) return *n1 < *n2; + else if (c1 == "" && n2) return true; else if (c1 == "pre" && c2 != "pre") return true; else if (c2 == "pre") return false; /* Assume that `2.3a' < `2.3.1'. */ - else if (c2Num) return true; - else if (c1Num) return false; + else if (n2) return true; + else if (n1) return false; else return c1 < c2; } diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc index 1427a0f98..784ebb719 100644 --- a/src/libstore/nar-accessor.cc +++ b/src/libstore/nar-accessor.cc @@ -203,7 +203,7 @@ struct NarAccessor : public FSAccessor return res; } - std::string readFile(const Path & path) override + std::string readFile(const Path & path, bool requireValidPath = true) override { auto i = get(path); if (i.type != FSAccessor::Type::tRegular) diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 8541cc51f..1d8d2d57e 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -109,8 +109,10 @@ public: SQLiteStmt(state->db, "delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))") .use() - (now - settings.ttlNegativeNarInfoCache) - (now - settings.ttlPositiveNarInfoCache) + // Use a minimum TTL to prevent --refresh from + // nuking the entire disk cache. + (now - std::max(settings.ttlNegativeNarInfoCache.get(), 3600U)) + (now - std::max(settings.ttlPositiveNarInfoCache.get(), 30 * 24 * 3600U)) .exec(); debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db)); diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc index 3454f34bb..49079388a 100644 --- a/src/libstore/nar-info.cc +++ b/src/libstore/nar-info.cc @@ -46,14 +46,18 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string & else if (name == "FileHash") fileHash = parseHashField(value); else if (name == "FileSize") { - if (!string2Int(value, fileSize)) throw corrupt(); + auto n = string2Int<decltype(fileSize)>(value); + if (!n) throw corrupt(); + fileSize = *n; } else if (name == "NarHash") { narHash = parseHashField(value); haveNarHash = true; } else if (name == "NarSize") { - if (!string2Int(value, narSize)) throw corrupt(); + auto n = string2Int<decltype(narSize)>(value); + if (!n) throw corrupt(); + narSize = *n; } else if (name == "References") { auto refs = tokenizeString<Strings>(value, " "); diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index e7b7202d4..c5c3ae3dc 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -101,6 +101,10 @@ bool ParsedDerivation::canBuildLocally(Store & localStore) const && !drv.isBuiltin()) return false; + if (settings.maxBuildJobs.get() == 0 + && !drv.isBuiltin()) + return false; + for (auto & feature : getRequiredSystemFeatures()) if (!localStore.systemFeatures.get().count(feature)) return false; diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc index ed10dd519..5d1723886 100644 --- a/src/libstore/profiles.cc +++ b/src/libstore/profiles.cc @@ -21,9 +21,8 @@ static std::optional<GenerationNumber> parseName(const string & profileName, con string s = string(name, profileName.size() + 1); string::size_type p = s.find("-link"); if (p == string::npos) return {}; - unsigned int n; - if (string2Int(string(s, 0, p), n) && n >= 0) - return n; + if (auto n = string2Int<unsigned int>(s.substr(0, p))) + return *n; else return {}; } @@ -214,12 +213,12 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b { time_t curTime = time(0); string strDays = string(timeSpec, 0, timeSpec.size() - 1); - int days; + auto days = string2Int<int>(strDays); - if (!string2Int(strDays, days) || days < 1) + if (!days || *days < 1) throw Error("invalid number of days specifier '%1%'", timeSpec); - time_t oldTime = curTime - days * 24 * 3600; + time_t oldTime = curTime - *days * 24 * 3600; deleteGenerationsOlderThan(profile, oldTime, dryRun); } diff --git a/src/libstore/references.cc b/src/libstore/references.cc index eb117b5ba..39c4970c6 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -88,9 +88,6 @@ PathSet scanForReferences(Sink & toTee, TeeSink sink { refsSink, toTee }; std::map<string, Path> backMap; - /* For efficiency (and a higher hit rate), just search for the - hash part of the file name. (This assumes that all references - have the form `HASH-bla'). */ for (auto & i : refs) { auto baseName = std::string(baseNameOf(i)); string::size_type pos = baseName.find('-'); diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 63bde92de..f43456f0b 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -43,13 +43,13 @@ void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string & } } -std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_) +std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath) { auto path = canonPath(path_); auto [storePath, restPath] = store->toStorePath(path); - if (!store->isValidPath(storePath)) + if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); auto i = nars.find(std::string(storePath.hashPart())); @@ -113,9 +113,9 @@ StringSet RemoteFSAccessor::readDirectory(const Path & path) return res.first->readDirectory(res.second); } -std::string RemoteFSAccessor::readFile(const Path & path) +std::string RemoteFSAccessor::readFile(const Path & path, bool requireValidPath) { - auto res = fetch(path); + auto res = fetch(path, requireValidPath); return res.first->readFile(res.second); } diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh index 347cf5764..594852d0e 100644 --- a/src/libstore/remote-fs-accessor.hh +++ b/src/libstore/remote-fs-accessor.hh @@ -14,7 +14,7 @@ class RemoteFSAccessor : public FSAccessor Path cacheDir; - std::pair<ref<FSAccessor>, Path> fetch(const Path & path_); + std::pair<ref<FSAccessor>, Path> fetch(const Path & path_, bool requireValidPath = true); friend class BinaryCacheStore; @@ -32,7 +32,7 @@ public: StringSet readDirectory(const Path & path) override; - std::string readFile(const Path & path) override; + std::string readFile(const Path & path, bool requireValidPath = true) override; std::string readLink(const Path & path) override; }; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index f1f4d0516..be07f02dc 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -77,8 +77,8 @@ void write(const Store & store, Sink & out, const std::optional<ContentAddress> /* TODO: Separate these store impls into different files, give them better names */ RemoteStore::RemoteStore(const Params & params) - : Store(params) - , RemoteStoreConfig(params) + : RemoteStoreConfig(params) + , Store(params) , connections(make_ref<Pool<Connection>>( std::max(1, (int) maxConnections), [this]() { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index fdd53e6ed..b3a9910a3 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -29,7 +29,7 @@ struct RemoteStoreConfig : virtual StoreConfig /* FIXME: RemoteStore is a misnomer - should be something like DaemonStore. */ -class RemoteStore : public virtual Store, public virtual RemoteStoreConfig +class RemoteStore : public virtual RemoteStoreConfig, public virtual Store { public: diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index d6edafd7e..6bfbee044 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -177,6 +177,11 @@ S3Helper::FileTransferResult S3Helper::getObject( return res; } +S3BinaryCacheStore::S3BinaryCacheStore(const Params & params) + : BinaryCacheStoreConfig(params) + , BinaryCacheStore(params) +{ } + struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig { using BinaryCacheStoreConfig::BinaryCacheStoreConfig; @@ -195,7 +200,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig const std::string name() override { return "S3 Binary Cache Store"; } }; -struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig +struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore { std::string bucketName; @@ -208,6 +213,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache const std::string & bucketName, const Params & params) : StoreConfig(params) + , BinaryCacheStoreConfig(params) + , S3BinaryCacheStoreConfig(params) + , Store(params) + , BinaryCacheStore(params) , S3BinaryCacheStore(params) , bucketName(bucketName) , s3Helper(profile, region, scheme, endpoint) diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh index 4d43fe4d2..bce828b11 100644 --- a/src/libstore/s3-binary-cache-store.hh +++ b/src/libstore/s3-binary-cache-store.hh @@ -6,13 +6,11 @@ namespace nix { -class S3BinaryCacheStore : public BinaryCacheStore +class S3BinaryCacheStore : public virtual BinaryCacheStore { protected: - S3BinaryCacheStore(const Params & params) - : BinaryCacheStore(params) - { } + S3BinaryCacheStore(const Params & params); public: diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index 08d0bd565..17c258201 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -20,12 +20,14 @@ struct SSHStoreConfig : virtual RemoteStoreConfig const std::string name() override { return "SSH Store"; } }; -class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig +class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore { public: SSHStore(const std::string & scheme, const std::string & host, const Params & params) : StoreConfig(params) + , RemoteStoreConfig(params) + , SSHStoreConfig(params) , Store(params) , RemoteStore(params) , host(host) diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 9d10ae76f..cce95e4cf 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -949,19 +949,20 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre getline(str, s); auto narHash = Hash::parseAny(s, htSHA256); getline(str, s); - uint64_t narSize; - if (!string2Int(s, narSize)) throw Error("number expected"); - hashGiven = { narHash, narSize }; + auto narSize = string2Int<uint64_t>(s); + if (!narSize) throw Error("number expected"); + hashGiven = { narHash, *narSize }; } ValidPathInfo info(store.parseStorePath(path), hashGiven->first); info.narSize = hashGiven->second; std::string deriver; getline(str, deriver); if (deriver != "") info.deriver = store.parseStorePath(deriver); - string s; int n; + string s; getline(str, s); - if (!string2Int(s, n)) throw Error("number expected"); - while (n--) { + auto n = string2Int<int>(s); + if (!n) throw Error("number expected"); + while ((*n)--) { getline(str, s); info.references.insert(store.parseStorePath(s)); } @@ -1066,26 +1067,23 @@ Derivation Store::derivationFromPath(const StorePath & drvPath) return readDerivation(drvPath); } - -Derivation Store::readDerivation(const StorePath & drvPath) +Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath) { - auto accessor = getFSAccessor(); + auto accessor = store.getFSAccessor(); try { - return parseDerivation(*this, - accessor->readFile(printStorePath(drvPath)), + return parseDerivation(store, + accessor->readFile(store.printStorePath(drvPath), requireValidPath), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { - throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg()); + throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); } } +Derivation Store::readDerivation(const StorePath & drvPath) +{ return readDerivationCommon(*this, drvPath, true); } + Derivation Store::readInvalidDerivation(const StorePath & drvPath) -{ - return parseDerivation( - *this, - readFile(Store::toRealPath(drvPath)), - Derivation::nameFromPath(drvPath)); -} +{ return readDerivationCommon(*this, drvPath, false); } } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 2b4672748..696a2d3fd 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -175,25 +175,7 @@ struct StoreConfig : public Config { using Config::Config; - /** - * When constructing a store implementation, we pass in a map `params` of - * parameters that's supposed to initialize the associated config. - * To do that, we must use the `StoreConfig(StringMap & params)` - * constructor, so we'd like to `delete` its default constructor to enforce - * it. - * - * However, actually deleting it means that all the subclasses of - * `StoreConfig` will have their default constructor deleted (because it's - * supposed to call the deleted default constructor of `StoreConfig`). But - * because we're always using virtual inheritance, the constructors of - * child classes will never implicitely call this one, so deleting it will - * be more painful than anything else. - * - * So we `assert(false)` here to ensure at runtime that the right - * constructor is always called without having to redefine a custom - * constructor for each `*Config` class. - */ - StoreConfig() { assert(false); } + StoreConfig() = delete; virtual ~StoreConfig() { } @@ -624,6 +606,11 @@ public: virtual ref<FSAccessor> getFSAccessor() { unsupported("getFSAccessor"); } + /* Repair the contents of the given path by redownloading it using + a substituter (if available). */ + virtual void repairPath(const StorePath & path) + { unsupported("repairPath"); } + /* Add signatures to the specified store path. The signatures are not verified. */ virtual void addSignatures(const StorePath & storePath, const StringSet & sigs) diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 24f3e9c6d..cac4fa036 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -15,6 +15,9 @@ namespace nix { UDSRemoteStore::UDSRemoteStore(const Params & params) : StoreConfig(params) + , LocalFSStoreConfig(params) + , RemoteStoreConfig(params) + , UDSRemoteStoreConfig(params) , Store(params) , LocalFSStore(params) , RemoteStore(params) diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh index e5de104c9..ddc7716cd 100644 --- a/src/libstore/uds-remote-store.hh +++ b/src/libstore/uds-remote-store.hh @@ -14,15 +14,10 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon { } - UDSRemoteStoreConfig() - : UDSRemoteStoreConfig(Store::Params({})) - { - } - const std::string name() override { return "Local Daemon Store"; } }; -class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig +class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore { public: diff --git a/src/libutil/args.cc b/src/libutil/args.cc index 61f9503ec..fb5cb80fb 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -254,6 +254,8 @@ nlohmann::json Args::toJSON() res["description"] = description(); res["flags"] = std::move(flags); res["args"] = std::move(args); + auto s = doc(); + if (s != "") res.emplace("doc", stripIndentation(s)); return res; } @@ -351,38 +353,6 @@ void printTable(std::ostream & out, const Table2 & table) } } -void Command::printHelp(const string & programName, std::ostream & out) -{ - Args::printHelp(programName, out); - - auto exs = examples(); - if (!exs.empty()) { - out << "\n" ANSI_BOLD "Examples:" ANSI_NORMAL "\n"; - for (auto & ex : exs) - out << "\n" - << " " << ex.description << "\n" // FIXME: wrap - << " $ " << ex.command << "\n"; - } -} - -nlohmann::json Command::toJSON() -{ - auto exs = nlohmann::json::array(); - - for (auto & example : examples()) { - auto ex = nlohmann::json::object(); - ex["description"] = example.description; - ex["command"] = chomp(stripIndentation(example.command)); - exs.push_back(std::move(ex)); - } - - auto res = Args::toJSON(); - res["examples"] = std::move(exs); - auto s = doc(); - if (s != "") res.emplace("doc", stripIndentation(s)); - return res; -} - MultiCommand::MultiCommand(const Commands & commands) : commands(commands) { diff --git a/src/libutil/args.hh b/src/libutil/args.hh index 8069fd70f..3783bc84f 100644 --- a/src/libutil/args.hh +++ b/src/libutil/args.hh @@ -25,6 +25,9 @@ public: /* Return a short one-line description of the command. */ virtual std::string description() { return ""; } + /* Return documentation about this command, in Markdown format. */ + virtual std::string doc() { return ""; } + protected: static const size_t ArityAny = std::numeric_limits<size_t>::max(); @@ -65,8 +68,12 @@ protected: , arity(ArityAny) { } - template<class T> - Handler(T * dest) + Handler(std::string * dest) + : fun([=](std::vector<std::string> ss) { *dest = ss[0]; }) + , arity(1) + { } + + Handler(std::optional<std::string> * dest) : fun([=](std::vector<std::string> ss) { *dest = ss[0]; }) , arity(1) { } @@ -76,6 +83,14 @@ protected: : fun([=](std::vector<std::string> ss) { *dest = val; }) , arity(0) { } + + template<class I> + Handler(I * dest) + : fun([=](std::vector<std::string> ss) { + *dest = string2IntWithUnitPrefix<I>(ss[0]); + }) + , arity(1) + { } }; /* Flags. */ @@ -127,19 +142,6 @@ public: /* Helper functions for constructing flags / positional arguments. */ - void mkFlag1(char shortName, const std::string & longName, - const std::string & label, const std::string & description, - std::function<void(std::string)> fun) - { - addFlag({ - .longName = longName, - .shortName = shortName, - .description = description, - .labels = {label}, - .handler = {[=](std::string s) { fun(s); }} - }); - } - void mkFlag(char shortName, const std::string & name, const std::string & description, bool * dest) { @@ -158,33 +160,6 @@ public: }); } - template<class I> - void mkIntFlag(char shortName, const std::string & longName, - const std::string & description, I * dest) - { - mkFlag<I>(shortName, longName, description, [=](I n) { - *dest = n; - }); - } - - template<class I> - void mkFlag(char shortName, const std::string & longName, - const std::string & description, std::function<void(I)> fun) - { - addFlag({ - .longName = longName, - .shortName = shortName, - .description = description, - .labels = {"N"}, - .handler = {[=](std::string s) { - I n; - if (!string2Int(s, n)) - throw UsageError("flag '--%s' requires a integer argument", longName); - fun(n); - }} - }); - } - void expectArgs(ExpectedArg && arg) { expectedArgs.emplace_back(std::move(arg)); @@ -225,28 +200,11 @@ struct Command : virtual Args virtual void prepare() { }; virtual void run() = 0; - /* Return documentation about this command, in Markdown format. */ - virtual std::string doc() { return ""; } - - struct Example - { - std::string description; - std::string command; - }; - - typedef std::list<Example> Examples; - - virtual Examples examples() { return Examples(); } - typedef int Category; static constexpr Category catDefault = 0; virtual Category category() { return catDefault; } - - void printHelp(const string & programName, std::ostream & out) override; - - nlohmann::json toJSON() override; }; typedef std::map<std::string, std::function<ref<Command>()>> Commands; diff --git a/src/libutil/config.cc b/src/libutil/config.cc index 7af3e7883..7467e5ac0 100644 --- a/src/libutil/config.cc +++ b/src/libutil/config.cc @@ -230,7 +230,9 @@ template<typename T> void BaseSetting<T>::set(const std::string & str, bool append) { static_assert(std::is_integral<T>::value, "Integer required."); - if (!string2Int(str, value)) + if (auto n = string2Int<T>(str)) + value = *n; + else throw UsageError("setting '%s' has invalid value '%s'", name, str); } diff --git a/src/libutil/error.cc b/src/libutil/error.cc index 803a72953..2a67a730a 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -61,36 +61,31 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos) if (errPos.origin == foFile) { LinesOfCode loc; try { + // FIXME: when running as the daemon, make sure we don't + // open a file to which the client doesn't have access. AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC); - if (!fd) { - logError(SysError("opening file '%1%'", errPos.file).info()); - return std::nullopt; - } - else + if (!fd) return {}; + + // count the newlines. + int count = 0; + string line; + int pl = errPos.line - 1; + do { - // count the newlines. - int count = 0; - string line; - int pl = errPos.line - 1; - do - { - line = readLine(fd.get()); - ++count; - if (count < pl) - { - ; - } - else if (count == pl) { - loc.prevLineOfCode = line; - } else if (count == pl + 1) { - loc.errLineOfCode = line; - } else if (count == pl + 2) { - loc.nextLineOfCode = line; - break; - } - } while (true); - return loc; - } + line = readLine(fd.get()); + ++count; + if (count < pl) + ; + else if (count == pl) + loc.prevLineOfCode = line; + else if (count == pl + 1) + loc.errLineOfCode = line; + else if (count == pl + 2) { + loc.nextLineOfCode = line; + break; + } + } while (true); + return loc; } catch (EndOfFile & eof) { if (loc.errLineOfCode.has_value()) @@ -99,7 +94,6 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos) return std::nullopt; } catch (std::exception & e) { - printError("error reading nix file: %s\n%s", errPos.file, e.what()); return std::nullopt; } } else { diff --git a/src/libutil/error.hh b/src/libutil/error.hh index aa4fadfcc..1e0bde7ea 100644 --- a/src/libutil/error.hh +++ b/src/libutil/error.hh @@ -38,7 +38,7 @@ namespace nix { ErrorInfo structs are sent to the logger as part of an exception, or directly with the logError or logWarning macros. - See the error-demo.cc program for usage examples. + See libutil/tests/logging.cc for usage examples. */ diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc index 7e53f17c6..5b32c84a4 100644 --- a/src/libutil/tests/logging.cc +++ b/src/libutil/tests/logging.cc @@ -49,7 +49,7 @@ namespace nix { }); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1mrandom.nix\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n"); + ASSERT_STREQ(str.c_str(), "@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n"); } TEST(logEI, appendingHintsToPreviousError) { @@ -208,7 +208,7 @@ namespace nix { }); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n"); } TEST(logError, logErrorWithOnlyHintAndName) { @@ -290,7 +290,7 @@ namespace nix { logError(e.info()); auto str = testing::internal::GetCapturedStderr(); - ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n 1| this is the other problem line of code\n | \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n"); + ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n 1| this is the other problem line of code\n | \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n"); } TEST(addTrace, hideTracesWithoutShowTrace) { diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc index 35a5d27bb..58df9c5ac 100644 --- a/src/libutil/tests/tests.cc +++ b/src/libutil/tests/tests.cc @@ -320,20 +320,15 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(string2Float, emptyString) { - double n; - ASSERT_EQ(string2Float("", n), false); + ASSERT_EQ(string2Float<double>(""), std::nullopt); } TEST(string2Float, trivialConversions) { - double n; - ASSERT_EQ(string2Float("1.0", n), true); - ASSERT_EQ(n, 1.0); + ASSERT_EQ(string2Float<double>("1.0"), 1.0); - ASSERT_EQ(string2Float("0.0", n), true); - ASSERT_EQ(n, 0.0); + ASSERT_EQ(string2Float<double>("0.0"), 0.0); - ASSERT_EQ(string2Float("-100.25", n), true); - ASSERT_EQ(n, (-100.25)); + ASSERT_EQ(string2Float<double>("-100.25"), -100.25); } /* ---------------------------------------------------------------------------- @@ -341,20 +336,15 @@ namespace nix { * --------------------------------------------------------------------------*/ TEST(string2Int, emptyString) { - double n; - ASSERT_EQ(string2Int("", n), false); + ASSERT_EQ(string2Int<int>(""), std::nullopt); } TEST(string2Int, trivialConversions) { - double n; - ASSERT_EQ(string2Int("1", n), true); - ASSERT_EQ(n, 1); + ASSERT_EQ(string2Int<int>("1"), 1); - ASSERT_EQ(string2Int("0", n), true); - ASSERT_EQ(n, 0); + ASSERT_EQ(string2Int<int>("0"), 0); - ASSERT_EQ(string2Int("-100", n), true); - ASSERT_EQ(n, (-100)); + ASSERT_EQ(string2Int<int>("-100"), -100); } /* ---------------------------------------------------------------------------- diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh index 5d21b8d1a..862d9fa6e 100644 --- a/src/libutil/url-parts.hh +++ b/src/libutil/url-parts.hh @@ -23,7 +23,7 @@ const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)"; const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)"; // A Git ref (i.e. branch or tag name). -const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check +const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.\\/-]*"; // FIXME: check extern std::regex refRegex; // Instead of defining what a good Git Ref is, we define what a bad Git Ref is diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 0f82bed78..ab0bd865a 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -397,21 +397,49 @@ bool statusOk(int status); /* Parse a string into an integer. */ -template<class N> bool string2Int(const string & s, N & n) +template<class N> +std::optional<N> string2Int(const std::string & s) { - if (string(s, 0, 1) == "-" && !std::numeric_limits<N>::is_signed) - return false; + if (s.substr(0, 1) == "-" && !std::numeric_limits<N>::is_signed) + return std::nullopt; std::istringstream str(s); + N n; str >> n; - return str && str.get() == EOF; + if (str && str.get() == EOF) return n; + return std::nullopt; +} + +/* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or + 'T' denoting a binary unit prefix. */ +template<class N> +N string2IntWithUnitPrefix(std::string s) +{ + N multiplier = 1; + if (!s.empty()) { + char u = std::toupper(*s.rbegin()); + if (std::isalpha(u)) { + if (u == 'K') multiplier = 1ULL << 10; + else if (u == 'M') multiplier = 1ULL << 20; + else if (u == 'G') multiplier = 1ULL << 30; + else if (u == 'T') multiplier = 1ULL << 40; + else throw UsageError("invalid unit specifier '%1%'", u); + s.resize(s.size() - 1); + } + } + if (auto n = string2Int<N>(s)) + return *n * multiplier; + throw UsageError("'%s' is not an integer", s); } /* Parse a string into a float. */ -template<class N> bool string2Float(const string & s, N & n) +template<class N> +std::optional<N> string2Float(const string & s) { std::istringstream str(s); + N n; str >> n; - return str && str.get() == EOF; + if (str && str.get() == EOF) return n; + return std::nullopt; } diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 74fafd426..38048da52 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -217,9 +217,9 @@ static void main_nix_build(int argc, char * * argv) // read the shebang to understand which packages to read from. Since // this is handled via nix-shell -p, we wrap our ruby script execution // in ruby -e 'load' which ignores the shebangs. - envCommand = (format("exec %1% %2% -e 'load(\"%3%\")' -- %4%") % execArgs % interpreter % script % joined.str()).str(); + envCommand = (format("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str(); } else { - envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % script % joined.str()).str(); + envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str(); } } diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index a4b5c9e2c..9963f05d9 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -1138,38 +1138,38 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs) i.queryName(), j) }); else { - if (v->type == tString) { + if (v->type() == nString) { attrs2["type"] = "string"; attrs2["value"] = v->string.s; xml.writeEmptyElement("meta", attrs2); - } else if (v->type == tInt) { + } else if (v->type() == nInt) { attrs2["type"] = "int"; attrs2["value"] = (format("%1%") % v->integer).str(); xml.writeEmptyElement("meta", attrs2); - } else if (v->type == tFloat) { + } else if (v->type() == nFloat) { attrs2["type"] = "float"; attrs2["value"] = (format("%1%") % v->fpoint).str(); xml.writeEmptyElement("meta", attrs2); - } else if (v->type == tBool) { + } else if (v->type() == nBool) { attrs2["type"] = "bool"; attrs2["value"] = v->boolean ? "true" : "false"; xml.writeEmptyElement("meta", attrs2); - } else if (v->isList()) { + } else if (v->type() == nList) { attrs2["type"] = "strings"; XMLOpenElement m(xml, "meta", attrs2); for (unsigned int j = 0; j < v->listSize(); ++j) { - if (v->listElems()[j]->type != tString) continue; + if (v->listElems()[j]->type() != nString) continue; XMLAttrs attrs3; attrs3["value"] = v->listElems()[j]->string.s; xml.writeEmptyElement("string", attrs3); } - } else if (v->type == tAttrs) { + } else if (v->type() == nAttrs) { attrs2["type"] = "strings"; XMLOpenElement m(xml, "meta", attrs2); Bindings & attrs = *v->attrs; for (auto &i : attrs) { Attr & a(*attrs.find(i.name)); - if(a.value->type != tString) continue; + if(a.value->type() != nString) continue; XMLAttrs attrs3; attrs3["type"] = i.name; attrs3["value"] = a.value->string.s; @@ -1250,11 +1250,10 @@ static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArg if (opArgs.size() != 1) throw UsageError("exactly one argument expected"); - GenerationNumber dstGen; - if (!string2Int(opArgs.front(), dstGen)) + if (auto dstGen = string2Int<GenerationNumber>(opArgs.front())) + switchGeneration(globals, *dstGen); + else throw UsageError("expected a generation number"); - - switchGeneration(globals, dstGen); } @@ -1308,17 +1307,17 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr if(opArgs.front().size() < 2) throw Error("invalid number of generations ‘%1%’", opArgs.front()); string str_max = string(opArgs.front(), 1, opArgs.front().size()); - GenerationNumber max; - if (!string2Int(str_max, max) || max == 0) + auto max = string2Int<GenerationNumber>(str_max); + if (!max || *max == 0) throw Error("invalid number of generations to keep ‘%1%’", opArgs.front()); - deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun); + deleteGenerationsGreaterThan(globals.profile, *max, globals.dryRun); } else { std::set<GenerationNumber> gens; for (auto & i : opArgs) { - GenerationNumber n; - if (!string2Int(i, n)) + if (auto n = string2Int<GenerationNumber>(i)) + gens.insert(*n); + else throw UsageError("invalid generation number '%1%'", i); - gens.insert(n); } deleteGenerations(globals.profile, gens, globals.dryRun); } diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc index 87387e794..168ac492b 100644 --- a/src/nix-env/user-env.cc +++ b/src/nix-env/user-env.cc @@ -53,10 +53,12 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, output paths, and optionally the derivation path, as well as the meta attributes. */ Path drvPath = keepDerivations ? i.queryDrvPath() : ""; + DrvInfo::Outputs outputs = i.queryOutputs(true); + StringSet metaNames = i.queryMetaNames(); Value & v(*state.allocValue()); manifest.listElems()[n++] = &v; - state.mkAttrs(v, 16); + state.mkAttrs(v, 7 + outputs.size()); mkString(*state.allocAttr(v, state.sType), "derivation"); mkString(*state.allocAttr(v, state.sName), i.queryName()); @@ -68,7 +70,6 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, mkString(*state.allocAttr(v, state.sDrvPath), i.queryDrvPath()); // Copy each output meant for installation. - DrvInfo::Outputs outputs = i.queryOutputs(true); Value & vOutputs = *state.allocAttr(v, state.sOutputs); state.mkList(vOutputs, outputs.size()); unsigned int m = 0; @@ -88,8 +89,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems, // Copy the meta attributes. Value & vMeta = *state.allocAttr(v, state.sMeta); - state.mkAttrs(vMeta, 16); - StringSet metaNames = i.queryMetaNames(); + state.mkAttrs(vMeta, metaNames.size()); for (auto & j : metaNames) { Value * v = i.queryMeta(j); if (!v) continue; diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc deleted file mode 100644 index 3bdee55a7..000000000 --- a/src/nix-prefetch-url/nix-prefetch-url.cc +++ /dev/null @@ -1,232 +0,0 @@ -#include "hash.hh" -#include "shared.hh" -#include "filetransfer.hh" -#include "store-api.hh" -#include "eval.hh" -#include "eval-inline.hh" -#include "common-eval-args.hh" -#include "attr-path.hh" -#include "finally.hh" -#include "../nix/legacy.hh" -#include "progress-bar.hh" -#include "tarfile.hh" - -#include <iostream> - -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> - -using namespace nix; - - -/* If ‘uri’ starts with ‘mirror://’, then resolve it using the list of - mirrors defined in Nixpkgs. */ -string resolveMirrorUri(EvalState & state, string uri) -{ - if (string(uri, 0, 9) != "mirror://") return uri; - - string s(uri, 9); - auto p = s.find('/'); - if (p == string::npos) throw Error("invalid mirror URI"); - string mirrorName(s, 0, p); - - Value vMirrors; - state.eval(state.parseExprFromString("import <nixpkgs/pkgs/build-support/fetchurl/mirrors.nix>", "."), vMirrors); - state.forceAttrs(vMirrors); - - auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); - if (mirrorList == vMirrors.attrs->end()) - throw Error("unknown mirror name '%1%'", mirrorName); - state.forceList(*mirrorList->value); - - if (mirrorList->value->listSize() < 1) - throw Error("mirror URI '%1%' did not expand to anything", uri); - - string mirror = state.forceString(*mirrorList->value->listElems()[0]); - return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); -} - - -static int main_nix_prefetch_url(int argc, char * * argv) -{ - { - HashType ht = htSHA256; - std::vector<string> args; - bool printPath = getEnv("PRINT_PATH") == "1"; - bool fromExpr = false; - string attrPath; - bool unpack = false; - bool executable = false; - string name; - - struct MyArgs : LegacyArgs, MixEvalArgs - { - using LegacyArgs::LegacyArgs; - }; - - MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { - if (*arg == "--help") - showManPage("nix-prefetch-url"); - else if (*arg == "--version") - printVersion("nix-prefetch-url"); - else if (*arg == "--type") { - string s = getArg(*arg, arg, end); - ht = parseHashType(s); - } - else if (*arg == "--print-path") - printPath = true; - else if (*arg == "--attr" || *arg == "-A") { - fromExpr = true; - attrPath = getArg(*arg, arg, end); - } - else if (*arg == "--unpack") - unpack = true; - else if (*arg == "--executable") - executable = true; - else if (*arg == "--name") - name = getArg(*arg, arg, end); - else if (*arg != "" && arg->at(0) == '-') - return false; - else - args.push_back(*arg); - return true; - }); - - myArgs.parseCmdline(argvToStrings(argc, argv)); - - initPlugins(); - - if (args.size() > 2) - throw UsageError("too many arguments"); - - Finally f([]() { stopProgressBar(); }); - - if (isatty(STDERR_FILENO)) - startProgressBar(); - - auto store = openStore(); - auto state = std::make_unique<EvalState>(myArgs.searchPath, store); - - Bindings & autoArgs = *myArgs.getAutoArgs(*state); - - /* If -A is given, get the URI from the specified Nix - expression. */ - string uri; - if (!fromExpr) { - if (args.empty()) - throw UsageError("you must specify a URI"); - uri = args[0]; - } else { - Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); - Value vRoot; - state->evalFile(path, vRoot); - Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); - state->forceAttrs(v); - - /* Extract the URI. */ - auto attr = v.attrs->find(state->symbols.create("urls")); - if (attr == v.attrs->end()) - throw Error("attribute set does not contain a 'urls' attribute"); - state->forceList(*attr->value); - if (attr->value->listSize() < 1) - throw Error("'urls' list is empty"); - uri = state->forceString(*attr->value->listElems()[0]); - - /* Extract the hash mode. */ - attr = v.attrs->find(state->symbols.create("outputHashMode")); - if (attr == v.attrs->end()) - printInfo("warning: this does not look like a fetchurl call"); - else - unpack = state->forceString(*attr->value) == "recursive"; - - /* Extract the name. */ - if (name.empty()) { - attr = v.attrs->find(state->symbols.create("name")); - if (attr != v.attrs->end()) - name = state->forceString(*attr->value); - } - } - - /* Figure out a name in the Nix store. */ - if (name.empty()) - name = baseNameOf(uri); - if (name.empty()) - throw Error("cannot figure out file name for '%1%'", uri); - - /* If an expected hash is given, the file may already exist in - the store. */ - std::optional<Hash> expectedHash; - Hash hash(ht); - std::optional<StorePath> storePath; - if (args.size() == 2) { - expectedHash = Hash::parseAny(args[1], ht); - const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - storePath = store->makeFixedOutputPath(recursive, *expectedHash, name); - if (store->isValidPath(*storePath)) - hash = *expectedHash; - else - storePath.reset(); - } - - if (!storePath) { - - auto actualUri = resolveMirrorUri(*state, uri); - - AutoDelete tmpDir(createTempDir(), true); - Path tmpFile = (Path) tmpDir + "/tmp"; - - /* Download the file. */ - { - auto mode = 0600; - if (executable) - mode = 0700; - - AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); - if (!fd) throw SysError("creating temporary file '%s'", tmpFile); - - FdSink sink(fd.get()); - - FileTransferRequest req(actualUri); - req.decompress = false; - getFileTransfer()->download(std::move(req), sink); - } - - /* Optionally unpack the file. */ - if (unpack) { - printInfo("unpacking..."); - Path unpacked = (Path) tmpDir + "/unpacked"; - createDirs(unpacked); - unpackTarfile(tmpFile, unpacked); - - /* If the archive unpacks to a single file/directory, then use - that as the top-level. */ - auto entries = readDirectory(unpacked); - if (entries.size() == 1) - tmpFile = unpacked + "/" + entries[0].name; - else - tmpFile = unpacked; - } - - const auto method = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; - - auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash); - storePath = info.path; - assert(info.ca); - hash = getContentAddressHash(*info.ca); - } - - stopProgressBar(); - - if (!printPath) - printInfo("path is '%s'", store->printStorePath(*storePath)); - - std::cout << printHash16or32(hash) << std::endl; - if (printPath) - std::cout << store->printStorePath(*storePath) << std::endl; - - return 0; - } -} - -static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 54394e921..b97f684a4 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -19,10 +19,6 @@ #include <sys/stat.h> #include <fcntl.h> -#if HAVE_SODIUM -#include <sodium.h> -#endif - namespace nix_store { @@ -761,7 +757,7 @@ static void opRepairPath(Strings opFlags, Strings opArgs) throw UsageError("no flags expected"); for (auto & i : opArgs) - ensureLocalStore()->repairPath(store->followLinksToStorePath(i)); + store->repairPath(store->followLinksToStorePath(i)); } /* Optimise the disk space usage of the Nix store by hard-linking @@ -980,21 +976,11 @@ static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs) string secretKeyFile = *i++; string publicKeyFile = *i++; -#if HAVE_SODIUM - if (sodium_init() == -1) - throw Error("could not initialise libsodium"); - - unsigned char pk[crypto_sign_PUBLICKEYBYTES]; - unsigned char sk[crypto_sign_SECRETKEYBYTES]; - if (crypto_sign_keypair(pk, sk) != 0) - throw Error("key generation failed"); + auto secretKey = SecretKey::generate(keyName); - writeFile(publicKeyFile, keyName + ":" + base64Encode(string((char *) pk, crypto_sign_PUBLICKEYBYTES))); + writeFile(publicKeyFile, secretKey.toPublicKey().to_string()); umask(0077); - writeFile(secretKeyFile, keyName + ":" + base64Encode(string((char *) sk, crypto_sign_SECRETKEYBYTES))); -#else - throw Error("Nix was not compiled with libsodium, required for signed binary cache support"); -#endif + writeFile(secretKeyFile, secretKey.to_string()); } diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index ea4bbbab9..2ae042789 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -19,7 +19,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand addFlag({ .longName = "name", .shortName = 'n', - .description = "name component of the store path", + .description = "Override the name component of the store path. It defaults to the base name of *path*.", .labels = {"name"}, .handler = {&namePart}, }); diff --git a/src/nix/build.cc b/src/nix/build.cc index 67be4024b..724ce9d79 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -19,7 +19,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile addFlag({ .longName = "out-link", .shortName = 'o', - .description = "path of the symlink to the build result", + .description = "Use *path* as prefix for the symlinks to the build results. It defaults to `result`.", .labels = {"path"}, .handler = {&outLink}, .completer = completePath @@ -27,13 +27,13 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile addFlag({ .longName = "no-link", - .description = "do not create a symlink to the build result", + .description = "Do not create symlinks to the build results.", .handler = {&outLink, Path("")}, }); addFlag({ .longName = "rebuild", - .description = "rebuild an already built package and compare the result to the existing store paths", + .description = "Rebuild an already built package and compare the result to the existing store paths.", .handler = {&buildMode, bmCheck}, }); } @@ -43,22 +43,11 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile return "build a derivation or fetch a store path"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To build and run GNU Hello from NixOS 17.03:", - "nix build -f channel:nixos-17.03 hello; ./result/bin/hello" - }, - Example{ - "To build the build.x86_64-linux attribute from release.nix:", - "nix build -f release.nix build.x86_64-linux" - }, - Example{ - "To make a profile point at GNU Hello:", - "nix build --profile /tmp/profile nixpkgs#hello" - }, - }; + return + #include "build.md" + ; } void run(ref<Store> store) override @@ -69,7 +58,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile if (outLink != "") if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) - for (size_t i = 0; i < buildables.size(); ++i) + for (const auto & [_i, buildable] : enumerate(buildables)) { + auto i = _i; std::visit(overloaded { [&](BuildableOpaque bo) { std::string symlink = outLink; @@ -85,7 +75,8 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile store2->addPermRoot(output.second, absPath(symlink)); } }, - }, buildables[i]); + }, buildable); + } updateProfile(buildables); diff --git a/src/nix/build.md b/src/nix/build.md new file mode 100644 index 000000000..c2f3e387a --- /dev/null +++ b/src/nix/build.md @@ -0,0 +1,92 @@ +R""( + +# Examples + +* Build the default package from the flake in the current directory: + + ```console + # nix build + ``` + +* Build and run GNU Hello from the `nixpkgs` flake: + + ```console + # nix build nixpkgs#hello + # ./result/bin/hello + Hello, world! + ``` + +* Build GNU Hello and Cowsay, leaving two result symlinks: + + ```console + # nix build nixpkgs#hello nixpkgs#cowsay + # ls -l result* + lrwxrwxrwx 1 … result -> /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10 + lrwxrwxrwx 1 … result-1 -> /nix/store/rkfrm0z6x6jmi7d3gsmma4j53h15mg33-cowsay-3.03+dfsg2 + ``` + +* Build a specific output: + + ```console + # nix build nixpkgs#glibc.dev + # ls -ld ./result-dev + lrwxrwxrwx 1 … ./result-dev -> /nix/store/dkm3gwl0xrx0wrw6zi5x3px3lpgjhlw4-glibc-2.32-dev + ``` + +* Build attribute `build.x86_64-linux` from (non-flake) Nix expression + `release.nix`: + + ```console + # nix build -f release.nix build.x86_64-linux + ``` + +* Build a NixOS system configuration from a flake, and make a profile + point to the result: + + ```console + # nix build --profile /nix/var/nix/profiles/system \ + ~/my-configurations#nixosConfigurations.machine.config.system.build.toplevel + ``` + + (This is essentially what `nixos-rebuild` does.) + +* Build an expression specified on the command line: + + ```console + # nix build --impure --expr \ + 'with import <nixpkgs> {}; + runCommand "foo" { + buildInputs = [ hello ]; + } + "hello > $out"' + # cat ./result + Hello, world! + ``` + + Note that `--impure` is needed because we're using `<nixpkgs>`, + which relies on the `$NIX_PATH` environment variable. + +* Fetch a store path from the configured substituters, if it doesn't + already exist: + + ```console + # nix build /nix/store/rkfrm0z6x6jmi7d3gsmma4j53h15mg33-cowsay-3.03+dfsg2 + ``` + +# Description + +`nix build` builds the specified *installables*. Installables that +resolve to derivations are built (or substituted if possible). Store +path installables are substituted. + +Unless `--no-link` is specified, after a successful build, it creates +symlinks to the store paths of the installables. These symlinks have +the prefix `./result` by default; this can be overriden using the +`--out-link` option. Each symlink has a suffix `-<N>-<outname>`, where +*N* is the index of the installable (with the left-most installable +having index 0), and *outname* is the symbolic derivation output name +(e.g. `bin`, `dev` or `lib`). `-<N>` is omitted if *N* = 0, and +`-<outname>` is omitted if *outname* = `out` (denoting the default +output). + +)"" diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index eddd82f40..1789e4598 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -16,7 +16,7 @@ struct CmdBundle : InstallableCommand { addFlag({ .longName = "bundler", - .description = "use custom bundler", + .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler), .labels = {"flake-url"}, .handler = {&bundler}, .completer = {[&](size_t, std::string_view prefix) { @@ -27,7 +27,7 @@ struct CmdBundle : InstallableCommand addFlag({ .longName = "out-link", .shortName = 'o', - .description = "path of the symlink to the build result", + .description = "Override the name of the symlink to the build result. It defaults to the base name of the app.", .labels = {"path"}, .handler = {&outLink}, .completer = completePath @@ -40,14 +40,11 @@ struct CmdBundle : InstallableCommand return "bundle an application so that it works outside of the Nix store"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To bundle Hello:", - "nix bundle hello" - }, - }; + return + #include "bundle.md" + ; } Category category() override { return catSecondary; } @@ -93,7 +90,7 @@ struct CmdBundle : InstallableCommand mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get()); arg->attrs->sort(); - + auto vRes = evalState->allocValue(); evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos); diff --git a/src/nix/bundle.md b/src/nix/bundle.md new file mode 100644 index 000000000..5e2298376 --- /dev/null +++ b/src/nix/bundle.md @@ -0,0 +1,36 @@ +R""( + +# Examples + +* Bundle Hello: + + ```console + # nix bundle nixpkgs#hello + # ./hello + Hello, world! + ``` + +* Bundle a specific version of Nix: + + ```console + # nix bundle github:NixOS/nix/e3ddffb27e5fc37a209cfd843c6f7f6a9460a8ec + # ./nix --version + nix (Nix) 2.4pre20201215_e3ddffb + ``` + +# Description + +`nix bundle` packs the closure of the [Nix app](./nix3-run.md) +*installable* into a single self-extracting executable. See the +[`nix-bundle` homepage](https://github.com/matthewbauer/nix-bundle) +for more details. + +> **Note** +> +> This command only works on Linux. + +# Bundler definitions + +TODO + +)"" diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 2ecffc9a5..e28ee3c50 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -37,6 +37,13 @@ struct CmdCatStore : StoreCommand, MixCat return "print the contents of a file in the Nix store on stdout"; } + std::string doc() override + { + return + #include "store-cat.md" + ; + } + void run(ref<Store> store) override { cat(store->getFSAccessor()); @@ -62,6 +69,13 @@ struct CmdCatNar : StoreCommand, MixCat return "print the contents of a file inside a NAR file on stdout"; } + std::string doc() override + { + return + #include "nar-cat.md" + ; + } + void run(ref<Store> store) override { cat(makeNarAccessor(make_ref<std::string>(readFile(narPath)))); diff --git a/src/nix/command.cc b/src/nix/command.cc index 596217775..ba58c7d6b 100644 --- a/src/nix/command.cc +++ b/src/nix/command.cc @@ -65,18 +65,18 @@ StorePathsCommand::StorePathsCommand(bool recursive) if (recursive) addFlag({ .longName = "no-recursive", - .description = "apply operation to specified paths only", + .description = "Apply operation to specified paths only.", .handler = {&this->recursive, false}, }); else addFlag({ .longName = "recursive", .shortName = 'r', - .description = "apply operation to closure of the specified paths", + .description = "Apply operation to closure of the specified paths.", .handler = {&this->recursive, true}, }); - mkFlag(0, "all", "apply operation to the entire store", &all); + mkFlag(0, "all", "Apply the operation to every store path.", &all); } void StorePathsCommand::run(ref<Store> store) @@ -133,7 +133,7 @@ MixProfile::MixProfile() { addFlag({ .longName = "profile", - .description = "profile to update", + .description = "The profile to update.", .labels = {"path"}, .handler = {&profile}, .completer = completePath @@ -190,14 +190,14 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false) addFlag({ .longName = "ignore-environment", .shortName = 'i', - .description = "clear the entire environment (except those specified with --keep)", + .description = "Clear the entire environment (except those specified with `--keep`).", .handler = {&ignoreEnvironment, true}, }); addFlag({ .longName = "keep", .shortName = 'k', - .description = "keep specified environment variable", + .description = "Keep the environment variable *name*.", .labels = {"name"}, .handler = {[&](std::string s) { keep.insert(s); }}, }); @@ -205,7 +205,7 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false) addFlag({ .longName = "unset", .shortName = 'u', - .description = "unset specified environment variable", + .description = "Unset the environment variable *name*.", .labels = {"name"}, .handler = {[&](std::string s) { unset.insert(s); }}, }); diff --git a/src/nix/command.hh b/src/nix/command.hh index 6882db195..f325cd906 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -13,6 +13,8 @@ namespace nix { extern std::string programPath; +extern char * * savedArgv; + class EvalState; struct Pos; class Store; @@ -261,6 +263,8 @@ void completeFlakeRefWithFragment( const Strings & defaultFlakeAttrPaths, std::string_view prefix); +std::string showVersions(const std::set<std::string> & versions); + void printClosureDiff( ref<Store> store, const StorePath & beforePath, diff --git a/src/nix/copy.cc b/src/nix/copy.cc index cb31aac8f..f15031a45 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -21,28 +21,28 @@ struct CmdCopy : StorePathsCommand { addFlag({ .longName = "from", - .description = "URI of the source Nix store", + .description = "URL of the source Nix store.", .labels = {"store-uri"}, .handler = {&srcUri}, }); addFlag({ .longName = "to", - .description = "URI of the destination Nix store", + .description = "URL of the destination Nix store.", .labels = {"store-uri"}, .handler = {&dstUri}, }); addFlag({ .longName = "no-check-sigs", - .description = "do not require that paths are signed by trusted keys", + .description = "Do not require that paths are signed by trusted keys.", .handler = {&checkSigs, NoCheckSigs}, }); addFlag({ .longName = "substitute-on-destination", .shortName = 's', - .description = "whether to try substitutes on the destination store (only supported by SSH)", + .description = "Whether to try substitutes on the destination store (only supported by SSH stores).", .handler = {&substitute, Substitute}, }); @@ -54,32 +54,11 @@ struct CmdCopy : StorePathsCommand return "copy paths between Nix stores"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To copy Firefox from the local store to a binary cache in file:///tmp/cache:", - "nix copy --to file:///tmp/cache $(type -p firefox)" - }, - Example{ - "To copy the entire current NixOS system closure to another machine via SSH:", - "nix copy --to ssh://server /run/current-system" - }, - Example{ - "To copy a closure from another machine via SSH:", - "nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2" - }, -#ifdef ENABLE_S3 - Example{ - "To copy Hello to an S3 binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs#hello" - }, - Example{ - "To copy Hello to an S3-compatible binary cache:", - "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs#hello" - }, -#endif - }; + return + #include "copy.md" + ; } Category category() override { return catSecondary; } diff --git a/src/nix/copy.md b/src/nix/copy.md new file mode 100644 index 000000000..25e0ddadc --- /dev/null +++ b/src/nix/copy.md @@ -0,0 +1,58 @@ +R""( + +# Examples + +* Copy Firefox from the local store to a binary cache in `/tmp/cache`: + + ```console + # nix copy --to file:///tmp/cache $(type -p firefox) + ``` + + Note the `file://` - without this, the destination is a chroot + store, not a binary cache. + +* Copy the entire current NixOS system closure to another machine via + SSH: + + ```console + # nix copy -s --to ssh://server /run/current-system + ``` + + The `-s` flag causes the remote machine to try to substitute missing + store paths, which may be faster if the link between the local and + remote machines is slower than the link between the remote machine + and its substituters (e.g. `https://cache.nixos.org`). + +* Copy a closure from another machine via SSH: + + ```console + # nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2 + ``` + +* Copy Hello to a binary cache in an Amazon S3 bucket: + + ```console + # nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs#hello + ``` + + or to an S3-compatible storage system: + + ```console + # nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs#hello + ``` + + Note that this only works if Nix is built with AWS support. + +* Copy a closure from `/nix/store` to the chroot store `/tmp/nix/nix/store`: + + ```console + # nix copy --to /tmp/nix nixpkgs#hello --no-check-sigs + ``` + +# Description + +`nix copy` copies store path closures between two Nix stores. The +source store is specified using `--from` and the destination using +`--to`. If one of these is omitted, it defaults to the local store. + +)"" diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix/daemon.cc index bd016bb0c..628e55f92 100644 --- a/src/nix-daemon/nix-daemon.cc +++ b/src/nix/daemon.cc @@ -1,3 +1,4 @@ +#include "command.hh" #include "shared.hh" #include "local-store.hh" #include "remote-store.hh" @@ -150,7 +151,7 @@ static ref<Store> openUncachedStore() } -static void daemonLoop(char * * argv) +static void daemonLoop() { if (chdir("/") == -1) throw SysError("cannot change current directory"); @@ -232,9 +233,9 @@ static void daemonLoop(char * * argv) setSigChldAction(false); // For debugging, stuff the pid into argv[1]. - if (peer.pidKnown && argv[1]) { + if (peer.pidKnown && savedArgv[1]) { string processName = std::to_string(peer.pid); - strncpy(argv[1], processName.c_str(), strlen(argv[1])); + strncpy(savedArgv[1], processName.c_str(), strlen(savedArgv[1])); } // Handle the connection. @@ -264,6 +265,57 @@ static void daemonLoop(char * * argv) } } +static void runDaemon(bool stdio, std::optional<TrustedFlag> isTrustedOpt = {}) +{ + auto ensureNoTrustedFlag = [&]() { + if (isTrustedOpt) + throw Error("--trust and --no-trust flags are only for use with --stdio when this nix-daemon process is not proxying another"); + }; + + if (stdio) { + if (auto store = openUncachedStore().dynamic_pointer_cast<RemoteStore>()) { + ensureNoTrustedFlag(); + auto conn = store->openConnectionWrapper(); + int from = conn->from.fd; + int to = conn->to.fd; + + auto nfds = std::max(from, STDIN_FILENO) + 1; + while (true) { + fd_set fds; + FD_ZERO(&fds); + FD_SET(from, &fds); + FD_SET(STDIN_FILENO, &fds); + if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) + throw SysError("waiting for data from client or server"); + if (FD_ISSET(from, &fds)) { + auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from daemon socket to stdout"); + else if (res == 0) + throw EndOfFile("unexpected EOF from daemon socket"); + } + if (FD_ISSET(STDIN_FILENO, &fds)) { + auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); + if (res == -1) + throw SysError("splicing data from stdin to daemon socket"); + else if (res == 0) + return; + } + } + } else { + FdSource from(STDIN_FILENO); + FdSink to(STDOUT_FILENO); + /* Auth hook is empty because in this mode we blindly trust the + standard streams. Limiting access to those is explicitly + not `nix-daemon`'s responsibility. */ + auto isTrusted = isTrustedOpt.value_or(Trusted); + processConnection(openUncachedStore(), from, to, isTrusted, NotRecursive, [&](Store & _){}); + } + } else { + ensureNoTrustedFlag(); + daemonLoop(); + } +} static int main_nix_daemon(int argc, char * * argv) { @@ -292,57 +344,34 @@ static int main_nix_daemon(int argc, char * * argv) initPlugins(); - auto ensureNoTrustedFlag = [&]() { - if (isTrustedOpt) - throw Error("--trust and --no-trust flags are only for use with --stdio when this nix-daemon process is not proxying another"); - }; - - if (stdio) { - if (auto store = openUncachedStore().dynamic_pointer_cast<RemoteStore>()) { - ensureNoTrustedFlag(); - auto conn = store->openConnectionWrapper(); - int from = conn->from.fd; - int to = conn->to.fd; - - auto nfds = std::max(from, STDIN_FILENO) + 1; - while (true) { - fd_set fds; - FD_ZERO(&fds); - FD_SET(from, &fds); - FD_SET(STDIN_FILENO, &fds); - if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1) - throw SysError("waiting for data from client or server"); - if (FD_ISSET(from, &fds)) { - auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from daemon socket to stdout"); - else if (res == 0) - throw EndOfFile("unexpected EOF from daemon socket"); - } - if (FD_ISSET(STDIN_FILENO, &fds)) { - auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE); - if (res == -1) - throw SysError("splicing data from stdin to daemon socket"); - else if (res == 0) - return 0; - } - } - } else { - FdSource from(STDIN_FILENO); - FdSink to(STDOUT_FILENO); - /* Auth hook is empty because in this mode we blindly trust the - standard streams. Limitting access to thoses is explicitly - not `nix-daemon`'s responsibility. */ - auto isTrusted = isTrustedOpt.value_or(Trusted); - processConnection(openUncachedStore(), from, to, isTrusted, NotRecursive, [&](Store & _){}); - } - } else { - ensureNoTrustedFlag(); - daemonLoop(argv); - } + runDaemon(stdio, isTrustedOpt); return 0; } } static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon); + +struct CmdDaemon : StoreCommand +{ + std::string description() override + { + return "daemon to perform store operations on behalf of non-root clients"; + } + + Category category() override { return catUtility; } + + std::string doc() override + { + return + #include "daemon.md" + ; + } + + void run(ref<Store> store) override + { + runDaemon(false); + } +}; + +static auto rCmdDaemon = registerCommand2<CmdDaemon>({"daemon"}); diff --git a/src/nix/daemon.md b/src/nix/daemon.md new file mode 100644 index 000000000..e97016a94 --- /dev/null +++ b/src/nix/daemon.md @@ -0,0 +1,21 @@ +R""( + +# Example + +* Run the daemon in the foreground: + + ```console + # nix daemon + ``` + +# Description + +This command runs the Nix daemon, which is a required component in +multi-user Nix installations. It performs build actions and other +operations on the Nix store on behalf of non-root users. Usually you +don't run the daemon directly; instead it's managed by a service +management framework such as `systemd`. + +Note that this daemon does not fork into the background. + +)"" diff --git a/src/nix/develop.cc b/src/nix/develop.cc index 457d94382..578258394 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -204,7 +204,7 @@ struct Common : InstallableCommand, MixProfile { addFlag({ .longName = "redirect", - .description = "redirect a store path to a mutable location", + .description = "Redirect a store path to a mutable location.", .labels = {"installable", "outputs-dir"}, .handler = {[&](std::string installable, std::string outputsDir) { redirects.push_back({installable, outputsDir}); @@ -334,7 +334,7 @@ struct CmdDevelop : Common, MixEnvironment addFlag({ .longName = "command", .shortName = 'c', - .description = "command and arguments to be executed instead of an interactive shell", + .description = "Instead of starting an interactive shell, start the specified command and arguments.", .labels = {"command", "args"}, .handler = {[&](std::vector<std::string> ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); @@ -344,38 +344,38 @@ struct CmdDevelop : Common, MixEnvironment addFlag({ .longName = "phase", - .description = "phase to run (e.g. `build` or `configure`)", + .description = "The stdenv phase to run (e.g. `build` or `configure`).", .labels = {"phase-name"}, .handler = {&phase}, }); addFlag({ .longName = "configure", - .description = "run the configure phase", + .description = "Run the `configure` phase.", .handler = {&phase, {"configure"}}, }); addFlag({ .longName = "build", - .description = "run the build phase", + .description = "Run the `build` phase.", .handler = {&phase, {"build"}}, }); addFlag({ .longName = "check", - .description = "run the check phase", + .description = "Run the `check` phase.", .handler = {&phase, {"check"}}, }); addFlag({ .longName = "install", - .description = "run the install phase", + .description = "Run the `install` phase.", .handler = {&phase, {"install"}}, }); addFlag({ .longName = "installcheck", - .description = "run the installcheck phase", + .description = "Run the `installcheck` phase.", .handler = {&phase, {"installCheck"}}, }); } @@ -385,30 +385,11 @@ struct CmdDevelop : Common, MixEnvironment return "run a bash shell that provides the build environment of a derivation"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To get the build environment of GNU hello:", - "nix develop nixpkgs#hello" - }, - Example{ - "To get the build environment of the default package of flake in the current directory:", - "nix develop" - }, - Example{ - "To store the build environment in a profile:", - "nix develop --profile /tmp/my-shell nixpkgs#hello" - }, - Example{ - "To use a build environment previously recorded in a profile:", - "nix develop /tmp/my-shell" - }, - Example{ - "To replace all occurences of a store path with a writable directory:", - "nix develop --redirect nixpkgs#glibc.dev ~/my-glibc/outputs/dev" - }, - }; + return + #include "develop.md" + ; } void run(ref<Store> store) override @@ -495,14 +476,11 @@ struct CmdPrintDevEnv : Common return "print shell code that can be sourced by bash to reproduce the build environment of a derivation"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To apply the build environment of GNU hello to the current shell:", - ". <(nix print-dev-env nixpkgs#hello)" - }, - }; + return + #include "print-dev-env.md" + ; } Category category() override { return catUtility; } diff --git a/src/nix/develop.md b/src/nix/develop.md new file mode 100644 index 000000000..e71d9f8aa --- /dev/null +++ b/src/nix/develop.md @@ -0,0 +1,94 @@ +R""( + +# Examples + +* Start a shell with the build environment of the default package of + the flake in the current directory: + + ```console + # nix develop + ``` + + Typical commands to run inside this shell are: + + ```console + # configurePhase + # buildPhase + # installPhase + ``` + + Alternatively, you can run whatever build tools your project uses + directly, e.g. for a typical Unix project: + + ```console + # ./configure --prefix=$out + # make + # make install + ``` + +* Run a particular build phase directly: + + ```console + # nix develop --configure + # nix develop --build + # nix develop --check + # nix develop --install + # nix develop --installcheck + ``` + +* Start a shell with the build environment of GNU Hello: + + ```console + # nix develop nixpkgs#hello + ``` + +* Record a build environment in a profile: + + ```console + # nix develop --profile /tmp/my-build-env nixpkgs#hello + ``` + +* Use a build environment previously recorded in a profile: + + ```console + # nix develop /tmp/my-build-env + ``` + +* Replace all occurences of the store path corresponding to + `glibc.dev` with a writable directory: + + ```console + # nix develop --redirect nixpkgs#glibc.dev ~/my-glibc/outputs/dev + ``` + + Note that this is useful if you're running a `nix develop` shell for + `nixpkgs#glibc` in `~/my-glibc` and want to compile another package + against it. + +# Description + +`nix develop` starts a `bash` shell that provides an interactive build +environment nearly identical to what Nix would use to build +*installable*. Inside this shell, environment variables and shell +functions are set up so that you can interactively and incrementally +build your package. + +Nix determines the build environment by building a modified version of +the derivation *installable* that just records the environment +initialised by `stdenv` and exits. This build environment can be +recorded into a profile using `--profile`. + +The prompt used by the `bash` shell can be customised by setting the +`bash-prompt` and `bash-prompt-suffix` settings in `nix.conf` or in +the flake's `nixConfig` attribute. + +# Flake output attributes + +If no flake output attribute is given, `nix run` tries the following +flake output attributes: + +* `devShell.<system>` + +* `defaultPackage.<system>` + +)"" diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index f72b5eff7..0c7d531c1 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -121,14 +121,11 @@ struct CmdDiffClosures : SourceExprCommand return "show what packages and versions were added and removed between two closures"; } - Examples examples() override + std::string doc() override { - return { - { - "To show what got added and removed between two versions of the NixOS system profile:", - "nix store diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link", - }, - }; + return + #include "diff-closures.md" + ; } void run(ref<Store> store) override diff --git a/src/nix/diff-closures.md b/src/nix/diff-closures.md new file mode 100644 index 000000000..0294c0d8d --- /dev/null +++ b/src/nix/diff-closures.md @@ -0,0 +1,51 @@ +R""( + +# Examples + +* Show what got added and removed between two versions of the NixOS + system profile: + + ```console + # nix store diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link + acpi-call: 2020-04-07-5.8.16 → 2020-04-07-5.8.18 + baloo-widgets: 20.08.1 → 20.08.2 + bluez-qt: +12.6 KiB + dolphin: 20.08.1 → 20.08.2, +13.9 KiB + kdeconnect: 20.08.2 → ∅, -6597.8 KiB + kdeconnect-kde: ∅ → 20.08.2, +6599.7 KiB + … + ``` + +# Description + +This command shows the differences between the two closures *before* +and *after* with respect to the addition, removal, or version change +of packages, as well as changes in store path sizes. + +For each package name in the two closures (where a package name is +defined as the name component of a store path excluding the version), +if there is a change in the set of versions of the package, or a +change in the size of the store paths of more than 8 KiB, it prints a +line like this: + +```console +dolphin: 20.08.1 → 20.08.2, +13.9 KiB +``` + +No size change is shown if it's below the threshold. If the package +does not exist in either the *before* or *after* closures, it is +represented using `∅` (empty set) on the appropriate side of the +arrow. If a package has an empty version string, the version is +rendered as `ε` (epsilon). + +There may be multiple versions of a package in each closure. In that +case, only the changed versions are shown. Thus, + +```console +libfoo: 1.2, 1.3 → 1.4 +``` + +leaves open the possibility that there are other versions (e.g. `1.1`) +that exist in both closures. + +)"" diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 256db64a9..c4edc894b 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -11,14 +11,11 @@ struct CmdDumpPath : StorePathCommand return "serialise a store path to stdout in NAR format"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To get a NAR from the binary cache https://cache.nixos.org/:", - "nix store dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25" - }, - }; + return + #include "store-dump-path.md" + ; } void run(ref<Store> store, const StorePath & storePath) override @@ -49,14 +46,11 @@ struct CmdDumpPath2 : Command return "serialise a path to stdout in NAR format"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To serialise directory 'foo' as a NAR:", - "nix nar dump-path ./foo" - }, - }; + return + #include "nar-dump-path.md" + ; } void run() override diff --git a/src/nix/edit.cc b/src/nix/edit.cc index 51c16f5a9..6472dd27a 100644 --- a/src/nix/edit.cc +++ b/src/nix/edit.cc @@ -15,14 +15,11 @@ struct CmdEdit : InstallableCommand return "open the Nix expression of a Nix package in $EDITOR"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To open the Nix expression of the GNU Hello package:", - "nix edit nixpkgs#hello" - }, - }; + return + #include "edit.md" + ; } Category category() override { return catSecondary; } diff --git a/src/nix/edit.md b/src/nix/edit.md new file mode 100644 index 000000000..80563d06b --- /dev/null +++ b/src/nix/edit.md @@ -0,0 +1,31 @@ +R""( + +# Examples + +* Open the Nix expression of the GNU Hello package: + + ```console + # nix edit nixpkgs#hello + ``` + +* Get the filename and line number used by `nix edit`: + + ```console + # nix eval --raw nixpkgs#hello.meta.position + /nix/store/fvafw0gvwayzdan642wrv84pzm5bgpmy-source/pkgs/applications/misc/hello/default.nix:15 + ``` + +# Description + +This command opens the Nix expression of a derivation in an +editor. The filename and line number of the derivation are taken from +its `meta.position` attribute. Nixpkgs' `stdenv.mkDerivation` sets +this attribute to the location of the definition of the +`meta.description`, `version` or `name` derivation attributes. + +The editor to invoke is specified by the `EDITOR` environment +variable. It defaults to `cat`. If the editor is `emacs`, `nano` or +`vim`, it is passed the line number of the derivation using the +argument `+<lineno>`. + +)"" diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 0f02919de..b5049ac65 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -18,18 +18,18 @@ struct CmdEval : MixJSON, InstallableCommand CmdEval() { - mkFlag(0, "raw", "print strings unquoted", &raw); + mkFlag(0, "raw", "Print strings without quotes or escaping.", &raw); addFlag({ .longName = "apply", - .description = "apply a function to each argument", + .description = "Apply the function *expr* to each argument.", .labels = {"expr"}, .handler = {&apply}, }); addFlag({ .longName = "write-to", - .description = "write a string or attrset of strings to 'path'", + .description = "Write a string or attrset of strings to *path*.", .labels = {"path"}, .handler = {&writeTo}, }); @@ -40,30 +40,11 @@ struct CmdEval : MixJSON, InstallableCommand return "evaluate a Nix expression"; } - Examples examples() override + std::string doc() override { - return { - { - "To evaluate a Nix expression given on the command line:", - "nix eval --expr '1 + 2'" - }, - { - "To evaluate a Nix expression from a file or URI:", - "nix eval -f ./my-nixpkgs hello.name" - }, - { - "To get the current version of Nixpkgs:", - "nix eval --raw nixpkgs#lib.version" - }, - { - "To print the store path of the Hello package:", - "nix eval --raw nixpkgs#hello" - }, - { - "To get a list of checks in the 'nix' flake:", - "nix eval nix#checks.x86_64-linux --apply builtins.attrNames" - }, - }; + return + #include "eval.md" + ; } Category category() override { return catSecondary; } @@ -97,10 +78,10 @@ struct CmdEval : MixJSON, InstallableCommand recurse = [&](Value & v, const Pos & pos, const Path & path) { state->forceValue(v); - if (v.type == tString) + if (v.type() == nString) // FIXME: disallow strings with contexts? writeFile(path, v.string.s); - else if (v.type == tAttrs) { + else if (v.type() == nAttrs) { if (mkdir(path.c_str(), 0777) == -1) throw SysError("creating directory '%s'", path); for (auto & attr : *v.attrs) diff --git a/src/nix/eval.md b/src/nix/eval.md new file mode 100644 index 000000000..61334cde1 --- /dev/null +++ b/src/nix/eval.md @@ -0,0 +1,74 @@ +R""( + +# Examples + +* Evaluate a Nix expression given on the command line: + + ```console + # nix eval --expr '1 + 2' + ``` + +* Evaluate a Nix expression to JSON: + + ```console + # nix eval --json --expr '{ x = 1; }' + {"x":1} + ``` + +* Evaluate a Nix expression from a file: + + ```console + # nix eval -f ./my-nixpkgs hello.name + ``` + +* Get the current version of the `nixpkgs` flake: + + ```console + # nix eval --raw nixpkgs#lib.version + ``` + +* Print the store path of the Hello package: + + ```console + # nix eval --raw nixpkgs#hello + ``` + +* Get a list of checks in the `nix` flake: + + ```console + # nix eval nix#checks.x86_64-linux --apply builtins.attrNames + ``` + +* Generate a directory with the specified contents: + + ```console + # nix eval --write-to ./out --expr '{ foo = "bar"; subdir.bla = "123"; }' + # cat ./out/foo + bar + # cat ./out/subdir/bla + 123 + +# Description + +This command evaluates the Nix expression *installable* and prints the +result on standard output. + +# Output format + +`nix eval` can produce output in several formats: + +* By default, the evaluation result is printed as a Nix expression. + +* With `--json`, the evaluation result is printed in JSON format. Note + that this fails if the result contains values that are not + representable as JSON, such as functions. + +* With `--raw`, the evaluation result must be a string, which is + printed verbatim, without any quoting. + +* With `--write-to` *path*, the evaluation result must be a string or + a nested attribute set whose leaf values are strings. These strings + are written to files named *path*/*attrpath*. *path* must not + already exist. + +)"" diff --git a/src/nix/flake-archive.md b/src/nix/flake-archive.md new file mode 100644 index 000000000..85bbeeb16 --- /dev/null +++ b/src/nix/flake-archive.md @@ -0,0 +1,29 @@ +R""( + +# Examples + +* Copy the `dwarffs` flake and its dependencies to a binary cache: + + ```console + # nix flake archive --to file:///tmp/my-cache dwarffs + ``` + +* Fetch the `dwarffs` flake and its dependencies to the local Nix + store: + + ```console + # nix flake archive dwarffs + ``` + +* Print the store paths of the flake sources of NixOps without + fetching them: + + ```console + # nix flake archive --json --dry-run nixops + ``` + +# Description + +FIXME + +)"" diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md new file mode 100644 index 000000000..dc079ba0c --- /dev/null +++ b/src/nix/flake-check.md @@ -0,0 +1,68 @@ +R""( + +# Examples + +* Evaluate the flake in the current directory, and build its checks: + + ```console + # nix flake check + ``` + +* Verify that the `patchelf` flake evaluates, but don't build its + checks: + + ```console + # nix flake check --no-build github:NixOS/patchelf + ``` + +# Description + +This command verifies that the flake specified by flake reference +*flake-url* can be evaluated successfully (as detailed below), and +that the derivations specified by the flake's `checks` output can be +built successfully. + +# Evaluation checks + +This following flake output attributes must be derivations: + +* `checks.`*system*`.`*name* +* `defaultPackage.`*system*` +* `devShell.`*system*` +* `nixosConfigurations.`*name*`.config.system.build.toplevel +* `packages.`*system*`.`*name* + +The following flake output attributes must be [app +definitions](./nix3-run.md): + +* `apps.`*system*`.`*name* +* `defaultApp.`*system*` + +The following flake output attributes must be [template +definitions](./nix3-flake-init.md): + +* `defaultTemplate` +* `templates`.`*name* + +The following flake output attributes must be *Nixpkgs overlays*: + +* `overlay` +* `overlays`.`*name* + +The following flake output attributes must be *NixOS modules*: + +* `nixosModule` +* `nixosModules`.`*name* + +The following flake output attributes must be +[bundlers](./nix3-bundle.md): + +* `bundlers`.`*name* +* `defaultBundler` + +In addition, the `hydraJobs` output is evaluated in the same way as +Hydra's `hydra-eval-jobs` (i.e. as a arbitrarily deeply nested +attribute set of derivations). Similarly, the +`legacyPackages`.*system* output is evaluated like `nix-env -qa`. + +)"" diff --git a/src/nix/flake-clone.md b/src/nix/flake-clone.md new file mode 100644 index 000000000..36cb96051 --- /dev/null +++ b/src/nix/flake-clone.md @@ -0,0 +1,18 @@ +R""( + +# Examples + +* Check out the source code of the `dwarffs` flake and build it: + + ```console + # nix flake clone dwarffs --dest dwarffs + # cd dwarffs + # nix build + ``` + +# Description + +This command performs a Git or Mercurial clone of the repository +containing the source code of the flake *flake-url*. + +)"" diff --git a/src/nix/flake-info.md b/src/nix/flake-info.md new file mode 100644 index 000000000..fda3171db --- /dev/null +++ b/src/nix/flake-info.md @@ -0,0 +1,99 @@ +R""( + +# Examples + +* Show what `nixpkgs` resolves to: + + ```console + # nix flake info nixpkgs + Resolved URL: github:NixOS/nixpkgs + Locked URL: github:NixOS/nixpkgs/b67ba0bfcc714453cdeb8d713e35751eb8b4c8f4 + Description: A collection of packages for the Nix package manager + Path: /nix/store/23qapccs6cfmwwrlq8kr41vz5vdmns3r-source + Revision: b67ba0bfcc714453cdeb8d713e35751eb8b4c8f4 + Last modified: 2020-12-23 12:36:12 + ``` + +* Show information about `dwarffs` in JSON format: + + ```console + # nix flake info dwarffs --json | jq . + { + "description": "A filesystem that fetches DWARF debug info from the Internet on demand", + "lastModified": 1597153508, + "locked": { + "lastModified": 1597153508, + "narHash": "sha256-VHg3MYVgQ12LeRSU2PSoDeKlSPD8PYYEFxxwkVVDRd0=", + "owner": "edolstra", + "repo": "dwarffs", + "rev": "d181d714fd36eb06f4992a1997cd5601e26db8f5", + "type": "github" + }, + "original": { + "id": "dwarffs", + "type": "indirect" + }, + "originalUrl": "flake:dwarffs", + "path": "/nix/store/hang3792qwdmm2n0d9nsrs5n6bsws6kv-source", + "resolved": { + "owner": "edolstra", + "repo": "dwarffs", + "type": "github" + }, + "resolvedUrl": "github:edolstra/dwarffs", + "revision": "d181d714fd36eb06f4992a1997cd5601e26db8f5", + "url": "github:edolstra/dwarffs/d181d714fd36eb06f4992a1997cd5601e26db8f5" + } + ``` + +# Description + +This command shows information about the flake specified by the flake +reference *flake-url*. It resolves the flake reference using the +[flake registry](./nix3-registry.md), fetches it, and prints some meta +data. This includes: + +* `Resolved URL`: If *flake-url* is a flake identifier, then this is + the flake reference that specifies its actual location, looked up in + the flake registry. + +* `Locked URL`: A flake reference that contains a commit or content + hash and thus uniquely identifies a specific flake version. + +* `Description`: A one-line description of the flake, taken from the + `description` field in `flake.nix`. + +* `Path`: The store path containing the source code of the flake. + +* `Revision`: The Git or Mercurial commit hash of the locked flake. + +* `Revisions`: The number of ancestors of the Git or Mercurial commit + of the locked flake. Note that this is not available for `github` + flakes. + +* `Last modified`: For Git or Mercurial flakes, this is the commit + time of the commit of the locked flake; for tarball flakes, it's the + most recent timestamp of any file inside the tarball. + +With `--json`, the output is a JSON object with the following fields: + +* `original` and `originalUrl`: The flake reference specified by the + user (*flake-url*) in attribute set and URL representation. + +* `resolved` and `resolvedUrl`: The resolved flake reference (see + above) in attribute set and URL representation. + +* `locked` and `lockedUrl`: The locked flake reference (see above) in + attribute set and URL representation. + +* `description`: See `Description` above. + +* `path`: See `Path` above. + +* `revision`: See `Revision` above. + +* `revCount`: See `Revisions` above. + +* `lastModified`: See `Last modified` above. + +)"" diff --git a/src/nix/flake-init.md b/src/nix/flake-init.md new file mode 100644 index 000000000..c66154ad5 --- /dev/null +++ b/src/nix/flake-init.md @@ -0,0 +1,54 @@ +R""( + +# Examples + +* Create a flake using the default template: + + ```console + # nix flake init + ``` + +* List available templates: + + ```console + # nix flake show templates + ``` + +* Create a flake from a specific template: + + ```console + # nix flake init -t templates#simpleContainer + ``` + +# Description + +This command creates a flake in the current directory by copying the +files of a template. It will not overwrite existing files. The default +template is `templates#defaultTemplate`, but this can be overriden +using `-t`. + +# Template definitions + +A flake can declare templates through its `templates` and +`defaultTemplate` output attributes. A template has two attributes: + +* `description`: A one-line description of the template, in CommonMark + syntax. + +* `path`: The path of the directory to be copied. + +Here is an example: + +``` +outputs = { self }: { + + templates.rust = { + path = ./rust; + description = "A simple Rust/Cargo project"; + }; + + templates.defaultTemplate = self.templates.rust; +} +``` + +)"" diff --git a/src/nix/flake-list-inputs.md b/src/nix/flake-list-inputs.md new file mode 100644 index 000000000..250e13be0 --- /dev/null +++ b/src/nix/flake-list-inputs.md @@ -0,0 +1,23 @@ +R""( + +# Examples + +* Show the inputs of the `hydra` flake: + + ```console + # nix flake list-inputs github:NixOS/hydra + github:NixOS/hydra/bde8d81876dfc02143e5070e42c78d8f0d83d6f7 + ├───nix: github:NixOS/nix/79aa7d95183cbe6c0d786965f0dbff414fd1aa67 + │ ├───lowdown-src: github:kristapsdz/lowdown/1705b4a26fbf065d9574dce47a94e8c7c79e052f + │ └───nixpkgs: github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31 + └───nixpkgs follows input 'nix/nixpkgs' + ``` + +# Description + +This command shows the inputs of the flake specified by the flake +referenced *flake-url*. Since it prints the locked inputs that result +from generating or updating the lock file, this command essentially +displays the contents of the flake's lock file in human-readable form. + +)"" diff --git a/src/nix/flake-new.md b/src/nix/flake-new.md new file mode 100644 index 000000000..725695c01 --- /dev/null +++ b/src/nix/flake-new.md @@ -0,0 +1,34 @@ +R""( + +# Examples + +* Create a flake using the default template in the directory `hello`: + + ```console + # nix flake new hello + ``` + +* List available templates: + + ```console + # nix flake show templates + ``` + +* Create a flake from a specific template in the directory `hello`: + + ```console + # nix flake new hello -t templates#trivial + ``` + +# Description + +This command creates a flake in the directory `dest-dir`, which must +not already exist. It's equivalent to: + +```console +# mkdir dest-dir +# cd dest-dir +# nix flake init +``` + +)"" diff --git a/src/nix/flake-prefetch.md b/src/nix/flake-prefetch.md new file mode 100644 index 000000000..a1cf0289a --- /dev/null +++ b/src/nix/flake-prefetch.md @@ -0,0 +1,28 @@ +R""( + +# Examples + +* Download a tarball and unpack it: + + ```console + # nix flake prefetch https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz + Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz?narHash=sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY=' + to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash + 'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY='). + ``` + +* Download the `dwarffs` flake (looked up in the flake registry): + + ```console + # nix flake prefetch dwarffs --json + {"hash":"sha256-VHg3MYVgQ12LeRSU2PSoDeKlSPD8PYYEFxxwkVVDRd0=" + ,"storePath":"/nix/store/hang3792qwdmm2n0d9nsrs5n6bsws6kv-source"} + ``` + +# Description + +This command downloads the source tree denoted by flake reference +*flake-url*. Note that this does not need to be a flake (i.e. it does +not have to contain a `flake.nix` file). + +)"" diff --git a/src/nix/flake-show.md b/src/nix/flake-show.md new file mode 100644 index 000000000..1a42c44a0 --- /dev/null +++ b/src/nix/flake-show.md @@ -0,0 +1,38 @@ +R""( + +# Examples + +* Show the output attributes provided by the `patchelf` flake: + + ```console + github:NixOS/patchelf/f34751b88bd07d7f44f5cd3200fb4122bf916c7e + ├───checks + │ ├───aarch64-linux + │ │ └───build: derivation 'patchelf-0.12.20201207.f34751b' + │ ├───i686-linux + │ │ └───build: derivation 'patchelf-0.12.20201207.f34751b' + │ └───x86_64-linux + │ └───build: derivation 'patchelf-0.12.20201207.f34751b' + ├───defaultPackage + │ ├───aarch64-linux: package 'patchelf-0.12.20201207.f34751b' + │ ├───i686-linux: package 'patchelf-0.12.20201207.f34751b' + │ └───x86_64-linux: package 'patchelf-0.12.20201207.f34751b' + ├───hydraJobs + │ ├───build + │ │ ├───aarch64-linux: derivation 'patchelf-0.12.20201207.f34751b' + │ │ ├───i686-linux: derivation 'patchelf-0.12.20201207.f34751b' + │ │ └───x86_64-linux: derivation 'patchelf-0.12.20201207.f34751b' + │ ├───coverage: derivation 'patchelf-coverage-0.12.20201207.f34751b' + │ ├───release: derivation 'patchelf-0.12.20201207.f34751b' + │ └───tarball: derivation 'patchelf-tarball-0.12.20201207.f34751b' + └───overlay: Nixpkgs overlay + ``` + +# Description + +This command shows the output attributes provided by the flake +specified by flake reference *flake-url*. These are the top-level +attributes in the `outputs` of the flake, as well as lower-level +attributes for some standard outputs (e.g. `packages` or `checks`). + +)"" diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md new file mode 100644 index 000000000..a2ffedd2a --- /dev/null +++ b/src/nix/flake-update.md @@ -0,0 +1,53 @@ +R""( + +# Examples + +* Update the `nixpkgs` and `nix` inputs of the flake in the current + directory: + + ```console + # nix flake update --update-input nixpkgs --update-input nix + * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c' + * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293' + ``` + +* Recreate the lock file (i.e. update all inputs) and commit the new + lock file: + + ```console + # nix flake update --recreate-lock-file --commit-lock-file + … + warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad' + ``` + +# Description + +This command updates the lock file of a flake (`flake.lock`) so that +it contains a lock for every flake input specified in +`flake.nix`. Note that every command that operates on a flake will +also update the lock file if needed, and supports the same +flags. Therefore, + +```console +# nix flake update --update-input nixpkgs +# nix build +``` + +is equivalent to: + +```console +# nix build --update-input nixpkgs +``` + +Thus, this command is only useful if you want to update the lock file +separately from any other action such as building. + +> **Note** +> +> This command does *not* update locks that are already present unless +> you explicitly ask for it using `--update-input` or +> `--recreate-lock-file`. Thus, if the lock file already has locks for +> every input, then `nix flake update` (without arguments) does +> nothing. + +)"" diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 7a7c71676..4cd7d77a0 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -104,6 +104,13 @@ struct CmdFlakeUpdate : FlakeCommand return "update flake lock file"; } + std::string doc() override + { + return + #include "flake-update.md" + ; + } + void run(nix::ref<nix::Store> store) override { /* Use --refresh by default for 'nix flake update'. */ @@ -134,6 +141,13 @@ struct CmdFlakeInfo : FlakeCommand, MixJSON return "list info about a given flake"; } + std::string doc() override + { + return + #include "flake-info.md" + ; + } + void run(nix::ref<nix::Store> store) override { auto flake = getFlake(); @@ -153,6 +167,13 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON return "list flake inputs"; } + std::string doc() override + { + return + #include "flake-list-inputs.md" + ; + } + void run(nix::ref<nix::Store> store) override { auto flake = lockFlake(); @@ -201,7 +222,7 @@ struct CmdFlakeCheck : FlakeCommand { addFlag({ .longName = "no-build", - .description = "do not build checks", + .description = "Do not build checks.", .handler = {&build, false} }); } @@ -211,6 +232,13 @@ struct CmdFlakeCheck : FlakeCommand return "check whether the flake evaluates and run its tests"; } + std::string doc() override + { + return + #include "flake-check.md" + ; + } + void run(nix::ref<nix::Store> store) override { settings.readOnlyMode = !build; @@ -260,7 +288,7 @@ struct CmdFlakeCheck : FlakeCommand auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v, pos); - if (v.type != tLambda || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final") + if (!v.isLambda() || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final") throw Error("overlay does not take an argument named 'final'"); auto body = dynamic_cast<ExprLambda *>(v.lambda.fun->body); if (!body || body->matchAttrs || std::string(body->arg) != "prev") @@ -276,10 +304,10 @@ struct CmdFlakeCheck : FlakeCommand auto checkModule = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v, pos); - if (v.type == tLambda) { + if (v.isLambda()) { if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis) throw Error("module must match an open attribute set ('{ config, ... }')"); - } else if (v.type == tAttrs) { + } else if (v.type() == nAttrs) { for (auto & attr : *v.attrs) try { state->forceValue(*attr.value, *attr.pos); @@ -371,7 +399,7 @@ struct CmdFlakeCheck : FlakeCommand auto checkBundler = [&](const std::string & attrPath, Value & v, const Pos & pos) { try { state->forceValue(v, pos); - if (v.type != tLambda) + if (!v.isLambda()) throw Error("bundler must be a function"); if (!v.lambda.fun->formals || v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() || @@ -545,7 +573,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand addFlag({ .longName = "template", .shortName = 't', - .description = "the template to use", + .description = "The template to use.", .labels = {"template"}, .handler = {&templateUrl}, .completer = {[&](size_t, std::string_view prefix) { @@ -631,22 +659,11 @@ struct CmdFlakeInit : CmdFlakeInitCommon return "create a flake in the current directory from a template"; } - Examples examples() override - { - return { - Example{ - "To create a flake using the default template:", - "nix flake init" - }, - Example{ - "To see available templates:", - "nix flake show templates" - }, - Example{ - "To create a flake from a specific template:", - "nix flake init -t templates#nixos-container" - }, - }; + std::string doc() override + { + return + #include "flake-init.md" + ; } CmdFlakeInit() @@ -662,6 +679,13 @@ struct CmdFlakeNew : CmdFlakeInitCommon return "create a flake in the specified directory from a template"; } + std::string doc() override + { + return + #include "flake-new.md" + ; + } + CmdFlakeNew() { expectArgs({ @@ -681,12 +705,19 @@ struct CmdFlakeClone : FlakeCommand return "clone flake repository"; } + std::string doc() override + { + return + #include "flake-clone.md" + ; + } + CmdFlakeClone() { addFlag({ .longName = "dest", .shortName = 'f', - .description = "destination path", + .description = "Clone the flake to path *dest*.", .labels = {"path"}, .handler = {&destDir} }); @@ -720,22 +751,11 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun return "copy a flake and all its inputs to a store"; } - Examples examples() override - { - return { - Example{ - "To copy the dwarffs flake and its dependencies to a binary cache:", - "nix flake archive --to file:///tmp/my-cache dwarffs" - }, - Example{ - "To fetch the dwarffs flake and its dependencies to the local Nix store:", - "nix flake archive dwarffs" - }, - Example{ - "To print the store paths of the flake sources of NixOps without fetching them:", - "nix flake archive --json --dry-run nixops" - }, - }; + std::string doc() override + { + return + #include "flake-archive.md" + ; } void run(nix::ref<nix::Store> store) override @@ -787,7 +807,7 @@ struct CmdFlakeShow : FlakeCommand { addFlag({ .longName = "legacy", - .description = "show the contents of the 'legacyPackages' output", + .description = "Show the contents of the `legacyPackages` output.", .handler = {&showLegacy, true} }); } @@ -797,6 +817,13 @@ struct CmdFlakeShow : FlakeCommand return "show the outputs provided by a flake"; } + std::string doc() override + { + return + #include "flake-show.md" + ; + } + void run(nix::ref<nix::Store> store) override { auto state = getEvalState(); @@ -933,6 +960,45 @@ struct CmdFlakeShow : FlakeCommand } }; +struct CmdFlakePrefetch : FlakeCommand, MixJSON +{ + CmdFlakePrefetch() + { + } + + std::string description() override + { + return "download the source tree denoted by a flake reference into the Nix store"; + } + + std::string doc() override + { + return + #include "flake-prefetch.md" + ; + } + + void run(ref<Store> store) override + { + auto originalRef = getFlakeRef(); + auto resolvedRef = originalRef.resolve(store); + auto [tree, lockedRef] = resolvedRef.fetchTree(store); + auto hash = store->queryPathInfo(tree.storePath)->narHash; + + if (json) { + auto res = nlohmann::json::object(); + res["storePath"] = store->printStorePath(tree.storePath); + res["hash"] = hash.to_string(SRI, true); + logger->cout(res.dump()); + } else { + notice("Downloaded '%s' to '%s' (hash '%s').", + lockedRef.to_string(), + store->printStorePath(tree.storePath), + hash.to_string(SRI, true)); + } + } +}; + struct CmdFlake : NixMultiCommand { CmdFlake() @@ -946,6 +1012,7 @@ struct CmdFlake : NixMultiCommand {"clone", []() { return make_ref<CmdFlakeClone>(); }}, {"archive", []() { return make_ref<CmdFlakeArchive>(); }}, {"show", []() { return make_ref<CmdFlakeShow>(); }}, + {"prefetch", []() { return make_ref<CmdFlakePrefetch>(); }}, }) { } @@ -955,6 +1022,13 @@ struct CmdFlake : NixMultiCommand return "manage Nix flakes"; } + std::string doc() override + { + return + #include "flake.md" + ; + } + void run() override { if (!command) diff --git a/src/nix/flake.md b/src/nix/flake.md new file mode 100644 index 000000000..440c45dd1 --- /dev/null +++ b/src/nix/flake.md @@ -0,0 +1,566 @@ +R""( + +# Description + +`nix flake` provides subcommands for creating, modifying and querying +*Nix flakes*. Flakes are the unit for packaging Nix code in a +reproducible and discoverable way. They can have dependencies on other +flakes, making it possible to have multi-repository Nix projects. + +A flake is a filesystem tree (typically fetched from a Git repository +or a tarball) that contains a file named `flake.nix` in the root +directory. `flake.nix` specifies some metadata about the flake such as +dependencies (called *inputs*), as well as its *outputs* (the Nix +values such as packages or NixOS modules provided by the flake). + +# Flake references + +Flake references (*flakerefs*) are a way to specify the location of a +flake. These have two different forms: + +* An attribute set representation, e.g. + + ```nix + { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + } + ``` + + The only required attribute is `type`. The supported types are + listed below. + +* A URL-like syntax, e.g. + + ``` + github:NixOS/nixpkgs + ``` + + These are used on the command line as a more convenient alternative + to the attribute set representation. For instance, in the command + + ```console + # nix build github:NixOS/nixpkgs#hello + ``` + + `github:NixOS/nixpkgs` is a flake reference (while `hello` is an + output attribute). They are also allowed in the `inputs` attribute + of a flake, e.g. + + ```nix + inputs.nixpkgs.url = github:NixOS/nixpkgs; + ``` + + is equivalent to + + ```nix + inputs.nixpkgs = { + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + }; + ``` + +## Examples + +Here are some examples of flake references in their URL-like representation: + +* `.`: The flake in the current directory. +* `/home/alice/src/patchelf`: A flake in some other directory. +* `nixpkgs`: The `nixpkgs` entry in the flake registry. +* `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: The `nixpkgs` + entry in the flake registry, with its Git revision overriden to a + specific value. +* `github:NixOS/nixpkgs`: The `master` branch of the `NixOS/nixpkgs` + repository on GitHub. +* `github:NixOS/nixpkgs/nixos-20.09`: The `nixos-20.09` branch of the + `nixpkgs` repository. +* `github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: A + specific revision of the `nixpkgs` repository. +* `github:edolstra/nix-warez?dir=blender`: A flake in a subdirectory + of a GitHub repository. +* `git+https://github.com/NixOS/patchelf`: A Git repository. +* `git+https://github.com/NixOS/patchelf?ref=master`: A specific + branch of a Git repository. +* `git+https://github.com/NixOS/patchelf?ref=master&rev=f34751b88bd07d7f44f5cd3200fb4122bf916c7e`: + A specific branch *and* revision of a Git repository. +* `https://github.com/NixOS/patchelf/archive/master.tar.gz`: A tarball + flake. + +## Flake reference attributes + +The following generic flake reference attributes are supported: + +* `dir`: The subdirectory of the flake in which `flake.nix` is + located. This parameter enables having multiple flakes in a + repository or tarball. The default is the root directory of the + flake. + +* `narHash`: The hash of the NAR serialisation (in SRI format) of the + contents of the flake. This is useful for flake types such as + tarballs that lack a unique content identifier such as a Git commit + hash. + +In addition, the following attributes are common to several flake +reference types: + +* `rev`: A Git or Mercurial commit hash. + +* `ref`: A Git or Mercurial branch or tag name. + +Finally, some attribute are typically not specified by the user, but +can occur in *locked* flake references and are available to Nix code: + +* `revCount`: The number of ancestors of the commit `rev`. + +* `lastModified`: The timestamp (in seconds since the Unix epoch) of + the last modification of this version of the flake. For + Git/Mercurial flakes, this is the commit time of commit *rev*, while + for tarball flakes, it's the most recent timestamp of any file + inside the tarball. + +## Types + +Currently the `type` attribute can be one of the following: + +* `path`: arbitrary local directories, or local Git trees. The + required attribute `path` specifies the path of the flake. The URL + form is + + ``` + [path:]<path>(\?<params)? + ``` + + where *path* is an absolute path. + + *path* must be a directory in the file system containing a file + named `flake.nix`. + + If the directory or any of its parents is a Git repository, then + this is essentially equivalent to `git+file://<path>` (see below), + except that the `dir` parameter is derived automatically. For + example, if `/foo/bar` is a Git repository, then the flake reference + `/foo/bar/flake` is equivalent to `/foo/bar?dir=flake`. + + If the directory is not inside a Git repository, then the flake + contents is the entire contents of *path*. + + *path* generally must be an absolute path. However, on the command + line, it can be a relative path (e.g. `.` or `./foo`) which is + interpreted as relative to the current directory. In this case, it + must start with `.` to avoid ambiguity with registry lookups + (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative + path). + +* `git`: Git repositories. The location of the repository is specified + by the attribute `url`. + + They have the URL form + + ``` + git(+http|+https|+ssh|+git|+file|):(//<server>)?<path>(\?<params>)? + ``` + + The `ref` attribute defaults to `master`. + + The `rev` attribute must denote a commit that exists in the branch + or tag specified by the `ref` attribute, since Nix doesn't do a full + clone of the remote repository by default (and the Git protocol + doesn't allow fetching a `rev` without a known `ref`). The default + is the commit currently pointed to by `ref`. + + For example, the following are valid Git flake references: + + * `git+https://example.org/my/repo` + * `git+https://example.org/my/repo?dir=flake1` + * `git+ssh://git@github.com/NixOS/nix?ref=v1.2.3` + * `git://github.com/edolstra/dwarffs?ref=unstable&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4` + * `git+file:///home/my-user/some-repo/some-repo` + +* `mercurial`: Mercurial repositories. The URL form is similar to the + `git` type, except that the URL schema must be one of `hg+http`, + `hg+https`, `hg+ssh` or `hg+file`. + +* `tarball`: Tarballs. The location of the tarball is specified by the + attribute `url`. + + In URL form, the schema must be `http://`, `https://` or `file://` + URLs and the extension must be `.zip`, `.tar`, `.tar.gz`, `.tar.xz` + or `.tar.bz2`. + +* `github`: A more efficient way to fetch repositories from + GitHub. The following attributes are required: + + * `owner`: The owner of the repository. + + * `repo`: The name of the repository. + + These are downloaded as tarball archives, rather than + through Git. This is often much faster and uses less disk space + since it doesn't require fetching the entire history of the + repository. On the other hand, it doesn't allow incremental fetching + (but full downloads are often faster than incremental fetches!). + + The URL syntax for `github` flakes is: + + ``` + github:<owner>/<repo>(/<rev-or-ref>)?(\?<params>)? + ``` + + `<rev-or-ref>` specifies the name of a branch or tag (`ref`), or a + commit hash (`rev`). Note that unlike Git, GitHub allows fetching by + commit hash without specifying a branch or tag. + + Some examples: + + * `github:edolstra/dwarffs` + * `github:edolstra/dwarffs/unstable` + * `github:edolstra/dwarffs/d3f2baba8f425779026c6ec04021b2e927f61e31` + +* `indirect`: Indirections through the flake registry. These have the + form + + ``` + [flake:]<flake-id>(/<rev-or-ref>(/rev)?)? + ``` + + These perform a lookup of `<flake-id>` in the flake registry. or + example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake + references. The specified `rev` and/or `ref` are merged with the + entry in the registry; see [nix registry](./nix3-registry.md) for + details. + +# Flake format + +As an example, here is a simple `flake.nix` that depends on the +Nixpkgs flake and provides a single package (i.e. an installable +derivation): + +```nix +{ + description = "A flake for building Hello World"; + + inputs.nixpkgs.url = github:NixOS/nixpkgs/nixos-20.03; + + outputs = { self, nixpkgs }: { + + defaultPackage.x86_64-linux = + # Notice the reference to nixpkgs here. + with import nixpkgs { system = "x86_64-linux"; }; + stdenv.mkDerivation { + name = "hello"; + src = self; + buildPhase = "gcc -o hello ./hello.c"; + installPhase = "mkdir -p $out/bin; install -t $out/bin hello"; + }; + + }; +} +``` + +The following attributes are supported in `flake.nix`: + +* `description`: A short, one-line description of the flake. + +* `inputs`: An attrset specifying the dependencies of the flake + (described below). + +* `outputs`: A function that, given an attribute set containing the + outputs of each of the input flakes keyed by their identifier, + yields the Nix values provided by this flake. Thus, in the example + above, `inputs.nixpkgs` contains the result of the call to the + `outputs` function of the `nixpkgs` flake. + + In addition to the outputs of each input, each input in `inputs` + also contains some metadata about the inputs. These are: + + * `outPath`: The path in the Nix store of the flake's source tree. + + * `rev`: The commit hash of the flake's repository, if applicable. + + * `revCount`: The number of ancestors of the revision `rev`. This is + not available for `github` repositories, since they're fetched as + tarballs rather than as Git repositories. + + * `lastModifiedDate`: The commit time of the revision `rev`, in the + format `%Y%m%d%H%M%S` (e.g. `20181231100934`). Unlike `revCount`, + this is available for both Git and GitHub repositories, so it's + useful for generating (hopefully) monotonically increasing version + strings. + + * `lastModified`: The commit time of the revision `rev` as an integer + denoting the number of seconds since 1970. + + * `narHash`: The SHA-256 (in SRI format) of the NAR serialization of + the flake's source tree. + + The value returned by the `outputs` function must be an attribute + set. The attributes can have arbitrary values; however, various + `nix` subcommands require specific attributes to have a specific + value (e.g. `packages.x86_64-linux` must be an attribute set of + derivations built for the `x86_64-linux` platform). + +## Flake inputs + +The attribute `inputs` specifies the dependencies of a flake, as an +attrset mapping input names to flake references. For example, the +following specifies a dependency on the `nixpkgs` and `import-cargo` +repositories: + +```nix +# A GitHub repository. +inputs.import-cargo = { + type = "github"; + owner = "edolstra"; + repo = "import-cargo"; +}; + +# An indirection through the flake registry. +inputs.nixpkgs = { + type = "indirect"; + id = "nixpkgs"; +}; +``` + +Alternatively, you can use the URL-like syntax: + +```nix +inputs.import-cargo.url = github:edolstra/import-cargo; +inputs.nixpkgs.url = "nixpkgs"; +``` + +Each input is fetched, evaluated and passed to the `outputs` function +as a set of attributes with the same name as the corresponding +input. The special input named `self` refers to the outputs and source +tree of *this* flake. Thus, a typical `outputs` function looks like +this: + +```nix +outputs = { self, nixpkgs, import-cargo }: { + ... outputs ... +}; +``` + +It is also possible to omit an input entirely and *only* list it as +expected function argument to `outputs`. Thus, + +```nix +outputs = { self, nixpkgs }: ...; +``` + +without an `inputs.nixpkgs` attribute is equivalent to + +```nix +inputs.nixpkgs = { + type = "indirect"; + id = "nixpkgs"; +}; +``` + +Repositories that don't contain a `flake.nix` can also be used as +inputs, by setting the input's `flake` attribute to `false`: + +```nix +inputs.grcov = { + type = "github"; + owner = "mozilla"; + repo = "grcov"; + flake = false; +}; + +outputs = { self, nixpkgs, grcov }: { + packages.x86_64-linux.grcov = stdenv.mkDerivation { + src = grcov; + ... + }; +}; +``` + +Transitive inputs can be overriden from a `flake.nix` file. For +example, the following overrides the `nixpkgs` input of the `nixops` +input: + +```nix +inputs.nixops.inputs.nixpkgs = { + type = "github"; + owner = "my-org"; + repo = "nixpkgs"; +}; +``` + +It is also possible to "inherit" an input from another input. This is +useful to minimize flake dependencies. For example, the following sets +the `nixpkgs` input of the top-level flake to be equal to the +`nixpkgs` input of the `dwarffs` input of the top-level flake: + +```nix +inputs.nixops.follows = "dwarffs/nixpkgs"; +``` + +The value of the `follows` attribute is a `/`-separated sequence of +input names denoting the path of inputs to be followed from the root +flake. + +Overrides and `follows` can be combined, e.g. + +```nix +inputs.nixops.inputs.nixpkgs.follows = "dwarffs/nixpkgs"; +``` + +sets the `nixpkgs` input of `nixops` to be the same as the `nixpkgs` +input of `dwarffs`. It is worth noting, however, that it is generally +not useful to eliminate transitive `nixpkgs` flake inputs in this +way. Most flakes provide their functionality through Nixpkgs overlays +or NixOS modules, which are composed into the top-level flake's +`nixpkgs` input; so their own `nixpkgs` input is usually irrelevant. + +# Lock files + +Inputs specified in `flake.nix` are typically "unlocked" in the sense +that they don't specify an exact revision. To ensure reproducibility, +Nix will automatically generate and use a *lock file* called +`flake.lock` in the flake's directory. The lock file contains a graph +structure isomorphic to the graph of dependencies of the root +flake. Each node in the graph (except the root node) maps the +(usually) unlocked input specifications in `flake.nix` to locked input +specifications. Each node also contains some metadata, such as the +dependencies (outgoing edges) of the node. + +For example, if `flake.nix` has the inputs in the example above, then +the resulting lock file might be: + +```json +{ + "version": 7, + "root": "n1", + "nodes": { + "n1": { + "inputs": { + "nixpkgs": "n2", + "import-cargo": "n3", + "grcov": "n4" + } + }, + "n2": { + "inputs": {}, + "locked": { + "owner": "edolstra", + "repo": "nixpkgs", + "rev": "7f8d4b088e2df7fdb6b513bc2d6941f1d422a013", + "type": "github", + "lastModified": 1580555482, + "narHash": "sha256-OnpEWzNxF/AU4KlqBXM2s5PWvfI5/BS6xQrPvkF5tO8=" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "n3": { + "inputs": {}, + "locked": { + "owner": "edolstra", + "repo": "import-cargo", + "rev": "8abf7b3a8cbe1c8a885391f826357a74d382a422", + "type": "github", + "lastModified": 1567183309, + "narHash": "sha256-wIXWOpX9rRjK5NDsL6WzuuBJl2R0kUCnlpZUrASykSc=" + }, + "original": { + "owner": "edolstra", + "repo": "import-cargo", + "type": "github" + } + }, + "n4": { + "inputs": {}, + "locked": { + "owner": "mozilla", + "repo": "grcov", + "rev": "989a84bb29e95e392589c4e73c29189fd69a1d4e", + "type": "github", + "lastModified": 1580729070, + "narHash": "sha256-235uMxYlHxJ5y92EXZWAYEsEb6mm+b069GAd+BOIOxI=" + }, + "original": { + "owner": "mozilla", + "repo": "grcov", + "type": "github" + }, + "flake": false + } + } +} +``` + +This graph has 4 nodes: the root flake, and its 3 dependencies. The +nodes have arbitrary labels (e.g. `n1`). The label of the root node of +the graph is specified by the `root` attribute. Nodes contain the +following fields: + +* `inputs`: The dependencies of this node, as a mapping from input + names (e.g. `nixpkgs`) to node labels (e.g. `n2`). + +* `original`: The original input specification from `flake.lock`, as a + set of `builtins.fetchTree` arguments. + +* `locked`: The locked input specification, as a set of + `builtins.fetchTree` arguments. Thus, in the example above, when we + build this flake, the input `nixpkgs` is mapped to revision + `7f8d4b088e2df7fdb6b513bc2d6941f1d422a013` of the `edolstra/nixpkgs` + repository on GitHub. + + It also includes the attribute `narHash`, specifying the expected + contents of the tree in the Nix store (as computed by `nix + hash-path`), and may include input-type-specific attributes such as + the `lastModified` or `revCount`. The main reason for these + attributes is to allow flake inputs to be substituted from a binary + cache: `narHash` allows the store path to be computed, while the + other attributes are necessary because they provide information not + stored in the store path. + +* `flake`: A Boolean denoting whether this is a flake or non-flake + dependency. Corresponds to the `flake` attribute in the `inputs` + attribute in `flake.nix`. + +The `original` and `locked` attributes are omitted for the root +node. This is because we cannot record the commit hash or content hash +of the root flake, since modifying `flake.lock` will invalidate these. + +The graph representation of lock files allows circular dependencies +between flakes. For example, here are two flakes that reference each +other: + +```nix +{ + inputs.b = ... location of flake B ...; + # Tell the 'b' flake not to fetch 'a' again, to ensure its 'a' is + # *this* 'a'. + inputs.b.inputs.a.follows = ""; + outputs = { self, b }: { + foo = 123 + b.bar; + xyzzy = 1000; + }; +} +``` + +and + +```nix +{ + inputs.a = ... location of flake A ...; + inputs.a.inputs.b.follows = ""; + outputs = { self, a }: { + bar = 456 + a.xyzzy; + }; +} +``` + +Lock files transitively lock direct as well as indirect +dependencies. That is, if a lock file exists and is up to date, Nix +will not look at the lock files of dependencies. However, lock file +generation itself *does* use the lock files of dependencies by +default. + +)"" diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 101b67e6a..79d506ace 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -19,15 +19,15 @@ struct CmdHashBase : Command CmdHashBase(FileIngestionMethod mode) : mode(mode) { - mkFlag(0, "sri", "print hash in SRI format", &base, SRI); - mkFlag(0, "base64", "print hash in base-64", &base, Base64); - mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32); - mkFlag(0, "base16", "print hash in base-16", &base, Base16); + mkFlag(0, "sri", "Print the hash in SRI format.", &base, SRI); + mkFlag(0, "base64", "Print the hash in base-64 format.", &base, Base64); + mkFlag(0, "base32", "Print the hash in base-32 (Nix-specific) format.", &base, Base32); + mkFlag(0, "base16", "Print the hash in base-16 format.", &base, Base16); addFlag(Flag::mkHashTypeFlag("type", &ht)); #if 0 mkFlag() .longName("modulo") - .description("compute hash modulo specified string") + .description("Compute the hash modulo specified the string.") .labels({"modulus"}) .dest(&modulus); #endif @@ -40,15 +40,14 @@ struct CmdHashBase : Command std::string description() override { - const char* d; switch (mode) { case FileIngestionMethod::Flat: - d = "print cryptographic hash of a regular file"; - break; + return "print cryptographic hash of a regular file"; case FileIngestionMethod::Recursive: - d = "print cryptographic hash of the NAR serialisation of a path"; + return "print cryptographic hash of the NAR serialisation of a path"; + default: + assert(false); }; - return d; } void run() override @@ -132,11 +131,6 @@ struct CmdHash : NixMultiCommand command->second->prepare(); command->second->run(); } - - void printHelp(const string & programName, std::ostream & out) override - { - MultiCommand::printHelp(programName, out); - } }; static auto rCmdHash = registerCommand<CmdHash>("hash"); diff --git a/src/nix/help.md b/src/nix/help.md new file mode 100644 index 000000000..734f35028 --- /dev/null +++ b/src/nix/help.md @@ -0,0 +1,17 @@ +R""( + +# Examples + +* Show help about `nix` in general: + + ```console + # nix help + ``` + +* Show help about a particular subcommand: + + ```console + # nix help flake info + ``` + +)"" diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 3506c3fcc..34ee238bf 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -60,37 +60,37 @@ MixFlakeOptions::MixFlakeOptions() { addFlag({ .longName = "recreate-lock-file", - .description = "recreate lock file from scratch", + .description = "Recreate the flake's lock file from scratch.", .handler = {&lockFlags.recreateLockFile, true} }); addFlag({ .longName = "no-update-lock-file", - .description = "do not allow any updates to the lock file", + .description = "Do not allow any updates to the flake's lock file.", .handler = {&lockFlags.updateLockFile, false} }); addFlag({ .longName = "no-write-lock-file", - .description = "do not write the newly generated lock file", + .description = "Do not write the flake's newly generated lock file.", .handler = {&lockFlags.writeLockFile, false} }); addFlag({ .longName = "no-registries", - .description = "don't use flake registries", + .description = "Don't allow lookups in the flake registries.", .handler = {&lockFlags.useRegistries, false} }); addFlag({ .longName = "commit-lock-file", - .description = "commit changes to the lock file", + .description = "Commit changes to the flake's lock file.", .handler = {&lockFlags.commitLockFile, true} }); addFlag({ .longName = "update-input", - .description = "update a specific flake input", + .description = "Update a specific flake input (ignoring its previous entry in the lock file).", .labels = {"input-path"}, .handler = {[&](std::string s) { lockFlags.inputUpdates.insert(flake::parseInputPath(s)); @@ -103,7 +103,7 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "override-input", - .description = "override a specific flake input (e.g. `dwarffs/nixpkgs`)", + .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`).", .labels = {"input-path", "flake-url"}, .handler = {[&](std::string inputPath, std::string flakeRef) { lockFlags.inputOverrides.insert_or_assign( @@ -114,7 +114,7 @@ MixFlakeOptions::MixFlakeOptions() addFlag({ .longName = "inputs-from", - .description = "use the inputs of the specified flake as registry entries", + .description = "Use the inputs of the specified flake as registry entries.", .labels = {"flake-url"}, .handler = {[&](std::string flakeRef) { auto evalState = getEvalState(); @@ -143,22 +143,22 @@ SourceExprCommand::SourceExprCommand() addFlag({ .longName = "file", .shortName = 'f', - .description = "evaluate *file* rather than the default", + .description = "Interpret installables as attribute paths relative to the Nix expression stored in *file*.", .labels = {"file"}, .handler = {&file}, .completer = completePath }); addFlag({ - .longName ="expr", - .description = "evaluate attributes from *expr*", + .longName = "expr", + .description = "Interpret installables as attribute paths relative to the Nix expression *expr*.", .labels = {"expr"}, .handler = {&expr} }); addFlag({ - .longName ="derivation", - .description = "operate on the store derivation rather than its outputs", + .longName = "derivation", + .description = "Operate on the store derivation rather than its outputs.", .handler = {&operateOn, OperateOn::Derivation}, }); } @@ -501,7 +501,7 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF auto drvInfo = DerivationInfo{ std::move(drvPath), - state->store->parseStorePath(attr->getAttr(state->sOutPath)->getString()), + state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()), attr->getAttr(state->sOutputName)->getString() }; diff --git a/src/nix/key-convert-secret-to-public.md b/src/nix/key-convert-secret-to-public.md new file mode 100644 index 000000000..3adc18502 --- /dev/null +++ b/src/nix/key-convert-secret-to-public.md @@ -0,0 +1,19 @@ +R""( + +# Examples + +* Convert a secret key to a public key: + + ```console + # echo cache.example.org-0:E7lAO+MsPwTFfPXsdPtW8GKui/5ho4KQHVcAGnX+Tti1V4dUxoVoqLyWJ4YESuZJwQ67GVIksDt47og+tPVUZw== \ + | nix key convert-secret-to-public + cache.example.org-0:tVeHVMaFaKi8lieGBErmScEOuxlSJLA7eO6IPrT1VGc= + ``` + +# Description + +This command reads a Ed25519 secret key from standard input, and +writes the corresponding public key to standard output. For more +details, see [nix key generate-secret](./nix3-key-generate-secret.md). + +)"" diff --git a/src/nix/key-generate-secret.md b/src/nix/key-generate-secret.md new file mode 100644 index 000000000..4938f637c --- /dev/null +++ b/src/nix/key-generate-secret.md @@ -0,0 +1,48 @@ +R""( + +# Examples + +* Generate a new secret key: + + ```console + # nix key generate-secret --key-name cache.example.org-1 > ./secret-key + ``` + + We can then use this key to sign the closure of the Hello package: + + ```console + # nix build nixpkgs#hello + # nix store sign --key-file ./secret-key --recursive ./result + ``` + + Finally, we can verify the store paths using the corresponding + public key: + + ``` + # nix store verify --trusted-public-keys $(nix key convert-secret-to-public < ./secret-key) ./result + ``` + +# Description + +This command generates a new Ed25519 secret key for signing store +paths and prints it on standard output. Use `nix key +convert-secret-to-public` to get the corresponding public key for +verifying signed store paths. + +The mandatory argument `--key-name` specifies a key name (such as +`cache.example.org-1). It is used to look up keys on the client when +it verifies signatures. It can be anything, but it’s suggested to use +the host name of your cache (e.g. `cache.example.org`) with a suffix +denoting the number of the key (to be incremented every time you need +to revoke a key). + +# Format + +Both secret and public keys are represented as the key name followed +by a base-64 encoding of the Ed25519 key data, e.g. + +``` +cache.example.org-0:E7lAO+MsPwTFfPXsdPtW8GKui/5ho4KQHVcAGnX+Tti1V4dUxoVoqLyWJ4YESuZJwQ67GVIksDt47og+tPVUZw== +``` + +)"" diff --git a/src/nix/local.mk b/src/nix/local.mk index f37b73384..23c08fc86 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -12,7 +12,6 @@ nix_SOURCES := \ $(wildcard src/nix-daemon/*.cc) \ $(wildcard src/nix-env/*.cc) \ $(wildcard src/nix-instantiate/*.cc) \ - $(wildcard src/nix-prefetch-url/*.cc) \ $(wildcard src/nix-store/*.cc) \ nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain diff --git a/src/nix/log.cc b/src/nix/log.cc index 33a3053f5..67d3742d6 100644 --- a/src/nix/log.cc +++ b/src/nix/log.cc @@ -13,22 +13,11 @@ struct CmdLog : InstallableCommand return "show the build log of the specified packages or paths, if available"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To get the build log of GNU Hello:", - "nix log nixpkgs#hello" - }, - Example{ - "To get the build log of a specific path:", - "nix log /nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1" - }, - Example{ - "To get a build log from a specific binary cache:", - "nix log --store https://cache.nixos.org nixpkgs#hello" - }, - }; + return + #include "log.md" + ; } Category category() override { return catSecondary; } diff --git a/src/nix/log.md b/src/nix/log.md new file mode 100644 index 000000000..1c76226a3 --- /dev/null +++ b/src/nix/log.md @@ -0,0 +1,40 @@ +R""( + +# Examples + +* Get the build log of GNU Hello: + + ```console + # nix log nixpkgs#hello + ``` + +* Get the build log of a specific store path: + + ```console + # nix log /nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1 + ``` + +* Get a build log from a specific binary cache: + + ```console + # nix log --store https://cache.nixos.org nixpkgs#hello + ``` + +# Description + +This command prints the log of a previous build of the derivation +*installable* on standard output. + +Nix looks for build logs in two places: + +* In the directory `/nix/var/log/nix/drvs`, which contains logs for + locally built derivations. + +* In the binary caches listed in the `substituters` setting. Logs + should be named `<cache>/log/<base-name-of-store-path>`, where + `store-path` is a derivation, + e.g. `https://cache.nixos.org/log/dvmig8jgrdapvbyxb1rprckdmdqx08kv-hello-2.10.drv`. + For non-derivation store paths, Nix will first try to determine the + deriver by fetching the `.narinfo` file for this store path. + +)"" diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 1f5ed6913..c0b1ecb32 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -17,9 +17,9 @@ struct MixLs : virtual Args, MixJSON MixLs() { - mkFlag('R', "recursive", "list subdirectories recursively", &recursive); - mkFlag('l', "long", "show more file information", &verbose); - mkFlag('d', "directory", "show directories rather than their contents", &showDirectory); + mkFlag('R', "recursive", "List subdirectories recursively.", &recursive); + mkFlag('l', "long", "Show detailed file information.", &verbose); + mkFlag('d', "directory", "Show directories rather than their contents.", &showDirectory); } void listText(ref<FSAccessor> accessor) @@ -75,6 +75,8 @@ struct MixLs : virtual Args, MixJSON if (json) { JSONPlaceholder jsonRoot(std::cout); + if (showDirectory) + throw UsageError("'--directory' is useless with '--json'"); listNar(jsonRoot, accessor, path, recursive); } else listText(accessor); @@ -92,19 +94,16 @@ struct CmdLsStore : StoreCommand, MixLs }); } - Examples examples() override + std::string description() override { - return { - Example{ - "To list the contents of a store path in a binary cache:", - "nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10" - }, - }; + return "show information about a path in the Nix store"; } - std::string description() override + std::string doc() override { - return "show information about a path in the Nix store"; + return + #include "store-ls.md" + ; } void run(ref<Store> store) override @@ -127,14 +126,11 @@ struct CmdLsNar : Command, MixLs expectArg("path", &path); } - Examples examples() override + std::string doc() override { - return { - Example{ - "To list a specific file in a NAR:", - "nix nar ls -l hello.nar /bin/hello" - }, - }; + return + #include "nar-ls.md" + ; } std::string description() override diff --git a/src/nix/main.cc b/src/nix/main.cc index 27b1d7257..80422bd24 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -52,6 +52,7 @@ static bool haveInternet() } std::string programPath; +char * * savedArgv; struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { @@ -69,15 +70,15 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "help", - .description = "show usage information", + .description = "Show usage information.", .handler = {[&]() { if (!completions) showHelpAndExit(); }}, }); addFlag({ .longName = "help-config", - .description = "show configuration options", + .description = "Show configuration settings.", .handler = {[&]() { - std::cout << "The following configuration options are available:\n\n"; + std::cout << "The following configuration settings are available:\n\n"; Table2 tbl; std::map<std::string, Config::SettingInfo> settings; globalConfig.getSettings(settings); @@ -91,25 +92,25 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs addFlag({ .longName = "print-build-logs", .shortName = 'L', - .description = "print full build logs on stderr", + .description = "Print full build logs on standard error.", .handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }}, }); addFlag({ .longName = "version", - .description = "show version information", + .description = "Show version information.", .handler = {[&]() { if (!completions) printVersion(programName); }}, }); addFlag({ .longName = "no-net", - .description = "disable substituters and consider all previously downloaded files up-to-date", + .description = "Disable substituters and consider all previously downloaded files up-to-date.", .handler = {[&]() { useNet = false; }}, }); addFlag({ .longName = "refresh", - .description = "consider all previously downloaded files out-of-date", + .description = "Consider all previously downloaded files out-of-date.", .handler = {[&]() { refresh = true; }}, }); } @@ -129,7 +130,7 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs {"make-content-addressable", {"store", "make-content-addressable"}}, {"optimise-store", {"store", "optimise"}}, {"ping-store", {"store", "ping"}}, - {"sign-paths", {"store", "sign-paths"}}, + {"sign-paths", {"store", "sign"}}, {"to-base16", {"hash", "to-base16"}}, {"to-base32", {"hash", "to-base32"}}, {"to-base64", {"hash", "to-base64"}}, @@ -184,6 +185,13 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs { return "a tool for reproducible and declarative configuration management"; } + + std::string doc() override + { + return + #include "nix.md" + ; + } }; static void showHelp(std::vector<std::string> subcommand) @@ -205,21 +213,14 @@ struct CmdHelp : Command std::string description() override { - return "show help about 'nix' or a particular subcommand"; + return "show help about `nix` or a particular subcommand"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show help about 'nix' in general:", - "nix help" - }, - Example{ - "To show help about a particular subcommand:", - "nix help run" - }, - }; + return + #include "help.md" + ; } void run() override @@ -232,6 +233,8 @@ static auto rCmdHelp = registerCommand<CmdHelp>("help"); void mainWrapped(int argc, char * * argv) { + savedArgv = argv; + /* The chroot helper needs to be run before any threads have been started. */ if (argc > 0 && argv[0] == chrootHelperName) { @@ -272,7 +275,7 @@ void mainWrapped(int argc, char * * argv) auto builtins = state.baseEnv.values[0]->attrs; for (auto & builtin : *builtins) { auto b = nlohmann::json::object(); - if (builtin.value->type != tPrimOp) continue; + if (!builtin.value->isPrimOp()) continue; auto primOp = builtin.value->primOp; if (!primOp->doc) continue; b["arity"] = primOp->arity; @@ -327,8 +330,11 @@ void mainWrapped(int argc, char * * argv) fileTransferSettings.connectTimeout = 1; } - if (args.refresh) + if (args.refresh) { settings.tarballTtl = 0; + settings.ttlNegativeNarInfoCache = 0; + settings.ttlPositiveNarInfoCache = 0; + } args.command->second->prepare(); args.command->second->run(); diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc index 5165c4804..f5bdc7e65 100644 --- a/src/nix/make-content-addressable.cc +++ b/src/nix/make-content-addressable.cc @@ -15,21 +15,14 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON std::string description() override { - return "rewrite a path or closure to content-addressable form"; + return "rewrite a path or closure to content-addressed form"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To create a content-addressable representation of GNU Hello (but not its dependencies):", - "nix store make-content-addressable nixpkgs#hello" - }, - Example{ - "To compute a content-addressable representation of the current NixOS system closure:", - "nix store make-content-addressable -r /run/current-system" - }, - }; + return + #include "make-content-addressable.md" + ; } void run(ref<Store> store, StorePaths storePaths) override diff --git a/src/nix/make-content-addressable.md b/src/nix/make-content-addressable.md new file mode 100644 index 000000000..3dd847edc --- /dev/null +++ b/src/nix/make-content-addressable.md @@ -0,0 +1,59 @@ +R""( + +# Examples + +* Create a content-addressed representation of the closure of GNU Hello: + + ```console + # nix store make-content-addressable -r nixpkgs#hello + … + rewrote '/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10' to '/nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10' + ``` + + Since the resulting paths are content-addressed, they are always + trusted and don't need signatures to copied to another store: + + ```console + # nix copy --to /tmp/nix --trusted-public-keys '' /nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10 + ``` + + By contrast, the original closure is input-addressed, so it does + need signatures to be trusted: + + ```console + # nix copy --to /tmp/nix --trusted-public-keys '' nixpkgs#hello + cannot add path '/nix/store/zy9wbxwcygrwnh8n2w9qbbcr6zk87m26-libunistring-0.9.10' because it lacks a valid signature + ``` + +* Create a content-addressed representation of the current NixOS + system closure: + + ```console + # nix store make-content-addressable -r /run/current-system + ``` + +# Description + +This command converts the closure of the store paths specified by +*installables* to content-addressed form. Nix store paths are usually +*input-addressed*, meaning that the hash part of the store path is +computed from the contents of the derivation (i.e., the build-time +dependency graph). Input-addressed paths need to be signed by a +trusted key if you want to import them into a store, because we need +to trust that the contents of the path were actually built by the +derivation. + +By contrast, in a *content-addressed* path, the hash part is computed +from the contents of the path. This allows the contents of the path to +be verified without any additional information such as +signatures. This means that a command like + +```console +# nix store build /nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10 \ + --substituters https://my-cache.example.org +``` + +will succeed even if the binary cache `https://my-cache.example.org` +doesn't present any signatures. + +)"" diff --git a/src/nix/nar-cat.md b/src/nix/nar-cat.md new file mode 100644 index 000000000..55c481a28 --- /dev/null +++ b/src/nix/nar-cat.md @@ -0,0 +1,19 @@ +R""( + +# Examples + +* List a file in a NAR and pipe it through `gunzip`: + + ```console + # nix nar cat ./hello.nar /share/man/man1/hello.1.gz | gunzip + .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.46.4. + .TH HELLO "1" "November 2014" "hello 2.10" "User Commands" + … + ``` + +# Description + +This command prints on standard output the contents of the regular +file *path* inside the NAR file *nar*. + +)"" diff --git a/src/nix/nar-dump-path.md b/src/nix/nar-dump-path.md new file mode 100644 index 000000000..26191ad25 --- /dev/null +++ b/src/nix/nar-dump-path.md @@ -0,0 +1,17 @@ +R""( + +# Examples + +* To serialise directory `foo` as a NAR: + + ```console + # nix nar dump-path ./foo > foo.nar + ``` + +# Description + +This command generates a NAR file containing the serialisation of +*path*, which must contain only regular files, directories and +symbolic links. The NAR is written to standard output. + +)"" diff --git a/src/nix/nar-ls.md b/src/nix/nar-ls.md new file mode 100644 index 000000000..d373f9715 --- /dev/null +++ b/src/nix/nar-ls.md @@ -0,0 +1,24 @@ +R""( + +# Examples + +* To list a specific file in a NAR: + + ```console + # nix nar ls -l ./hello.nar /bin/hello + -r-xr-xr-x 38184 hello + ``` + +* To recursively list the contents of a directory inside a NAR, in JSON + format: + + ```console + # nix nar ls --json -R ./hello.nar /bin + {"type":"directory","entries":{"hello":{"type":"regular","size":38184,"executable":true,"narOffset":400}}} + ``` + +# Description + +This command shows information about a *path* inside NAR file *nar*. + +)"" diff --git a/src/nix/nar.cc b/src/nix/nar.cc index e239ce96a..0775d3c25 100644 --- a/src/nix/nar.cc +++ b/src/nix/nar.cc @@ -9,7 +9,14 @@ struct CmdNar : NixMultiCommand std::string description() override { - return "query the contents of NAR files"; + return "create or inspect NAR files"; + } + + std::string doc() override + { + return + #include "nar.md" + ; } Category category() override { return catUtility; } diff --git a/src/nix/nar.md b/src/nix/nar.md new file mode 100644 index 000000000..a83b5c764 --- /dev/null +++ b/src/nix/nar.md @@ -0,0 +1,13 @@ +R""( + +# Description + +`nix nar` provides several subcommands for creating and inspecting +*Nix Archives* (NARs). + +# File format + +For the definition of the NAR file format, see Figure 5.2 in +https://edolstra.github.io/pubs/phd-thesis.pdf. + +)"" diff --git a/src/nix/nix.md b/src/nix/nix.md new file mode 100644 index 000000000..d10de7c01 --- /dev/null +++ b/src/nix/nix.md @@ -0,0 +1,119 @@ +R""( + +# Examples + +* Create a new flake: + + ```console + # nix flake new hello + # cd hello + ``` + +* Build the flake in the current directory: + + ```console + # nix build + # ./result/bin/hello + Hello, world! + ``` + +* Run the flake in the current directory: + + ```console + # nix run + Hello, world! + ``` + +* Start a development shell for hacking on this flake: + + ```console + # nix develop + # unpackPhase + # cd hello-* + # configurePhase + # buildPhase + # ./hello + Hello, world! + # installPhase + # ../outputs/out/bin/hello + Hello, world! + ``` + +# Description + +Nix is a tool for building software, configurations and other +artifacts in a reproducible and declarative way. For more information, +see the [Nix homepage](https://nixos.org/) or the [Nix +manual](https://nixos.org/manual/nix/stable/). + +# Installables + +Many `nix` subcommands operate on one or more *installables*. These are +command line arguments that represent something that can be built in +the Nix store. Here are the recognised types of installables: + +* **Flake output attributes**: `nixpkgs#hello` + + These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a + flake reference and *attrpath* is an optional attribute path. For + more information on flakes, see [the `nix flake` manual + page](./nix3-flake.md). Flake references are most commonly a flake + identifier in the flake registry (e.g. `nixpkgs`) or a path + (e.g. `/path/to/my-flake` or `.`). + + If *attrpath* is omitted, Nix tries some default values; for most + subcommands, the default is `defaultPackage.`*system* + (e.g. `defaultPackage.x86_64-linux`), but some subcommands have + other defaults. If *attrpath* *is* specified, *attrpath* is + interpreted as relative to one or more prefixes; for most + subcommands, these are `packages.`*system*, + `legacyPackages.*system*` and the empty prefix. Thus, on + `x86_64-linux` `nix build nixpkgs#hello` will try to build the + attributes `packages.x86_64-linux.hello`, + `legacyPackages.x86_64-linux.hello` and `hello`. + +* **Store paths**: `/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10` + + These are paths inside the Nix store, or symlinks that resolve to a + path in the Nix store. + +* **Store derivations**: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv` + + Store derivations are store paths with extension `.drv` and are a + low-level representation of a build-time dependency graph used + internally by Nix. By default, if you pass a store derivation to a + `nix` subcommand, it will operate on the *output paths* of the + derivation. For example, `nix path-info` prints information about + the output paths: + + ```console + # nix path-info --json /nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv + [{"path":"/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10",…}] + ``` + + If you want to operate on the store derivation itself, pass the + `--derivation` flag. + +* **Nix attributes**: `--file /path/to/nixpkgs hello` + + When the `-f` / `--file` *path* option is given, installables are + interpreted as attribute paths referencing a value returned by + evaluating the Nix file *path*. + +* **Nix expressions**: `--expr '(import <nixpkgs> {}).hello.overrideDerivation (prev: { name = "my-hello"; })'`. + + When the `--expr` option is given, all installables are interpreted + as Nix expressions. You may need to specify `--impure` if the + expression references impure inputs (such as `<nixpkgs>`). + +For most commands, if no installable is specified, the default is `.`, +i.e. Nix will operate on the default flake output attribute of the +flake in the current directory. + +# Nix stores + +Most `nix` subcommands operate on a *Nix store*. + +TODO: list store types, options + +)"" diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc index bc7f175ac..985006e5a 100644 --- a/src/nix/optimise-store.cc +++ b/src/nix/optimise-store.cc @@ -13,14 +13,11 @@ struct CmdOptimiseStore : StoreCommand return "replace identical files in the store by hard links"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To optimise the Nix store:", - "nix store optimise" - }, - }; + return + #include "optimise-store.md" + ; } void run(ref<Store> store) override diff --git a/src/nix/optimise-store.md b/src/nix/optimise-store.md new file mode 100644 index 000000000..f6fb66f97 --- /dev/null +++ b/src/nix/optimise-store.md @@ -0,0 +1,23 @@ +R""( + +# Examples + +* Optimise the Nix store: + + ```console + nix store optimise + ``` + +# Description + +This command deduplicates the Nix store: it scans the store for +regular files with identical contents, and replaces them with hard +links to a single instance. + +Note that you can also set `auto-optimise-store` to `true` in +`nix.conf` to perform this optimisation incrementally whenever a new +path is added to the Nix store. To make this efficient, Nix maintains +a content-addressed index of all the files in the Nix store in the +directory `/nix/store/.links/`. + +)"" diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc index 63cf885f9..0fa88f1bf 100644 --- a/src/nix/path-info.cc +++ b/src/nix/path-info.cc @@ -18,10 +18,10 @@ struct CmdPathInfo : StorePathsCommand, MixJSON CmdPathInfo() { - mkFlag('s', "size", "print size of the NAR dump of each path", &showSize); - mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize); - mkFlag('h', "human-readable", "with -s and -S, print sizes like 1K 234M 5.67G etc.", &humanReadable); - mkFlag(0, "sigs", "show signatures", &showSigs); + mkFlag('s', "size", "Print the size of the NAR serialisation of each path.", &showSize); + mkFlag('S', "closure-size", "Print the sum of the sizes of the NAR serialisations of the closure of each path.", &showClosureSize); + mkFlag('h', "human-readable", "With `-s` and `-S`, print sizes in a human-friendly format such as `5.67G`.", &humanReadable); + mkFlag(0, "sigs", "Show signatures.", &showSigs); } std::string description() override @@ -29,38 +29,15 @@ struct CmdPathInfo : StorePathsCommand, MixJSON return "query information about store paths"; } - Category category() override { return catSecondary; } - - Examples examples() override + std::string doc() override { - return { - Example{ - "To show the closure sizes of every path in the current NixOS system closure, sorted by size:", - "nix path-info -rS /run/current-system | sort -nk2" - }, - Example{ - "To show a package's closure size and all its dependencies with human readable sizes:", - "nix path-info -rsSh nixpkgs#rust" - }, - Example{ - "To check the existence of a path in a binary cache:", - "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store https://cache.nixos.org/" - }, - Example{ - "To print the 10 most recently added paths (using --json and the jq(1) command):", - "nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path'" - }, - Example{ - "To show the size of the entire Nix store:", - "nix path-info --json --all | jq 'map(.narSize) | add'" - }, - Example{ - "To show every path whose closure is bigger than 1 GB, sorted by closure size:", - "nix path-info --json --all -S | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])'" - }, - }; + return + #include "path-info.md" + ; } + Category category() override { return catSecondary; } + void printSize(uint64_t value) { if (!humanReadable) { diff --git a/src/nix/path-info.md b/src/nix/path-info.md new file mode 100644 index 000000000..76a83e39d --- /dev/null +++ b/src/nix/path-info.md @@ -0,0 +1,94 @@ +R""( + +# Examples + +* Print the store path produced by `nixpkgs#hello`: + + ```console + # nix path-info nixpkgs#hello + /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10 + ``` + +* Show the closure sizes of every path in the current NixOS system + closure, sorted by size: + + ```console + # nix path-info -rS /run/current-system | sort -nk2 + /nix/store/hl5xwp9kdrd1zkm0idm3kkby9q66z404-empty 96 + /nix/store/27324qvqhnxj3rncazmxc4mwy79kz8ha-nameservers 112 + … + /nix/store/539jkw9a8dyry7clcv60gk6na816j7y8-etc 5783255504 + /nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65 5887562256 + ``` + +* Show a package's closure size and all its dependencies with human + readable sizes: + + ```console + # nix path-info -rsSh nixpkgs#rustc + /nix/store/01rrgsg5zk3cds0xgdsq40zpk6g51dz9-ncurses-6.2-dev 386.7K 69.1M + /nix/store/0q783wnvixpqz6dxjp16nw296avgczam-libpfm-4.11.0 5.9M 37.4M + … + ``` + +* Check the existence of a path in a binary cache: + + ```console + # nix path-info -r /nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1 --store https://cache.nixos.org/ + path '/nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1' is not valid + + ``` + +* Print the 10 most recently added paths (using --json and the jq(1) + command): + + ```console + # nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path' + ``` + +* Show the size of the entire Nix store: + + ```console + # nix path-info --json --all | jq 'map(.narSize) | add' + 49812020936 + ``` + +* Show every path whose closure is bigger than 1 GB, sorted by closure + size: + + ```console + # nix path-info --json --all -S \ + | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])' + [ + …, + [ + "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65", + 5887562256 + ] + ] + ``` + +* Print the path of the store derivation produced by `nixpkgs#hello`: + + ```console + # nix path-info --derivation nixpkgs#hello + /nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv + ``` + +# Description + +This command shows information about the store paths produced by +*installables*, or about all paths in the store if you pass `--all`. + +By default, this command only prints the store paths. You can get +additional information by passing flags such as `--closure-size`, +--size`, `--sigs` or `--json`. + +> **Warning** +> +> Note that `nix path-info` does not build or substitute the +> *installables* you specify. Thus, if the corresponding store paths +> don't already exist, this command will fail. You can use `nix build` +> to ensure that they exist. + +)"" diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc index 19b1a55c8..62b645b06 100644 --- a/src/nix/ping-store.cc +++ b/src/nix/ping-store.cc @@ -8,17 +8,14 @@ struct CmdPingStore : StoreCommand { std::string description() override { - return "test whether a store can be opened"; + return "test whether a store can be accessed"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To test whether connecting to a remote Nix store via SSH works:", - "nix store ping --store ssh://mac1" - }, - }; + return + #include "ping-store.md" + ; } void run(ref<Store> store) override diff --git a/src/nix/ping-store.md b/src/nix/ping-store.md new file mode 100644 index 000000000..8c846791b --- /dev/null +++ b/src/nix/ping-store.md @@ -0,0 +1,33 @@ +R""( + +# Examples + +* Test whether connecting to a remote Nix store via SSH works: + + ```console + # nix store ping --store ssh://mac1 + ``` + +* Test whether a URL is a valid binary cache: + + ```console + # nix store ping --store https://cache.nixos.org + ``` + +* Test whether the Nix daemon is up and running: + + ```console + # nix store ping --store daemon + ``` + +# Description + +This command tests whether a particular Nix store (specified by the +argument `--store` *url*) can be accessed. What this means is +dependent on the type of the store. For instance, for an SSH store it +means that Nix can connect to the specified machine. + +If the command succeeds, Nix returns a exit code of 0 and does not +print any output. + +)"" diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc new file mode 100644 index 000000000..a831dcd15 --- /dev/null +++ b/src/nix/prefetch.cc @@ -0,0 +1,319 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "filetransfer.hh" +#include "finally.hh" +#include "progress-bar.hh" +#include "tarfile.hh" +#include "attr-path.hh" +#include "eval-inline.hh" +#include "legacy.hh" + +#include <nlohmann/json.hpp> + +using namespace nix; + +/* If ‘url’ starts with ‘mirror://’, then resolve it using the list of + mirrors defined in Nixpkgs. */ +string resolveMirrorUrl(EvalState & state, string url) +{ + if (url.substr(0, 9) != "mirror://") return url; + + std::string s(url, 9); + auto p = s.find('/'); + if (p == std::string::npos) throw Error("invalid mirror URL '%s'", url); + std::string mirrorName(s, 0, p); + + Value vMirrors; + // FIXME: use nixpkgs flake + state.eval(state.parseExprFromString("import <nixpkgs/pkgs/build-support/fetchurl/mirrors.nix>", "."), vMirrors); + state.forceAttrs(vMirrors); + + auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName)); + if (mirrorList == vMirrors.attrs->end()) + throw Error("unknown mirror name '%s'", mirrorName); + state.forceList(*mirrorList->value); + + if (mirrorList->value->listSize() < 1) + throw Error("mirror URL '%s' did not expand to anything", url); + + auto mirror = state.forceString(*mirrorList->value->listElems()[0]); + return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1); +} + +std::tuple<StorePath, Hash> prefetchFile( + ref<Store> store, + std::string_view url, + std::optional<std::string> name, + HashType hashType, + std::optional<Hash> expectedHash, + bool unpack, + bool executable) +{ + auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat; + + /* Figure out a name in the Nix store. */ + if (!name) { + name = baseNameOf(url); + if (name->empty()) + throw Error("cannot figure out file name for '%s'", url); + } + + std::optional<StorePath> storePath; + std::optional<Hash> hash; + + /* If an expected hash is given, the file may already exist in + the store. */ + if (expectedHash) { + hashType = expectedHash->type; + storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name); + if (store->isValidPath(*storePath)) + hash = expectedHash; + else + storePath.reset(); + } + + if (!storePath) { + + AutoDelete tmpDir(createTempDir(), true); + Path tmpFile = (Path) tmpDir + "/tmp"; + + /* Download the file. */ + { + auto mode = 0600; + if (executable) + mode = 0700; + + AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode); + if (!fd) throw SysError("creating temporary file '%s'", tmpFile); + + FdSink sink(fd.get()); + + FileTransferRequest req(url); + req.decompress = false; + getFileTransfer()->download(std::move(req), sink); + } + + /* Optionally unpack the file. */ + if (unpack) { + Activity act(*logger, lvlChatty, actUnknown, + fmt("unpacking '%s'", url)); + Path unpacked = (Path) tmpDir + "/unpacked"; + createDirs(unpacked); + unpackTarfile(tmpFile, unpacked); + + /* If the archive unpacks to a single file/directory, then use + that as the top-level. */ + auto entries = readDirectory(unpacked); + if (entries.size() == 1) + tmpFile = unpacked + "/" + entries[0].name; + else + tmpFile = unpacked; + } + + Activity act(*logger, lvlChatty, actUnknown, + fmt("adding '%s' to the store", url)); + + auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash); + storePath = info.path; + assert(info.ca); + hash = getContentAddressHash(*info.ca); + } + + return {storePath.value(), hash.value()}; +} + +static int main_nix_prefetch_url(int argc, char * * argv) +{ + { + HashType ht = htSHA256; + std::vector<string> args; + bool printPath = getEnv("PRINT_PATH") == "1"; + bool fromExpr = false; + string attrPath; + bool unpack = false; + bool executable = false; + std::optional<std::string> name; + + struct MyArgs : LegacyArgs, MixEvalArgs + { + using LegacyArgs::LegacyArgs; + }; + + MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) { + if (*arg == "--help") + showManPage("nix-prefetch-url"); + else if (*arg == "--version") + printVersion("nix-prefetch-url"); + else if (*arg == "--type") { + string s = getArg(*arg, arg, end); + ht = parseHashType(s); + } + else if (*arg == "--print-path") + printPath = true; + else if (*arg == "--attr" || *arg == "-A") { + fromExpr = true; + attrPath = getArg(*arg, arg, end); + } + else if (*arg == "--unpack") + unpack = true; + else if (*arg == "--executable") + executable = true; + else if (*arg == "--name") + name = getArg(*arg, arg, end); + else if (*arg != "" && arg->at(0) == '-') + return false; + else + args.push_back(*arg); + return true; + }); + + myArgs.parseCmdline(argvToStrings(argc, argv)); + + initPlugins(); + + if (args.size() > 2) + throw UsageError("too many arguments"); + + Finally f([]() { stopProgressBar(); }); + + if (isatty(STDERR_FILENO)) + startProgressBar(); + + auto store = openStore(); + auto state = std::make_unique<EvalState>(myArgs.searchPath, store); + + Bindings & autoArgs = *myArgs.getAutoArgs(*state); + + /* If -A is given, get the URL from the specified Nix + expression. */ + string url; + if (!fromExpr) { + if (args.empty()) + throw UsageError("you must specify a URL"); + url = args[0]; + } else { + Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0])); + Value vRoot; + state->evalFile(path, vRoot); + Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first); + state->forceAttrs(v); + + /* Extract the URL. */ + auto attr = v.attrs->find(state->symbols.create("urls")); + if (attr == v.attrs->end()) + throw Error("attribute set does not contain a 'urls' attribute"); + state->forceList(*attr->value); + if (attr->value->listSize() < 1) + throw Error("'urls' list is empty"); + url = state->forceString(*attr->value->listElems()[0]); + + /* Extract the hash mode. */ + attr = v.attrs->find(state->symbols.create("outputHashMode")); + if (attr == v.attrs->end()) + printInfo("warning: this does not look like a fetchurl call"); + else + unpack = state->forceString(*attr->value) == "recursive"; + + /* Extract the name. */ + if (!name) { + attr = v.attrs->find(state->symbols.create("name")); + if (attr != v.attrs->end()) + name = state->forceString(*attr->value); + } + } + + std::optional<Hash> expectedHash; + if (args.size() == 2) + expectedHash = Hash::parseAny(args[1], ht); + + auto [storePath, hash] = prefetchFile( + store, resolveMirrorUrl(*state, url), name, ht, expectedHash, unpack, executable); + + stopProgressBar(); + + if (!printPath) + printInfo("path is '%s'", store->printStorePath(storePath)); + + std::cout << printHash16or32(hash) << std::endl; + if (printPath) + std::cout << store->printStorePath(storePath) << std::endl; + + return 0; + } +} + +static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url); + +struct CmdStorePrefetchFile : StoreCommand, MixJSON +{ + std::string url; + bool executable = false; + std::optional<std::string> name; + HashType hashType = htSHA256; + std::optional<Hash> expectedHash; + + CmdStorePrefetchFile() + { + addFlag({ + .longName = "name", + .description = "Override the name component of the resulting store path. It defaults to the base name of *url*.", + .labels = {"name"}, + .handler = {&name} + }); + + addFlag({ + .longName = "expected-hash", + .description = "The expected hash of the file.", + .labels = {"hash"}, + .handler = {[&](std::string s) { + expectedHash = Hash::parseAny(s, hashType); + }} + }); + + addFlag(Flag::mkHashTypeFlag("hash-type", &hashType)); + + addFlag({ + .longName = "executable", + .description = + "Make the resulting file executable. Note that this causes the " + "resulting hash to be a NAR hash rather than a flat file hash.", + .handler = {&executable, true}, + }); + + expectArg("url", &url); + } + + Category category() override { return catUtility; } + + std::string description() override + { + return "download a file into the Nix store"; + } + + std::string doc() override + { + return + #include "store-prefetch-file.md" + ; + } + void run(ref<Store> store) override + { + auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, false, executable); + + if (json) { + auto res = nlohmann::json::object(); + res["storePath"] = store->printStorePath(storePath); + res["hash"] = hash.to_string(SRI, true); + logger->cout(res.dump()); + } else { + notice("Downloaded '%s' to '%s' (hash '%s').", + url, + store->printStorePath(storePath), + hash.to_string(SRI, true)); + } + } +}; + +static auto rCmdStorePrefetchFile = registerCommand2<CmdStorePrefetchFile>({"store", "prefetch-file"}); diff --git a/src/nix/print-dev-env.md b/src/nix/print-dev-env.md new file mode 100644 index 000000000..b80252acf --- /dev/null +++ b/src/nix/print-dev-env.md @@ -0,0 +1,19 @@ +R""( + +# Examples + +* Apply the build environment of GNU hello to the current shell: + + ```console + # . <(nix print-dev-env nixpkgs#hello) + ``` + +# Description + +This command prints a shell script that can be sourced by `b`ash and +that sets the environment variables and shell functions defined by the +build process of *installable*. This allows you to get a similar build +environment in your current shell rather than in a subshell (as with +`nix develop`). + +)"" diff --git a/src/nix/profile-diff-closures.md b/src/nix/profile-diff-closures.md new file mode 100644 index 000000000..295d1252b --- /dev/null +++ b/src/nix/profile-diff-closures.md @@ -0,0 +1,28 @@ +R""( + +# Examples + +* Show what changed between each version of the NixOS system + profile: + + ```console + # nix profile diff-closures --profile /nix/var/nix/profiles/system + Version 13 -> 14: + acpi-call: 2020-04-07-5.8.13 → 2020-04-07-5.8.14 + aws-sdk-cpp: -6723.1 KiB + … + + Version 14 -> 15: + acpi-call: 2020-04-07-5.8.14 → 2020-04-07-5.8.16 + attica: -996.2 KiB + breeze-icons: -78713.5 KiB + brotli: 1.0.7 → 1.0.9, +44.2 KiB + ``` + +# Description + +This command shows the difference between the closures of subsequent +versions of a profile. See [`nix store +diff-closures`](nix3-store-diff-closures.md) for details. + +)"" diff --git a/src/nix/profile-history.md b/src/nix/profile-history.md new file mode 100644 index 000000000..d0fe40c82 --- /dev/null +++ b/src/nix/profile-history.md @@ -0,0 +1,26 @@ +R""( + +# Examples + +* Show the changes between each version of your default profile: + + ```console + # nix profile history + Version 508 -> 509: + flake:nixpkgs#legacyPackages.x86_64-linux.awscli: ∅ -> 1.17.13 + + Version 509 -> 510: + flake:nixpkgs#legacyPackages.x86_64-linux.awscli: 1.17.13 -> 1.18.211 + ``` + +# Description + +This command shows what packages were added, removed or upgraded +between subsequent versions of a profile. It only shows top-level +packages, not dependencies; for that, use [`nix profile +diff-closures`](./nix3-profile-diff-closures.md). + +The addition of a package to a profile is denoted by the string `∅ ->` +*version*, whereas the removal is denoted by *version* `-> ∅`. + +)"" diff --git a/src/nix/profile-install.md b/src/nix/profile-install.md new file mode 100644 index 000000000..e3009491e --- /dev/null +++ b/src/nix/profile-install.md @@ -0,0 +1,27 @@ +R""( + +# Examples + +* Install a package from Nixpkgs: + + ```console + # nix profile install nixpkgs#hello + ``` + +* Install a package from a specific branch of Nixpkgs: + + ```console + # nix profile install nixpkgs/release-20.09#hello + ``` + +* Install a package from a specific revision of Nixpkgs: + + ```console + # nix profile install nixpkgs/d73407e8e6002646acfdef0e39ace088bacc83da#hello + ``` + +# Description + +This command adds *installables* to a Nix profile. + +)"" diff --git a/src/nix/profile-list.md b/src/nix/profile-list.md new file mode 100644 index 000000000..5c29c0b02 --- /dev/null +++ b/src/nix/profile-list.md @@ -0,0 +1,31 @@ +R""( + +# Examples + +* Show what packages are installed in the default profile: + + ```console + # nix profile list + 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1 + 1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027 + 2 flake:blender-bin#defaultPackage.x86_64-linux github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#defaultPackage.x86_64-linux /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0 + ``` + +# Description + +This command shows what packages are currently installed in a +profile. The output consists of one line per package, with the +following fields: + +* An integer that can be used to unambiguously identify the package in + invocations of `nix profile remove` and `nix profile upgrade`. + +* The original ("mutable") flake reference and output attribute path + used at installation time. + +* The immutable flake reference to which the mutable flake reference + was resolved. + +* The store path(s) of the package. + +)"" diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md new file mode 100644 index 000000000..dcf825da9 --- /dev/null +++ b/src/nix/profile-remove.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Remove a package by position: + + ```console + # nix profile remove 3 + ``` + +* Remove a package by attribute path: + + ```console + # nix profile remove packages.x86_64-linux.hello + ``` + +* Remove all packages: + ```console + # nix profile remove '.*' + ``` + +* Remove a package by store path: + + ```console + # nix profile remove /nix/store/rr3y0c6zyk7kjjl8y19s4lsrhn4aiq1z-hello-2.10 + ``` + +# Description + +This command removes a package from a profile. + +)"" diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md new file mode 100644 index 000000000..2bd5d256d --- /dev/null +++ b/src/nix/profile-upgrade.md @@ -0,0 +1,41 @@ +R""( + +# Examples + +* Upgrade all packages that were installed using a mutable flake + reference: + + ```console + # nix profile upgrade '.*' + ``` + +* Upgrade a specific package: + + ```console + # nix profile upgrade packages.x86_64-linux.hello + ``` + +* Upgrade a specific profile element by number: + + ```console + # nix profile info + 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify … + + # nix profile upgrade 0 + ``` + +# Description + +This command upgrades a previously installed package in a Nix profile, +by fetching and evaluating the latest version of the flake from which +the package was installed. + +> **Warning** +> +> This only works if you used a *mutable* flake reference at +> installation time, e.g. `nixpkgs#hello`. It does not work if you +> used an *immutable* flake reference +> (e.g. `github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a#hello`), +> since in that case the "latest version" is always the same. + +)"" diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 8cf5ccd62..765d6866e 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -8,6 +8,7 @@ #include "flake/flakeref.hh" #include "../nix-env/user-env.hh" #include "profiles.hh" +#include "names.hh" #include <nlohmann/json.hpp> #include <regex> @@ -21,6 +22,13 @@ struct ProfileElementSource FlakeRef resolvedRef; std::string attrPath; // FIXME: output names + + bool operator < (const ProfileElementSource & other) const + { + return + std::pair(originalRef.to_string(), attrPath) < + std::pair(other.originalRef.to_string(), other.attrPath); + } }; struct ProfileElement @@ -29,6 +37,29 @@ struct ProfileElement std::optional<ProfileElementSource> source; bool active = true; // FIXME: priority + + std::string describe() const + { + if (source) + return fmt("%s#%s", source->originalRef, source->attrPath); + StringSet names; + for (auto & path : storePaths) + names.insert(DrvName(path.name()).name); + return concatStringsSep(", ", names); + } + + std::string versions() const + { + StringSet versions; + for (auto & path : storePaths) + versions.insert(DrvName(path.name()).version); + return showVersions(versions); + } + + bool operator < (const ProfileElement & other) const + { + return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths); + } }; struct ProfileManifest @@ -142,6 +173,46 @@ struct ProfileManifest return std::move(info.path); } + + static void printDiff(const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent) + { + auto prevElems = prev.elements; + std::sort(prevElems.begin(), prevElems.end()); + + auto curElems = cur.elements; + std::sort(curElems.begin(), curElems.end()); + + auto i = prevElems.begin(); + auto j = curElems.begin(); + + bool changes = false; + + while (i != prevElems.end() || j != curElems.end()) { + if (j != curElems.end() && (i == prevElems.end() || i->describe() > j->describe())) { + std::cout << fmt("%s%s: ∅ -> %s\n", indent, j->describe(), j->versions()); + changes = true; + ++j; + } + else if (i != prevElems.end() && (j == curElems.end() || i->describe() < j->describe())) { + std::cout << fmt("%s%s: %s -> ∅\n", indent, i->describe(), i->versions()); + changes = true; + ++i; + } + else { + auto v1 = i->versions(); + auto v2 = j->versions(); + if (v1 != v2) { + std::cout << fmt("%s%s: %s -> %s\n", indent, i->describe(), v1, v2); + changes = true; + } + ++i; + ++j; + } + } + + if (!changes) + std::cout << fmt("%sNo changes.\n", indent); + } }; struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile @@ -151,22 +222,11 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile return "install a package into a profile"; } - Examples examples() override - { - return { - Example{ - "To install a package from Nixpkgs:", - "nix profile install nixpkgs#hello" - }, - Example{ - "To install a package from a specific branch of Nixpkgs:", - "nix profile install nixpkgs/release-19.09#hello" - }, - Example{ - "To install a package from a specific revision of Nixpkgs:", - "nix profile install nixpkgs/1028bb33859f8dfad7f98e1c8d185f3d1aaa7340#hello" - }, - }; + std::string doc() override + { + return + #include "profile-install.md" + ; } void run(ref<Store> store) override @@ -192,8 +252,28 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME manifest.elements.emplace_back(std::move(element)); - } else - throw UnimplementedError("'nix profile install' does not support argument '%s'", installable->what()); + } else { + auto buildables = build(store, Realise::Outputs, {installable}, bmNormal); + + for (auto & buildable : buildables) { + ProfileElement element; + + std::visit(overloaded { + [&](BuildableOpaque bo) { + pathsToBuild.push_back({bo.path, {}}); + element.storePaths.insert(bo.path); + }, + [&](BuildableFromDrv bfd) { + for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) { + pathsToBuild.push_back({bfd.drvPath, {output.first}}); + element.storePaths.insert(output.second); + } + }, + }, buildable); + + manifest.elements.emplace_back(std::move(element)); + } + } } store->buildPaths(pathsToBuild); @@ -220,9 +300,8 @@ public: std::vector<Matcher> res; for (auto & s : _matchers) { - size_t n; - if (string2Int(s, n)) - res.push_back(n); + if (auto n = string2Int<size_t>(s)) + res.push_back(*n); else if (store->isStorePath(s)) res.push_back(s); else @@ -257,26 +336,11 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem return "remove packages from a profile"; } - Examples examples() override - { - return { - Example{ - "To remove a package by attribute path:", - "nix profile remove packages.x86_64-linux.hello" - }, - Example{ - "To remove all packages:", - "nix profile remove '.*'" - }, - Example{ - "To remove a package by store path:", - "nix profile remove /nix/store/rr3y0c6zyk7kjjl8y19s4lsrhn4aiq1z-hello-2.10" - }, - Example{ - "To remove a package by position:", - "nix profile remove 3" - }, - }; + std::string doc() override + { + return + #include "profile-remove.md" + ; } void run(ref<Store> store) override @@ -310,18 +374,11 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf return "upgrade packages using their most recent flake"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To upgrade all packages that were installed using a mutable flake reference:", - "nix profile upgrade '.*'" - }, - Example{ - "To upgrade a specific package:", - "nix profile upgrade packages.x86_64-linux.hello" - }, - }; + return + #include "profile-upgrade.md" + ; } void run(ref<Store> store) override @@ -370,21 +427,18 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf } }; -struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile +struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile { std::string description() override { return "list installed packages"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show what packages are installed in the default profile:", - "nix profile info" - }, - }; + return + #include "profile-list.md" + ; } void run(ref<Store> store) override @@ -405,17 +459,14 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile { std::string description() override { - return "show the closure difference between each generation of a profile"; + return "show the closure difference between each version of a profile"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show what changed between each generation of the NixOS system profile:", - "nix profile diff-closures --profile /nix/var/nix/profiles/system" - }, - }; + return + #include "profile-diff-closures.md" + ; } void run(ref<Store> store) override @@ -429,7 +480,7 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile if (prevGen) { if (!first) std::cout << "\n"; first = false; - std::cout << fmt("Generation %d -> %d:\n", prevGen->number, gen.number); + std::cout << fmt("Version %d -> %d:\n", prevGen->number, gen.number); printClosureDiff(store, store->followLinksToStorePath(prevGen->path), store->followLinksToStorePath(gen.path), @@ -441,6 +492,48 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile } }; +struct CmdProfileHistory : virtual StoreCommand, EvalCommand, MixDefaultProfile +{ + std::string description() override + { + return "show all versions of a profile"; + } + + std::string doc() override + { + return + #include "profile-history.md" + ; + } + + void run(ref<Store> store) override + { + auto [gens, curGen] = findGenerations(*profile); + + std::optional<std::pair<Generation, ProfileManifest>> prevGen; + bool first = true; + + for (auto & gen : gens) { + ProfileManifest manifest(*getEvalState(), gen.path); + + if (!first) std::cout << "\n"; + first = false; + + if (prevGen) + std::cout << fmt("Version %d -> %d:\n", prevGen->first.number, gen.number); + else + std::cout << fmt("Version %d:\n", gen.number); + + ProfileManifest::printDiff( + prevGen ? prevGen->second : ProfileManifest(), + manifest, + " "); + + prevGen = {gen, std::move(manifest)}; + } + } +}; + struct CmdProfile : NixMultiCommand { CmdProfile() @@ -448,8 +541,9 @@ struct CmdProfile : NixMultiCommand {"install", []() { return make_ref<CmdProfileInstall>(); }}, {"remove", []() { return make_ref<CmdProfileRemove>(); }}, {"upgrade", []() { return make_ref<CmdProfileUpgrade>(); }}, - {"info", []() { return make_ref<CmdProfileInfo>(); }}, + {"list", []() { return make_ref<CmdProfileList>(); }}, {"diff-closures", []() { return make_ref<CmdProfileDiffClosures>(); }}, + {"history", []() { return make_ref<CmdProfileHistory>(); }}, }) { } @@ -458,6 +552,13 @@ struct CmdProfile : NixMultiCommand return "manage Nix profiles"; } + std::string doc() override + { + return + #include "profile.md" + ; + } + void run() override { if (!command) diff --git a/src/nix/profile.md b/src/nix/profile.md new file mode 100644 index 000000000..d3ddcd3d1 --- /dev/null +++ b/src/nix/profile.md @@ -0,0 +1,107 @@ +R""( + +# Description + +`nix profile` allows you to create and manage *Nix profiles*. A Nix +profile is a set of packages that can be installed and upgraded +independently from each other. Nix profiles are versioned, allowing +them to be rolled back easily. + +# Default profile + +The default profile used by `nix profile` is `$HOME/.nix-profile`, +which, if it does not exist, is created as a symlink to +`/nix/var/nix/profiles/per-user/default` if Nix is invoked by the +`root` user, or `/nix/var/nix/profiles/per-user/`*username* otherwise. + +You can specify another profile location using `--profile` *path*. + +# Filesystem layout + +Profiles are versioned as follows. When using profile *path*, *path* +is a symlink to *path*`-`*N*, where *N* is the current *version* of +the profile. In turn, *path*`-`*N* is a symlink to a path in the Nix +store. For example: + +```console +$ ls -l /nix/var/nix/profiles/per-user/alice/profile* +lrwxrwxrwx 1 alice users 14 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile -> profile-7-link +lrwxrwxrwx 1 alice users 51 Oct 28 16:18 /nix/var/nix/profiles/per-user/alice/profile-5-link -> /nix/store/q69xad13ghpf7ir87h0b2gd28lafjj1j-profile +lrwxrwxrwx 1 alice users 51 Oct 29 13:20 /nix/var/nix/profiles/per-user/alice/profile-6-link -> /nix/store/6bvhpysd7vwz7k3b0pndn7ifi5xr32dg-profile +lrwxrwxrwx 1 alice users 51 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile-7-link -> /nix/store/mp0x6xnsg0b8qhswy6riqvimai4gm677-profile +``` + +Each of these symlinks is a root for the Nix garbage collector. + +The contents of the store path corresponding to each version of the +profile is a tree of symlinks to the files of the installed packages, +e.g. + +```console +$ ll -R /nix/var/nix/profiles/per-user/eelco/profile-7-link/ +/nix/var/nix/profiles/per-user/eelco/profile-7-link/: +total 20 +dr-xr-xr-x 2 root root 4096 Jan 1 1970 bin +-r--r--r-- 2 root root 1402 Jan 1 1970 manifest.json +dr-xr-xr-x 4 root root 4096 Jan 1 1970 share + +/nix/var/nix/profiles/per-user/eelco/profile-7-link/bin: +total 20 +lrwxrwxrwx 5 root root 79 Jan 1 1970 chromium -> /nix/store/ijm5k0zqisvkdwjkc77mb9qzb35xfi4m-chromium-86.0.4240.111/bin/chromium +lrwxrwxrwx 7 root root 87 Jan 1 1970 spotify -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/bin/spotify +lrwxrwxrwx 3 root root 79 Jan 1 1970 zoom-us -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/bin/zoom-us + +/nix/var/nix/profiles/per-user/eelco/profile-7-link/share/applications: +total 12 +lrwxrwxrwx 4 root root 120 Jan 1 1970 chromium-browser.desktop -> /nix/store/4cf803y4vzfm3gyk3vzhzb2327v0kl8a-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop +lrwxrwxrwx 7 root root 110 Jan 1 1970 spotify.desktop -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/share/applications/spotify.desktop +lrwxrwxrwx 3 root root 107 Jan 1 1970 us.zoom.Zoom.desktop -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/share/applications/us.zoom.Zoom.desktop + +… +``` + +The file `manifest.json` records the provenance of the packages that +are installed in this version of the profile. It looks like this: + +```json +{ + "version": 1, + "elements": [ + { + "active": true, + "attrPath": "legacyPackages.x86_64-linux.zoom-us", + "originalUri": "flake:nixpkgs", + "storePaths": [ + "/nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927" + ], + "uri": "github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a" + }, + … + ] +} +``` + +Each object in the array `elements` denotes an installed package and +has the following fields: + +* `originalUri`: The [flake reference](./nix3-flake.md) specified by + the user at the time of installation (e.g. `nixpkgs`). This is also + the flake reference that will be used by `nix profile upgrade`. + +* `uri`: The immutable flake reference to which `originalUri` + resolved. + +* `attrPath`: The flake output attribute that provided this + package. Note that this is not necessarily the attribute that the + user specified, but the one resulting from applying the default + attribute paths and prefixes; for instance, `hello` might resolve to + `packages.x86_64-linux.hello` and the empty string to + `defaultPackage.x86_64-linux`. + +* `storePath`: The paths in the Nix store containing the package. + +* `active`: Whether the profile contains symlinks to the files of this + package. If set to false, the package is kept in the Nix store, but + is not "visible" in the profile's symlink tree. + +)"" diff --git a/src/nix/registry-add.md b/src/nix/registry-add.md new file mode 100644 index 000000000..80a31996a --- /dev/null +++ b/src/nix/registry-add.md @@ -0,0 +1,33 @@ +R""( + +# Examples + +* Set the `nixpkgs` flake identifier to a specific branch of Nixpkgs: + + ```console + # nix registry add nixpkgs github:NixOS/nixpkgs/nixos-20.03 + ``` + +* Pin `nixpkgs` to a specific revision: + + ```console + # nix registry add nixpkgs github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a + ``` + +* Add an entry that redirects a specific branch of `nixpkgs` to + another fork: + + ```console + # nix registry add nixpkgs/nixos-20.03 ~/Dev/nixpkgs + ``` + +# Description + +This command adds an entry to the user registry that maps flake +reference *from-url* to flake reference *to-url*. If an entry for +*from-url* already exists, it is overwritten. + +Entries can be removed using [`nix registry +remove`](./nix3-registry-remove.md). + +)"" diff --git a/src/nix/registry-list.md b/src/nix/registry-list.md new file mode 100644 index 000000000..30b6e29d8 --- /dev/null +++ b/src/nix/registry-list.md @@ -0,0 +1,29 @@ +R""( + +# Examples + +* Show the contents of all registries: + + ```console + # nix registry list + user flake:dwarffs github:edolstra/dwarffs/d181d714fd36eb06f4992a1997cd5601e26db8f5 + system flake:nixpkgs path:/nix/store/fxl9mrm5xvzam0lxi9ygdmksskx4qq8s-source?lastModified=1605220118&narHash=sha256-Und10ixH1WuW0XHYMxxuHRohKYb45R%2fT8CwZuLd2D2Q=&rev=3090c65041104931adda7625d37fa874b2b5c124 + global flake:blender-bin github:edolstra/nix-warez?dir=blender + global flake:dwarffs github:edolstra/dwarffs + … + ``` + +# Description + +This command displays the contents of all registries on standard +output. Each line represents one registry entry in the format *type* +*from* *to*, where *type* denotes the registry containing the entry: + +* `flags`: entries specified on the command line using `--override-flake`. +* `user`: the user registry. +* `system`: the system registry. +* `global`: the global registry. + +See the [`nix registry` manual page](./nix3-registry.md) for more details. + +)"" diff --git a/src/nix/registry-pin.md b/src/nix/registry-pin.md new file mode 100644 index 000000000..6e97e003e --- /dev/null +++ b/src/nix/registry-pin.md @@ -0,0 +1,38 @@ +R""( + +# Examples + +* Pin `nixpkgs` to its most recent Git revision: + + ```console + # nix registry pin nixpkgs + ``` + + Afterwards the user registry will have an entry like this: + + ```console + nix registry list | grep '^user ' + user flake:nixpkgs github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a + ``` + + and `nix flake info` will say: + + ```console + # nix flake info nixpkgs + Resolved URL: github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a + Locked URL: github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a + … + ``` + +# Description + +This command adds an entry to the user registry that maps flake +reference *url* to the corresponding *locked* flake reference, that +is, a flake reference that specifies an exact revision or content +hash. This ensures that until this registry entry is removed, all uses +of *url* will resolve to exactly the same flake. + +Entries can be removed using [`nix registry +remove`](./nix3-registry-remove.md). + +)"" diff --git a/src/nix/registry-remove.md b/src/nix/registry-remove.md new file mode 100644 index 000000000..4c0eb4947 --- /dev/null +++ b/src/nix/registry-remove.md @@ -0,0 +1,16 @@ +R""( + +# Examples + +* Remove the entry `nixpkgs` from the user registry: + + ```console + # nix registry remove nixpkgs + ``` + +# Description + +This command removes from the user registry any entry for flake +reference *url*. + +)"" diff --git a/src/nix/registry.cc b/src/nix/registry.cc index 9352e00a7..f9719600f 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -17,6 +17,13 @@ struct CmdRegistryList : StoreCommand return "list available Nix flakes"; } + std::string doc() override + { + return + #include "registry-list.md" + ; + } + void run(nix::ref<nix::Store> store) override { using namespace fetchers; @@ -47,6 +54,13 @@ struct CmdRegistryAdd : MixEvalArgs, Command return "add/replace flake in user flake registry"; } + std::string doc() override + { + return + #include "registry-add.md" + ; + } + CmdRegistryAdd() { expectArg("from-url", &fromUrl); @@ -75,6 +89,13 @@ struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command return "remove flake from user flake registry"; } + std::string doc() override + { + return + #include "registry-remove.md" + ; + } + CmdRegistryRemove() { expectArg("url", &url); @@ -97,6 +118,13 @@ struct CmdRegistryPin : virtual Args, EvalCommand return "pin a flake to its current version in user flake registry"; } + std::string doc() override + { + return + #include "registry-pin.md" + ; + } + CmdRegistryPin() { expectArg("url", &url); @@ -132,6 +160,13 @@ struct CmdRegistry : virtual NixMultiCommand return "manage the flake registry"; } + std::string doc() override + { + return + #include "registry.md" + ; + } + Category category() override { return catSecondary; } void run() override diff --git a/src/nix/registry.md b/src/nix/registry.md new file mode 100644 index 000000000..557e5795b --- /dev/null +++ b/src/nix/registry.md @@ -0,0 +1,98 @@ +R""( + +# Description + +`nix flake` provides subcommands for managing *flake +registries*. Flake registries are a convenience feature that allows +you to refer to flakes using symbolic identifiers such as `nixpkgs`, +rather than full URLs such as `git://github.com/NixOS/nixpkgs`. You +can use these identifiers on the command line (e.g. when you do `nix +run nixpkgs#hello`) or in flake input specifications in `flake.nix` +files. The latter are automatically resolved to full URLs and recorded +in the flake's `flake.lock` file. + +In addition, the flake registry allows you to redirect arbitrary flake +references (e.g. `github:NixOS/patchelf`) to another location, such as +a local fork. + +There are multiple registries. These are, in order from lowest to +highest precedence: + +* The global registry, which is a file downloaded from the URL + specified by the setting `flake-registry`. It is cached locally and + updated automatically when it's older than `tarball-ttl` + seconds. The default global registry is kept in [a GitHub + repository](https://github.com/NixOS/flake-registry). + +* The system registry, which is shared by all users. The default + location is `/etc/nix/registry.json`. On NixOS, the system registry + can be specified using the NixOS option `nix.registry`. + +* The user registry `~/.config/nix/registry.json`. This registry can + be modified by commands such as `nix flake pin`. + +* Overrides specified on the command line using the option + `--override-flake`. + +# Registry format + +A registry is a JSON file with the following format: + +```json +{ + "version": 2, + [ + { + "from": { + "type": "indirect", + "id": "nixpkgs" + }, + "to": { + "type": "github", + "owner": "NixOS", + "repo": "nixpkgs" + } + }, + ... + ] +} +``` + +That is, it contains a list of objects with attributes `from` and +`to`, both of which contain a flake reference in attribute +representation. (For example, `{"type": "indirect", "id": "nixpkgs"}` +is the attribute representation of `nixpkgs`, while `{"type": +"github", "owner": "NixOS", "repo": "nixpkgs"}` is the attribute +representation of `github:NixOS/nixpkgs`.) + +Given some flake reference *R*, a registry entry is used if its +`from` flake reference *matches* *R*. *R* is then replaced by the +*unification* of the `to` flake reference with *R*. + +# Matching + +The `from` flake reference in a registry entry *matches* some flake +reference *R* if the attributes in `from` are the same as the +attributes in `R`. For example: + +* `nixpkgs` matches with `nixpkgs`. + +* `nixpkgs` matches with `nixpkgs/nixos-20.09`. + +* `nixpkgs/nixos-20.09` does not match with `nixpkgs`. + +* `nixpkgs` does not match with `git://github.com/NixOS/patchelf`. + +# Unification + +The `to` flake reference in a registry entry is *unified* with some flake +reference *R* by taking `to` and applying the `rev` and `ref` +attributes from *R*, if specified. For example: + +* `github:NixOS/nixpkgs` unified with `nixpkgs` produces `github:NixOS/nixpkgs`. + +* `github:NixOS/nixpkgs` unified with `nixpkgs/nixos-20.09` produces `github:NixOS/nixpkgs/nixos-20.09`. + +* `github:NixOS/nixpkgs/master` unified with `nixpkgs/nixos-20.09` produces `github:NixOS/nixpkgs/nixos-20.09`. + +)"" diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 71794a309..bce8d31dc 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -405,6 +405,7 @@ bool NixRepl::processLine(string line) } if (command == ":?" || command == ":help") { + // FIXME: convert to Markdown, include in the 'nix repl' manpage. std::cout << "The following commands are available:\n" << "\n" @@ -446,11 +447,11 @@ bool NixRepl::processLine(string line) Pos pos; - if (v.type == tPath || v.type == tString) { + if (v.type() == nPath || v.type() == nString) { PathSet context; auto filename = state->coerceToString(noPos, v, context); pos.file = state->symbols.create(filename); - } else if (v.type == tLambda) { + } else if (v.isLambda()) { pos = v.lambda.fun->pos; } else { // assume it's a derivation @@ -551,9 +552,7 @@ bool NixRepl::processLine(string line) { Expr * e = parseString(string(line, p + 1)); Value & v(*state->allocValue()); - v.type = tThunk; - v.thunk.env = env; - v.thunk.expr = e; + v.mkThunk(env, e); addVarToScope(state->symbols.create(name), v); } else { Value v; @@ -669,31 +668,31 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m state->forceValue(v); - switch (v.type) { + switch (v.type()) { - case tInt: + case nInt: str << ANSI_CYAN << v.integer << ANSI_NORMAL; break; - case tBool: + case nBool: str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL; break; - case tString: + case nString: str << ANSI_YELLOW; printStringValue(str, v.string.s); str << ANSI_NORMAL; break; - case tPath: + case nPath: str << ANSI_GREEN << v.path << ANSI_NORMAL; // !!! escaping? break; - case tNull: + case nNull: str << ANSI_CYAN "null" ANSI_NORMAL; break; - case tAttrs: { + case nAttrs: { seen.insert(&v); bool isDrv = state->isDerivation(v); @@ -738,9 +737,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m break; } - case tList1: - case tList2: - case tListN: + case nList: seen.insert(&v); str << "[ "; @@ -761,22 +758,21 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m str << "]"; break; - case tLambda: { - std::ostringstream s; - s << v.lambda.fun->pos; - str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL; - break; - } - - case tPrimOp: - str << ANSI_MAGENTA "«primop»" ANSI_NORMAL; - break; - - case tPrimOpApp: - str << ANSI_BLUE "«primop-app»" ANSI_NORMAL; + case nFunction: + if (v.isLambda()) { + std::ostringstream s; + s << v.lambda.fun->pos; + str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL; + } else if (v.isPrimOp()) { + str << ANSI_MAGENTA "«primop»" ANSI_NORMAL; + } else if (v.isPrimOpApp()) { + str << ANSI_BLUE "«primop-app»" ANSI_NORMAL; + } else { + abort(); + } break; - case tFloat: + case nFloat: str << v.fpoint; break; @@ -806,14 +802,11 @@ struct CmdRepl : StoreCommand, MixEvalArgs return "start an interactive environment for evaluating Nix expressions"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "Display all special commands within the REPL:", - "nix repl\nnix-repl> :?" - } - }; + return + #include "repl.md" + ; } void run(ref<Store> store) override diff --git a/src/nix/repl.md b/src/nix/repl.md new file mode 100644 index 000000000..bba60f871 --- /dev/null +++ b/src/nix/repl.md @@ -0,0 +1,57 @@ +R""( + +# Examples + +* Display all special commands within the REPL: + + ```console + # nix repl + nix-repl> :? + ``` + +* Evaluate some simple Nix expressions: + + ```console + # nix repl + + nix-repl> 1 + 2 + 3 + + nix-repl> map (x: x * 2) [1 2 3] + [ 2 4 6 ] + ``` + +* Interact with Nixpkgs in the REPL: + + ```console + # nix repl '<nixpkgs>' + + Loading '<nixpkgs>'... + Added 12428 variables. + + nix-repl> emacs.name + "emacs-27.1" + + nix-repl> emacs.drvPath + "/nix/store/lp0sjrhgg03y2n0l10n70rg0k7hhyz0l-emacs-27.1.drv" + + nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello > $out" + + nix-repl> :b x + this derivation produced the following outputs: + out -> /nix/store/0njwbgwmkwls0w5dv9mpc1pq5fj39q0l-hello + + nix-repl> builtins.readFile drv + "Hello, world!\n" + ``` + +# Description + +This command provides an interactive environment for evaluating Nix +expressions. (REPL stands for 'read–eval–print loop'.) + +On startup, it loads the Nix expressions named *files* and adds them +into the lexical scope. You can load addition files using the `:l +<filename>` command, or reload all files using `:r`. + +)"" diff --git a/src/nix/run.cc b/src/nix/run.cc index 92a52c6cd..ec9388234 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -72,7 +72,7 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment addFlag({ .longName = "command", .shortName = 'c', - .description = "command and arguments to be executed; defaults to '$SHELL'", + .description = "Command and arguments to be executed, defaulting to `$SHELL`", .labels = {"command", "args"}, .handler = {[&](std::vector<std::string> ss) { if (ss.empty()) throw UsageError("--command requires at least one argument"); @@ -86,26 +86,11 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment return "run a shell in which the specified packages are available"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To start a shell providing GNU Hello from NixOS 20.03:", - "nix shell nixpkgs/nixos-20.03#hello" - }, - Example{ - "To start a shell providing youtube-dl from your 'nixpkgs' channel:", - "nix shell nixpkgs#youtube-dl" - }, - Example{ - "To run GNU Hello:", - "nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!'" - }, - Example{ - "To run GNU Hello in a chroot store:", - "nix shell --store ~/my-nix nixpkgs#hello -c hello" - }, - }; + return + #include "shell.md" + ; } void run(ref<Store> store) override @@ -168,22 +153,11 @@ struct CmdRun : InstallableCommand, RunCommon return "run a Nix application"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To run Blender:", - "nix run blender-bin" - }, - Example{ - "To run vim from nixpkgs:", - "nix run nixpkgs#vim" - }, - Example{ - "To run vim from nixpkgs with arguments:", - "nix run nixpkgs#vim -- --help" - }, - }; + return + #include "run.md" + ; } Strings getDefaultFlakeAttrPaths() override @@ -258,14 +232,16 @@ void chrootHelper(int argc, char * * argv) for (auto entry : readDirectory("/")) { auto src = "/" + entry.name; - auto st = lstat(src); - if (!S_ISDIR(st.st_mode)) continue; Path dst = tmpDir + "/" + entry.name; if (pathExists(dst)) continue; - if (mkdir(dst.c_str(), 0700) == -1) - throw SysError("creating directory '%s'", dst); - if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) - throw SysError("mounting '%s' on '%s'", src, dst); + auto st = lstat(src); + if (S_ISDIR(st.st_mode)) { + if (mkdir(dst.c_str(), 0700) == -1) + throw SysError("creating directory '%s'", dst); + if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1) + throw SysError("mounting '%s' on '%s'", src, dst); + } else if (S_ISLNK(st.st_mode)) + createSymlink(readLink(src), dst); } char * cwd = getcwd(0, 0); diff --git a/src/nix/run.md b/src/nix/run.md new file mode 100644 index 000000000..c178e8b13 --- /dev/null +++ b/src/nix/run.md @@ -0,0 +1,87 @@ +R""( + +# Examples + +* Run the default app from the `blender-bin` flake: + + ```console + # nix run blender-bin + ``` + +* Run a non-default app from the `blender-bin` flake: + + ```console + # nix run blender-bin#blender_2_83 + ``` + + Tip: you can find apps provided by this flake by running `nix flake + show blender-bin`. + +* Run `vim` from the `nixpkgs` flake: + + ```console + # nix run nixpkgs#vim + ``` + + Note that `vim` (as of the time of writing of this page) is not an + app but a package. Thus, Nix runs the eponymous file from the `vim` + package. + +* Run `vim` with arguments: + + ```console + # nix run nixpkgs#vim -- --help + ``` + +# Description + +`nix run` builds and runs *installable*, which must evaluate to an +*app* or a regular Nix derivation. + +If *installable* evaluates to an *app* (see below), it executes the +program specified by the app definition. + +If *installable* evaluates to a derivation, it will try to execute the +program `<out>/bin/<name>`, where *out* is the primary output store +path of the derivation and *name* is the name part of the value of the +`name` attribute of the derivation (e.g. if `name` is set to +`hello-1.10`, it will run `$out/bin/hello`). + +# Flake output attributes + +If no flake output attribute is given, `nix run` tries the following +flake output attributes: + +* `defaultApp.<system>` + +* `defaultPackage.<system>` + +If an attribute *name* is given, `nix run` tries the following flake +output attributes: + +* `apps.<system>.<name>` + +* `packages.<system>.<name>` + +* `legacyPackages.<system>.<name>` + +# Apps + +An app is specified by a flake output attribute named +`apps.<system>.<name>` or `defaultApp.<system>`. It looks like this: + +```nix +apps.x86_64-linux.blender_2_79 = { + type = "app"; + program = "${self.packages.x86_64-linux.blender_2_79}/bin/blender"; +}; +``` + +The only supported attributes are: + +* `type` (required): Must be set to `app`. + +* `program` (required): The full path of the executable to run. It + must reside in the Nix store. + +)"" diff --git a/src/nix/search.cc b/src/nix/search.cc index 47770e128..9f864b3a4 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -41,29 +41,14 @@ struct CmdSearch : InstallableCommand, MixJSON std::string description() override { - return "query available packages"; + return "search for packages"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show all packages in the flake in the current directory:", - "nix search" - }, - Example{ - "To show packages in the 'nixpkgs' flake containing 'blender' in its name or description:", - "nix search nixpkgs blender" - }, - Example{ - "To search for Firefox or Chromium:", - "nix search nixpkgs 'firefox|chromium'" - }, - Example{ - "To search for packages containing 'git' and either 'frontend' or 'gui':", - "nix search nixpkgs git 'frontend|gui'" - } - }; + return + #include "search.md" + ; } Strings getDefaultFlakeAttrPaths() override diff --git a/src/nix/search.md b/src/nix/search.md new file mode 100644 index 000000000..d182788a6 --- /dev/null +++ b/src/nix/search.md @@ -0,0 +1,72 @@ +R""( + +# Examples + +* Show all packages in the `nixpkgs` flake: + + ```console + # nix search nixpkgs + * legacyPackages.x86_64-linux.AMB-plugins (0.8.1) + A set of ambisonics ladspa plugins + + * legacyPackages.x86_64-linux.ArchiSteamFarm (4.3.1.0) + Application with primary purpose of idling Steam cards from multiple accounts simultaneously + … + ``` + +* Show packages in the `nixpkgs` flake containing `blender` in its + name or description: + + ```console + # nix search nixpkgs blender + * legacyPackages.x86_64-linux.blender (2.91.0) + 3D Creation/Animation/Publishing System + ``` + +* Search for packages underneath the attribute `gnome3` in Nixpkgs: + + ```console + # nix search nixpkgs#gnome3 vala + * legacyPackages.x86_64-linux.gnome3.vala (0.48.9) + Compiler for GObject type system + ``` + +* Show all packages in the flake in the current directory: + + ```console + # nix search + ``` + +* Search for Firefox or Chromium: + + ```console + # nix search nixpkgs 'firefox|chromium' + ``` + +* Search for packages containing `git'`and either `frontend` or `gui`: + + ```console + # nix search nixpkgs git 'frontend|gui' + ``` + +# Description + +`nix search` searches *installable* (which must be evaluatable, e.g. a +flake) for packages whose name or description matches all of the +regular expressions *regex*. For each matching package, It prints the +full attribute name (from the root of the installable), the version +and the `meta.description` field, highlighting the substrings that +were matched by the regular expressions. If no regular expressions are +specified, all packages are shown. + +# Flake output attributes + +If no flake output attribute is given, `nix search` searches for +packages: + +* Directly underneath `packages.<system>`. + +* Underneath `legacyPackages.<system>`, recursing into attribute sets + that contain an attribute `recurseForDerivations = true`. + +)"" diff --git a/src/nix/shell.md b/src/nix/shell.md new file mode 100644 index 000000000..2a379e03f --- /dev/null +++ b/src/nix/shell.md @@ -0,0 +1,48 @@ +R""( + +# Examples + +* Start a shell providing `youtube-dl` from the `nixpkgs` flake: + + ```console + # nix shell nixpkgs#youtube-dl + # youtube-dl --version + 2020.11.01.1 + ``` + +* Start a shell providing GNU Hello from NixOS 20.03: + + ```console + # nix shell nixpkgs/nixos-20.03#hello + ``` + +* Run GNU Hello: + + ```console + # nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!' + Hi everybody! + ``` + +* Run GNU Hello in a chroot store: + + ```console + # nix shell --store ~/my-nix nixpkgs#hello -c hello + ``` + +* Start a shell providing GNU Hello in a chroot store: + + ```console + # nix shell --store ~/my-nix nixpkgs#hello nixpkgs#bashInteractive -c bash + ``` + + Note that it's necessary to specify `bash` explicitly because your + default shell (e.g. `/bin/bash`) generally will not exist in the + chroot. + +# Description + +`nix shell` runs a command in an environment in which the `$PATH` +variable provides the specified *installables*. If not command is +specified, it starts the default shell of your user account. + +)"" diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc index 8e1a58ac2..2588a011d 100644 --- a/src/nix/show-derivation.cc +++ b/src/nix/show-derivation.cc @@ -19,7 +19,7 @@ struct CmdShowDerivation : InstallablesCommand addFlag({ .longName = "recursive", .shortName = 'r', - .description = "include the dependencies of the specified derivations", + .description = "Include the dependencies of the specified derivations.", .handler = {&recursive, true} }); } @@ -29,18 +29,11 @@ struct CmdShowDerivation : InstallablesCommand return "show the contents of a store derivation"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show the store derivation that results from evaluating the Hello package:", - "nix show-derivation nixpkgs#hello" - }, - Example{ - "To show the full derivation graph (if available) that produced your NixOS system:", - "nix show-derivation -r /run/current-system" - }, - }; + return + #include "show-derivation.md" + ; } Category category() override { return catUtility; } @@ -103,7 +96,7 @@ struct CmdShowDerivation : InstallablesCommand } } - drvObj.attr("platform", drv.platform); + drvObj.attr("system", drv.platform); drvObj.attr("builder", drv.builder); { diff --git a/src/nix/show-derivation.md b/src/nix/show-derivation.md new file mode 100644 index 000000000..aa863899c --- /dev/null +++ b/src/nix/show-derivation.md @@ -0,0 +1,103 @@ +R""( + +# Examples + +* Show the store derivation that results from evaluating the Hello + package: + + ```console + # nix show-derivation nixpkgs#hello + { + "/nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv": { + … + } + } + ``` + +* Show the full derivation graph (if available) that produced your + NixOS system: + + ```console + # nix show-derivation -r /run/current-system + ``` + +* Print all files fetched using `fetchurl` by Firefox's dependency + graph: + + ```console + # nix show-derivation -r nixpkgs#firefox \ + | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' \ + | uniq | sort + ``` + + Note that `.outputs.out.hash` selects *fixed-output derivations* + (derivations that produce output with a specified content hash), + while `.env.urls` selects derivations with a `urls` attribute. + +# Description + +This command prints on standard output a JSON representation of the +store derivations to which *installables* evaluate. Store derivations +are used internally by Nix. They are store paths with extension `.drv` +that represent the build-time dependency graph to which a Nix +expression evaluates. + +By default, this command only shows top-level derivations, but with +`--recursive`, it also shows their dependencies. + +The JSON output is a JSON object whose keys are the store paths of the +derivations, and whose values are a JSON object with the following +fields: + +* `outputs`: Information about the output paths of the + derivation. This is a JSON object with one member per output, where + the key is the output name and the value is a JSON object with these + fields: + + * `path`: The output path. + * `hashAlgo`: For fixed-output derivations, the hashing algorithm + (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a + NAR hash rather than a flat file hash. + * `hash`: For fixed-output derivations, the expected content hash in + base-16. + + Example: + + ```json + "outputs": { + "out": { + "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", + "hashAlgo": "r:sha256", + "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" + } + } + ``` + +* `inputSrcs`: A list of store paths on which this derivation depends. + +* `inputDrvs`: A JSON object specifying the derivations on which this + derivation depends, and what outputs of those derivations. For + example, + + ```json + "inputDrvs": { + "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], + "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] + } + ``` + + specifies that this derivation depends on the `dev` output of + `curl`, and the `out` output of `unzip`. + +* `system`: The system type on which this derivation is to be built + (e.g. `x86_64-linux`). + +* `builder`: The absolute path of the program to be executed to run + the build. Typically this is the `bash` shell + (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`). + +* `args`: The command-line arguments passed to the `builder`. + +* `env`: The environment passed to the `builder`. + +)"" diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 37b8a6712..3445182f2 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -16,7 +16,7 @@ struct CmdCopySigs : StorePathsCommand addFlag({ .longName = "substituter", .shortName = 's', - .description = "use signatures from specified store", + .description = "Use signatures from specified store.", .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }}, }); @@ -92,16 +92,16 @@ struct CmdCopySigs : StorePathsCommand static auto rCmdCopySigs = registerCommand2<CmdCopySigs>({"store", "copy-sigs"}); -struct CmdSignPaths : StorePathsCommand +struct CmdSign : StorePathsCommand { Path secretKeyFile; - CmdSignPaths() + CmdSign() { addFlag({ .longName = "key-file", .shortName = 'k', - .description = "file containing the secret signing key", + .description = "File containing the secret signing key.", .labels = {"file"}, .handler = {&secretKeyFile}, .completer = completePath @@ -140,4 +140,89 @@ struct CmdSignPaths : StorePathsCommand } }; -static auto rCmdSignPaths = registerCommand2<CmdSignPaths>({"store", "sign-paths"}); +static auto rCmdSign = registerCommand2<CmdSign>({"store", "sign"}); + +struct CmdKeyGenerateSecret : Command +{ + std::optional<std::string> keyName; + + CmdKeyGenerateSecret() + { + addFlag({ + .longName = "key-name", + .description = "Identifier of the key (e.g. `cache.example.org-1`).", + .labels = {"name"}, + .handler = {&keyName}, + }); + } + + std::string description() override + { + return "generate a secret key for signing store paths"; + } + + std::string doc() override + { + return + #include "key-generate-secret.md" + ; + } + + void run() override + { + if (!keyName) + throw UsageError("required argument '--key-name' is missing"); + + std::cout << SecretKey::generate(*keyName).to_string(); + } +}; + +struct CmdKeyConvertSecretToPublic : Command +{ + std::string description() override + { + return "generate a public key for verifying store paths from a secret key read from standard input"; + } + + std::string doc() override + { + return + #include "key-convert-secret-to-public.md" + ; + } + + void run() override + { + SecretKey secretKey(drainFD(STDIN_FILENO)); + std::cout << secretKey.toPublicKey().to_string(); + } +}; + +struct CmdKey : NixMultiCommand +{ + CmdKey() + : MultiCommand({ + {"generate-secret", []() { return make_ref<CmdKeyGenerateSecret>(); }}, + {"convert-secret-to-public", []() { return make_ref<CmdKeyConvertSecretToPublic>(); }}, + }) + { + } + + std::string description() override + { + return "generate and convert Nix signing keys"; + } + + Category category() override { return catUtility; } + + void run() override + { + if (!command) + throw UsageError("'nix flake' requires a sub-command."); + settings.requireExperimentalFeature("flakes"); + command->second->prepare(); + command->second->run(); + } +}; + +static auto rCmdKey = registerCommand<CmdKey>("key"); diff --git a/src/nix/store-cat.md b/src/nix/store-cat.md new file mode 100644 index 000000000..da2073473 --- /dev/null +++ b/src/nix/store-cat.md @@ -0,0 +1,19 @@ +R""( + +# Examples + +* Show the contents of a file in a binary cache: + + ```console + # nix store cat --store https://cache.nixos.org/ \ + /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello | hexdump -C | head -n1 + 00000000 7f 45 4c 46 02 01 01 00 00 00 00 00 00 00 00 00 |.ELF............| + ``` + +# Description + +This command prints on standard output the contents of the regular +file *path* in a Nix store. *path* can be a top-level store path or +any file inside a store path. + +)"" diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc new file mode 100644 index 000000000..10245978e --- /dev/null +++ b/src/nix/store-delete.cc @@ -0,0 +1,44 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreDelete : StorePathsCommand +{ + GCOptions options { .action = GCOptions::gcDeleteSpecific }; + + CmdStoreDelete() + { + addFlag({ + .longName = "ignore-liveness", + .description = "Do not check whether the paths are reachable from a root.", + .handler = {&options.ignoreLiveness, true} + }); + } + + std::string description() override + { + return "delete paths from the Nix store"; + } + + std::string doc() override + { + return + #include "store-delete.md" + ; + } + + void run(ref<Store> store, std::vector<StorePath> storePaths) override + { + for (auto & path : storePaths) + options.pathsToDelete.insert(path); + + GCResults results; + PrintFreed freed(true, results); + store->collectGarbage(options, results); + } +}; + +static auto rCmdStoreDelete = registerCommand2<CmdStoreDelete>({"store", "delete"}); diff --git a/src/nix/store-delete.md b/src/nix/store-delete.md new file mode 100644 index 000000000..db535f87c --- /dev/null +++ b/src/nix/store-delete.md @@ -0,0 +1,24 @@ +R""( + +# Examples + +* Delete a specific store path: + + ```console + # nix store delete /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + ``` + +# Description + +This command deletes the store paths specified by *installables*. , +but only if it is safe to do so; that is, when the path is not +reachable from a root of the garbage collector. This means that you +can only delete paths that would also be deleted by `nix store +gc`. Thus, `nix store delete` is a more targeted version of `nix store +gc`. + +With the option `--ignore-liveness`, reachability from the roots is +ignored. However, the path still won't be deleted if there are other +paths in the store that refer to it (i.e., depend on it). + +)"" diff --git a/src/nix/store-dump-path.md b/src/nix/store-dump-path.md new file mode 100644 index 000000000..4ef563526 --- /dev/null +++ b/src/nix/store-dump-path.md @@ -0,0 +1,23 @@ +R""( + +# Examples + +* To get a NAR containing the GNU Hello package: + + ```console + # nix store dump-path nixpkgs#hello > hello.nar + ``` + +* To get a NAR from the binary cache https://cache.nixos.org/: + + ```console + # nix store dump-path --store https://cache.nixos.org/ \ + /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25 > glibc.nar + ``` + +# Description + +This command generates a NAR file containing the serialisation of the +store path *installable*. The NAR is written to standard output. + +)"" diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc new file mode 100644 index 000000000..a2d74066e --- /dev/null +++ b/src/nix/store-gc.cc @@ -0,0 +1,43 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreGC : StoreCommand, MixDryRun +{ + GCOptions options; + + CmdStoreGC() + { + addFlag({ + .longName = "max", + .description = "Stop after freeing *n* bytes of disk space.", + .labels = {"n"}, + .handler = {&options.maxFreed} + }); + } + + std::string description() override + { + return "perform garbage collection on a Nix store"; + } + + std::string doc() override + { + return + #include "store-gc.md" + ; + } + + void run(ref<Store> store) override + { + options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead; + GCResults results; + PrintFreed freed(options.action == GCOptions::gcDeleteDead, results); + store->collectGarbage(options, results); + } +}; + +static auto rCmdStoreGC = registerCommand2<CmdStoreGC>({"store", "gc"}); diff --git a/src/nix/store-gc.md b/src/nix/store-gc.md new file mode 100644 index 000000000..956b3c872 --- /dev/null +++ b/src/nix/store-gc.md @@ -0,0 +1,21 @@ +R""( + +# Examples + +* Delete unreachable paths in the Nix store: + + ```console + # nix store gc + ``` + +* Delete up to 1 gigabyte of garbage: + + ```console + # nix store gc --max 1G + ``` + +# Description + +This command deletes unreachable paths in the Nix store. + +)"" diff --git a/src/nix/store-ls.md b/src/nix/store-ls.md new file mode 100644 index 000000000..836efce42 --- /dev/null +++ b/src/nix/store-ls.md @@ -0,0 +1,27 @@ +R""( + +# Examples + +* To list the contents of a store path in a binary cache: + + ```console + # nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10 + dr-xr-xr-x 0 ./bin + -r-xr-xr-x 38184 ./bin/hello + dr-xr-xr-x 0 ./share + … + ``` + +* To show information about a specific file in a binary cache: + + ```console + # nix store ls --store https://cache.nixos.org/ -l /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello + -r-xr-xr-x 38184 hello + ``` + +# Description + +This command shows information about *path* in a Nix store. *path* can +be a top-level store path or any file inside a store path. + +)"" diff --git a/src/nix/store-prefetch-file.md b/src/nix/store-prefetch-file.md new file mode 100644 index 000000000..1663b847b --- /dev/null +++ b/src/nix/store-prefetch-file.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Download a file to the Nix store: + + ```console + # nix store prefetch-file https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz + Downloaded 'https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz' to + '/nix/store/vbdbi42hgnc4h7pyqzp6h2yf77kw93aw-source' (hash + 'sha256-qKheVd5D0BervxMDbt+1hnTKE2aRWC8XCAwc0SeHt6s='). + ``` + +* Download a file and get the SHA-512 hash: + + ```console + # nix store prefetch-file --json --hash-type sha512 \ + https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz \ + | jq -r .hash + sha512-6XJxfym0TNH9knxeH4ZOvns6wElFy3uahunl2hJgovACCMEMXSy42s69zWVyGJALXTI+86tpDJGlIcAySEKBbA== + ``` + +# Description + +This command downloads the file *url* to the Nix store. It prints out +the resulting store path and the cryptographic hash of the contents of +the file. + +The name component of the store path defaults to the last component of +*url*, but this can be overriden using `--name`. + +)"" diff --git a/src/nix/store-repair.cc b/src/nix/store-repair.cc new file mode 100644 index 000000000..1c7a4392e --- /dev/null +++ b/src/nix/store-repair.cc @@ -0,0 +1,27 @@ +#include "command.hh" +#include "store-api.hh" + +using namespace nix; + +struct CmdStoreRepair : StorePathsCommand +{ + std::string description() override + { + return "repair store paths"; + } + + std::string doc() override + { + return + #include "store-repair.md" + ; + } + + void run(ref<Store> store, std::vector<StorePath> storePaths) override + { + for (auto & path : storePaths) + store->repairPath(path); + } +}; + +static auto rStoreRepair = registerCommand2<CmdStoreRepair>({"store", "repair"}); diff --git a/src/nix/store-repair.md b/src/nix/store-repair.md new file mode 100644 index 000000000..92d2205a9 --- /dev/null +++ b/src/nix/store-repair.md @@ -0,0 +1,32 @@ +R""( + +# Examples + +* Repair a store path, after determining that it is corrupt: + + ```console + # nix store verify /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + path '/nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10' was + modified! expected hash + 'sha256:1hd5vnh6xjk388gdk841vflicy8qv7qzj2hb7xlyh8lpb43j921l', got + 'sha256:1a25lf78x5wi6pfkrxalf0n13kdaca0bqmjqnp7wfjza2qz5ssgl' + + # nix store repair /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10 + ``` + +# Description + +This command attempts to "repair" the store paths specified by +*installables* by redownloading them using the available +substituters. If no substitutes are available, then repair is not +possible. + +> **Warning** +> +> During repair, there is a very small time window during which the old +> path (if it exists) is moved out of the way and replaced with the new +> path. If repair is interrupted in between, then the system may be left +> in a broken state (e.g., if the path contains a critical system +> component like the GNU C Library). + +)"" diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index 66ecc5b34..299ea40aa 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -19,14 +19,14 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand addFlag({ .longName = "profile", .shortName = 'p', - .description = "the Nix profile to upgrade", + .description = "The path to the Nix profile to upgrade.", .labels = {"profile-dir"}, .handler = {&profileDir} }); addFlag({ .longName = "nix-store-paths-url", - .description = "URL of the file that contains the store paths of the latest Nix release", + .description = "The URL of the file that contains the store paths of the latest Nix release.", .labels = {"url"}, .handler = {&storePathsUrl} }); @@ -37,18 +37,11 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand return "upgrade Nix to the latest stable version"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To upgrade Nix to the latest stable version:", - "nix upgrade-nix" - }, - Example{ - "To upgrade Nix in a specific profile:", - "nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile" - }, - }; + return + #include "upgrade-nix.md" + ; } Category category() override { return catNixInstallation; } diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md new file mode 100644 index 000000000..4d27daad9 --- /dev/null +++ b/src/nix/upgrade-nix.md @@ -0,0 +1,28 @@ +R""( + +# Examples + +* Upgrade Nix to the latest stable version: + + ```console + # nix upgrade-nix + ``` + +* Upgrade Nix in a specific profile: + + ```console + # nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile + ``` + +# Description + +This command upgrades Nix to the latest version. By default, it +locates the directory containing the `nix` binary in the `$PATH` +environment variable. If that directory is a Nix profile, it will +upgrade the `nix` package in that profile to the latest stable binary +release. + +You cannot use this command to upgrade Nix in the system profile of a +NixOS system (that is, if `nix` is found in `/run/current-system`). + +)"" diff --git a/src/nix/verify.cc b/src/nix/verify.cc index bcf85d7dd..b2963cf74 100644 --- a/src/nix/verify.cc +++ b/src/nix/verify.cc @@ -18,16 +18,24 @@ struct CmdVerify : StorePathsCommand CmdVerify() { - mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents); - mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust); + mkFlag(0, "no-contents", "Do not verify the contents of each store path.", &noContents); + mkFlag(0, "no-trust", "Do not verify whether each store path is trusted.", &noTrust); + addFlag({ .longName = "substituter", .shortName = 's', - .description = "use signatures from specified store", + .description = "Use signatures from the specified store.", .labels = {"store-uri"}, .handler = {[&](std::string s) { substituterUris.push_back(s); }} }); - mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded); + + addFlag({ + .longName = "sigs-needed", + .shortName = 'n', + .description = "Require that each path has at least *n* valid signatures.", + .labels = {"n"}, + .handler = {&sigsNeeded} + }); } std::string description() override @@ -35,18 +43,11 @@ struct CmdVerify : StorePathsCommand return "verify the integrity of store paths"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To verify the entire Nix store:", - "nix store verify --all" - }, - Example{ - "To check whether each path in the closure of Firefox has at least 2 signatures:", - "nix store verify -r -n2 --no-contents $(type -p firefox)" - }, - }; + return + #include "verify.md" + ; } void run(ref<Store> store, StorePaths storePaths) override diff --git a/src/nix/verify.md b/src/nix/verify.md new file mode 100644 index 000000000..1c43792e7 --- /dev/null +++ b/src/nix/verify.md @@ -0,0 +1,49 @@ +R""( + +# Examples + +* Verify the entire Nix store: + + ```console + # nix store verify --all + ``` + +* Check whether each path in the closure of Firefox has at least 2 + signatures: + + ```console + # nix store verify -r -n2 --no-contents $(type -p firefox) + ``` + +* Verify a store path in the binary cache `https://cache.nixos.org/`: + + ```console + # nix store verify --store https://cache.nixos.org/ \ + /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10 + ``` + +# Description + +This command verifies the integrity of the store paths *installables*, +or, if `--all` is given, the entire Nix store. For each path, it +checks that + +* its contents match the NAR hash recorded in the Nix database; and + +* it is *trusted*, that is, it is signed by at least one trusted + signing key, is content-addressed, or is built locally ("ultimately + trusted"). + +# Exit status + +The exit status of this command is the sum of the following values: + +* **1** if any path is corrupted (i.e. its contents don't match the + recorded NAR hash). + +* **2** if any path is untrusted. + +* **4** if any path couldn't be verified for any other reason (such as + an I/O error). + +)"" diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 57b9a2208..7a4ca5172 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -40,7 +40,7 @@ struct CmdWhyDepends : SourceExprCommand addFlag({ .longName = "all", .shortName = 'a', - .description = "show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path", + .description = "Show all edges in the dependency graph leading from *package* to *dependency*, rather than just a shortest path.", .handler = {&all, true}, }); } @@ -50,22 +50,11 @@ struct CmdWhyDepends : SourceExprCommand return "show why a package has another package in its closure"; } - Examples examples() override + std::string doc() override { - return { - Example{ - "To show one path through the dependency graph leading from Hello to Glibc:", - "nix why-depends nixpkgs#hello nixpkgs#glibc" - }, - Example{ - "To show all files and paths in the dependency graph leading from Thunderbird to libX11:", - "nix why-depends --all nixpkgs#thunderbird nixpkgs#xorg.libX11" - }, - Example{ - "To show why Glibc depends on itself:", - "nix why-depends nixpkgs#glibc nixpkgs#glibc" - }, - }; + return + #include "why-depends.md" + ; } Category category() override { return catSecondary; } diff --git a/src/nix/why-depends.md b/src/nix/why-depends.md new file mode 100644 index 000000000..dc13619e1 --- /dev/null +++ b/src/nix/why-depends.md @@ -0,0 +1,80 @@ +R""( + +# Examples + +* Show one path through the dependency graph leading from Hello to + Glibc: + + ```console + # nix why-depends nixpkgs#hello nixpkgs#glibc + /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10 + └───bin/hello: …...................../nix/store/9l06v7fc38c1x3r2iydl15ksgz0ysb82-glibc-2.32/lib/ld-linux-x86-64.… + → /nix/store/9l06v7fc38c1x3r2iydl15ksgz0ysb82-glibc-2.32 + ``` + +* Show all files and paths in the dependency graph leading from + Thunderbird to libX11: + + ```console + # nix why-depends --all nixpkgs#thunderbird nixpkgs#xorg.libX11 + /nix/store/qfc8729nzpdln1h0hvi1ziclsl3m84sr-thunderbird-78.5.1 + ├───lib/thunderbird/libxul.so: …6wrw-libxcb-1.14/lib:/nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0/lib:/nix/store/ssf… + │ → /nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0 + ├───lib/thunderbird/libxul.so: …pxyc-libXt-1.2.0/lib:/nix/store/1qj29ipxl2fyi2b13l39hdircq17gnk0-libXdamage-1.1.5/lib:/nix/store… + │ → /nix/store/1qj29ipxl2fyi2b13l39hdircq17gnk0-libXdamage-1.1.5 + │ ├───lib/libXdamage.so.1.1.0: …-libXfixes-5.0.3/lib:/nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0/lib:/nix/store/9l0… + │ │ → /nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0 + … + ``` + +* Show why Glibc depends on itself: + + ```console + # nix why-depends nixpkgs#glibc nixpkgs#glibc + /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31 + └───lib/ld-2.31.so: …che Do not use /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31/etc/ld.so.cache. --… + → /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31 + ``` + +* Show why Geeqie has a build-time dependency on `systemd`: + + ```console + # nix why-depends --derivation nixpkgs#geeqie nixpkgs#systemd + /nix/store/drrpq2fqlrbj98bmazrnww7hm1in3wgj-geeqie-1.4.drv + └───/: …atch.drv",["out"]),("/nix/store/qzh8dyq3lfbk3i1acbp7x9wh3il2imiv-gtk+3-3.24.21.drv",["dev"]),("/… + → /nix/store/qzh8dyq3lfbk3i1acbp7x9wh3il2imiv-gtk+3-3.24.21.drv + └───/: …16.0.drv",["dev"]),("/nix/store/8kp79fyslf3z4m3dpvlh6w46iaadz5c2-cups-2.3.3.drv",["dev"]),("/nix… + → /nix/store/8kp79fyslf3z4m3dpvlh6w46iaadz5c2-cups-2.3.3.drv + └───/: ….3.1.drv",["out"]),("/nix/store/yd3ihapyi5wbz1kjacq9dbkaq5v5hqjg-systemd-246.4.drv",["dev"]),("/… + → /nix/store/yd3ihapyi5wbz1kjacq9dbkaq5v5hqjg-systemd-246.4.drv + ``` + +# Description + +Nix automatically determines potential runtime dependencies between +store paths by scanning for the *hash parts* of store paths. For +instance, if there exists a store path +`/nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31`, and a file +inside another store path contains the string `9df65igw…`, then the +latter store path *refers* to the former, and thus might need it at +runtime. Nix always maintains the existence of the transitive closure +of a store path under the references relationship; it is therefore not +possible to install a store path without having all of its references +present. + +Sometimes Nix packages end up with unexpected runtime dependencies; +for instance, a reference to a compiler might accidentally end up in a +binary, causing the former to be in the latter's closure. This kind of +*closure size bloat* is undesirable. + +`nix why-depends` allows you to diagnose the cause of such issues. It +shows why the store path *package* depends on the store path +*dependency*, by showing a shortest sequence in the references graph +from the former to the latter. Also, for each node along this path, it +shows a file fragment containing a reference to the next store path in +the sequence. + +To show why derivation *package* has a build-time rather than runtime +dependency on derivation *dependency*, use `--derivation`. + +)"" |