diff options
Diffstat (limited to 'src')
53 files changed, 3986 insertions, 201 deletions
diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc index 10c9c16bb..47a57f91c 100644 --- a/src/libexpr/common-eval-args.cc +++ b/src/libexpr/common-eval-args.cc @@ -4,6 +4,8 @@ #include "util.hh" #include "eval.hh" #include "fetchers.hh" +#include "registry.hh" +#include "flake/flakeref.hh" #include "store-api.hh" namespace nix { @@ -28,6 +30,26 @@ MixEvalArgs::MixEvalArgs() .description("add a path to the list of locations used to look up <...> file names") .label("path") .handler([&](std::string s) { searchPath.push_back(s); }); + + mkFlag() + .longName("impure") + .description("allow access to mutable paths and repositories") + .handler([&](std::vector<std::string> ss) { + evalSettings.pureEval = false; + }); + + mkFlag() + .longName("override-flake") + .labels({"original-ref", "resolved-ref"}) + .description("override a flake registry value") + .arity(2) + .handler([&](std::vector<std::string> ss) { + auto from = parseFlakeRef(ss[0], absPath(".")); + auto to = parseFlakeRef(ss[1], absPath(".")); + fetchers::Attrs extraAttrs; + if (to.subdir != "") extraAttrs["dir"] = to.subdir; + fetchers::overrideRegistry(from.input, to.input, extraAttrs); + }); } Bindings * MixEvalArgs::getAutoArgs(EvalState & state) diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f963a42ca..c5945adf0 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -8,6 +8,7 @@ #include "filetransfer.hh" #include "json.hh" #include "function-trace.hh" +#include "flake/flake.hh" #include <algorithm> #include <chrono> @@ -153,12 +154,12 @@ const Value *getPrimOp(const Value &v) { } -string showType(const Value & v) +string showType(ValueType type) { - switch (v.type) { + switch (type) { case tInt: return "an integer"; - case tBool: return "a boolean"; - case tString: return v.string.context ? "a string with context" : "a string"; + case tBool: return "a Boolean"; + case tString: return "a string"; case tPath: return "a path"; case tNull: return "null"; case tAttrs: return "a set"; @@ -167,14 +168,39 @@ string showType(const Value & v) case tApp: return "a function application"; case tLambda: return "a function"; case tBlackhole: return "a black hole"; + case tPrimOp: return "a built-in function"; + case tPrimOpApp: return "a partially applied built-in function"; + case tExternal: return "an external value"; + case tFloat: return "a float"; + } + abort(); +} + + +string showType(const Value & v) +{ + switch (v.type) { + case tString: return v.string.context ? "a string with context" : "a string"; case tPrimOp: return fmt("the built-in function '%s'", string(v.primOp->name)); case tPrimOpApp: return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name)); case tExternal: return v.external->showType(); - case tFloat: return "a float"; + default: + return showType(v.type); } - abort(); +} + + +bool Value::isTrivial() const +{ + return + type != tApp + && type != tPrimOpApp + && (type != tThunk + || (dynamic_cast<ExprAttrs *>(thunk.expr) + && ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty()) + || dynamic_cast<ExprLambda *>(thunk.expr)); } @@ -315,6 +341,8 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store) , sOutputHash(symbols.create("outputHash")) , sOutputHashAlgo(symbols.create("outputHashAlgo")) , sOutputHashMode(symbols.create("outputHashMode")) + , sDescription(symbols.create("description")) + , sSelf(symbols.create("self")) , repair(NoRepair) , store(store) , baseEnv(allocEnv(128)) @@ -463,14 +491,21 @@ Value * EvalState::addConstant(const string & name, Value & v) Value * EvalState::addPrimOp(const string & name, size_t arity, PrimOpFun primOp) { + auto name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; + Symbol sym = symbols.create(name2); + + /* Hack to make constants lazy: turn them into a application of + the primop to a dummy value. */ if (arity == 0) { + auto vPrimOp = allocValue(); + vPrimOp->type = tPrimOp; + vPrimOp->primOp = new PrimOp(primOp, 1, sym); Value v; - primOp(*this, noPos, nullptr, v); + mkApp(v, *vPrimOp, *vPrimOp); return addConstant(name, v); } + Value * v = allocValue(); - string name2 = string(name, 0, 2) == "__" ? string(name, 2) : name; - Symbol sym = symbols.create(name2); v->type = tPrimOp; v->primOp = new PrimOp(primOp, arity, sym); staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl; @@ -736,7 +771,7 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env) } -void EvalState::evalFile(const Path & path_, Value & v) +void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial) { auto path = checkSourcePath(path_); @@ -765,6 +800,11 @@ void EvalState::evalFile(const Path & path_, Value & v) fileParseCache[path2] = e; try { + // Enforce that 'flake.nix' is a direct attrset, not a + // computation. + if (mustBeTrivial && + !(dynamic_cast<ExprAttrs *>(e))) + throw Error("file '%s' must be an attribute set", path); eval(e, v); } catch (Error & e) { addErrorPrefix(e, "while evaluating the file '%1%':\n", path2); diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index 1485dc7fe..ab4d2bc4b 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -4,13 +4,13 @@ #include "value.hh" #include "nixexpr.hh" #include "symbol-table.hh" -#include "hash.hh" #include "config.hh" #include <regex> #include <map> #include <optional> #include <unordered_map> +#include <mutex> namespace nix { @@ -74,7 +74,8 @@ public: sSystem, sOverrides, sOutputs, sOutputName, sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sBuilder, sArgs, - sOutputHash, sOutputHashAlgo, sOutputHashMode; + sOutputHash, sOutputHashAlgo, sOutputHashMode, + sDescription, sSelf; Symbol sDerivationNix; /* If set, force copying files to the Nix store even if they @@ -89,6 +90,7 @@ public: const ref<Store> store; + private: SrcToStore srcToStore; @@ -151,8 +153,9 @@ public: Expr * parseStdin(); /* Evaluate an expression read from the given file to normal - form. */ - void evalFile(const Path & path, Value & v); + form. Optionally enforce that the top-level expression is + trivial (i.e. doesn't require arbitrary computation). */ + void evalFile(const Path & path, Value & v, bool mustBeTrivial = false); void resetFileCache(); @@ -324,6 +327,7 @@ private: /* Return a string representing the type of the value `v'. */ +string showType(ValueType type); string showType(const Value & v); /* Decode a context string ‘!<name>!<path>’ into a pair <path, diff --git a/src/libexpr/flake/call-flake.nix b/src/libexpr/flake/call-flake.nix new file mode 100644 index 000000000..8ee17b8f4 --- /dev/null +++ b/src/libexpr/flake/call-flake.nix @@ -0,0 +1,29 @@ +lockFileStr: rootSrc: rootSubdir: + +let + + lockFile = builtins.fromJSON lockFileStr; + + allNodes = + builtins.mapAttrs + (key: node: + let + sourceInfo = + if key == lockFile.root + then rootSrc + else fetchTree ({ inherit (node.info) narHash; } // removeAttrs node.locked ["dir"]); + subdir = if key == lockFile.root then rootSubdir else node.locked.dir or ""; + flake = import (sourceInfo + (if subdir != "" then "/" else "") + subdir + "/flake.nix"); + inputs = builtins.mapAttrs (inputName: key: allNodes.${key}) (node.inputs or {}); + outputs = flake.outputs (inputs // { self = result; }); + result = outputs // sourceInfo // { inherit inputs; inherit outputs; inherit sourceInfo; }; + in + if node.flake or true then + assert builtins.isFunction flake.outputs; + result + else + sourceInfo + ) + lockFile.nodes; + +in allNodes.${lockFile.root} diff --git a/src/libexpr/flake/eval-cache.cc b/src/libexpr/flake/eval-cache.cc new file mode 100644 index 000000000..8d01ef0fc --- /dev/null +++ b/src/libexpr/flake/eval-cache.cc @@ -0,0 +1,116 @@ +#include "eval-cache.hh" +#include "sqlite.hh" +#include "eval.hh" + +#include <set> + +namespace nix::flake { + +static const char * schema = R"sql( + +create table if not exists Fingerprints ( + fingerprint blob primary key not null, + timestamp integer not null +); + +create table if not exists Attributes ( + fingerprint blob not null, + attrPath text not null, + type integer, + value text, + primary key (fingerprint, attrPath), + foreign key (fingerprint) references Fingerprints(fingerprint) on delete cascade +); +)sql"; + +struct EvalCache::State +{ + SQLite db; + SQLiteStmt insertFingerprint; + SQLiteStmt insertAttribute; + SQLiteStmt queryAttribute; + std::set<Fingerprint> fingerprints; +}; + +EvalCache::EvalCache() + : _state(std::make_unique<Sync<State>>()) +{ + auto state(_state->lock()); + + Path dbPath = getCacheDir() + "/nix/eval-cache-v1.sqlite"; + createDirs(dirOf(dbPath)); + + state->db = SQLite(dbPath); + state->db.isCache(); + state->db.exec(schema); + + state->insertFingerprint.create(state->db, + "insert or ignore into Fingerprints(fingerprint, timestamp) values (?, ?)"); + + state->insertAttribute.create(state->db, + "insert or replace into Attributes(fingerprint, attrPath, type, value) values (?, ?, ?, ?)"); + + state->queryAttribute.create(state->db, + "select type, value from Attributes where fingerprint = ? and attrPath = ?"); +} + +enum ValueType { + Derivation = 1, +}; + +void EvalCache::addDerivation( + const Fingerprint & fingerprint, + const std::string & attrPath, + const Derivation & drv) +{ + if (!evalSettings.pureEval) return; + + auto state(_state->lock()); + + if (state->fingerprints.insert(fingerprint).second) + // FIXME: update timestamp + state->insertFingerprint.use() + (fingerprint.hash, fingerprint.hashSize) + (time(0)).exec(); + + state->insertAttribute.use() + (fingerprint.hash, fingerprint.hashSize) + (attrPath) + (ValueType::Derivation) + (std::string(drv.drvPath.to_string()) + " " + std::string(drv.outPath.to_string()) + " " + drv.outputName).exec(); +} + +std::optional<EvalCache::Derivation> EvalCache::getDerivation( + const Fingerprint & fingerprint, + const std::string & attrPath) +{ + if (!evalSettings.pureEval) return {}; + + auto state(_state->lock()); + + auto queryAttribute(state->queryAttribute.use() + (fingerprint.hash, fingerprint.hashSize) + (attrPath)); + if (!queryAttribute.next()) return {}; + + // FIXME: handle negative results + + auto type = (ValueType) queryAttribute.getInt(0); + auto s = queryAttribute.getStr(1); + + if (type != ValueType::Derivation) return {}; + + auto ss = tokenizeString<std::vector<std::string>>(s, " "); + + debug("evaluation cache hit for '%s'", attrPath); + + return Derivation { StorePath::fromBaseName(ss[0]), StorePath::fromBaseName(ss[1]), ss[2] }; +} + +EvalCache & EvalCache::singleton() +{ + static std::unique_ptr<EvalCache> evalCache(new EvalCache()); + return *evalCache; +} + +} diff --git a/src/libexpr/flake/eval-cache.hh b/src/libexpr/flake/eval-cache.hh new file mode 100644 index 000000000..f81d48ba5 --- /dev/null +++ b/src/libexpr/flake/eval-cache.hh @@ -0,0 +1,40 @@ +#pragma once + +#include "sync.hh" +#include "flake.hh" +#include "path.hh" + +namespace nix { struct SQLite; struct SQLiteStmt; } + +namespace nix::flake { + +class EvalCache +{ + struct State; + + std::unique_ptr<Sync<State>> _state; + + EvalCache(); + +public: + + struct Derivation + { + StorePath drvPath; + StorePath outPath; + std::string outputName; + }; + + void addDerivation( + const Fingerprint & fingerprint, + const std::string & attrPath, + const Derivation & drv); + + std::optional<Derivation> getDerivation( + const Fingerprint & fingerprint, + const std::string & attrPath); + + static EvalCache & singleton(); +}; + +} diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc new file mode 100644 index 000000000..212eceeac --- /dev/null +++ b/src/libexpr/flake/flake.cc @@ -0,0 +1,666 @@ +#include "flake.hh" +#include "lockfile.hh" +#include "primops.hh" +#include "eval-inline.hh" +#include "store-api.hh" +#include "fetchers.hh" +#include "finally.hh" + +namespace nix { + +using namespace flake; + +namespace flake { + +/* If 'allowLookup' is true, then resolve 'flakeRef' using the + registries. */ +static FlakeRef maybeLookupFlake( + ref<Store> store, + const FlakeRef & flakeRef, + bool allowLookup) +{ + if (!flakeRef.input->isDirect()) { + if (allowLookup) + return flakeRef.resolve(store); + else + throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", flakeRef); + } else + return flakeRef; +} + +typedef std::vector<std::pair<FlakeRef, FlakeRef>> FlakeCache; + +static FlakeRef lookupInFlakeCache( + const FlakeCache & flakeCache, + const FlakeRef & flakeRef) +{ + // FIXME: inefficient. + for (auto & i : flakeCache) { + if (flakeRef == i.first) { + debug("mapping '%s' to previously seen input '%s' -> '%s", + flakeRef, i.first, i.second); + return i.second; + } + } + + return flakeRef; +} + +static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree( + EvalState & state, + const FlakeRef & originalRef, + std::optional<TreeInfo> treeInfo, + bool allowLookup, + FlakeCache & flakeCache) +{ + /* The tree may already be in the Nix store, or it could be + substituted (which is often faster than fetching from the + original source). So check that. */ + if (treeInfo && originalRef.input->isDirect() && originalRef.input->isImmutable()) { + try { + auto storePath = treeInfo->computeStorePath(*state.store); + + state.store->ensurePath(storePath); + + debug("using substituted/cached input '%s' in '%s'", + originalRef, state.store->printStorePath(storePath)); + + auto actualPath = state.store->toRealPath(storePath); + + if (state.allowedPaths) + state.allowedPaths->insert(actualPath); + + return { + Tree { + .actualPath = actualPath, + .storePath = std::move(storePath), + .info = *treeInfo, + }, + originalRef, + originalRef + }; + } catch (Error & e) { + debug("substitution of input '%s' failed: %s", originalRef, e.what()); + } + } + + auto resolvedRef = lookupInFlakeCache(flakeCache, + maybeLookupFlake(state.store, + lookupInFlakeCache(flakeCache, originalRef), allowLookup)); + + auto [tree, lockedRef] = resolvedRef.fetchTree(state.store); + + debug("got tree '%s' from '%s'", + state.store->printStorePath(tree.storePath), lockedRef); + + flakeCache.push_back({originalRef, lockedRef}); + flakeCache.push_back({resolvedRef, lockedRef}); + + if (state.allowedPaths) + state.allowedPaths->insert(tree.actualPath); + + if (treeInfo) + assert(tree.storePath == treeInfo->computeStorePath(*state.store)); + + return {std::move(tree), resolvedRef, lockedRef}; +} + +static void expectType(EvalState & state, ValueType type, + Value & value, const Pos & pos) +{ + if (value.type == tThunk && value.isTrivial()) + state.forceValue(value, pos); + if (value.type != type) + throw Error("expected %s but got %s at %s", + showType(type), showType(value.type), pos); +} + +static std::map<FlakeId, FlakeInput> parseFlakeInputs( + EvalState & state, Value * value, const Pos & pos); + +static FlakeInput parseFlakeInput(EvalState & state, + const std::string & inputName, Value * value, const Pos & pos) +{ + expectType(state, tAttrs, *value, pos); + + FlakeInput input { + .ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}}) + }; + + auto sInputs = state.symbols.create("inputs"); + auto sUrl = state.symbols.create("url"); + auto sUri = state.symbols.create("uri"); // FIXME: remove soon + auto sFlake = state.symbols.create("flake"); + auto sFollows = state.symbols.create("follows"); + + fetchers::Attrs attrs; + std::optional<std::string> url; + + for (nix::Attr attr : *(value->attrs)) { + try { + if (attr.name == sUrl || attr.name == sUri) { + expectType(state, tString, *attr.value, *attr.pos); + url = attr.value->string.s; + attrs.emplace("url", *url); + } else if (attr.name == sFlake) { + expectType(state, tBool, *attr.value, *attr.pos); + input.isFlake = attr.value->boolean; + } else if (attr.name == sInputs) { + input.overrides = parseFlakeInputs(state, attr.value, *attr.pos); + } else if (attr.name == sFollows) { + expectType(state, tString, *attr.value, *attr.pos); + input.follows = parseInputPath(attr.value->string.s); + } else { + state.forceValue(*attr.value); + if (attr.value->type == tString) + attrs.emplace(attr.name, attr.value->string.s); + else + throw TypeError("flake input attribute '%s' is %s while a string is expected", + attr.name, showType(*attr.value)); + } + } catch (Error & e) { + e.addPrefix(fmt("in flake attribute '%s' at '%s':\n", attr.name, *attr.pos)); + throw; + } + } + + if (attrs.count("type")) + try { + input.ref = FlakeRef::fromAttrs(attrs); + } catch (Error & e) { + e.addPrefix(fmt("in flake input at '%s':\n", pos)); + throw; + } + else { + attrs.erase("url"); + if (!attrs.empty()) + throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos); + if (url) + input.ref = parseFlakeRef(*url); + } + + return input; +} + +static std::map<FlakeId, FlakeInput> parseFlakeInputs( + EvalState & state, Value * value, const Pos & pos) +{ + std::map<FlakeId, FlakeInput> inputs; + + expectType(state, tAttrs, *value, pos); + + for (nix::Attr & inputAttr : *(*value).attrs) { + inputs.emplace(inputAttr.name, + parseFlakeInput(state, + inputAttr.name, + inputAttr.value, + *inputAttr.pos)); + } + + return inputs; +} + +static Flake getFlake( + EvalState & state, + const FlakeRef & originalRef, + std::optional<TreeInfo> treeInfo, + bool allowLookup, + FlakeCache & flakeCache) +{ + auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + state, originalRef, treeInfo, allowLookup, flakeCache); + + // Guard against symlink attacks. + auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix"); + if (!isInDir(flakeFile, sourceInfo.actualPath)) + throw Error("'flake.nix' file of flake '%s' escapes from '%s'", + lockedRef, state.store->printStorePath(sourceInfo.storePath)); + + Flake flake { + .originalRef = originalRef, + .resolvedRef = resolvedRef, + .lockedRef = lockedRef, + .sourceInfo = std::make_shared<fetchers::Tree>(std::move(sourceInfo)) + }; + + if (!pathExists(flakeFile)) + throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); + + Value vInfo; + state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack + + expectType(state, tAttrs, vInfo, Pos(state.symbols.create(flakeFile), 0, 0)); + + auto sEdition = state.symbols.create("edition"); // FIXME: remove soon + auto sEpoch = state.symbols.create("epoch"); // FIXME: remove soon + + if (vInfo.attrs->get(sEdition)) + warn("flake '%s' has deprecated attribution 'edition'", lockedRef); + + if (vInfo.attrs->get(sEpoch)) + warn("flake '%s' has deprecated attribution 'epoch'", lockedRef); + + if (auto description = vInfo.attrs->get(state.sDescription)) { + expectType(state, tString, *description->value, *description->pos); + flake.description = description->value->string.s; + } + + auto sInputs = state.symbols.create("inputs"); + + if (auto inputs = vInfo.attrs->get(sInputs)) + flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos); + + auto sOutputs = state.symbols.create("outputs"); + + if (auto outputs = vInfo.attrs->get(sOutputs)) { + expectType(state, tLambda, *outputs->value, *outputs->pos); + flake.vOutputs = outputs->value; + + if (flake.vOutputs->lambda.fun->matchAttrs) { + for (auto & formal : flake.vOutputs->lambda.fun->formals->formals) { + if (formal.name != state.sSelf) + flake.inputs.emplace(formal.name, FlakeInput { + .ref = parseFlakeRef(formal.name) + }); + } + } + + } else + throw Error("flake '%s' lacks attribute 'outputs'", lockedRef); + + for (auto & attr : *vInfo.attrs) { + if (attr.name != sEdition && + attr.name != sEpoch && + attr.name != state.sDescription && + attr.name != sInputs && + attr.name != sOutputs) + throw Error("flake '%s' has an unsupported attribute '%s', at %s", + lockedRef, attr.name, *attr.pos); + } + + return flake; +} + +Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup) +{ + FlakeCache flakeCache; + return getFlake(state, originalRef, {}, allowLookup, flakeCache); +} + +/* Compute an in-memory lock file for the specified top-level flake, + and optionally write it to file, it the flake is writable. */ +LockedFlake lockFlake( + EvalState & state, + const FlakeRef & topRef, + const LockFlags & lockFlags) +{ + settings.requireExperimentalFeature("flakes"); + + FlakeCache flakeCache; + + auto flake = getFlake(state, topRef, {}, lockFlags.useRegistries, flakeCache); + + // FIXME: symlink attack + auto oldLockFile = LockFile::read( + flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"); + + debug("old lock file: %s", oldLockFile); + + // FIXME: check whether all overrides are used. + std::map<InputPath, FlakeInput> overrides; + std::set<InputPath> overridesUsed, updatesUsed; + + for (auto & i : lockFlags.inputOverrides) + overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); + + LockFile newLockFile; + + std::vector<FlakeRef> parents; + std::map<InputPath, InputPath> follows; + + std::function<void( + const FlakeInputs & flakeInputs, + std::shared_ptr<Node> node, + const InputPath & inputPathPrefix, + std::shared_ptr<const Node> oldNode)> + computeLocks; + + computeLocks = [&]( + const FlakeInputs & flakeInputs, + std::shared_ptr<Node> node, + const InputPath & inputPathPrefix, + std::shared_ptr<const Node> oldNode) + { + debug("computing lock file node '%s'", concatStringsSep("/", inputPathPrefix)); + + /* Get the overrides (i.e. attributes of the form + 'inputs.nixops.inputs.nixpkgs.url = ...'). */ + // FIXME: check this + for (auto & [id, input] : flake.inputs) { + for (auto & [idOverride, inputOverride] : input.overrides) { + auto inputPath(inputPathPrefix); + inputPath.push_back(id); + inputPath.push_back(idOverride); + overrides.insert_or_assign(inputPath, inputOverride); + } + } + + /* Go over the flake inputs, resolve/fetch them if + necessary (i.e. if they're new or the flakeref changed + from what's in the lock file). */ + for (auto & [id, input2] : flakeInputs) { + auto inputPath(inputPathPrefix); + inputPath.push_back(id); + auto inputPathS = concatStringsSep("/", inputPath); + debug("computing input '%s'", concatStringsSep("/", inputPath)); + + /* Do we have an override for this input from one of the + ancestors? */ + auto i = overrides.find(inputPath); + bool hasOverride = i != overrides.end(); + if (hasOverride) overridesUsed.insert(inputPath); + auto & input = hasOverride ? i->second : input2; + + /* Resolve 'follows' later (since it may refer to an input + path we haven't processed yet. */ + if (input.follows) { + if (hasOverride) + /* 'follows' from an override is relative to the + root of the graph. */ + follows.insert_or_assign(inputPath, *input.follows); + else { + /* Otherwise, it's relative to the current flake. */ + InputPath path(inputPathPrefix); + for (auto & i : *input.follows) path.push_back(i); + follows.insert_or_assign(inputPath, path); + } + continue; + } + + /* Do we have an entry in the existing lock file? And we + don't have a --update-input flag for this input? */ + std::shared_ptr<const LockedNode> oldLock; + + updatesUsed.insert(inputPath); + + if (oldNode && !lockFlags.inputUpdates.count(inputPath)) { + auto oldLockIt = oldNode->inputs.find(id); + if (oldLockIt != oldNode->inputs.end()) + oldLock = std::dynamic_pointer_cast<const LockedNode>(oldLockIt->second); + } + + if (oldLock + && oldLock->originalRef == input.ref + && !hasOverride) + { + debug("keeping existing input '%s'", inputPathS); + + /* Copy the input from the old lock since its flakeref + didn't change and there is no override from a + higher level flake. */ + auto childNode = std::make_shared<LockedNode>( + oldLock->lockedRef, oldLock->originalRef, oldLock->info, oldLock->isFlake); + + node->inputs.insert_or_assign(id, childNode); + + /* If we have an --update-input flag for an input + of this input, then we must fetch the flake to + to update it. */ + auto lb = lockFlags.inputUpdates.lower_bound(inputPath); + + auto hasChildUpdate = + lb != lockFlags.inputUpdates.end() + && lb->size() > inputPath.size() + && std::equal(inputPath.begin(), inputPath.end(), lb->begin()); + + if (hasChildUpdate) { + auto inputFlake = getFlake( + state, oldLock->lockedRef, oldLock->info, false, flakeCache); + computeLocks(inputFlake.inputs, childNode, inputPath, oldLock); + } else { + /* No need to fetch this flake, we can be + lazy. However there may be new overrides on the + inputs of this flake, so we need to check + those. */ + FlakeInputs fakeInputs; + + for (auto & i : oldLock->inputs) { + auto lockedNode = std::dynamic_pointer_cast<LockedNode>(i.second); + // Note: this node is not locked in case + // of a circular reference back to the root. + if (lockedNode) + fakeInputs.emplace(i.first, FlakeInput { + .ref = lockedNode->originalRef + }); + else { + InputPath path(inputPath); + path.push_back(i.first); + follows.insert_or_assign(path, InputPath()); + } + } + + computeLocks(fakeInputs, childNode, inputPath, oldLock); + } + + } else { + /* We need to create a new lock file entry. So fetch + this input. */ + + if (!lockFlags.allowMutable && !input.ref.input->isImmutable()) + throw Error("cannot update flake input '%s' in pure mode", inputPathS); + + if (input.isFlake) { + auto inputFlake = getFlake(state, input.ref, {}, lockFlags.useRegistries, flakeCache); + + /* Note: in case of an --override-input, we use + the *original* ref (input2.ref) for the + "original" field, rather than the + override. This ensures that the override isn't + nuked the next time we update the lock + file. That is, overrides are sticky unless you + use --no-write-lock-file. */ + auto childNode = std::make_shared<LockedNode>( + inputFlake.lockedRef, input2.ref, inputFlake.sourceInfo->info); + + node->inputs.insert_or_assign(id, childNode); + + /* Guard against circular flake imports. */ + for (auto & parent : parents) + if (parent == input.ref) + throw Error("found circular import of flake '%s'", parent); + parents.push_back(input.ref); + Finally cleanup([&]() { parents.pop_back(); }); + + /* Recursively process the inputs of this + flake. Also, unless we already have this flake + in the top-level lock file, use this flake's + own lock file. */ + computeLocks( + inputFlake.inputs, childNode, inputPath, + oldLock + ? std::dynamic_pointer_cast<const Node>(oldLock) + : LockFile::read( + inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root); + } + + else { + auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree( + state, input.ref, {}, lockFlags.useRegistries, flakeCache); + node->inputs.insert_or_assign(id, + std::make_shared<LockedNode>(lockedRef, input.ref, sourceInfo.info, false)); + } + } + } + }; + + computeLocks( + flake.inputs, newLockFile.root, {}, + lockFlags.recreateLockFile ? nullptr : oldLockFile.root); + + /* Insert edges for 'follows' overrides. */ + for (auto & [from, to] : follows) { + debug("adding 'follows' node from '%s' to '%s'", + concatStringsSep("/", from), + concatStringsSep("/", to)); + + assert(!from.empty()); + + InputPath fromParent(from); + fromParent.pop_back(); + + auto fromParentNode = newLockFile.root->findInput(fromParent); + assert(fromParentNode); + + auto toNode = newLockFile.root->findInput(to); + if (!toNode) + throw Error("flake input '%s' follows non-existent flake input '%s'", + concatStringsSep("/", from), + concatStringsSep("/", to)); + + fromParentNode->inputs.insert_or_assign(from.back(), toNode); + } + + for (auto & i : lockFlags.inputOverrides) + if (!overridesUsed.count(i.first)) + warn("the flag '--override-input %s %s' does not match any input", + concatStringsSep("/", i.first), i.second); + + for (auto & i : lockFlags.inputUpdates) + if (!updatesUsed.count(i)) + warn("the flag '--update-input %s' does not match any input", concatStringsSep("/", i)); + + debug("new lock file: %s", newLockFile); + + /* Check whether we need to / can write the new lock file. */ + if (!(newLockFile == oldLockFile)) { + + auto diff = diffLockFiles(oldLockFile, newLockFile); + + if (!(oldLockFile == LockFile())) + printInfo("inputs of flake '%s' changed:\n%s", topRef, chomp(diff)); + + if (lockFlags.writeLockFile) { + if (auto sourcePath = topRef.input->getSourcePath()) { + if (!newLockFile.isImmutable()) { + if (settings.warnDirty) + warn("will not write lock file of flake '%s' because it has a mutable input", topRef); + } else { + if (!lockFlags.updateLockFile) + throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); + + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + + auto path = *sourcePath + "/" + relPath; + + bool lockFileExists = pathExists(path); + + if (lockFileExists) + warn("updating lock file '%s'", path); + else + warn("creating lock file '%s'", path); + + newLockFile.write(path); + + topRef.input->markChangedFile( + (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", + lockFlags.commitLockFile + ? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s", + relPath, lockFileExists ? "Update" : "Add", diff)) + : std::nullopt); + + /* Rewriting the lockfile changed the top-level + repo, so we should re-read it. FIXME: we could + also just clear the 'rev' field... */ + auto prevLockedRef = flake.lockedRef; + FlakeCache dummyCache; + flake = getFlake(state, topRef, {}, lockFlags.useRegistries, dummyCache); + + if (lockFlags.commitLockFile && + flake.lockedRef.input->getRev() && + prevLockedRef.input->getRev() != flake.lockedRef.input->getRev()) + warn("committed new revision '%s'", flake.lockedRef.input->getRev()->gitRev()); + + /* Make sure that we picked up the change, + i.e. the tree should usually be dirty + now. Corner case: we could have reverted from a + dirty to a clean tree! */ + if (flake.lockedRef.input == prevLockedRef.input + && !flake.lockedRef.input->isImmutable()) + throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef); + } + } else + throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef); + } else + warn("not writing modified lock file of flake '%s'", topRef); + } + + return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) }; +} + +void callFlake(EvalState & state, + const LockedFlake & lockedFlake, + Value & vRes) +{ + auto vLocks = state.allocValue(); + auto vRootSrc = state.allocValue(); + auto vRootSubdir = state.allocValue(); + auto vTmp1 = state.allocValue(); + auto vTmp2 = state.allocValue(); + + mkString(*vLocks, lockedFlake.lockFile.to_string()); + + emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc); + + mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir); + + static Value * vCallFlake = nullptr; + + if (!vCallFlake) { + vCallFlake = state.allocValue(); + state.eval(state.parseExprFromString( + #include "call-flake.nix.gen.hh" + , "/"), *vCallFlake); + } + + state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos); + state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); + state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); +} + +static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto flakeRefS = state.forceStringNoCtx(*args[0], pos); + auto flakeRef = parseFlakeRef(flakeRefS); + if (evalSettings.pureEval && !flakeRef.input->isImmutable()) + throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos); + + callFlake(state, + lockFlake(state, flakeRef, + LockFlags { + .updateLockFile = false, + .useRegistries = !evalSettings.pureEval, + .allowMutable = !evalSettings.pureEval, + }), + v); +} + +static RegisterPrimOp r2("getFlake", 1, prim_getFlake); + +} + +Fingerprint LockedFlake::getFingerprint() const +{ + // FIXME: as an optimization, if the flake contains a lock file + // and we haven't changed it, then it's sufficient to use + // flake.sourceInfo.storePath for the fingerprint. + return hashString(htSHA256, + fmt("%s;%d;%d;%s", + flake.sourceInfo->storePath.to_string(), + flake.sourceInfo->info.revCount.value_or(0), + flake.sourceInfo->info.lastModified.value_or(0), + lockFile)); +} + +Flake::~Flake() { } + +} diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh new file mode 100644 index 000000000..107498c1b --- /dev/null +++ b/src/libexpr/flake/flake.hh @@ -0,0 +1,110 @@ +#pragma once + +#include "types.hh" +#include "flakeref.hh" +#include "lockfile.hh" + +namespace nix { + +struct Value; +class EvalState; + +namespace fetchers { struct Tree; } + +namespace flake { + +struct FlakeInput; + +typedef std::map<FlakeId, FlakeInput> FlakeInputs; + +struct FlakeInput +{ + FlakeRef ref; + bool isFlake = true; + std::optional<InputPath> follows; + FlakeInputs overrides; +}; + +struct Flake +{ + FlakeRef originalRef; + FlakeRef resolvedRef; + FlakeRef lockedRef; + std::optional<std::string> description; + std::shared_ptr<const fetchers::Tree> sourceInfo; + FlakeInputs inputs; + Value * vOutputs; // FIXME: gc + ~Flake(); +}; + +Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool allowLookup); + +/* Fingerprint of a locked flake; used as a cache key. */ +typedef Hash Fingerprint; + +struct LockedFlake +{ + Flake flake; + LockFile lockFile; + + Fingerprint getFingerprint() const; +}; + +struct LockFlags +{ + /* Whether to ignore the existing lock file, creating a new one + from scratch. */ + bool recreateLockFile = false; + + /* Whether to update the lock file at all. If set to false, if any + change to the lock file is needed (e.g. when an input has been + added to flake.nix), you get a fatal error. */ + bool updateLockFile = true; + + /* Whether to write the lock file to disk. If set to true, if the + any changes to the lock file are needed and the flake is not + writable (i.e. is not a local Git working tree or similar), you + get a fatal error. If set to false, Nix will use the modified + lock file in memory only, without writing it to disk. */ + bool writeLockFile = true; + + /* Whether to use the registries to lookup indirect flake + references like 'nixpkgs'. */ + bool useRegistries = true; + + /* Whether mutable flake references (i.e. those without a Git + revision or similar) without a corresponding lock are + allowed. Mutable flake references with a lock are always + allowed. */ + bool allowMutable = true; + + /* Whether to commit changes to flake.lock. */ + bool commitLockFile = false; + + /* Flake inputs to be overriden. */ + std::map<InputPath, FlakeRef> inputOverrides; + + /* Flake inputs to be updated. This means that any existing lock + for those inputs will be ignored. */ + std::set<InputPath> inputUpdates; +}; + +LockedFlake lockFlake( + EvalState & state, + const FlakeRef & flakeRef, + const LockFlags & lockFlags); + +void callFlake( + EvalState & state, + const LockedFlake & lockedFlake, + Value & v); + +} + +void emitTreeAttrs( + EvalState & state, + const fetchers::Tree & tree, + std::shared_ptr<const fetchers::Input> input, + Value & v); + +} diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc new file mode 100644 index 000000000..de91f2eed --- /dev/null +++ b/src/libexpr/flake/flakeref.cc @@ -0,0 +1,188 @@ +#include "flakeref.hh" +#include "store-api.hh" +#include "url.hh" +#include "fetchers.hh" +#include "registry.hh" + +namespace nix { + +#if 0 +// 'dir' path elements cannot start with a '.'. We also reject +// potentially dangerous characters like ';'. +const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)"; +const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*"; +#endif + +std::string FlakeRef::to_string() const +{ + auto url = input->toURL(); + if (subdir != "") + url.query.insert_or_assign("dir", subdir); + return url.to_string(); +} + +fetchers::Attrs FlakeRef::toAttrs() const +{ + auto attrs = input->toAttrs(); + if (subdir != "") + attrs.emplace("dir", subdir); + return attrs; +} + +std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) +{ + str << flakeRef.to_string(); + return str; +} + +bool FlakeRef::operator ==(const FlakeRef & other) const +{ + return *input == *other.input && subdir == other.subdir; +} + +FlakeRef FlakeRef::resolve(ref<Store> store) const +{ + auto [input2, extraAttrs] = lookupInRegistries(store, input); + return FlakeRef(input2, fetchers::maybeGetStrAttr(extraAttrs, "dir").value_or(subdir)); +} + +FlakeRef parseFlakeRef( + const std::string & url, const std::optional<Path> & baseDir) +{ + auto [flakeRef, fragment] = parseFlakeRefWithFragment(url, baseDir); + if (fragment != "") + throw Error("unexpected fragment '%s' in flake reference '%s'", fragment, url); + return flakeRef; +} + +std::optional<FlakeRef> maybeParseFlakeRef( + const std::string & url, const std::optional<Path> & baseDir) +{ + try { + return parseFlakeRef(url, baseDir); + } catch (Error &) { + return {}; + } +} + +std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( + const std::string & url, const std::optional<Path> & baseDir) +{ + using namespace fetchers; + + static std::string fnRegex = "[0-9a-zA-Z-._~!$&'\"()*+,;=]+"; + + static std::regex pathUrlRegex( + "(/?" + fnRegex + "(?:/" + fnRegex + ")*/?)" + + "(?:\\?(" + queryRegex + "))?" + + "(?:#(" + queryRegex + "))?", + std::regex::ECMAScript); + + static std::regex flakeRegex( + "((" + flakeIdRegexS + ")(?:/(?:" + refAndOrRevRegex + "))?)" + + "(?:#(" + queryRegex + "))?", + std::regex::ECMAScript); + + std::smatch match; + + /* Check if 'url' is a flake ID. This is an abbreviated syntax for + 'flake:<flake-id>?ref=<ref>&rev=<rev>'. */ + + if (std::regex_match(url, match, flakeRegex)) { + auto parsedURL = ParsedURL{ + .url = url, + .base = "flake:" + std::string(match[1]), + .scheme = "flake", + .authority = "", + .path = match[1], + }; + + return std::make_pair( + FlakeRef(inputFromURL(parsedURL), ""), + percentDecode(std::string(match[6]))); + } + + /* Check if 'url' is a path (either absolute or relative to + 'baseDir'). If so, search upward to the root of the repo + (i.e. the directory containing .git). */ + + else if (std::regex_match(url, match, pathUrlRegex)) { + std::string path = match[1]; + if (!baseDir && !hasPrefix(path, "/")) + throw BadURL("flake reference '%s' is not an absolute path", url); + path = absPath(path, baseDir, true); + + if (!S_ISDIR(lstat(path).st_mode)) + throw BadURL("path '%s' is not a flake (because it's not a directory)", path); + + auto flakeRoot = path; + std::string subdir; + + while (true) { + if (pathExists(flakeRoot + "/.git")) break; + subdir = std::string(baseNameOf(flakeRoot)) + (subdir.empty() ? "" : "/" + subdir); + flakeRoot = dirOf(flakeRoot); + if (flakeRoot == "/") + throw BadURL("path '%s' is not a flake (because it does not reference a Git repository)", path); + } + + auto base = std::string("git+file://") + flakeRoot; + + auto parsedURL = ParsedURL{ + .url = base, // FIXME + .base = base, + .scheme = "git+file", + .authority = "", + .path = flakeRoot, + .query = decodeQuery(match[2]), + }; + + auto fragment = percentDecode(std::string(match[3])); + + if (subdir != "") { + if (parsedURL.query.count("dir")) + throw Error("flake URL '%s' has an inconsistent 'dir' parameter", url); + parsedURL.query.insert_or_assign("dir", subdir); + } + + return std::make_pair( + FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + fragment); + } + + else { + auto parsedURL = parseURL(url); + std::string fragment; + std::swap(fragment, parsedURL.fragment); + return std::make_pair( + FlakeRef(inputFromURL(parsedURL), get(parsedURL.query, "dir").value_or("")), + fragment); + } +} + +std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment( + const std::string & url, const std::optional<Path> & baseDir) +{ + try { + return parseFlakeRefWithFragment(url, baseDir); + } catch (Error & e) { + return {}; + } +} + +FlakeRef FlakeRef::fromAttrs(const fetchers::Attrs & attrs) +{ + auto attrs2(attrs); + attrs2.erase("dir"); + return FlakeRef( + fetchers::inputFromAttrs(attrs2), + fetchers::maybeGetStrAttr(attrs, "dir").value_or("")); +} + +std::pair<fetchers::Tree, FlakeRef> FlakeRef::fetchTree(ref<Store> store) const +{ + auto [tree, lockedInput] = input->fetchTree(store); + return {std::move(tree), FlakeRef(lockedInput, subdir)}; +} + +} diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh new file mode 100644 index 000000000..1dbb6e54d --- /dev/null +++ b/src/libexpr/flake/flakeref.hh @@ -0,0 +1,55 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" +#include "fetchers.hh" + +#include <variant> + +namespace nix { + +class Store; + +typedef std::string FlakeId; + +struct FlakeRef +{ + std::shared_ptr<const fetchers::Input> input; + + Path subdir; + + bool operator==(const FlakeRef & other) const; + + FlakeRef(const std::shared_ptr<const fetchers::Input> & input, const Path & subdir) + : input(input), subdir(subdir) + { + assert(input); + } + + // FIXME: change to operator <<. + std::string to_string() const; + + fetchers::Attrs toAttrs() const; + + FlakeRef resolve(ref<Store> store) const; + + static FlakeRef fromAttrs(const fetchers::Attrs & attrs); + + std::pair<fetchers::Tree, FlakeRef> fetchTree(ref<Store> store) const; +}; + +std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); + +FlakeRef parseFlakeRef( + const std::string & url, const std::optional<Path> & baseDir = {}); + +std::optional<FlakeRef> maybeParseFlake( + const std::string & url, const std::optional<Path> & baseDir = {}); + +std::pair<FlakeRef, std::string> parseFlakeRefWithFragment( + const std::string & url, const std::optional<Path> & baseDir = {}); + +std::optional<std::pair<FlakeRef, std::string>> maybeParseFlakeRefWithFragment( + const std::string & url, const std::optional<Path> & baseDir = {}); + +} diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc new file mode 100644 index 000000000..a40637824 --- /dev/null +++ b/src/libexpr/flake/lockfile.cc @@ -0,0 +1,284 @@ +#include "lockfile.hh" +#include "store-api.hh" + +#include <nlohmann/json.hpp> + +namespace nix::flake { + +FlakeRef flakeRefFromJson(const nlohmann::json & json) +{ + return FlakeRef::fromAttrs(jsonToAttrs(json)); +} + +FlakeRef getFlakeRef( + const nlohmann::json & json, + const char * version3Attr1, + const char * version3Attr2, + const char * version4Attr) +{ + auto i = json.find(version4Attr); + if (i != json.end()) + return flakeRefFromJson(*i); + + // FIXME: remove these. + i = json.find(version3Attr1); + if (i != json.end()) + return parseFlakeRef(*i); + + i = json.find(version3Attr2); + if (i != json.end()) + return parseFlakeRef(*i); + + throw Error("attribute '%s' missing in lock file", version4Attr); +} + +LockedNode::LockedNode(const nlohmann::json & json) + : lockedRef(getFlakeRef(json, "url", "uri", "locked")) + , originalRef(getFlakeRef(json, "originalUrl", "originalUri", "original")) + , info(TreeInfo::fromJson(json)) + , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) +{ + if (!lockedRef.input->isImmutable()) + throw Error("lockfile contains mutable flakeref '%s'", lockedRef); +} + +StorePath LockedNode::computeStorePath(Store & store) const +{ + return info.computeStorePath(store); +} + +std::shared_ptr<Node> Node::findInput(const InputPath & path) +{ + auto pos = shared_from_this(); + + for (auto & elem : path) { + auto i = pos->inputs.find(elem); + if (i == pos->inputs.end()) + return {}; + pos = i->second; + } + + return pos; +} + +LockFile::LockFile(const nlohmann::json & json, const Path & path) +{ + auto version = json.value("version", 0); + if (version < 3 || version > 5) + throw Error("lock file '%s' has unsupported version %d", path, version); + + if (version < 5) { + std::function<void(Node & node, const nlohmann::json & json)> getInputs; + + getInputs = [&](Node & node, const nlohmann::json & json) + { + for (auto & i : json["inputs"].items()) { + auto input = std::make_shared<LockedNode>(i.value()); + getInputs(*input, i.value()); + node.inputs.insert_or_assign(i.key(), input); + } + }; + + getInputs(*root, json); + } + + else { + std::unordered_map<std::string, std::shared_ptr<Node>> nodeMap; + + std::function<void(Node & node, const nlohmann::json & jsonNode)> getInputs; + + getInputs = [&](Node & node, const nlohmann::json & jsonNode) + { + if (jsonNode.find("inputs") == jsonNode.end()) return; + for (auto & i : jsonNode["inputs"].items()) { + std::string inputKey = i.value(); + auto k = nodeMap.find(inputKey); + if (k == nodeMap.end()) { + auto jsonNode2 = json["nodes"][inputKey]; + auto input = std::make_shared<LockedNode>(jsonNode2); + k = nodeMap.insert_or_assign(inputKey, input).first; + getInputs(*input, jsonNode2); + } + node.inputs.insert_or_assign(i.key(), k->second); + } + }; + + std::string rootKey = json["root"]; + nodeMap.insert_or_assign(rootKey, root); + getInputs(*root, json["nodes"][rootKey]); + } +} + +nlohmann::json LockFile::toJson() const +{ + nlohmann::json nodes; + std::unordered_map<std::shared_ptr<const Node>, std::string> nodeKeys; + std::unordered_set<std::string> keys; + + std::function<std::string(const std::string & key, std::shared_ptr<const Node> node)> dumpNode; + + dumpNode = [&](std::string key, std::shared_ptr<const Node> node) -> std::string + { + auto k = nodeKeys.find(node); + if (k != nodeKeys.end()) + return k->second; + + if (!keys.insert(key).second) { + for (int n = 2; ; ++n) { + auto k = fmt("%s_%d", key, n); + if (keys.insert(k).second) { + key = k; + break; + } + } + } + + nodeKeys.insert_or_assign(node, key); + + auto n = nlohmann::json::object(); + + if (!node->inputs.empty()) { + auto inputs = nlohmann::json::object(); + for (auto & i : node->inputs) + inputs[i.first] = dumpNode(i.first, i.second); + n["inputs"] = std::move(inputs); + } + + if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) { + n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs()); + n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs()); + n["info"] = lockedNode->info.toJson(); + if (!lockedNode->isFlake) n["flake"] = false; + } + + nodes[key] = std::move(n); + + return key; + }; + + nlohmann::json json; + json["version"] = 5; + json["root"] = dumpNode("root", root); + json["nodes"] = std::move(nodes); + + return json; +} + +std::string LockFile::to_string() const +{ + return toJson().dump(2); +} + +LockFile LockFile::read(const Path & path) +{ + if (!pathExists(path)) return LockFile(); + return LockFile(nlohmann::json::parse(readFile(path)), path); +} + +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) +{ + stream << lockFile.toJson().dump(2); + return stream; +} + +void LockFile::write(const Path & path) const +{ + createDirs(dirOf(path)); + writeFile(path, fmt("%s\n", *this)); +} + +bool LockFile::isImmutable() const +{ + std::unordered_set<std::shared_ptr<const Node>> nodes; + + std::function<void(std::shared_ptr<const Node> node)> visit; + + visit = [&](std::shared_ptr<const Node> node) + { + if (!nodes.insert(node).second) return; + for (auto & i : node->inputs) visit(i.second); + }; + + visit(root); + + for (auto & i : nodes) { + if (i == root) continue; + auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(i); + if (lockedNode && !lockedNode->lockedRef.input->isImmutable()) return false; + } + + return true; +} + +bool LockFile::operator ==(const LockFile & other) const +{ + // FIXME: slow + return toJson() == other.toJson(); +} + +InputPath parseInputPath(std::string_view s) +{ + InputPath path; + + for (auto & elem : tokenizeString<std::vector<std::string>>(s, "/")) { + if (!std::regex_match(elem, flakeIdRegex)) + throw Error("invalid flake input path element '%s'", elem); + path.push_back(elem); + } + + return path; +} + +static void flattenLockFile( + std::shared_ptr<const Node> node, + const InputPath & prefix, + std::unordered_set<std::shared_ptr<const Node>> & done, + std::map<InputPath, std::shared_ptr<const LockedNode>> & res) +{ + if (!done.insert(node).second) return; + + for (auto &[id, input] : node->inputs) { + auto inputPath(prefix); + inputPath.push_back(id); + if (auto lockedInput = std::dynamic_pointer_cast<const LockedNode>(input)) + res.emplace(inputPath, lockedInput); + flattenLockFile(input, inputPath, done, res); + } +} + +std::string diffLockFiles(const LockFile & oldLocks, const LockFile & newLocks) +{ + std::unordered_set<std::shared_ptr<const Node>> done; + std::map<InputPath, std::shared_ptr<const LockedNode>> oldFlat, newFlat; + flattenLockFile(oldLocks.root, {}, done, oldFlat); + done.clear(); + flattenLockFile(newLocks.root, {}, done, newFlat); + + auto i = oldFlat.begin(); + auto j = newFlat.begin(); + std::string res; + + while (i != oldFlat.end() || j != newFlat.end()) { + if (j != newFlat.end() && (i == oldFlat.end() || i->first > j->first)) { + res += fmt("* Added '%s': '%s'\n", concatStringsSep("/", j->first), j->second->lockedRef); + ++j; + } else if (i != oldFlat.end() && (j == newFlat.end() || i->first < j->first)) { + res += fmt("* Removed '%s'\n", concatStringsSep("/", i->first)); + ++i; + } else { + if (!(i->second->lockedRef == j->second->lockedRef)) { + assert(i->second->lockedRef.to_string() != j->second->lockedRef.to_string()); + res += fmt("* Updated '%s': '%s' -> '%s'\n", + concatStringsSep("/", i->first), + i->second->lockedRef, + j->second->lockedRef); + } + ++i; + ++j; + } + } + + return res; +} + +} diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh new file mode 100644 index 000000000..c34939ebc --- /dev/null +++ b/src/libexpr/flake/lockfile.hh @@ -0,0 +1,77 @@ +#pragma once + +#include "flakeref.hh" + +#include <nlohmann/json_fwd.hpp> + +namespace nix { +class Store; +struct StorePath; +} + +namespace nix::flake { + +using namespace fetchers; + +typedef std::vector<FlakeId> InputPath; + +/* A node in the lock file. It has outgoing edges to other nodes (its + inputs). Only the root node has this type; all other nodes have + type LockedNode. */ +struct Node : std::enable_shared_from_this<Node> +{ + std::map<FlakeId, std::shared_ptr<Node>> inputs; + + virtual ~Node() { } + + std::shared_ptr<Node> findInput(const InputPath & path); +}; + +/* A non-root node in the lock file. */ +struct LockedNode : Node +{ + FlakeRef lockedRef, originalRef; + TreeInfo info; + bool isFlake = true; + + LockedNode( + const FlakeRef & lockedRef, + const FlakeRef & originalRef, + const TreeInfo & info, + bool isFlake = true) + : lockedRef(lockedRef), originalRef(originalRef), info(info), isFlake(isFlake) + { } + + LockedNode(const nlohmann::json & json); + + StorePath computeStorePath(Store & store) const; +}; + +struct LockFile +{ + std::shared_ptr<Node> root = std::make_shared<Node>(); + + LockFile() {}; + LockFile(const nlohmann::json & json, const Path & path); + + nlohmann::json toJson() const; + + std::string to_string() const; + + static LockFile read(const Path & path); + + void write(const Path & path) const; + + bool isImmutable() const; + + bool operator ==(const LockFile & other) const; +}; + +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile); + +InputPath parseInputPath(std::string_view s); + +std::string diffLockFiles(const LockFile & oldLocks, const LockFile & newLocks); + +} + diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index 917e8a1c7..f9460e821 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -4,7 +4,12 @@ libexpr_NAME = libnixexpr libexpr_DIR := $(d) -libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc +libexpr_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/primops/*.cc) \ + $(wildcard $(d)/flake/*.cc) \ + $(d)/lexer-tab.cc \ + $(d)/parser-tab.cc libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libmain -I src/libexpr @@ -34,4 +39,9 @@ dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer $(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644)) +$(foreach i, $(wildcard src/libexpr/flake/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644))) + $(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh + +$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index fc6c8296b..08a1adf3b 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -50,20 +50,20 @@ void EvalState::realiseContext(const PathSet & context) std::vector<StorePathWithOutputs> drvs; for (auto & i : context) { - std::pair<string, string> decoded = decodeContext(i); - auto ctx = store->parseStorePath(decoded.first); + auto [ctxS, outputName] = decodeContext(i); + auto ctx = store->parseStorePath(ctxS); if (!store->isValidPath(ctx)) throw InvalidPathError(store->printStorePath(ctx)); - if (!decoded.second.empty() && ctx.isDerivation()) { - drvs.push_back(StorePathWithOutputs{ctx.clone(), {decoded.second}}); + if (!outputName.empty() && ctx.isDerivation()) { + drvs.push_back(StorePathWithOutputs{ctx.clone(), {outputName}}); /* Add the output of this derivation to the allowed paths. */ if (allowedPaths) { - auto drv = store->derivationFromPath(store->parseStorePath(decoded.first)); - DerivationOutputs::iterator i = drv.outputs.find(decoded.second); + auto drv = store->derivationFromPath(ctx); + DerivationOutputs::iterator i = drv.outputs.find(outputName); if (i == drv.outputs.end()) - throw Error("derivation '%s' does not have an output named '%s'", decoded.first, decoded.second); + throw Error("derivation '%s' does not have an output named '%s'", ctxS, outputName); allowedPaths->insert(store->printStorePath(i->second.path)); } } @@ -79,6 +79,7 @@ void EvalState::realiseContext(const PathSet & context) StorePathSet willBuild, willSubstitute, unknown; unsigned long long downloadSize, narSize; store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize); + store->buildPaths(drvs); } diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 43c58485a..9d1f5b8d4 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -3,6 +3,7 @@ #include "store-api.hh" #include "fetchers.hh" #include "filetransfer.hh" +#include "registry.hh" #include <ctime> #include <iomanip> @@ -33,9 +34,11 @@ void emitTreeAttrs( if (tree.info.revCount) mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *tree.info.revCount); - if (tree.info.lastModified) - mkString(*state.allocAttr(v, state.symbols.create("lastModified")), + if (tree.info.lastModified) { + mkInt(*state.allocAttr(v, state.symbols.create("lastModified")), *tree.info.lastModified); + mkString(*state.allocAttr(v, state.symbols.create("lastModifiedDate")), fmt("%s", std::put_time(std::gmtime(&*tree.info.lastModified), "%Y%m%d%H%M%S"))); + } v.attrs->sort(); } @@ -60,8 +63,10 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V attrs.emplace(attr.name, attr.value->string.s); else if (attr.value->type == tBool) attrs.emplace(attr.name, attr.value->boolean); + else if (attr.value->type == tInt) + attrs.emplace(attr.name, attr.value->integer); else - throw TypeError("fetchTree argument '%s' is %s while a string or Boolean is expected", + throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected", attr.name, showType(*attr.value)); } @@ -72,8 +77,11 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V } else input = fetchers::inputFromURL(state.coerceToString(pos, *args[0], context, false, false)); + if (!evalSettings.pureEval && !input->isDirect()) + input = lookupInRegistries(state.store, input).first; + if (evalSettings.pureEval && !input->isImmutable()) - throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input"); + throw Error("in pure evaluation mode, 'fetchTree' requires an immutable input, at %s", pos); // FIXME: use fetchOrSubstituteTree auto [tree, input2] = input->fetchTree(state.store); diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh index 689373873..60de60c67 100644 --- a/src/libexpr/value.hh +++ b/src/libexpr/value.hh @@ -166,6 +166,11 @@ struct Value { return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size; } + + /* Check whether forcing this value requires a trivial amount of + computation. In particular, function applications are + non-trivial. */ + bool isTrivial() const; }; diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 94ac30e38..83268b4bf 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -72,4 +72,15 @@ std::pair<Tree, std::shared_ptr<const Input>> Input::fetchTree(ref<Store> store) return {std::move(tree), input}; } +std::shared_ptr<const Input> Input::applyOverrides( + std::optional<std::string> ref, + std::optional<Hash> rev) const +{ + if (ref) + throw Error("don't know how to apply '%s' to '%s'", *ref, to_string()); + if (rev) + throw Error("don't know how to apply '%s' to '%s'", rev->to_string(Base16, false), to_string()); + return shared_from_this(); +} + } diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index 59a58ae67..b75dcffa5 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -57,6 +57,22 @@ struct Input : std::enable_shared_from_this<Input> std::pair<Tree, std::shared_ptr<const Input>> fetchTree(ref<Store> store) const; + virtual std::shared_ptr<const Input> applyOverrides( + std::optional<std::string> ref, + std::optional<Hash> rev) const; + + virtual std::optional<Path> getSourcePath() const { return {}; } + + virtual void markChangedFile( + std::string_view file, + std::optional<std::string> commitMsg) const + { assert(false); } + + virtual void clone(const Path & destDir) const + { + throw Error("do not know how to clone input '%s'", to_string()); + } + private: virtual std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(ref<Store> store) const = 0; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index 7c18cf67f..210e29193 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -79,6 +79,63 @@ struct GitInput : Input return attrs; } + void clone(const Path & destDir) const override + { + auto [isLocal, actualUrl] = getActualUrl(); + + Strings args = {"clone"}; + + args.push_back(actualUrl); + + if (ref) { + args.push_back("--branch"); + args.push_back(*ref); + } + + if (rev) throw Error("cloning a specific revision is not implemented"); + + args.push_back(destDir); + + runProgram("git", true, args); + } + + std::shared_ptr<const Input> applyOverrides( + std::optional<std::string> ref, + std::optional<Hash> rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared<GitInput>(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + if (!res->ref && res->rev) + throw Error("Git input '%s' has a commit hash but no branch/tag name", res->to_string()); + + return res; + } + + std::optional<Path> getSourcePath() const override + { + if (url.scheme == "file" && !ref && !rev) + return url.path; + return {}; + } + + void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override + { + auto sourcePath = getSourcePath(); + assert(sourcePath); + + runProgram("git", true, + { "-C", *sourcePath, "add", "--force", "--intent-to-add", std::string(file) }); + + if (commitMsg) + runProgram("git", true, + { "-C", *sourcePath, "commit", std::string(file), "-m", *commitMsg }); + } + std::pair<bool, std::string> getActualUrl() const { // Don't clone file:// URIs (but otherwise treat them the diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 8675a5a66..c01917dcc 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -64,6 +64,13 @@ struct GitHubInput : Input return attrs; } + void clone(const Path & destDir) const override + { + std::shared_ptr<const Input> input = inputFromURL(fmt("git+ssh://git@github.com/%s/%s.git", owner, repo)); + input = input->applyOverrides(ref.value_or("master"), rev); + input->clone(destDir); + } + std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override { auto rev = this->rev; @@ -126,6 +133,20 @@ struct GitHubInput : Input return {std::move(tree), input}; } + + std::shared_ptr<const Input> applyOverrides( + std::optional<std::string> ref, + std::optional<Hash> rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared<GitHubInput>(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + return res; + } }; struct GitHubInputScheme : InputScheme diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc new file mode 100644 index 000000000..380b69fe0 --- /dev/null +++ b/src/libfetchers/indirect.cc @@ -0,0 +1,140 @@ +#include "fetchers.hh" + +namespace nix::fetchers { + +std::regex flakeRegex("[a-zA-Z][a-zA-Z0-9_-]*", std::regex::ECMAScript); + +struct IndirectInput : Input +{ + std::string id; + std::optional<Hash> rev; + std::optional<std::string> ref; + + std::string type() const override { return "indirect"; } + + bool operator ==(const Input & other) const override + { + auto other2 = dynamic_cast<const IndirectInput *>(&other); + return + other2 + && id == other2->id + && rev == other2->rev + && ref == other2->ref; + } + + bool isDirect() const override + { + return false; + } + + std::optional<std::string> getRef() const override { return ref; } + + std::optional<Hash> getRev() const override { return rev; } + + bool contains(const Input & other) const override + { + auto other2 = dynamic_cast<const IndirectInput *>(&other); + return + other2 + && id == other2->id + && (!ref || ref == other2->ref) + && (!rev || rev == other2->rev); + } + + ParsedURL toURL() const override + { + ParsedURL url; + url.scheme = "flake"; + url.path = id; + if (ref) { url.path += '/'; url.path += *ref; }; + if (rev) { url.path += '/'; url.path += rev->gitRev(); }; + return url; + } + + Attrs toAttrsInternal() const override + { + Attrs attrs; + attrs.emplace("id", id); + if (ref) + attrs.emplace("ref", *ref); + if (rev) + attrs.emplace("rev", rev->gitRev()); + return attrs; + } + + std::shared_ptr<const Input> applyOverrides( + std::optional<std::string> ref, + std::optional<Hash> rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared<IndirectInput>(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + return res; + } + + std::pair<Tree, std::shared_ptr<const Input>> fetchTreeInternal(nix::ref<Store> store) const override + { + throw Error("indirect input '%s' cannot be fetched directly", to_string()); + } +}; + +struct IndirectInputScheme : InputScheme +{ + std::unique_ptr<Input> inputFromURL(const ParsedURL & url) override + { + if (url.scheme != "flake") return nullptr; + + auto path = tokenizeString<std::vector<std::string>>(url.path, "/"); + auto input = std::make_unique<IndirectInput>(); + + if (path.size() == 1) { + } else if (path.size() == 2) { + if (std::regex_match(path[1], revRegex)) + input->rev = Hash(path[1], htSHA1); + else if (std::regex_match(path[1], refRegex)) + input->ref = path[1]; + else + throw BadURL("in flake URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[1]); + } else if (path.size() == 3) { + if (!std::regex_match(path[1], refRegex)) + throw BadURL("in flake URL '%s', '%s' is not a branch/tag name", url.url, path[1]); + input->ref = path[1]; + if (!std::regex_match(path[2], revRegex)) + throw BadURL("in flake URL '%s', '%s' is not a commit hash", url.url, path[2]); + input->rev = Hash(path[2], htSHA1); + } else + throw BadURL("GitHub URL '%s' is invalid", url.url); + + // FIXME: forbid query params? + + input->id = path[0]; + if (!std::regex_match(input->id, flakeRegex)) + throw BadURL("'%s' is not a valid flake ID", input->id); + + return input; + } + + std::unique_ptr<Input> inputFromAttrs(const Attrs & attrs) override + { + if (maybeGetStrAttr(attrs, "type") != "indirect") return {}; + + for (auto & [name, value] : attrs) + if (name != "type" && name != "id" && name != "ref" && name != "rev") + throw Error("unsupported indirect input attribute '%s'", name); + + auto input = std::make_unique<IndirectInput>(); + input->id = getStrAttr(attrs, "id"); + input->ref = maybeGetStrAttr(attrs, "ref"); + if (auto rev = maybeGetStrAttr(attrs, "rev")) + input->rev = Hash(*rev, htSHA1); + return input; + } +}; + +static auto r1 = OnStartup([] { registerInputScheme(std::make_unique<IndirectInputScheme>()); }); + +} diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 1d6571571..5abb00172 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -60,6 +60,41 @@ struct MercurialInput : Input return attrs; } + std::shared_ptr<const Input> applyOverrides( + std::optional<std::string> ref, + std::optional<Hash> rev) const override + { + if (!ref && !rev) return shared_from_this(); + + auto res = std::make_shared<MercurialInput>(*this); + + if (ref) res->ref = ref; + if (rev) res->rev = rev; + + return res; + } + + std::optional<Path> getSourcePath() const + { + if (url.scheme == "file" && !ref && !rev) + return url.path; + return {}; + } + + void markChangedFile(std::string_view file, std::optional<std::string> commitMsg) const override + { + auto sourcePath = getSourcePath(); + assert(sourcePath); + + // FIXME: shut up if file is already tracked. + runProgram("hg", true, + { "add", *sourcePath + "/" + std::string(file) }); + + if (commitMsg) + runProgram("hg", true, + { "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg }); + } + std::pair<bool, std::string> getActualUrl() const { bool isLocal = url.scheme == "file"; diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc new file mode 100644 index 000000000..77d3b3378 --- /dev/null +++ b/src/libfetchers/registry.cc @@ -0,0 +1,222 @@ +#include "registry.hh" +#include "fetchers.hh" +#include "util.hh" +#include "globals.hh" +#include "store-api.hh" + +#include <nlohmann/json.hpp> + +namespace nix::fetchers { + +std::shared_ptr<Registry> Registry::read( + const Path & path, RegistryType type) +{ + auto registry = std::make_shared<Registry>(type); + + if (!pathExists(path)) + return std::make_shared<Registry>(type); + + try { + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + + // FIXME: remove soon + if (version == 1) { + auto flakes = json["flakes"]; + for (auto i = flakes.begin(); i != flakes.end(); ++i) { + auto url = i->value("url", i->value("uri", "")); + if (url.empty()) + throw Error("flake registry '%s' lacks a 'url' attribute for entry '%s'", + path, i.key()); + registry->entries.push_back( + {inputFromURL(i.key()), inputFromURL(url), {}}); + } + } + + else if (version == 2) { + for (auto & i : json["flakes"]) { + auto toAttrs = jsonToAttrs(i["to"]); + Attrs extraAttrs; + auto j = toAttrs.find("dir"); + if (j != toAttrs.end()) { + extraAttrs.insert(*j); + toAttrs.erase(j); + } + auto exact = i.find("exact"); + registry->entries.push_back( + Entry { + .from = inputFromAttrs(jsonToAttrs(i["from"])), + .to = inputFromAttrs(toAttrs), + .extraAttrs = extraAttrs, + .exact = exact != i.end() && exact.value() + }); + } + } + + else + throw Error("flake registry '%s' has unsupported version %d", path, version); + + } catch (nlohmann::json::exception & e) { + warn("cannot parse flake registry '%s': %s", path, e.what()); + } catch (Error & e) { + warn("cannot read flake registry '%s': %s", path, e.what()); + } + + return registry; +} + +void Registry::write(const Path & path) +{ + nlohmann::json arr; + for (auto & entry : entries) { + nlohmann::json obj; + obj["from"] = attrsToJson(entry.from->toAttrs()); + obj["to"] = attrsToJson(entry.to->toAttrs()); + if (!entry.extraAttrs.empty()) + obj["to"].update(attrsToJson(entry.extraAttrs)); + if (entry.exact) + obj["exact"] = true; + arr.emplace_back(std::move(obj)); + } + + nlohmann::json json; + json["version"] = 2; + json["flakes"] = std::move(arr); + + createDirs(dirOf(path)); + writeFile(path, json.dump(2)); +} + +void Registry::add( + const std::shared_ptr<const Input> & from, + const std::shared_ptr<const Input> & to, + const Attrs & extraAttrs) +{ + entries.emplace_back( + Entry { + .from = from, + .to = to, + .extraAttrs = extraAttrs + }); +} + +void Registry::remove(const std::shared_ptr<const Input> & input) +{ + // FIXME: use C++20 std::erase. + for (auto i = entries.begin(); i != entries.end(); ) + if (*i->from == *input) + i = entries.erase(i); + else + ++i; +} + +static Path getSystemRegistryPath() +{ + return settings.nixConfDir + "/registry.json"; +} + +static std::shared_ptr<Registry> getSystemRegistry() +{ + static auto systemRegistry = + Registry::read(getSystemRegistryPath(), Registry::System); + return systemRegistry; +} + +Path getUserRegistryPath() +{ + return getHome() + "/.config/nix/registry.json"; +} + +std::shared_ptr<Registry> getUserRegistry() +{ + static auto userRegistry = + Registry::read(getUserRegistryPath(), Registry::User); + return userRegistry; +} + +static std::shared_ptr<Registry> flagRegistry = + std::make_shared<Registry>(Registry::Flag); + +std::shared_ptr<Registry> getFlagRegistry() +{ + return flagRegistry; +} + +void overrideRegistry( + const std::shared_ptr<const Input> & from, + const std::shared_ptr<const Input> & to, + const Attrs & extraAttrs) +{ + flagRegistry->add(from, to, extraAttrs); +} + +static std::shared_ptr<Registry> getGlobalRegistry(ref<Store> store) +{ + static auto reg = [&]() { + auto path = settings.flakeRegistry; + + if (!hasPrefix(path, "/")) { + auto storePath = downloadFile(store, path, "flake-registry.json", false).storePath; + if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) + store2->addPermRoot(storePath, getCacheDir() + "/nix/flake-registry.json", true); + path = store->toRealPath(storePath); + } + + return Registry::read(path, Registry::Global); + }(); + + return reg; +} + +Registries getRegistries(ref<Store> store) +{ + Registries registries; + registries.push_back(getFlagRegistry()); + registries.push_back(getUserRegistry()); + registries.push_back(getSystemRegistry()); + registries.push_back(getGlobalRegistry(store)); + return registries; +} + +std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries( + ref<Store> store, + std::shared_ptr<const Input> input) +{ + Attrs extraAttrs; + int n = 0; + + restart: + + n++; + if (n > 100) throw Error("cycle detected in flake registr for '%s'", input); + + for (auto & registry : getRegistries(store)) { + // FIXME: O(n) + for (auto & entry : registry->entries) { + if (entry.exact) { + if (*entry.from == *input) { + input = entry.to; + extraAttrs = entry.extraAttrs; + goto restart; + } + } else { + if (entry.from->contains(*input)) { + input = entry.to->applyOverrides( + !entry.from->getRef() && input->getRef() ? input->getRef() : std::optional<std::string>(), + !entry.from->getRev() && input->getRev() ? input->getRev() : std::optional<Hash>()); + extraAttrs = entry.extraAttrs; + goto restart; + } + } + } + } + + if (!input->isDirect()) + throw Error("cannot find flake '%s' in the flake registries", input->to_string()); + + return {input, extraAttrs}; +} + +} diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh new file mode 100644 index 000000000..c3ce948a8 --- /dev/null +++ b/src/libfetchers/registry.hh @@ -0,0 +1,65 @@ +#pragma once + +#include "types.hh" +#include "fetchers.hh" + +namespace nix { class Store; } + +namespace nix::fetchers { + +struct Registry +{ + enum RegistryType { + Flag = 0, + User = 1, + System = 2, + Global = 3, + }; + + RegistryType type; + + struct Entry + { + std::shared_ptr<const Input> from; + std::shared_ptr<const Input> to; + Attrs extraAttrs; + bool exact = false; + }; + + std::vector<Entry> entries; + + Registry(RegistryType type) + : type(type) + { } + + static std::shared_ptr<Registry> read( + const Path & path, RegistryType type); + + void write(const Path & path); + + void add( + const std::shared_ptr<const Input> & from, + const std::shared_ptr<const Input> & to, + const Attrs & extraAttrs); + + void remove(const std::shared_ptr<const Input> & input); +}; + +typedef std::vector<std::shared_ptr<Registry>> Registries; + +std::shared_ptr<Registry> getUserRegistry(); + +Path getUserRegistryPath(); + +Registries getRegistries(ref<Store> store); + +void overrideRegistry( + const std::shared_ptr<const Input> & from, + const std::shared_ptr<const Input> & to, + const Attrs & extraAttrs); + +std::pair<std::shared_ptr<const Input>, Attrs> lookupInRegistries( + ref<Store> store, + std::shared_ptr<const Input> input); + +} diff --git a/src/libfetchers/tree-info.cc b/src/libfetchers/tree-info.cc index 5788e94a1..42a19cbc8 100644 --- a/src/libfetchers/tree-info.cc +++ b/src/libfetchers/tree-info.cc @@ -11,4 +11,50 @@ StorePath TreeInfo::computeStorePath(Store & store) const return store.makeFixedOutputPath(true, narHash, "source"); } +TreeInfo TreeInfo::fromJson(const nlohmann::json & json) +{ + TreeInfo info; + + auto i = json.find("info"); + if (i != json.end()) { + const nlohmann::json & i2(*i); + + auto j = i2.find("narHash"); + if (j != i2.end()) + info.narHash = Hash((std::string) *j); + else + throw Error("attribute 'narHash' missing in lock file"); + + j = i2.find("revCount"); + if (j != i2.end()) + info.revCount = *j; + + j = i2.find("lastModified"); + if (j != i2.end()) + info.lastModified = *j; + + return info; + } + + i = json.find("narHash"); + if (i != json.end()) { + info.narHash = Hash((std::string) *i); + return info; + } + + throw Error("attribute 'info' missing in lock file"); +} + +nlohmann::json TreeInfo::toJson() const +{ + nlohmann::json json; + assert(narHash); + json["narHash"] = narHash.to_string(SRI); + if (revCount) + json["revCount"] = *revCount; + if (lastModified) + json["lastModified"] = *lastModified; + return json; +} + } diff --git a/src/libfetchers/tree-info.hh b/src/libfetchers/tree-info.hh index 2c7347281..3b62151c6 100644 --- a/src/libfetchers/tree-info.hh +++ b/src/libfetchers/tree-info.hh @@ -24,6 +24,10 @@ struct TreeInfo } StorePath computeStorePath(Store & store) const; + + static TreeInfo fromJson(const nlohmann::json & json); + + nlohmann::json toJson() const; }; } diff --git a/src/libstore/builtins/buildenv.hh b/src/libstore/builtins/buildenv.hh index 0a37459b0..73c0f5f7f 100644 --- a/src/libstore/builtins/buildenv.hh +++ b/src/libstore/builtins/buildenv.hh @@ -9,7 +9,7 @@ struct Package { Path path; bool active; int priority; - Package(Path path, bool active, int priority) : path{path}, active{active}, priority{priority} {} + Package(const Path & path, bool active, int priority) : path{path}, active{active}, priority{priority} {} }; typedef std::vector<Package> Packages; diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh index da95fd3ae..8099aa150 100644 --- a/src/libstore/globals.hh +++ b/src/libstore/globals.hh @@ -369,6 +369,9 @@ public: Setting<bool> warnDirty{this, true, "warn-dirty", "Whether to warn about dirty Git/Mercurial trees."}; + + Setting<std::string> flakeRegistry{this, "https://github.com/NixOS/flake-registry/raw/master/flake-registry.json", "flake-registry", + "Path or URI of the global flake registry."}; }; diff --git a/src/libstore/local.mk b/src/libstore/local.mk index 91acef368..636f74b65 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -61,3 +61,6 @@ $(d)/build.cc: clean-files += $(d)/schema.sql.gen.hh $(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644)) + +$(foreach i, $(wildcard src/libstore/builtins/*.hh), \ + $(eval $(call install-file-in, $(i), $(includedir)/nix/builtins, 0644))) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index bb437cf1c..3cc4ef8f1 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -3,6 +3,7 @@ #include <atomic> #include <nlohmann/json.hpp> +#include <iostream> namespace nix { @@ -24,6 +25,11 @@ void Logger::warn(const std::string & msg) log(lvlWarn, ANSI_YELLOW "warning:" ANSI_NORMAL " " + msg); } +void Logger::writeToStdout(std::string_view s) +{ + std::cout << s << "\n"; +} + class SimpleLogger : public Logger { public: diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh index 108b5dcb0..18c24d508 100644 --- a/src/libutil/logging.hh +++ b/src/libutil/logging.hh @@ -78,6 +78,16 @@ public: virtual void stopActivity(ActivityId act) { }; virtual void result(ActivityId act, ResultType type, const Fields & fields) { }; + + virtual void writeToStdout(std::string_view s); + + template<typename... Args> + inline void stdout(const std::string & fs, const Args & ... args) + { + boost::format f(fs); + formatHelper(f, args...); + writeToStdout(f.str()); + } }; ActivityId getCurActivity(); diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 332c1c43a..4266a5474 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -23,6 +23,7 @@ #include <sys/types.h> #include <sys/socket.h> #include <sys/wait.h> +#include <sys/time.h> #include <sys/un.h> #include <unistd.h> @@ -97,7 +98,7 @@ void replaceEnv(std::map<std::string, std::string> newEnv) } -Path absPath(Path path, std::optional<Path> dir) +Path absPath(Path path, std::optional<Path> dir, bool resolveSymlinks) { if (path[0] != '/') { if (!dir) { @@ -118,7 +119,7 @@ Path absPath(Path path, std::optional<Path> dir) } path = *dir + "/" + path; } - return canonPath(path); + return canonPath(path, resolveSymlinks); } @@ -360,7 +361,6 @@ void writeFile(const Path & path, Source & source, mode_t mode) } } - string readLine(int fd) { string s; @@ -569,20 +569,31 @@ Paths createDirs(const Path & path) } -void createSymlink(const Path & target, const Path & link) +void createSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime) { if (symlink(target.c_str(), link.c_str())) throw SysError(format("creating symlink from '%1%' to '%2%'") % link % target); + if (mtime) { + struct timeval times[2]; + times[0].tv_sec = *mtime; + times[0].tv_usec = 0; + times[1].tv_sec = *mtime; + times[1].tv_usec = 0; + if (lutimes(link.c_str(), times)) + throw SysError("setting time of symlink '%s'", link); + } } -void replaceSymlink(const Path & target, const Path & link) +void replaceSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime) { for (unsigned int n = 0; true; n++) { Path tmp = canonPath(fmt("%s/.%d_%s", dirOf(link), n, baseNameOf(link))); try { - createSymlink(target, tmp); + createSymlink(target, tmp, mtime); } catch (SysError & e) { if (e.errNo == EEXIST) continue; throw; @@ -994,12 +1005,14 @@ std::vector<char *> stringsToCharPtrs(const Strings & ss) return res; } - +// Output = "standard out" output stream string runProgram(Path program, bool searchPath, const Strings & args, const std::optional<std::string> & input) { RunOptions opts(program, args); opts.searchPath = searchPath; + // This allows you to refer to a program with a pathname relative to the + // PATH variable. opts.input = input; auto res = runProgram(opts); @@ -1010,6 +1023,7 @@ string runProgram(Path program, bool searchPath, const Strings & args, return res.second; } +// Output = error code + "standard out" output stream std::pair<int, std::string> runProgram(const RunOptions & options_) { RunOptions options(options_); @@ -1082,6 +1096,8 @@ void runProgram2(const RunOptions & options) if (options.searchPath) execvp(options.program.c_str(), stringsToCharPtrs(args_).data()); + // This allows you to refer to a program with a pathname relative + // to the PATH variable. else execv(options.program.c_str(), stringsToCharPtrs(args_).data()); diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 815b1f288..d4d0b9f7e 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -47,7 +47,9 @@ void clearEnv(); /* Return an absolutized path, resolving paths relative to the specified directory, or the current directory otherwise. The path is also canonicalised. */ -Path absPath(Path path, std::optional<Path> dir = {}); +Path absPath(Path path, + std::optional<Path> dir = {}, + bool resolveSymlinks = false); /* Canonicalise a path by removing all `.' or `..' components and double or trailing slashes. Optionally resolves all symlink @@ -145,10 +147,12 @@ Path getDataDir(); Paths createDirs(const Path & path); /* Create a symlink. */ -void createSymlink(const Path & target, const Path & link); +void createSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime = {}); /* Atomically create or replace a symlink. */ -void replaceSymlink(const Path & target, const Path & link); +void replaceSymlink(const Path & target, const Path & link, + std::optional<time_t> mtime = {}); /* Wrappers arount read()/write() that read/write exactly the diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc index 139db3657..ed02227db 100644 --- a/src/nix/add-to-store.cc +++ b/src/nix/add-to-store.cc @@ -50,7 +50,7 @@ struct CmdAddToStore : MixDryRun, StoreCommand if (!dryRun) store->addToStore(info, sink.s); - std::cout << fmt("%s\n", store->printStorePath(info.path)); + logger->stdout("%s", store->printStorePath(info.path)); } }; diff --git a/src/nix/build.cc b/src/nix/build.cc index 0b0762836..613bd9efb 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -1,3 +1,4 @@ +#include "eval.hh" #include "command.hh" #include "common-args.hh" #include "shared.hh" @@ -42,7 +43,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile }, Example{ "To make a profile point at GNU Hello:", - "nix build --profile /tmp/profile nixpkgs.hello" + "nix build --profile /tmp/profile nixpkgs#hello" }, }; } diff --git a/src/nix/command.hh b/src/nix/command.hh index 2c2303208..8b910ba78 100644 --- a/src/nix/command.hh +++ b/src/nix/command.hh @@ -4,12 +4,18 @@ #include "args.hh" #include "common-eval-args.hh" #include "path.hh" -#include "eval.hh" +#include "flake/lockfile.hh" + +#include <optional> namespace nix { extern std::string programPath; +class EvalState; +struct Pos; +class Store; + /* A command that requires a Nix store. */ struct StoreCommand : virtual Command { @@ -23,25 +29,36 @@ private: std::shared_ptr<Store> _store; }; -struct SourceExprCommand : virtual StoreCommand, MixEvalArgs +struct EvalCommand : virtual StoreCommand, MixEvalArgs { - Path file; + ref<EvalState> getEvalState(); - SourceExprCommand(); + std::shared_ptr<EvalState> evalState; +}; - /* Return a value representing the Nix expression from which we - are installing. This is either the file specified by ‘--file’, - or an attribute set constructed from $NIX_PATH, e.g. ‘{ nixpkgs - = import ...; bla = import ...; }’. */ - Value * getSourceExpr(EvalState & state); +struct MixFlakeOptions : virtual Args +{ + flake::LockFlags lockFlags; - ref<EvalState> getEvalState(); + MixFlakeOptions(); +}; -private: +struct SourceExprCommand : virtual Args, EvalCommand, MixFlakeOptions +{ + std::optional<Path> file; + std::optional<std::string> expr; - std::shared_ptr<EvalState> evalState; + SourceExprCommand(); - std::shared_ptr<Value> vSourceExpr; + std::vector<std::shared_ptr<Installable>> parseInstallables( + ref<Store> store, std::vector<std::string> ss); + + std::shared_ptr<Installable> parseInstallable( + ref<Store> store, const std::string & installable); + + virtual Strings getDefaultFlakeAttrPaths(); + + virtual Strings getDefaultFlakeAttrPathPrefixes(); }; enum RealiseMode { Build, NoBuild, DryRun }; @@ -73,14 +90,14 @@ struct InstallableCommand : virtual Args, SourceExprCommand InstallableCommand() { - expectArg("installable", &_installable); + expectArg("installable", &_installable, true); } void prepare() override; private: - std::string _installable; + std::string _installable{"."}; }; /* A command that operates on zero or more store paths. */ @@ -137,10 +154,6 @@ static RegisterCommand registerCommand(const std::string & name) return RegisterCommand(name, [](){ return make_ref<T>(); }); } -std::shared_ptr<Installable> parseInstallable( - SourceExprCommand & cmd, ref<Store> store, const std::string & installable, - bool useDefaultInstallables); - Buildables build(ref<Store> store, RealiseMode mode, std::vector<std::shared_ptr<Installable>> installables); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 6398fc58e..96ca83325 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -28,7 +28,7 @@ struct CmdEval : MixJSON, InstallableCommand return { Example{ "To evaluate a Nix expression given on the command line:", - "nix eval '(1 + 2)'" + "nix eval --expr '1 + 2'" }, Example{ "To evaluate a Nix expression from a file or URI:", @@ -55,16 +55,15 @@ struct CmdEval : MixJSON, InstallableCommand auto v = installable->toValue(*state).first; PathSet context; - stopProgressBar(); - if (raw) { + stopProgressBar(); std::cout << state->coerceToString(noPos, *v, context); } else if (json) { JSONPlaceholder jsonOut(std::cout); printValueAsJSON(*state, true, *v, jsonOut, context); } else { state->forceValueDeep(*v); - std::cout << *v << "\n"; + logger->stdout("%s", *v); } } }; diff --git a/src/nix/flake-template.nix b/src/nix/flake-template.nix new file mode 100644 index 000000000..60d7171f3 --- /dev/null +++ b/src/nix/flake-template.nix @@ -0,0 +1,9 @@ +{ + description = "A flake for building Hello World"; + + outputs = { self, nixpkgs }: { + + packages.x86_64-linux.hello = nixpkgs.legacyPackages.x86_64-linux.hello; + + }; +} diff --git a/src/nix/flake.cc b/src/nix/flake.cc new file mode 100644 index 000000000..a8518a543 --- /dev/null +++ b/src/nix/flake.cc @@ -0,0 +1,709 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "eval.hh" +#include "eval-inline.hh" +#include "flake/flake.hh" +#include "get-drvs.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "attr-path.hh" +#include "fetchers.hh" +#include "registry.hh" +#include "json.hh" + +#include <nlohmann/json.hpp> +#include <queue> +#include <iomanip> + +using namespace nix; +using namespace nix::flake; + +class FlakeCommand : virtual Args, public EvalCommand, public MixFlakeOptions +{ + std::string flakeUrl = "."; + +public: + + FlakeCommand() + { + expectArg("flake-url", &flakeUrl, true); + } + + FlakeRef getFlakeRef() + { + return parseFlakeRef(flakeUrl, absPath(".")); //FIXME + } + + Flake getFlake() + { + auto evalState = getEvalState(); + return flake::getFlake(*evalState, getFlakeRef(), lockFlags.useRegistries); + } + + LockedFlake lockFlake() + { + return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags); + } +}; + +struct CmdFlakeList : EvalCommand +{ + std::string description() override + { + return "list available Nix flakes"; + } + + void run(nix::ref<nix::Store> store) override + { + using namespace fetchers; + + auto registries = getRegistries(store); + + for (auto & registry : registries) { + for (auto & entry : registry->entries) { + // FIXME: format nicely + logger->stdout("%s %s %s", + registry->type == Registry::Flag ? "flags " : + registry->type == Registry::User ? "user " : + registry->type == Registry::System ? "system" : + "global", + entry.from->to_string(), + entry.to->to_string()); + } + } + } +}; + +static void printFlakeInfo(const Store & store, const Flake & flake) +{ + logger->stdout("Resolved URL: %s", flake.resolvedRef.to_string()); + logger->stdout("Locked URL: %s", flake.lockedRef.to_string()); + if (flake.description) + logger->stdout("Description: %s", *flake.description); + logger->stdout("Path: %s", store.printStorePath(flake.sourceInfo->storePath)); + if (auto rev = flake.lockedRef.input->getRev()) + logger->stdout("Revision: %s", rev->to_string(Base16, false)); + if (flake.sourceInfo->info.revCount) + logger->stdout("Revisions: %s", *flake.sourceInfo->info.revCount); + if (flake.sourceInfo->info.lastModified) + logger->stdout("Last modified: %s", + std::put_time(std::localtime(&*flake.sourceInfo->info.lastModified), "%F %T")); +} + +static nlohmann::json flakeToJson(const Store & store, const Flake & flake) +{ + nlohmann::json j; + if (flake.description) + j["description"] = *flake.description; + j["originalUrl"] = flake.originalRef.to_string(); + j["original"] = attrsToJson(flake.originalRef.toAttrs()); + j["resolvedUrl"] = flake.resolvedRef.to_string(); + j["resolved"] = attrsToJson(flake.resolvedRef.toAttrs()); + j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl + j["locked"] = attrsToJson(flake.lockedRef.toAttrs()); + j["info"] = flake.sourceInfo->info.toJson(); + if (auto rev = flake.lockedRef.input->getRev()) + j["revision"] = rev->to_string(Base16, false); + if (flake.sourceInfo->info.revCount) + j["revCount"] = *flake.sourceInfo->info.revCount; + if (flake.sourceInfo->info.lastModified) + j["lastModified"] = *flake.sourceInfo->info.lastModified; + j["path"] = store.printStorePath(flake.sourceInfo->storePath); + return j; +} + +struct CmdFlakeUpdate : FlakeCommand +{ + std::string description() override + { + return "update flake lock file"; + } + + void run(nix::ref<nix::Store> store) override + { + /* Use --refresh by default for 'nix flake update'. */ + settings.tarballTtl = 0; + + lockFlake(); + } +}; + +static void enumerateOutputs(EvalState & state, Value & vFlake, + std::function<void(const std::string & name, Value & vProvide, const Pos & pos)> callback) +{ + state.forceAttrs(vFlake); + + auto aOutputs = vFlake.attrs->get(state.symbols.create("outputs")); + assert(aOutputs); + + state.forceAttrs(*aOutputs->value); + + for (auto & attr : *aOutputs->value->attrs) + callback(attr.name, *attr.value, *attr.pos); +} + +struct CmdFlakeInfo : FlakeCommand, MixJSON +{ + std::string description() override + { + return "list info about a given flake"; + } + + void run(nix::ref<nix::Store> store) override + { + auto flake = getFlake(); + + if (json) { + auto json = flakeToJson(*store, flake); + logger->stdout(json.dump()); + } else + printFlakeInfo(*store, flake); + } +}; + +struct CmdFlakeListInputs : FlakeCommand, MixJSON +{ + std::string description() override + { + return "list flake inputs"; + } + + void run(nix::ref<nix::Store> store) override + { + auto flake = lockFlake(); + + if (json) + logger->stdout(flake.lockFile.toJson()); + else { + logger->stdout("%s", flake.flake.lockedRef); + + std::function<void(const Node & node, const std::string & prefix)> recurse; + + recurse = [&](const Node & node, const std::string & prefix) + { + for (const auto & [i, input] : enumerate(node.inputs)) { + //auto tree2 = tree.child(i + 1 == inputs.inputs.size()); + bool last = i + 1 == node.inputs.size(); + logger->stdout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s", + prefix + (last ? treeLast : treeConn), input.first, + std::dynamic_pointer_cast<const LockedNode>(input.second)->lockedRef); + recurse(*input.second, prefix + (last ? treeNull : treeLine)); + } + }; + + recurse(*flake.lockFile.root, ""); + } + } +}; + +struct CmdFlakeCheck : FlakeCommand +{ + bool build = true; + + CmdFlakeCheck() + { + mkFlag() + .longName("no-build") + .description("do not build checks") + .set(&build, false); + } + + std::string description() override + { + return "check whether the flake evaluates and run its tests"; + } + + void run(nix::ref<nix::Store> store) override + { + settings.readOnlyMode = !build; + + auto state = getEvalState(); + auto flake = lockFlake(); + + auto checkSystemName = [&](const std::string & system, const Pos & pos) { + // FIXME: what's the format of "system"? + if (system.find('-') == std::string::npos) + throw Error("'%s' is not a valid system type, at %s", system, pos); + }; + + auto checkDerivation = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + auto drvInfo = getDerivation(*state, v, false); + if (!drvInfo) + throw Error("flake attribute '%s' is not a derivation", attrPath); + // FIXME: check meta attributes + return store->parseStorePath(drvInfo->queryDrvPath()); + } catch (Error & e) { + e.addPrefix(fmt("while checking the derivation '" ANSI_BOLD "%s" ANSI_NORMAL "' at %s:\n", attrPath, pos)); + throw; + } + }; + + std::vector<StorePathWithOutputs> drvPaths; + + auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + auto app = App(*state, v); + for (auto & i : app.context) { + auto [drvPathS, outputName] = decodeContext(i); + auto drvPath = store->parseStorePath(drvPathS); + if (!outputName.empty() && drvPath.isDerivation()) + drvPaths.emplace_back(drvPath); + } + } catch (Error & e) { + e.addPrefix(fmt("while checking the app definition '" ANSI_BOLD "%s" ANSI_NORMAL "' at %s:\n", attrPath, pos)); + throw; + } + }; + + auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceValue(v, pos); + if (v.type != tLambda || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final") + throw Error("overlay does not take an argument named 'final'"); + auto body = dynamic_cast<ExprLambda *>(v.lambda.fun->body); + if (!body || body->matchAttrs || std::string(body->arg) != "prev") + throw Error("overlay does not take an argument named 'prev'"); + // FIXME: if we have a 'nixpkgs' input, use it to + // evaluate the overlay. + } catch (Error & e) { + e.addPrefix(fmt("while checking the overlay '" ANSI_BOLD "%s" ANSI_NORMAL "' at %s:\n", attrPath, pos)); + throw; + } + }; + + auto checkModule = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceValue(v, pos); + if (v.type == tLambda) { + if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis) + throw Error("module must match an open attribute set ('{ config, ... }')"); + } else if (v.type == tAttrs) { + for (auto & attr : *v.attrs) + try { + state->forceValue(*attr.value, *attr.pos); + } catch (Error & e) { + e.addPrefix(fmt("while evaluating the option '" ANSI_BOLD "%s" ANSI_NORMAL "' at %s:\n", attr.name, *attr.pos)); + throw; + } + } else + throw Error("module must be a function or an attribute set"); + // FIXME: if we have a 'nixpkgs' input, use it to + // check the module. + } catch (Error & e) { + e.addPrefix(fmt("while checking the NixOS module '" ANSI_BOLD "%s" ANSI_NORMAL "' at %s:\n", attrPath, pos)); + throw; + } + }; + + std::function<void(const std::string & attrPath, Value & v, const Pos & pos)> checkHydraJobs; + + checkHydraJobs = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + state->forceAttrs(v, pos); + + if (state->isDerivation(v)) + throw Error("jobset should not be a derivation at top-level"); + + for (auto & attr : *v.attrs) { + state->forceAttrs(*attr.value, *attr.pos); + if (!state->isDerivation(*attr.value)) + checkHydraJobs(attrPath + "." + (std::string) attr.name, + *attr.value, *attr.pos); + } + + } catch (Error & e) { + e.addPrefix(fmt("while checking the Hydra jobset '" ANSI_BOLD "%s" ANSI_NORMAL "' at %s:\n", attrPath, pos)); + throw; + } + }; + + auto checkNixOSConfiguration = [&](const std::string & attrPath, Value & v, const Pos & pos) { + try { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking NixOS configuration '%s'", attrPath)); + Bindings & bindings(*state->allocBindings(0)); + auto vToplevel = findAlongAttrPath(*state, "config.system.build.toplevel", bindings, v).first; + state->forceAttrs(*vToplevel, pos); + if (!state->isDerivation(*vToplevel)) + throw Error("attribute 'config.system.build.toplevel' is not a derivation"); + } catch (Error & e) { + e.addPrefix(fmt("while checking the NixOS configuration '" ANSI_BOLD "%s" ANSI_NORMAL "' at %s:\n", attrPath, pos)); + throw; + } + }; + + { + Activity act(*logger, lvlInfo, actUnknown, "evaluating flake"); + + auto vFlake = state->allocValue(); + flake::callFlake(*state, flake, *vFlake); + + enumerateOutputs(*state, + *vFlake, + [&](const std::string & name, Value & vOutput, const Pos & pos) { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking flake output '%s'", name)); + + try { + state->forceValue(vOutput, pos); + + if (name == "checks") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) { + auto drvPath = checkDerivation( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + if ((std::string) attr.name == settings.thisSystem.get()) + drvPaths.emplace_back(drvPath); + } + } + } + + else if (name == "packages") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) + checkDerivation( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + } + } + + else if (name == "apps") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + state->forceAttrs(*attr.value, *attr.pos); + for (auto & attr2 : *attr.value->attrs) + checkApp( + fmt("%s.%s.%s", name, attr.name, attr2.name), + *attr2.value, *attr2.pos); + } + } + + else if (name == "defaultPackage" || name == "devShell") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + checkDerivation( + fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + } + + else if (name == "defaultApp") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + checkApp( + fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + } + + else if (name == "legacyPackages") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) { + checkSystemName(attr.name, *attr.pos); + // FIXME: do getDerivations? + } + } + + else if (name == "overlay") + checkOverlay(name, vOutput, pos); + + else if (name == "overlays") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkOverlay(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else if (name == "nixosModule") + checkModule(name, vOutput, pos); + + else if (name == "nixosModules") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkModule(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else if (name == "nixosConfigurations") { + state->forceAttrs(vOutput, pos); + for (auto & attr : *vOutput.attrs) + checkNixOSConfiguration(fmt("%s.%s", name, attr.name), + *attr.value, *attr.pos); + } + + else if (name == "hydraJobs") + checkHydraJobs(name, vOutput, pos); + + else + warn("unknown flake output '%s'", name); + + } catch (Error & e) { + e.addPrefix(fmt("while checking flake output '" ANSI_BOLD "%s" ANSI_NORMAL "':\n", name)); + throw; + } + }); + } + + if (build && !drvPaths.empty()) { + Activity act(*logger, lvlInfo, actUnknown, "running flake checks"); + store->buildPaths(drvPaths); + } + } +}; + +struct CmdFlakeAdd : MixEvalArgs, Command +{ + std::string fromUrl, toUrl; + + std::string description() override + { + return "upsert flake in user flake registry"; + } + + CmdFlakeAdd() + { + expectArg("from-url", &fromUrl); + expectArg("to-url", &toUrl); + } + + void run() override + { + auto fromRef = parseFlakeRef(fromUrl); + auto toRef = parseFlakeRef(toUrl); + fetchers::Attrs extraAttrs; + if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir; + auto userRegistry = fetchers::getUserRegistry(); + userRegistry->remove(fromRef.input); + userRegistry->add(fromRef.input, toRef.input, extraAttrs); + userRegistry->write(fetchers::getUserRegistryPath()); + } +}; + +struct CmdFlakeRemove : virtual Args, MixEvalArgs, Command +{ + std::string url; + + std::string description() override + { + return "remove flake from user flake registry"; + } + + CmdFlakeRemove() + { + expectArg("url", &url); + } + + void run() override + { + auto userRegistry = fetchers::getUserRegistry(); + userRegistry->remove(parseFlakeRef(url).input); + userRegistry->write(fetchers::getUserRegistryPath()); + } +}; + +struct CmdFlakePin : virtual Args, EvalCommand +{ + std::string url; + + std::string description() override + { + return "pin a flake to its current version in user flake registry"; + } + + CmdFlakePin() + { + expectArg("url", &url); + } + + void run(nix::ref<nix::Store> store) override + { + auto ref = parseFlakeRef(url); + auto userRegistry = fetchers::getUserRegistry(); + userRegistry->remove(ref.input); + auto [tree, resolved] = ref.resolve(store).input->fetchTree(store); + fetchers::Attrs extraAttrs; + if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; + userRegistry->add(ref.input, resolved, extraAttrs); + } +}; + +struct CmdFlakeInit : virtual Args, Command +{ + std::string description() override + { + return "create a skeleton 'flake.nix' file in the current directory"; + } + + void run() override + { + Path flakeDir = absPath("."); + + if (!pathExists(flakeDir + "/.git")) + throw Error("the directory '%s' is not a Git repository", flakeDir); + + Path flakePath = flakeDir + "/flake.nix"; + + if (pathExists(flakePath)) + throw Error("file '%s' already exists", flakePath); + + writeFile(flakePath, +#include "flake-template.nix.gen.hh" + ); + } +}; + +struct CmdFlakeClone : FlakeCommand +{ + Path destDir; + + std::string description() override + { + return "clone flake repository"; + } + + CmdFlakeClone() + { + mkFlag() + .shortName('f') + .longName("dest") + .label("path") + .description("destination path") + .dest(&destDir); + } + + void run(nix::ref<nix::Store> store) override + { + if (destDir.empty()) + throw Error("missing flag '--dest'"); + + getFlakeRef().resolve(store).input->clone(destDir); + } +}; + +struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun +{ + std::string dstUri; + + CmdFlakeArchive() + { + mkFlag() + .longName("to") + .labels({"store-uri"}) + .description("URI of the destination Nix store") + .dest(&dstUri); + } + + std::string description() override + { + return "copy a flake and all its inputs to a store"; + } + + Examples examples() override + { + return { + Example{ + "To copy the dwarffs flake and its dependencies to a binary cache:", + "nix flake archive --to file:///tmp/my-cache dwarffs" + }, + Example{ + "To fetch the dwarffs flake and its dependencies to the local Nix store:", + "nix flake archive dwarffs" + }, + Example{ + "To print the store paths of the flake sources of NixOps without fetching them:", + "nix flake archive --json --dry-run nixops" + }, + }; + } + + void run(nix::ref<nix::Store> store) override + { + auto flake = lockFlake(); + + auto jsonRoot = json ? std::optional<JSONObject>(std::cout) : std::nullopt; + + StorePathSet sources; + + sources.insert(flake.flake.sourceInfo->storePath.clone()); + if (jsonRoot) + jsonRoot->attr("path", store->printStorePath(flake.flake.sourceInfo->storePath)); + + // FIXME: use graph output, handle cycles. + std::function<void(const Node & node, std::optional<JSONObject> & jsonObj)> traverse; + traverse = [&](const Node & node, std::optional<JSONObject> & jsonObj) + { + auto jsonObj2 = jsonObj ? jsonObj->object("inputs") : std::optional<JSONObject>(); + for (auto & input : node.inputs) { + auto lockedInput = std::dynamic_pointer_cast<const LockedNode>(input.second); + assert(lockedInput); + auto jsonObj3 = jsonObj2 ? jsonObj2->object(input.first) : std::optional<JSONObject>(); + if (!dryRun) + lockedInput->lockedRef.input->fetchTree(store); + auto storePath = lockedInput->computeStorePath(*store); + if (jsonObj3) + jsonObj3->attr("path", store->printStorePath(storePath)); + sources.insert(std::move(storePath)); + traverse(*lockedInput, jsonObj3); + } + }; + + traverse(*flake.lockFile.root, jsonRoot); + + if (!dryRun && !dstUri.empty()) { + ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri); + copyPaths(store, dstStore, sources); + } + } +}; + +struct CmdFlake : virtual MultiCommand, virtual Command +{ + CmdFlake() + : MultiCommand({ + {"list", []() { return make_ref<CmdFlakeList>(); }}, + {"update", []() { return make_ref<CmdFlakeUpdate>(); }}, + {"info", []() { return make_ref<CmdFlakeInfo>(); }}, + {"list-inputs", []() { return make_ref<CmdFlakeListInputs>(); }}, + {"check", []() { return make_ref<CmdFlakeCheck>(); }}, + {"add", []() { return make_ref<CmdFlakeAdd>(); }}, + {"remove", []() { return make_ref<CmdFlakeRemove>(); }}, + {"pin", []() { return make_ref<CmdFlakePin>(); }}, + {"init", []() { return make_ref<CmdFlakeInit>(); }}, + {"clone", []() { return make_ref<CmdFlakeClone>(); }}, + {"archive", []() { return make_ref<CmdFlakeArchive>(); }}, + }) + { + } + + std::string description() override + { + return "manage Nix flakes"; + } + + void run() override + { + if (!command) + throw UsageError("'nix flake' requires a sub-command."); + command->prepare(); + command->run(); + } + + void printHelp(const string & programName, std::ostream & out) override + { + MultiCommand::printHelp(programName, out); + } +}; + +static auto r1 = registerCommand<CmdFlake>("flake"); diff --git a/src/nix/hash.cc b/src/nix/hash.cc index 0cc523f50..01628cf6c 100644 --- a/src/nix/hash.cc +++ b/src/nix/hash.cc @@ -60,8 +60,7 @@ struct CmdHash : Command Hash h = hashSink->finish().first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); - std::cout << format("%1%\n") % - h.to_string(base, base == SRI); + logger->stdout(h.to_string(base, base == SRI)); } } }; @@ -95,7 +94,7 @@ struct CmdToBase : Command void run() override { for (auto s : args) - std::cout << fmt("%s\n", Hash(s, ht).to_string(base, base == SRI)); + logger->stdout(Hash(s, ht).to_string(base, base == SRI)); } }; diff --git a/src/nix/installables.cc b/src/nix/installables.cc index 902383bff..fece68b17 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -1,3 +1,4 @@ +#include "installables.hh" #include "command.hh" #include "attr-path.hh" #include "common-eval-args.hh" @@ -7,78 +8,98 @@ #include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" +#include "flake/flake.hh" +#include "flake/eval-cache.hh" +#include "url.hh" #include <gc/gc.h> #include <regex> +#include <queue> namespace nix { - -SourceExprCommand::SourceExprCommand() +MixFlakeOptions::MixFlakeOptions() { mkFlag() - .shortName('f') - .longName("file") - .label("file") - .description("evaluate FILE rather than the default") - .dest(&file); -} + .longName("recreate-lock-file") + .description("recreate lock file from scratch") + .set(&lockFlags.recreateLockFile, true); -Value * SourceExprCommand::getSourceExpr(EvalState & state) -{ - if (vSourceExpr) return vSourceExpr.get(); - - auto sToplevel = state.symbols.create("_toplevel"); - - // Allocate the vSourceExpr Value as uncollectable. Boehm GC doesn't - // consider the member variable "alive" during execution causing it to be - // GC'ed in the middle of evaluation. - vSourceExpr = std::allocate_shared<Value>(traceable_allocator<Value>()); - - if (file != "") - state.evalFile(lookupFileArg(state, file), *vSourceExpr); - - else { + mkFlag() + .longName("no-update-lock-file") + .description("do not allow any updates to the lock file") + .set(&lockFlags.updateLockFile, false); - /* Construct the installation source from $NIX_PATH. */ + mkFlag() + .longName("no-write-lock-file") + .description("do not write the newly generated lock file") + .set(&lockFlags.writeLockFile, false); - auto searchPath = state.getSearchPath(); + mkFlag() + .longName("no-registries") + .description("don't use flake registries") + .set(&lockFlags.useRegistries, false); - state.mkAttrs(*vSourceExpr, 1024); + mkFlag() + .longName("commit-lock-file") + .description("commit changes to the lock file") + .set(&lockFlags.commitLockFile, true); - mkBool(*state.allocAttr(*vSourceExpr, sToplevel), true); + mkFlag() + .longName("update-input") + .description("update a specific flake input") + .label("input-path") + .handler([&](std::vector<std::string> ss) { + lockFlags.inputUpdates.insert(flake::parseInputPath(ss[0])); + }); - std::unordered_set<std::string> seen; + mkFlag() + .longName("override-input") + .description("override a specific flake input (e.g. 'dwarffs/nixpkgs')") + .arity(2) + .labels({"input-path", "flake-url"}) + .handler([&](std::vector<std::string> ss) { + lockFlags.inputOverrides.insert_or_assign( + flake::parseInputPath(ss[0]), + parseFlakeRef(ss[1], absPath("."))); + }); +} - auto addEntry = [&](const std::string & name) { - if (name == "") return; - if (!seen.insert(name).second) return; - Value * v1 = state.allocValue(); - mkPrimOpApp(*v1, state.getBuiltin("findFile"), state.getBuiltin("nixPath")); - Value * v2 = state.allocValue(); - mkApp(*v2, *v1, mkString(*state.allocValue(), name)); - mkApp(*state.allocAttr(*vSourceExpr, state.symbols.create(name)), - state.getBuiltin("import"), *v2); - }; +SourceExprCommand::SourceExprCommand() +{ + mkFlag() + .shortName('f') + .longName("file") + .label("file") + .description("evaluate attributes from FILE") + .dest(&file); - for (auto & i : searchPath) - /* Hack to handle channels. */ - if (i.first.empty() && pathExists(i.second + "/manifest.nix")) { - for (auto & j : readDirectory(i.second)) - if (j.name != "manifest.nix" - && pathExists(fmt("%s/%s/default.nix", i.second, j.name))) - addEntry(j.name); - } else - addEntry(i.first); + mkFlag() + .longName("expr") + .label("expr") + .description("evaluate attributes from EXPR") + .dest(&expr); +} - vSourceExpr->attrs->sort(); - } +Strings SourceExprCommand::getDefaultFlakeAttrPaths() +{ + return {"defaultPackage." + settings.thisSystem.get()}; +} - return vSourceExpr.get(); +Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes() +{ + return { + // As a convenience, look for the attribute in + // 'outputs.packages'. + "packages." + settings.thisSystem.get() + ".", + // As a temporary hack until Nixpkgs is properly converted + // to provide a clean 'packages' set, look in 'legacyPackages'. + "legacyPackages." + settings.thisSystem.get() + "." + }; } -ref<EvalState> SourceExprCommand::getEvalState() +ref<EvalState> EvalCommand::getEvalState() { if (!evalState) evalState = std::make_shared<EvalState>(searchPath, getStore()); @@ -93,6 +114,27 @@ Buildable Installable::toBuildable() return std::move(buildables[0]); } +App::App(EvalState & state, Value & vApp) +{ + state.forceAttrs(vApp); + + auto aType = vApp.attrs->need(state.sType); + if (state.forceStringNoCtx(*aType.value, *aType.pos) != "app") + throw Error("value does not have type 'app', at %s", *aType.pos); + + auto aProgram = vApp.attrs->need(state.symbols.create("program")); + program = state.forceString(*aProgram.value, context, *aProgram.pos); + + // FIXME: check that 'program' is in the closure of 'context'. + if (!state.store->isInStore(program)) + throw Error("app program '%s' is not in the Nix store", program); +} + +App Installable::toApp(EvalState & state) +{ + return App(state, *toValue(state).first); +} + struct InstallableStorePath : Installable { ref<Store> store; @@ -122,54 +164,63 @@ struct InstallableStorePath : Installable } }; -struct InstallableValue : Installable +std::vector<flake::EvalCache::Derivation> InstallableValue::toDerivations() { - SourceExprCommand & cmd; + auto state = cmd.getEvalState(); - InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { } + auto v = toValue(*state).first; - Buildables toBuildables() override - { - auto state = cmd.getEvalState(); + Bindings & autoArgs = *cmd.getAutoArgs(*state); - auto v = toValue(*state).first; + DrvInfos drvInfos; + getDerivations(*state, *v, "", autoArgs, drvInfos, false); - Bindings & autoArgs = *cmd.getAutoArgs(*state); + std::vector<flake::EvalCache::Derivation> res; + for (auto & drvInfo : drvInfos) { + res.push_back({ + state->store->parseStorePath(drvInfo.queryDrvPath()), + state->store->parseStorePath(drvInfo.queryOutPath()), + drvInfo.queryOutputName() + }); + } - DrvInfos drvs; - getDerivations(*state, *v, "", autoArgs, drvs, false); + return res; +} - Buildables res; +Buildables InstallableValue::toBuildables() +{ + auto state = cmd.getEvalState(); - StorePathSet drvPaths; + Buildables res; - for (auto & drv : drvs) { - Buildable b{.drvPath = state->store->parseStorePath(drv.queryDrvPath())}; - drvPaths.insert(b.drvPath->clone()); + StorePathSet drvPaths; - auto outputName = drv.queryOutputName(); - if (outputName == "") - throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(*b.drvPath)); + for (auto & drv : toDerivations()) { + Buildable b{.drvPath = drv.drvPath.clone()}; + drvPaths.insert(drv.drvPath.clone()); - b.outputs.emplace(outputName, state->store->parseStorePath(drv.queryOutPath())); + auto outputName = drv.outputName; + if (outputName == "") + throw Error("derivation '%s' lacks an 'outputName' attribute", state->store->printStorePath(*b.drvPath)); - res.push_back(std::move(b)); - } + b.outputs.emplace(outputName, drv.outPath.clone()); - // Hack to recognize .all: if all drvs have the same drvPath, - // merge the buildables. - if (drvPaths.size() == 1) { - Buildable b{.drvPath = drvPaths.begin()->clone()}; - for (auto & b2 : res) - for (auto & output : b2.outputs) - b.outputs.insert_or_assign(output.first, output.second.clone()); - Buildables bs; - bs.push_back(std::move(b)); - return bs; - } else - return res; + res.push_back(std::move(b)); } -}; + + // Hack to recognize .all: if all drvs have the same drvPath, + // merge the buildables. + if (drvPaths.size() == 1) { + Buildable b{.drvPath = drvPaths.begin()->clone()}; + for (auto & b2 : res) + for (auto & output : b2.outputs) + b.outputs.insert_or_assign(output.first, output.second.clone()); + Buildables bs; + bs.push_back(std::move(b)); + return bs; + } else + return res; +} struct InstallableExpr : InstallableValue { @@ -190,70 +241,201 @@ struct InstallableExpr : InstallableValue struct InstallableAttrPath : InstallableValue { + Value * v; std::string attrPath; - InstallableAttrPath(SourceExprCommand & cmd, const std::string & attrPath) - : InstallableValue(cmd), attrPath(attrPath) + InstallableAttrPath(SourceExprCommand & cmd, Value * v, const std::string & attrPath) + : InstallableValue(cmd), v(v), attrPath(attrPath) { } std::string what() override { return attrPath; } std::pair<Value *, Pos> toValue(EvalState & state) override { - auto source = cmd.getSourceExpr(state); + auto [vRes, pos] = findAlongAttrPath(state, attrPath, *cmd.getAutoArgs(state), *v); + state.forceValue(*vRes); + return {vRes, pos}; + } +}; - Bindings & autoArgs = *cmd.getAutoArgs(state); +std::vector<std::string> InstallableFlake::getActualAttrPaths() +{ + std::vector<std::string> res; - auto v = findAlongAttrPath(state, attrPath, autoArgs, *source).first; - state.forceValue(*v); + for (auto & prefix : prefixes) + res.push_back(prefix + *attrPaths.begin()); - return {v, noPos}; + for (auto & s : attrPaths) + res.push_back(s); + + return res; +} + +Value * InstallableFlake::getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake) +{ + auto vFlake = state.allocValue(); + + callFlake(state, lockedFlake, *vFlake); + + auto aOutputs = vFlake->attrs->get(state.symbols.create("outputs")); + assert(aOutputs); + + state.forceValue(*aOutputs->value); + + return aOutputs->value; +} + +std::tuple<std::string, FlakeRef, flake::EvalCache::Derivation> InstallableFlake::toDerivation() +{ + auto state = cmd.getEvalState(); + + auto lockedFlake = lockFlake(*state, flakeRef, cmd.lockFlags); + + Value * vOutputs = nullptr; + + auto emptyArgs = state->allocBindings(0); + + auto & evalCache = flake::EvalCache::singleton(); + + auto fingerprint = lockedFlake.getFingerprint(); + + for (auto & attrPath : getActualAttrPaths()) { + auto drv = evalCache.getDerivation(fingerprint, attrPath); + if (drv) { + if (state->store->isValidPath(drv->drvPath)) + return {attrPath, lockedFlake.flake.lockedRef, std::move(*drv)}; + } + + if (!vOutputs) + vOutputs = getFlakeOutputs(*state, lockedFlake); + + try { + auto * v = findAlongAttrPath(*state, attrPath, *emptyArgs, *vOutputs).first; + state->forceValue(*v); + + auto drvInfo = getDerivation(*state, *v, false); + if (!drvInfo) + throw Error("flake output attribute '%s' is not a derivation", attrPath); + + auto drv = flake::EvalCache::Derivation{ + state->store->parseStorePath(drvInfo->queryDrvPath()), + state->store->parseStorePath(drvInfo->queryOutPath()), + drvInfo->queryOutputName() + }; + + evalCache.addDerivation(fingerprint, attrPath, drv); + + return {attrPath, lockedFlake.flake.lockedRef, std::move(drv)}; + } catch (AttrPathNotFound & e) { + } } -}; + + throw Error("flake '%s' does not provide attribute %s", + flakeRef, concatStringsSep(", ", quoteStrings(attrPaths))); +} + +std::vector<flake::EvalCache::Derivation> InstallableFlake::toDerivations() +{ + std::vector<flake::EvalCache::Derivation> res; + res.push_back(std::get<2>(toDerivation())); + return res; +} + +std::pair<Value *, Pos> InstallableFlake::toValue(EvalState & state) +{ + auto lockedFlake = lockFlake(state, flakeRef, cmd.lockFlags); + + auto vOutputs = getFlakeOutputs(state, lockedFlake); + + auto emptyArgs = state.allocBindings(0); + + for (auto & attrPath : getActualAttrPaths()) { + try { + auto [v, pos] = findAlongAttrPath(state, attrPath, *emptyArgs, *vOutputs); + state.forceValue(*v); + return {v, pos}; + } catch (AttrPathNotFound & e) { + } + } + + throw Error("flake '%s' does not provide attribute %s", + flakeRef, concatStringsSep(", ", quoteStrings(attrPaths))); +} // FIXME: extend std::string attrRegex = R"([A-Za-z_][A-Za-z0-9-_+]*)"; static std::regex attrPathRegex(fmt(R"(%1%(\.%1%)*)", attrRegex)); -static std::vector<std::shared_ptr<Installable>> parseInstallables( - SourceExprCommand & cmd, ref<Store> store, std::vector<std::string> ss, bool useDefaultInstallables) +std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables( + ref<Store> store, std::vector<std::string> ss) { std::vector<std::shared_ptr<Installable>> result; - if (ss.empty() && useDefaultInstallables) { - if (cmd.file == "") - cmd.file = "."; - ss = {""}; - } + if (file || expr) { + if (file && expr) + throw UsageError("'--file' and '--expr' are exclusive"); - for (auto & s : ss) { + // FIXME: backward compatibility hack + if (file) evalSettings.pureEval = false; - if (s.compare(0, 1, "(") == 0) - result.push_back(std::make_shared<InstallableExpr>(cmd, s)); + auto state = getEvalState(); + auto vFile = state->allocValue(); - else if (s.find("/") != std::string::npos) { + if (file) + state->evalFile(lookupFileArg(*state, *file), *vFile); + else { + auto e = state->parseExprFromString(*expr, absPath(".")); + state->eval(e, *vFile); + } - auto path = store->toStorePath(store->followLinksToStore(s)); + for (auto & s : ss) + result.push_back(std::make_shared<InstallableAttrPath>(*this, vFile, s == "." ? "" : s)); - if (store->isStorePath(path)) - result.push_back(std::make_shared<InstallableStorePath>(store, path)); - } + } else { - else if (s == "" || std::regex_match(s, attrPathRegex)) - result.push_back(std::make_shared<InstallableAttrPath>(cmd, s)); + auto follow = [&](const std::string & s) -> std::optional<StorePath> { + try { + return store->followLinksToStorePath(s); + } catch (NotInStore &) { + return {}; + } + }; - else - throw UsageError("don't know what to do with argument '%s'", s); + for (auto & s : ss) { + if (hasPrefix(s, "nixpkgs.")) { + bool static warned; + warnOnce(warned, "the syntax 'nixpkgs.<attr>' is deprecated; use 'nixpkgs#<attr>' instead"); + result.push_back(std::make_shared<InstallableFlake>(*this, + FlakeRef::fromAttrs({{"type", "indirect"}, {"id", "nixpkgs"}}), + Strings{"legacyPackages." + settings.thisSystem.get() + "." + std::string(s, 8)}, Strings{})); + } + + else { + auto res = maybeParseFlakeRefWithFragment(s, absPath(".")); + if (res) { + auto &[flakeRef, fragment] = *res; + result.push_back(std::make_shared<InstallableFlake>( + *this, std::move(flakeRef), + fragment == "" ? getDefaultFlakeAttrPaths() : Strings{fragment}, + getDefaultFlakeAttrPathPrefixes())); + } else { + std::optional<StorePath> storePath; + if (s.find('/') != std::string::npos && (storePath = follow(s))) + result.push_back(std::make_shared<InstallableStorePath>(store, store->printStorePath(*storePath))); + else + throw Error("unrecognized argument '%s'", s); + } + } + } } return result; } -std::shared_ptr<Installable> parseInstallable( - SourceExprCommand & cmd, ref<Store> store, const std::string & installable, - bool useDefaultInstallables) +std::shared_ptr<Installable> SourceExprCommand::parseInstallable( + ref<Store> store, const std::string & installable) { - auto installables = parseInstallables(cmd, store, {installable}, false); + auto installables = parseInstallables(store, {installable}); assert(installables.size() == 1); return installables.front(); } @@ -308,7 +490,7 @@ StorePath toStorePath(ref<Store> store, RealiseMode mode, auto paths = toStorePaths(store, mode, {installable}); if (paths.size() != 1) - throw Error("argument '%s' should evaluate to one store path", installable->what()); + throw Error("argument '%s' should evaluate to one store path", installable->what()); return paths.begin()->clone(); } @@ -339,12 +521,16 @@ StorePathSet toDerivations(ref<Store> store, void InstallablesCommand::prepare() { - installables = parseInstallables(*this, getStore(), _installables, useDefaultInstallables()); + if (_installables.empty() && useDefaultInstallables()) + // FIXME: commands like "nix install" should not have a + // default, probably. + _installables.push_back("."); + installables = parseInstallables(getStore(), _installables); } void InstallableCommand::prepare() { - installable = parseInstallable(*this, getStore(), _installable, false); + installable = parseInstallable(getStore(), _installable); } } diff --git a/src/nix/installables.hh b/src/nix/installables.hh index 503984220..8f2d50077 100644 --- a/src/nix/installables.hh +++ b/src/nix/installables.hh @@ -3,11 +3,15 @@ #include "util.hh" #include "path.hh" #include "eval.hh" +#include "flake/eval-cache.hh" #include <optional> namespace nix { +struct DrvInfo; +struct SourceExprCommand; + struct Buildable { std::optional<StorePath> drvPath; @@ -16,6 +20,15 @@ struct Buildable typedef std::vector<Buildable> Buildables; +struct App +{ + PathSet context; + Path program; + // FIXME: add args, sandbox settings, metadata, ... + + App(EvalState & state, Value & vApp); +}; + struct Installable { virtual ~Installable() { } @@ -29,6 +42,8 @@ struct Installable Buildable toBuildable(); + App toApp(EvalState & state); + virtual std::pair<Value *, Pos> toValue(EvalState & state) { throw Error("argument '%s' cannot be evaluated", what()); @@ -42,4 +57,40 @@ struct Installable } }; +struct InstallableValue : Installable +{ + SourceExprCommand & cmd; + + InstallableValue(SourceExprCommand & cmd) : cmd(cmd) { } + + virtual std::vector<flake::EvalCache::Derivation> toDerivations(); + + Buildables toBuildables() override; +}; + +struct InstallableFlake : InstallableValue +{ + FlakeRef flakeRef; + Strings attrPaths; + Strings prefixes; + + InstallableFlake(SourceExprCommand & cmd, FlakeRef && flakeRef, + Strings && attrPaths, Strings && prefixes) + : InstallableValue(cmd), flakeRef(flakeRef), attrPaths(attrPaths), + prefixes(prefixes) + { } + + std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); } + + std::vector<std::string> getActualAttrPaths(); + + Value * getFlakeOutputs(EvalState & state, const flake::LockedFlake & lockedFlake); + + std::tuple<std::string, FlakeRef, flake::EvalCache::Derivation> toDerivation(); + + std::vector<flake::EvalCache::Derivation> toDerivations() override; + + std::pair<Value *, Pos> toValue(EvalState & state) override; +}; + } diff --git a/src/nix/local.mk b/src/nix/local.mk index 033675e89..3fcd15dc6 100644 --- a/src/nix/local.mk +++ b/src/nix/local.mk @@ -27,3 +27,5 @@ $(foreach name, \ $(eval $(call install-symlink, $(bindir)/nix, $(libexecdir)/nix/build-remote)) src/nix-env/user-env.cc: src/nix-env/buildenv.nix.gen.hh + +$(d)/flake.cc: $(d)/flake-template.nix.gen.hh diff --git a/src/nix/ls.cc b/src/nix/ls.cc index 3ef1f2750..8590199d7 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -34,16 +34,14 @@ struct MixLs : virtual Args, MixJSON (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : st.type == FSAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - std::cout << - (format("%s %20d %s") % tp % st.fileSize % relPath); + auto line = fmt("%s %20d %s", tp, st.fileSize, relPath); if (st.type == FSAccessor::Type::tSymlink) - std::cout << " -> " << accessor->readLink(curPath) - ; - std::cout << "\n"; + line += " -> " + accessor->readLink(curPath); + logger->stdout(line); if (recursive && st.type == FSAccessor::Type::tDirectory) doPath(st, curPath, relPath, false); } else { - std::cout << relPath << "\n"; + logger->stdout(relPath); if (recursive) { auto st = accessor->stat(curPath); if (st.type == FSAccessor::Type::tDirectory) diff --git a/src/nix/main.cc b/src/nix/main.cc index 2c64c7476..15f412dae 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -150,6 +150,7 @@ void mainWrapped(int argc, char * * argv) verbosity = lvlWarn; settings.verboseBuild = false; + evalSettings.pureEval = true; NixArgs args; diff --git a/src/nix/profile.cc b/src/nix/profile.cc new file mode 100644 index 000000000..e473be47f --- /dev/null +++ b/src/nix/profile.cc @@ -0,0 +1,427 @@ +#include "command.hh" +#include "common-args.hh" +#include "shared.hh" +#include "store-api.hh" +#include "derivations.hh" +#include "archive.hh" +#include "builtins/buildenv.hh" +#include "flake/flakeref.hh" +#include "../nix-env/user-env.hh" + +#include <nlohmann/json.hpp> +#include <regex> + +using namespace nix; + +struct ProfileElementSource +{ + FlakeRef originalRef; + // FIXME: record original attrpath. + FlakeRef resolvedRef; + std::string attrPath; + // FIXME: output names +}; + +struct ProfileElement +{ + StorePathSet storePaths; + std::optional<ProfileElementSource> source; + bool active = true; + // FIXME: priority +}; + +struct ProfileManifest +{ + std::vector<ProfileElement> elements; + + ProfileManifest() { } + + ProfileManifest(EvalState & state, const Path & profile) + { + auto manifestPath = profile + "/manifest.json"; + + if (pathExists(manifestPath)) { + auto json = nlohmann::json::parse(readFile(manifestPath)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("profile manifest '%s' has unsupported version %d", manifestPath, version); + + for (auto & e : json["elements"]) { + ProfileElement element; + for (auto & p : e["storePaths"]) + element.storePaths.insert(state.store->parseStorePath((std::string) p)); + element.active = e["active"]; + if (e.value("uri", "") != "") { + element.source = ProfileElementSource{ + parseFlakeRef(e["originalUri"]), + parseFlakeRef(e["uri"]), + e["attrPath"] + }; + } + elements.emplace_back(std::move(element)); + } + } + + else if (pathExists(profile + "/manifest.nix")) { + // FIXME: needed because of pure mode; ugly. + if (state.allowedPaths) { + state.allowedPaths->insert(state.store->followLinksToStore(profile)); + state.allowedPaths->insert(state.store->followLinksToStore(profile + "/manifest.nix")); + } + + auto drvInfos = queryInstalled(state, state.store->followLinksToStore(profile)); + + for (auto & drvInfo : drvInfos) { + ProfileElement element; + element.storePaths = singleton(state.store->parseStorePath(drvInfo.queryOutPath())); + elements.emplace_back(std::move(element)); + } + } + } + + std::string toJSON(Store & store) const + { + auto array = nlohmann::json::array(); + for (auto & element : elements) { + auto paths = nlohmann::json::array(); + for (auto & path : element.storePaths) + paths.push_back(store.printStorePath(path)); + nlohmann::json obj; + obj["storePaths"] = paths; + obj["active"] = element.active; + if (element.source) { + obj["originalUri"] = element.source->originalRef.to_string(); + obj["uri"] = element.source->resolvedRef.to_string(); + obj["attrPath"] = element.source->attrPath; + } + array.push_back(obj); + } + nlohmann::json json; + json["version"] = 1; + json["elements"] = array; + return json.dump(); + } + + StorePath build(ref<Store> store) + { + auto tempDir = createTempDir(); + + StorePathSet references; + + Packages pkgs; + for (auto & element : elements) { + for (auto & path : element.storePaths) { + if (element.active) + pkgs.emplace_back(store->printStorePath(path), true, 5); + references.insert(path.clone()); + } + } + + buildProfile(tempDir, std::move(pkgs)); + + writeFile(tempDir + "/manifest.json", toJSON(*store)); + + /* Add the symlink tree to the store. */ + StringSink sink; + dumpPath(tempDir, sink); + + auto narHash = hashString(htSHA256, *sink.s); + + ValidPathInfo info(store->makeFixedOutputPath(true, narHash, "profile", references)); + info.references = std::move(references); + info.narHash = narHash; + info.narSize = sink.s->size(); + info.ca = makeFixedOutputCA(true, info.narHash); + + store->addToStore(info, sink.s); + + return std::move(info.path); + } +}; + +struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile +{ + std::string description() override + { + return "install a package into a profile"; + } + + Examples examples() override + { + return { + Example{ + "To install a package from Nixpkgs:", + "nix profile install nixpkgs#hello" + }, + Example{ + "To install a package from a specific branch of Nixpkgs:", + "nix profile install nixpkgs/release-19.09#hello" + }, + Example{ + "To install a package from a specific revision of Nixpkgs:", + "nix profile install nixpkgs/1028bb33859f8dfad7f98e1c8d185f3d1aaa7340#hello" + }, + }; + } + + void run(ref<Store> store) override + { + ProfileManifest manifest(*getEvalState(), *profile); + + std::vector<StorePathWithOutputs> pathsToBuild; + + for (auto & installable : installables) { + if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) { + auto [attrPath, resolvedRef, drv] = installable2->toDerivation(); + + ProfileElement element; + element.storePaths = singleton(drv.outPath.clone()); // FIXME + element.source = ProfileElementSource{ + installable2->flakeRef, + resolvedRef, + attrPath, + }; + + pathsToBuild.emplace_back(drv.drvPath.clone(), StringSet{"out"}); // FIXME + + manifest.elements.emplace_back(std::move(element)); + } else + throw Error("'nix profile install' does not support argument '%s'", installable->what()); + } + + store->buildPaths(pathsToBuild); + + updateProfile(manifest.build(store)); + } +}; + +class MixProfileElementMatchers : virtual Args +{ + std::vector<std::string> _matchers; + +public: + + MixProfileElementMatchers() + { + expectArgs("elements", &_matchers); + } + + typedef std::variant<size_t, Path, std::regex> Matcher; + + std::vector<Matcher> getMatchers(ref<Store> store) + { + std::vector<Matcher> res; + + for (auto & s : _matchers) { + size_t n; + if (string2Int(s, n)) + res.push_back(n); + else if (store->isStorePath(s)) + res.push_back(s); + else + res.push_back(std::regex(s, std::regex::extended | std::regex::icase)); + } + + return res; + } + + bool matches(const Store & store, const ProfileElement & element, size_t pos, const std::vector<Matcher> & matchers) + { + for (auto & matcher : matchers) { + if (auto n = std::get_if<size_t>(&matcher)) { + if (*n == pos) return true; + } else if (auto path = std::get_if<Path>(&matcher)) { + if (element.storePaths.count(store.parseStorePath(*path))) return true; + } else if (auto regex = std::get_if<std::regex>(&matcher)) { + if (element.source + && std::regex_match(element.source->attrPath, *regex)) + return true; + } + } + + return false; + } +}; + +struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElementMatchers +{ + std::string description() override + { + return "remove packages from a profile"; + } + + Examples examples() override + { + return { + Example{ + "To remove a package by attribute path:", + "nix profile remove packages.x86_64-linux.hello" + }, + Example{ + "To remove all packages:", + "nix profile remove '.*'" + }, + Example{ + "To remove a package by store path:", + "nix profile remove /nix/store/rr3y0c6zyk7kjjl8y19s4lsrhn4aiq1z-hello-2.10" + }, + Example{ + "To remove a package by position:", + "nix profile remove 3" + }, + }; + } + + void run(ref<Store> store) override + { + ProfileManifest oldManifest(*getEvalState(), *profile); + + auto matchers = getMatchers(store); + + ProfileManifest newManifest; + + for (size_t i = 0; i < oldManifest.elements.size(); ++i) { + auto & element(oldManifest.elements[i]); + if (!matches(*store, element, i, matchers)) + newManifest.elements.push_back(std::move(element)); + } + + // FIXME: warn about unused matchers? + + printInfo("removed %d packages, kept %d packages", + oldManifest.elements.size() - newManifest.elements.size(), + newManifest.elements.size()); + + updateProfile(newManifest.build(store)); + } +}; + +struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProfileElementMatchers +{ + std::string description() override + { + return "upgrade packages using their most recent flake"; + } + + Examples examples() override + { + return { + Example{ + "To upgrade all packages that were installed using a mutable flake reference:", + "nix profile upgrade '.*'" + }, + Example{ + "To upgrade a specific package:", + "nix profile upgrade packages.x86_64-linux.hello" + }, + }; + } + + void run(ref<Store> store) override + { + ProfileManifest manifest(*getEvalState(), *profile); + + auto matchers = getMatchers(store); + + // FIXME: code duplication + std::vector<StorePathWithOutputs> pathsToBuild; + + for (size_t i = 0; i < manifest.elements.size(); ++i) { + auto & element(manifest.elements[i]); + if (element.source + && !element.source->originalRef.input->isImmutable() + && matches(*store, element, i, matchers)) + { + Activity act(*logger, lvlChatty, actUnknown, + fmt("checking '%s' for updates", element.source->attrPath)); + + InstallableFlake installable(*this, FlakeRef(element.source->originalRef), {element.source->attrPath}, {}); + + auto [attrPath, resolvedRef, drv] = installable.toDerivation(); + + if (element.source->resolvedRef == resolvedRef) continue; + + printInfo("upgrading '%s' from flake '%s' to '%s'", + element.source->attrPath, element.source->resolvedRef, resolvedRef); + + element.storePaths = singleton(drv.outPath.clone()); // FIXME + element.source = ProfileElementSource{ + installable.flakeRef, + resolvedRef, + attrPath, + }; + + pathsToBuild.emplace_back(drv.drvPath, StringSet{"out"}); // FIXME + } + } + + store->buildPaths(pathsToBuild); + + updateProfile(manifest.build(store)); + } +}; + +struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile +{ + std::string description() override + { + return "list installed packages"; + } + + Examples examples() override + { + return { + Example{ + "To show what packages are installed in the default profile:", + "nix profile info" + }, + }; + } + + void run(ref<Store> store) override + { + ProfileManifest manifest(*getEvalState(), *profile); + + for (size_t i = 0; i < manifest.elements.size(); ++i) { + auto & element(manifest.elements[i]); + logger->stdout("%d %s %s %s", i, + element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath : "-", + element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath : "-", + concatStringsSep(" ", store->printStorePathSet(element.storePaths))); + } + } +}; + +struct CmdProfile : virtual MultiCommand, virtual Command +{ + CmdProfile() + : MultiCommand({ + {"install", []() { return make_ref<CmdProfileInstall>(); }}, + {"remove", []() { return make_ref<CmdProfileRemove>(); }}, + {"upgrade", []() { return make_ref<CmdProfileUpgrade>(); }}, + {"info", []() { return make_ref<CmdProfileInfo>(); }}, + }) + { } + + std::string description() override + { + return "manage Nix profiles"; + } + + void run() override + { + if (!command) + throw UsageError("'nix profile' requires a sub-command."); + command->prepare(); + command->run(); + } + + void printHelp(const string & programName, std::ostream & out) override + { + MultiCommand::printHelp(programName, out); + } +}; + +static auto r1 = registerCommand<CmdProfile>("profile"); + diff --git a/src/nix/progress-bar.cc b/src/nix/progress-bar.cc index adc9b9a5d..8e7ba95a3 100644 --- a/src/nix/progress-bar.cc +++ b/src/nix/progress-bar.cc @@ -7,6 +7,7 @@ #include <atomic> #include <map> #include <thread> +#include <iostream> namespace nix { @@ -442,6 +443,18 @@ public: return res; } + + void writeToStdout(std::string_view s) override + { + auto state(state_.lock()); + if (state->active) { + std::cerr << "\r\e[K"; + Logger::writeToStdout(s); + draw(*state); + } else { + Logger::writeToStdout(s); + } + } }; void startProgressBar(bool printBuildLogs) diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 27727bd25..a2632cff8 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -811,6 +811,7 @@ struct CmdRepl : StoreCommand, MixEvalArgs void run(ref<Store> store) override { + evalSettings.pureEval = false; auto repl = std::make_unique<NixRepl>(searchPath, openStore()); repl->autoArgs = getAutoArgs(*repl->state); repl->mainLoop(files); diff --git a/src/nix/run.cc b/src/nix/run.cc index 5334531fd..901b87fbb 100644 --- a/src/nix/run.cc +++ b/src/nix/run.cc @@ -89,15 +89,15 @@ struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment }, Example{ "To start a shell providing youtube-dl from your 'nixpkgs' channel:", - "nix run nixpkgs.youtube-dl" + "nix run nixpkgs#youtube-dl" }, Example{ "To run GNU Hello:", - "nix run nixpkgs.hello -c hello --greeting 'Hi everybody!'" + "nix run nixpkgs#hello -c hello --greeting 'Hi everybody!'" }, Example{ "To run GNU Hello in a chroot store:", - "nix run --store ~/my-nix nixpkgs.hello -c hello" + "nix run --store ~/my-nix nixpkgs#hello -c hello" }, }; } @@ -143,6 +143,57 @@ struct CmdRun : InstallablesCommand, RunCommon, MixEnvironment static auto r1 = registerCommand<CmdRun>("run"); +struct CmdApp : InstallableCommand, RunCommon +{ + std::vector<std::string> args; + + CmdApp() + { + expectArgs("args", &args); + } + + std::string description() override + { + return "run a Nix application"; + } + + Examples examples() override + { + return { + Example{ + "To run Blender:", + "nix app blender-bin" + }, + }; + } + + Strings getDefaultFlakeAttrPaths() override + { + return {"defaultApp." + settings.thisSystem.get()}; + } + + Strings getDefaultFlakeAttrPathPrefixes() override + { + return {"apps." + settings.thisSystem.get() + "."}; + } + + void run(ref<Store> store) override + { + auto state = getEvalState(); + + auto app = installable->toApp(*state); + + state->realiseContext(app.context); + + Strings allArgs{app.program}; + for (auto & i : args) allArgs.push_back(i); + + runProgram(store, app.program, allArgs); + } +}; + +static auto r2 = registerCommand<CmdApp>("app"); + void chrootHelper(int argc, char * * argv) { int p = 1; diff --git a/src/nix/search.cc b/src/nix/search.cc index 769274543..caea25cdc 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -248,7 +248,9 @@ struct CmdSearch : SourceExprCommand, MixJSON auto cache = writeCache ? std::make_unique<JSONObject>(jsonCacheFile, false) : nullptr; - doExpr(getSourceExpr(*state), "", true, cache.get()); + // FIXME + throw Error("NOT IMPLEMENTED"); + //doExpr(getSourceExpr(*state), "", true, cache.get()); } catch (std::exception &) { /* Fun fact: catching std::ios::failure does not work diff --git a/src/nix/shell.cc b/src/nix/shell.cc index 71e640667..439ef02ed 100644 --- a/src/nix/shell.cc +++ b/src/nix/shell.cc @@ -182,6 +182,11 @@ struct Common : InstallableCommand, MixProfile out << "eval \"$shellHook\"\n"; } + Strings getDefaultFlakeAttrPaths() override + { + return {"devShell." + settings.thisSystem.get(), "defaultPackage." + settings.thisSystem.get()}; + } + StorePath getShellOutPath(ref<Store> store) { auto path = installable->getStorePath(); @@ -238,11 +243,15 @@ struct CmdDevShell : Common, MixEnvironment return { Example{ "To get the build environment of GNU hello:", - "nix dev-shell nixpkgs.hello" + "nix dev-shell nixpkgs#hello" + }, + Example{ + "To get the build environment of the default package of flake in the current directory:", + "nix dev-shell" }, Example{ "To store the build environment in a profile:", - "nix dev-shell --profile /tmp/my-shell nixpkgs.hello" + "nix dev-shell --profile /tmp/my-shell nixpkgs#hello" }, Example{ "To use a build environment previously recorded in a profile:", @@ -300,7 +309,7 @@ struct CmdPrintDevEnv : Common return { Example{ "To apply the build environment of GNU hello to the current shell:", - ". <(nix print-dev-env nixpkgs.hello)" + ". <(nix print-dev-env nixpkgs#hello)" }, }; } diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc index 87544f937..55a3255d2 100644 --- a/src/nix/show-config.cc +++ b/src/nix/show-config.cc @@ -23,7 +23,7 @@ struct CmdShowConfig : Command, MixJSON std::map<std::string, Config::SettingInfo> settings; globalConfig.getSettings(settings); for (auto & s : settings) - std::cout << s.first + " = " + s.second.value + "\n"; + logger->stdout(s.first + " = " + s.second.value); } } }; diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index d3b7a674a..6037f37fe 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -69,9 +69,9 @@ struct CmdWhyDepends : SourceExprCommand void run(ref<Store> store) override { - auto package = parseInstallable(*this, store, _package, false); + auto package = parseInstallable(store, _package); auto packagePath = toStorePath(store, Build, package); - auto dependency = parseInstallable(*this, store, _dependency, false); + auto dependency = parseInstallable(store, _dependency); auto dependencyPath = toStorePath(store, NoBuild, dependency); auto dependencyPathHash = storePathToHash(store->printStorePath(dependencyPath)); @@ -149,7 +149,7 @@ struct CmdWhyDepends : SourceExprCommand auto pathS = store->printStorePath(node.path); assert(node.dist != inf); - std::cout << fmt("%s%s%s%s" ANSI_NORMAL "\n", + logger->stdout("%s%s%s%s" ANSI_NORMAL, firstPad, node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "→ " : "", |