diff options
Diffstat (limited to 'src/libexpr/flake/flake.cc')
| -rw-r--r-- | src/libexpr/flake/flake.cc | 672 |
1 files changed, 672 insertions, 0 deletions
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc new file mode 100644 index 000000000..f23c53a82 --- /dev/null +++ b/src/libexpr/flake/flake.cc @@ -0,0 +1,672 @@ +#include "flake.hh" +#include "lockfile.hh" +#include "primops.hh" +#include "eval-inline.hh" +#include "store-api.hh" +#include "fetchers.hh" +#include "finally.hh" + +namespace nix { + +using namespace flake; + +namespace flake { + +/* If 'allowLookup' is true, then resolve 'flakeRef' using the + registries. */ +static FlakeRef maybeLookupFlake( + ref<Store> store, + const FlakeRef & flakeRef, + bool allowLookup) +{ + if (!flakeRef.input->isDirect()) { + if (allowLookup) + return flakeRef.resolve(store); + else + throw Error("'%s' is an indirect flake reference, but registry lookups are not allowed", flakeRef); + } else + return flakeRef; +} + +typedef std::vector<std::pair<FlakeRef, FlakeRef>> FlakeCache; + +static FlakeRef lookupInFlakeCache( + const FlakeCache & flakeCache, + const FlakeRef & flakeRef) +{ + // FIXME: inefficient. + for (auto & i : flakeCache) { + if (flakeRef == i.first) { + debug("mapping '%s' to previously seen input '%s' -> '%s", + flakeRef, i.first, i.second); + return i.second; + } + } + + return flakeRef; +} + +static std::pair<fetchers::Tree, FlakeRef> fetchOrSubstituteTree( + EvalState & state, + const FlakeRef & originalRef, + std::optional<TreeInfo> treeInfo, + bool allowLookup, + FlakeCache & flakeCache) +{ + /* The tree may already be in the Nix store, or it could be + substituted (which is often faster than fetching from the + original source). So check that. */ + if (treeInfo && originalRef.input->isDirect() && originalRef.input->isImmutable()) { + try { + auto storePath = treeInfo->computeStorePath(*state.store); + + state.store->ensurePath(storePath); + + debug("using substituted/cached input '%s' in '%s'", + originalRef, state.store->printStorePath(storePath)); + + auto actualPath = state.store->toRealPath(storePath); + + if (state.allowedPaths) + state.allowedPaths->insert(actualPath); + + return { + Tree { + .actualPath = actualPath, + .storePath = std::move(storePath), + .info = *treeInfo, + }, + originalRef + }; + } catch (Error & e) { + debug("substitution of input '%s' failed: %s", originalRef, e.what()); + } + } + + auto resolvedRef = lookupInFlakeCache(flakeCache, + maybeLookupFlake(state.store, + lookupInFlakeCache(flakeCache, originalRef), allowLookup)); + + auto [tree, lockedRef] = resolvedRef.fetchTree(state.store); + + debug("got tree '%s' from '%s'", + state.store->printStorePath(tree.storePath), lockedRef); + + flakeCache.push_back({originalRef, lockedRef}); + flakeCache.push_back({resolvedRef, lockedRef}); + + if (state.allowedPaths) + state.allowedPaths->insert(tree.actualPath); + + if (treeInfo) + assert(tree.storePath == treeInfo->computeStorePath(*state.store)); + + return {std::move(tree), lockedRef}; +} + +static void expectType(EvalState & state, ValueType type, + Value & value, const Pos & pos) +{ + if (value.type == tThunk && value.isTrivial()) + state.forceValue(value, pos); + if (value.type != type) + throw Error("expected %s but got %s at %s", + showType(type), showType(value.type), pos); +} + +static std::map<FlakeId, FlakeInput> parseFlakeInputs( + EvalState & state, Value * value, const Pos & pos); + +static FlakeInput parseFlakeInput(EvalState & state, + const std::string & inputName, Value * value, const Pos & pos) +{ + expectType(state, tAttrs, *value, pos); + + FlakeInput input { + .ref = FlakeRef::fromAttrs({{"type", "indirect"}, {"id", inputName}}) + }; + + auto sInputs = state.symbols.create("inputs"); + auto sUrl = state.symbols.create("url"); + auto sUri = state.symbols.create("uri"); // FIXME: remove soon + auto sFlake = state.symbols.create("flake"); + auto sFollows = state.symbols.create("follows"); + + fetchers::Attrs attrs; + std::optional<std::string> url; + + for (nix::Attr attr : *(value->attrs)) { + try { + if (attr.name == sUrl || attr.name == sUri) { + expectType(state, tString, *attr.value, *attr.pos); + url = attr.value->string.s; + attrs.emplace("url", *url); + } else if (attr.name == sFlake) { + expectType(state, tBool, *attr.value, *attr.pos); + input.isFlake = attr.value->boolean; + } else if (attr.name == sInputs) { + input.overrides = parseFlakeInputs(state, attr.value, *attr.pos); + } else if (attr.name == sFollows) { + expectType(state, tString, *attr.value, *attr.pos); + input.follows = parseInputPath(attr.value->string.s); + } else { + state.forceValue(*attr.value); + if (attr.value->type == tString) + attrs.emplace(attr.name, attr.value->string.s); + else + throw TypeError("flake input attribute '%s' is %s while a string is expected", + attr.name, showType(*attr.value)); + } + } catch (Error & e) { + e.addPrefix(fmt("in flake attribute '%s' at '%s':\n", attr.name, *attr.pos)); + throw; + } + } + + if (attrs.count("type")) + try { + input.ref = FlakeRef::fromAttrs(attrs); + } catch (Error & e) { + e.addPrefix(fmt("in flake input at '%s':\n", pos)); + throw; + } + else { + attrs.erase("url"); + if (!attrs.empty()) + throw Error("unexpected flake input attribute '%s', at %s", attrs.begin()->first, pos); + if (url) + input.ref = parseFlakeRef(*url); + } + + return input; +} + +static std::map<FlakeId, FlakeInput> parseFlakeInputs( + EvalState & state, Value * value, const Pos & pos) +{ + std::map<FlakeId, FlakeInput> inputs; + + expectType(state, tAttrs, *value, pos); + + for (nix::Attr & inputAttr : *(*value).attrs) { + inputs.emplace(inputAttr.name, + parseFlakeInput(state, + inputAttr.name, + inputAttr.value, + *inputAttr.pos)); + } + + return inputs; +} + +static Flake getFlake( + EvalState & state, + const FlakeRef & originalRef, + std::optional<TreeInfo> treeInfo, + bool allowLookup, + FlakeCache & flakeCache) +{ + auto [sourceInfo, lockedRef] = fetchOrSubstituteTree( + state, originalRef, treeInfo, allowLookup, flakeCache); + + // Guard against symlink attacks. + auto flakeFile = canonPath(sourceInfo.actualPath + "/" + lockedRef.subdir + "/flake.nix"); + if (!isInDir(flakeFile, sourceInfo.actualPath)) + throw Error("'flake.nix' file of flake '%s' escapes from '%s'", + lockedRef, state.store->printStorePath(sourceInfo.storePath)); + + Flake flake { + .originalRef = originalRef, + .lockedRef = lockedRef, + .sourceInfo = std::make_shared<fetchers::Tree>(std::move(sourceInfo)) + }; + + if (!pathExists(flakeFile)) + throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", lockedRef, lockedRef.subdir); + + Value vInfo; + state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack + + expectType(state, tAttrs, vInfo, Pos(state.symbols.create(flakeFile), 0, 0)); + + auto sEdition = state.symbols.create("edition"); + auto sEpoch = state.symbols.create("epoch"); // FIXME: remove soon + + auto edition = vInfo.attrs->get(sEdition); + if (!edition) + edition = vInfo.attrs->get(sEpoch); + + if (edition) { + expectType(state, tInt, *edition->value, *edition->pos); + flake.edition = edition->value->integer; + if (flake.edition > 201909) + throw Error("flake '%s' requires unsupported edition %d; please upgrade Nix", lockedRef, flake.edition); + if (flake.edition < 201909) + throw Error("flake '%s' has illegal edition %d", lockedRef, flake.edition); + } else + throw Error("flake '%s' lacks attribute 'edition'", lockedRef); + + if (auto description = vInfo.attrs->get(state.sDescription)) { + expectType(state, tString, *description->value, *description->pos); + flake.description = description->value->string.s; + } + + auto sInputs = state.symbols.create("inputs"); + + if (auto inputs = vInfo.attrs->get(sInputs)) + flake.inputs = parseFlakeInputs(state, inputs->value, *inputs->pos); + + auto sOutputs = state.symbols.create("outputs"); + + if (auto outputs = vInfo.attrs->get(sOutputs)) { + expectType(state, tLambda, *outputs->value, *outputs->pos); + flake.vOutputs = outputs->value; + + if (flake.vOutputs->lambda.fun->matchAttrs) { + for (auto & formal : flake.vOutputs->lambda.fun->formals->formals) { + if (formal.name != state.sSelf) + flake.inputs.emplace(formal.name, FlakeInput { + .ref = parseFlakeRef(formal.name) + }); + } + } + + } else + throw Error("flake '%s' lacks attribute 'outputs'", lockedRef); + + for (auto & attr : *vInfo.attrs) { + if (attr.name != sEdition && + attr.name != sEpoch && + attr.name != state.sDescription && + attr.name != sInputs && + attr.name != sOutputs) + throw Error("flake '%s' has an unsupported attribute '%s', at %s", + lockedRef, attr.name, *attr.pos); + } + + return flake; +} + +Flake getFlake(EvalState & state, const FlakeRef & originalRef, bool allowLookup) +{ + FlakeCache flakeCache; + return getFlake(state, originalRef, {}, allowLookup, flakeCache); +} + +/* Compute an in-memory lock file for the specified top-level flake, + and optionally write it to file, it the flake is writable. */ +LockedFlake lockFlake( + EvalState & state, + const FlakeRef & topRef, + const LockFlags & lockFlags) +{ + settings.requireExperimentalFeature("flakes"); + + FlakeCache flakeCache; + + auto flake = getFlake(state, topRef, {}, lockFlags.useRegistries, flakeCache); + + // FIXME: symlink attack + auto oldLockFile = LockFile::read( + flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock"); + + debug("old lock file: %s", oldLockFile); + + // FIXME: check whether all overrides are used. + std::map<InputPath, FlakeInput> overrides; + std::set<InputPath> overridesUsed, updatesUsed; + + for (auto & i : lockFlags.inputOverrides) + overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second }); + + LockFile newLockFile; + + std::vector<FlakeRef> parents; + std::map<InputPath, InputPath> follows; + + std::function<void( + const FlakeInputs & flakeInputs, + std::shared_ptr<Node> node, + const InputPath & inputPathPrefix, + std::shared_ptr<const Node> oldNode)> + computeLocks; + + computeLocks = [&]( + const FlakeInputs & flakeInputs, + std::shared_ptr<Node> node, + const InputPath & inputPathPrefix, + std::shared_ptr<const Node> oldNode) + { + debug("computing lock file node '%s'", concatStringsSep("/", inputPathPrefix)); + + /* Get the overrides (i.e. attributes of the form + 'inputs.nixops.inputs.nixpkgs.url = ...'). */ + // FIXME: check this + for (auto & [id, input] : flake.inputs) { + for (auto & [idOverride, inputOverride] : input.overrides) { + auto inputPath(inputPathPrefix); + inputPath.push_back(id); + inputPath.push_back(idOverride); + overrides.insert_or_assign(inputPath, inputOverride); + } + } + + /* Go over the flake inputs, resolve/fetch them if + necessary (i.e. if they're new or the flakeref changed + from what's in the lock file). */ + for (auto & [id, input2] : flakeInputs) { + auto inputPath(inputPathPrefix); + inputPath.push_back(id); + auto inputPathS = concatStringsSep("/", inputPath); + debug("computing input '%s'", concatStringsSep("/", inputPath)); + + /* Do we have an override for this input from one of the + ancestors? */ + auto i = overrides.find(inputPath); + bool hasOverride = i != overrides.end(); + if (hasOverride) overridesUsed.insert(inputPath); + auto & input = hasOverride ? i->second : input2; + + /* Resolve 'follows' later (since it may refer to an input + path we haven't processed yet. */ + if (input.follows) { + if (hasOverride) + /* 'follows' from an override is relative to the + root of the graph. */ + follows.insert_or_assign(inputPath, *input.follows); + else { + /* Otherwise, it's relative to the current flake. */ + InputPath path(inputPathPrefix); + for (auto & i : *input.follows) path.push_back(i); + follows.insert_or_assign(inputPath, path); + } + continue; + } + + /* Do we have an entry in the existing lock file? And we + don't have a --update-input flag for this input? */ + std::shared_ptr<const LockedNode> oldLock; + + updatesUsed.insert(inputPath); + + if (oldNode && !lockFlags.inputUpdates.count(inputPath)) { + auto oldLockIt = oldNode->inputs.find(id); + if (oldLockIt != oldNode->inputs.end()) + oldLock = std::dynamic_pointer_cast<const LockedNode>(oldLockIt->second); + } + + if (oldLock + && oldLock->originalRef == input.ref + && !hasOverride) + { + debug("keeping existing input '%s'", inputPathS); + + /* Copy the input from the old lock since its flakeref + didn't change and there is no override from a + higher level flake. */ + auto childNode = std::make_shared<LockedNode>( + oldLock->lockedRef, oldLock->originalRef, oldLock->info, oldLock->isFlake); + + node->inputs.insert_or_assign(id, childNode); + + /* If we have an --update-input flag for an input + of this input, then we must fetch the flake to + to update it. */ + auto lb = lockFlags.inputUpdates.lower_bound(inputPath); + + auto hasChildUpdate = + lb != lockFlags.inputUpdates.end() + && lb->size() > inputPath.size() + && std::equal(inputPath.begin(), inputPath.end(), lb->begin()); + + if (hasChildUpdate) { + auto inputFlake = getFlake( + state, oldLock->lockedRef, oldLock->info, false, flakeCache); + computeLocks(inputFlake.inputs, childNode, inputPath, oldLock); + } else { + /* No need to fetch this flake, we can be + lazy. However there may be new overrides on the + inputs of this flake, so we need to check + those. */ + FlakeInputs fakeInputs; + + for (auto & i : oldLock->inputs) { + auto lockedNode = std::dynamic_pointer_cast<LockedNode>(i.second); + // Note: this node is not locked in case + // of a circular reference back to the root. + if (lockedNode) + fakeInputs.emplace(i.first, FlakeInput { + .ref = lockedNode->originalRef + }); + else { + InputPath path(inputPath); + path.push_back(i.first); + follows.insert_or_assign(path, InputPath()); + } + } + + computeLocks(fakeInputs, childNode, inputPath, oldLock); + } + + } else { + /* We need to create a new lock file entry. So fetch + this input. */ + + if (!lockFlags.allowMutable && !input.ref.input->isImmutable()) + throw Error("cannot update flake input '%s' in pure mode", inputPathS); + + if (input.isFlake) { + auto inputFlake = getFlake(state, input.ref, {}, lockFlags.useRegistries, flakeCache); + + /* Note: in case of an --override-input, we use + the *original* ref (input2.ref) for the + "original" field, rather than the + override. This ensures that the override isn't + nuked the next time we update the lock + file. That is, overrides are sticky unless you + use --no-write-lock-file. */ + auto childNode = std::make_shared<LockedNode>( + inputFlake.lockedRef, input2.ref, inputFlake.sourceInfo->info); + + node->inputs.insert_or_assign(id, childNode); + + /* Guard against circular flake imports. */ + for (auto & parent : parents) + if (parent == input.ref) + throw Error("found circular import of flake '%s'", parent); + parents.push_back(input.ref); + Finally cleanup([&]() { parents.pop_back(); }); + + /* Recursively process the inputs of this + flake. Also, unless we already have this flake + in the top-level lock file, use this flake's + own lock file. */ + computeLocks( + inputFlake.inputs, childNode, inputPath, + oldLock + ? std::dynamic_pointer_cast<const Node>(oldLock) + : LockFile::read( + inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root); + } + + else { + auto [sourceInfo, lockedRef] = fetchOrSubstituteTree( + state, input.ref, {}, lockFlags.useRegistries, flakeCache); + node->inputs.insert_or_assign(id, + std::make_shared<LockedNode>(lockedRef, input.ref, sourceInfo.info, false)); + } + } + } + }; + + computeLocks( + flake.inputs, newLockFile.root, {}, + lockFlags.recreateLockFile ? nullptr : oldLockFile.root); + + /* Insert edges for 'follows' overrides. */ + for (auto & [from, to] : follows) { + debug("adding 'follows' node from '%s' to '%s'", + concatStringsSep("/", from), + concatStringsSep("/", to)); + + assert(!from.empty()); + + InputPath fromParent(from); + fromParent.pop_back(); + + auto fromParentNode = newLockFile.root->findInput(fromParent); + assert(fromParentNode); + + auto toNode = newLockFile.root->findInput(to); + if (!toNode) + throw Error("flake input '%s' follows non-existent flake input '%s'", + concatStringsSep("/", from), + concatStringsSep("/", to)); + + fromParentNode->inputs.insert_or_assign(from.back(), toNode); + } + + for (auto & i : lockFlags.inputOverrides) + if (!overridesUsed.count(i.first)) + warn("the flag '--override-input %s %s' does not match any input", + concatStringsSep("/", i.first), i.second); + + for (auto & i : lockFlags.inputUpdates) + if (!updatesUsed.count(i)) + warn("the flag '--update-input %s' does not match any input", concatStringsSep("/", i)); + + debug("new lock file: %s", newLockFile); + + /* Check whether we need to / can write the new lock file. */ + if (!(newLockFile == oldLockFile)) { + + auto diff = diffLockFiles(oldLockFile, newLockFile); + + if (!(oldLockFile == LockFile())) + printInfo("inputs of flake '%s' changed:\n%s", topRef, chomp(diff)); + + if (lockFlags.writeLockFile) { + if (auto sourcePath = topRef.input->getSourcePath()) { + if (!newLockFile.isImmutable()) { + if (settings.warnDirty) + warn("will not write lock file of flake '%s' because it has a mutable input", topRef); + } else { + if (!lockFlags.updateLockFile) + throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef); + + auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock"; + + auto path = *sourcePath + "/" + relPath; + + bool lockFileExists = pathExists(path); + + if (lockFileExists) + warn("updating lock file '%s'", path); + else + warn("creating lock file '%s'", path); + + newLockFile.write(path); + + topRef.input->markChangedFile( + (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock", + lockFlags.commitLockFile + ? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s", + relPath, lockFileExists ? "Update" : "Add", diff)) + : std::nullopt); + + /* Rewriting the lockfile changed the top-level + repo, so we should re-read it. FIXME: we could + also just clear the 'rev' field... */ + auto prevLockedRef = flake.lockedRef; + FlakeCache dummyCache; + flake = getFlake(state, topRef, {}, lockFlags.useRegistries, dummyCache); + + if (lockFlags.commitLockFile && + flake.lockedRef.input->getRev() && + prevLockedRef.input->getRev() != flake.lockedRef.input->getRev()) + warn("committed new revision '%s'", flake.lockedRef.input->getRev()->gitRev()); + + /* Make sure that we picked up the change, + i.e. the tree should usually be dirty + now. Corner case: we could have reverted from a + dirty to a clean tree! */ + if (flake.lockedRef.input == prevLockedRef.input + && !flake.lockedRef.input->isImmutable()) + throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef); + } + } else + throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef); + } else + warn("not writing modified lock file of flake '%s'", topRef); + } + + return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) }; +} + +void callFlake(EvalState & state, + const LockedFlake & lockedFlake, + Value & vRes) +{ + auto vLocks = state.allocValue(); + auto vRootSrc = state.allocValue(); + auto vRootSubdir = state.allocValue(); + auto vTmp1 = state.allocValue(); + auto vTmp2 = state.allocValue(); + + mkString(*vLocks, lockedFlake.lockFile.to_string()); + + emitTreeAttrs(state, *lockedFlake.flake.sourceInfo, lockedFlake.flake.lockedRef.input, *vRootSrc); + + mkString(*vRootSubdir, lockedFlake.flake.lockedRef.subdir); + + static Value * vCallFlake = nullptr; + + if (!vCallFlake) { + vCallFlake = state.allocValue(); + state.eval(state.parseExprFromString( + #include "call-flake.nix.gen.hh" + , "/"), *vCallFlake); + } + + state.callFunction(*vCallFlake, *vLocks, *vTmp1, noPos); + state.callFunction(*vTmp1, *vRootSrc, *vTmp2, noPos); + state.callFunction(*vTmp2, *vRootSubdir, vRes, noPos); +} + +static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto flakeRefS = state.forceStringNoCtx(*args[0], pos); + auto flakeRef = parseFlakeRef(flakeRefS); + if (evalSettings.pureEval && !flakeRef.input->isImmutable()) + throw Error("cannot call 'getFlake' on mutable flake reference '%s', at %s (use --impure to override)", flakeRefS, pos); + + callFlake(state, + lockFlake(state, flakeRef, + LockFlags { + .updateLockFile = false, + .useRegistries = !evalSettings.pureEval, + .allowMutable = !evalSettings.pureEval, + }), + v); +} + +static RegisterPrimOp r2("getFlake", 1, prim_getFlake); + +} + +Fingerprint LockedFlake::getFingerprint() const +{ + // FIXME: as an optimization, if the flake contains a lock file + // and we haven't changed it, then it's sufficient to use + // flake.sourceInfo.storePath for the fingerprint. + return hashString(htSHA256, + fmt("%s;%d;%d;%s", + flake.sourceInfo->storePath.to_string(), + flake.sourceInfo->info.revCount.value_or(0), + flake.sourceInfo->info.lastModified.value_or(0), + lockFile)); +} + +Flake::~Flake() { } + +} |