diff options
Diffstat (limited to 'src/libexpr/flake/flake.cc')
| -rw-r--r-- | src/libexpr/flake/flake.cc | 604 |
1 files changed, 604 insertions, 0 deletions
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc new file mode 100644 index 000000000..bb0543541 --- /dev/null +++ b/src/libexpr/flake/flake.cc @@ -0,0 +1,604 @@ +#include "flake.hh" +#include "lockfile.hh" +#include "primops.hh" +#include "eval-inline.hh" +#include "primops/fetchGit.hh" +#include "download.hh" +#include "args.hh" + +#include <iostream> +#include <queue> +#include <regex> +#include <ctime> +#include <iomanip> +#include <nlohmann/json.hpp> + +namespace nix { + +using namespace flake; + +namespace flake { + +/* Read a registry. */ +std::shared_ptr<FlakeRegistry> readRegistry(const Path & path) +{ + auto registry = std::make_shared<FlakeRegistry>(); + + if (!pathExists(path)) + return std::make_shared<FlakeRegistry>(); + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("flake registry '%s' has unsupported version %d", path, version); + + auto flakes = json["flakes"]; + for (auto i = flakes.begin(); i != flakes.end(); ++i) + registry->entries.emplace(i.key(), FlakeRef(i->value("uri", ""))); + + return registry; +} + +/* Write a registry to a file. */ +void writeRegistry(const FlakeRegistry & registry, const Path & path) +{ + nlohmann::json json; + json["version"] = 2; + for (auto elem : registry.entries) + json["flakes"][elem.first.to_string()] = { {"uri", elem.second.to_string()} }; + createDirs(dirOf(path)); + writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. +} + +Path getUserRegistryPath() +{ + return getHome() + "/.config/nix/registry.json"; +} + +std::shared_ptr<FlakeRegistry> getUserRegistry() +{ + return readRegistry(getUserRegistryPath()); +} + +std::shared_ptr<FlakeRegistry> getFlagRegistry(RegistryOverrides registryOverrides) +{ + auto flagRegistry = std::make_shared<FlakeRegistry>(); + for (auto const & x : registryOverrides) { + flagRegistry->entries.insert_or_assign(FlakeRef(x.first), FlakeRef(x.second)); + } + return flagRegistry; +} + +static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, + std::vector<FlakeRef> pastSearches = {}); + +FlakeRef updateFlakeRef(EvalState & state, const FlakeRef & newRef, const Registries & registries, std::vector<FlakeRef> pastSearches) +{ + std::string errorMsg = "found cycle in flake registries: "; + for (FlakeRef oldRef : pastSearches) { + errorMsg += oldRef.to_string(); + if (oldRef == newRef) + throw Error(errorMsg); + errorMsg += " - "; + } + pastSearches.push_back(newRef); + return lookupFlake(state, newRef, registries, pastSearches); +} + +static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, + std::vector<FlakeRef> pastSearches) +{ + if (registries.empty() && !flakeRef.isDirect()) + throw Error("indirect flake reference '%s' is not allowed", flakeRef); + + for (std::shared_ptr<FlakeRegistry> registry : registries) { + auto i = registry->entries.find(flakeRef); + if (i != registry->entries.end()) { + auto newRef = i->second; + return updateFlakeRef(state, newRef, registries, pastSearches); + } + + auto j = registry->entries.find(flakeRef.baseRef()); + if (j != registry->entries.end()) { + auto newRef = j->second; + newRef.ref = flakeRef.ref; + newRef.rev = flakeRef.rev; + return updateFlakeRef(state, newRef, registries, pastSearches); + } + } + + if (!flakeRef.isDirect()) + throw Error("could not resolve flake reference '%s'", flakeRef); + + return flakeRef; +} + +// Lookups happen here too +static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) +{ + FlakeRef resolvedRef = lookupFlake(state, flakeRef, + impureIsAllowed ? state.getFlakeRegistries() : std::vector<std::shared_ptr<FlakeRegistry>>()); + + if (evalSettings.pureEval && !impureIsAllowed && !resolvedRef.isImmutable()) + throw Error("requested to fetch mutable flake '%s' in pure mode", resolvedRef); + + auto doGit = [&](const GitInfo & gitInfo) { + FlakeRef ref(resolvedRef.baseRef()); + ref.ref = gitInfo.ref; + ref.rev = gitInfo.rev; + SourceInfo info(ref); + info.storePath = gitInfo.storePath; + info.revCount = gitInfo.revCount; + info.narHash = state.store->queryPathInfo(info.storePath)->narHash; + info.lastModified = gitInfo.lastModified; + return info; + }; + + // This only downloads only one revision of the repo, not the entire history. + if (auto refData = std::get_if<FlakeRef::IsGitHub>(&resolvedRef.data)) { + + // FIXME: use regular /archive URLs instead? api.github.com + // might have stricter rate limits. + + auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", + refData->owner, refData->repo, + resolvedRef.rev ? resolvedRef.rev->to_string(Base16, false) + : resolvedRef.ref ? *resolvedRef.ref : "master"); + + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken; + + CachedDownloadRequest request(url); + request.unpack = true; + request.name = "source"; + request.ttl = resolvedRef.rev ? 1000000000 : settings.tarballTtl; + request.getLastModified = true; + auto result = getDownloader()->downloadCached(state.store, request); + + if (!result.etag) + throw Error("did not receive an ETag header from '%s'", url); + + if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"') + throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url); + + FlakeRef ref(resolvedRef.baseRef()); + ref.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); + SourceInfo info(ref); + info.storePath = result.storePath; + info.narHash = state.store->queryPathInfo(info.storePath)->narHash; + info.lastModified = result.lastModified; + + return info; + } + + // This downloads the entire git history + else if (auto refData = std::get_if<FlakeRef::IsGit>(&resolvedRef.data)) { + return doGit(exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source")); + } + + else if (auto refData = std::get_if<FlakeRef::IsPath>(&resolvedRef.data)) { + if (!pathExists(refData->path + "/.git")) + throw Error("flake '%s' does not reference a Git repository", refData->path); + return doGit(exportGit(state.store, refData->path, {}, {}, "source")); + } + + else abort(); +} + +// This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within `fetchFlake`, which is used here. +Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) +{ + SourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); + debug("got flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); + + FlakeRef resolvedRef = sourceInfo.resolvedRef; + + state.store->assertStorePath(sourceInfo.storePath); + + if (state.allowedPaths) + state.allowedPaths->insert(state.store->toRealPath(sourceInfo.storePath)); + + // Guard against symlink attacks. + Path flakeFile = canonPath(sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix"); + Path realFlakeFile = state.store->toRealPath(flakeFile); + if (!isInDir(realFlakeFile, state.store->toRealPath(sourceInfo.storePath))) + throw Error("'flake.nix' file of flake '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath); + + Flake flake(flakeRef, sourceInfo); + + if (!pathExists(realFlakeFile)) + throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir); + + Value vInfo; + state.evalFile(realFlakeFile, vInfo); // FIXME: symlink attack + + state.forceAttrs(vInfo); + + auto sEpoch = state.symbols.create("epoch"); + + if (auto epoch = vInfo.attrs->get(sEpoch)) { + flake.epoch = state.forceInt(*(**epoch).value, *(**epoch).pos); + if (flake.epoch > 201906) + throw Error("flake '%s' requires unsupported epoch %d; please upgrade Nix", flakeRef, flake.epoch); + } else + throw Error("flake '%s' lacks attribute 'epoch'", flakeRef); + + if (auto name = vInfo.attrs->get(state.sName)) + flake.id = state.forceStringNoCtx(*(**name).value, *(**name).pos); + else + throw Error("flake '%s' lacks attribute 'name'", flakeRef); + + if (auto description = vInfo.attrs->get(state.sDescription)) + flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos); + + auto sInputs = state.symbols.create("inputs"); + + if (auto inputs = vInfo.attrs->get(sInputs)) { + state.forceList(*(**inputs).value, *(**inputs).pos); + for (unsigned int n = 0; n < (**inputs).value->listSize(); ++n) + flake.inputs.push_back(FlakeRef(state.forceStringNoCtx( + *(**inputs).value->listElems()[n], *(**inputs).pos))); + } + + auto sNonFlakeInputs = state.symbols.create("nonFlakeInputs"); + + if (std::optional<Attr *> nonFlakeInputs = vInfo.attrs->get(sNonFlakeInputs)) { + state.forceAttrs(*(**nonFlakeInputs).value, *(**nonFlakeInputs).pos); + for (Attr attr : *(*(**nonFlakeInputs).value).attrs) { + std::string myNonFlakeUri = state.forceStringNoCtx(*attr.value, *attr.pos); + FlakeRef nonFlakeRef = FlakeRef(myNonFlakeUri); + flake.nonFlakeInputs.insert_or_assign(attr.name, nonFlakeRef); + } + } + + auto sOutputs = state.symbols.create("outputs"); + + if (auto outputs = vInfo.attrs->get(sOutputs)) { + state.forceFunction(*(**outputs).value, *(**outputs).pos); + flake.vOutputs = (**outputs).value; + } else + throw Error("flake '%s' lacks attribute 'outputs'", flakeRef); + + for (auto & attr : *vInfo.attrs) { + if (attr.name != sEpoch && + attr.name != state.sName && + attr.name != state.sDescription && + attr.name != sInputs && + attr.name != sNonFlakeInputs && + attr.name != sOutputs) + throw Error("flake '%s' has an unsupported attribute '%s', at %s", + flakeRef, attr.name, *attr.pos); + } + + return flake; +} + +// Get the `NonFlake` corresponding to a `FlakeRef`. +NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) +{ + auto sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); + debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); + + FlakeRef resolvedRef = sourceInfo.resolvedRef; + + NonFlake nonFlake(flakeRef, sourceInfo); + + state.store->assertStorePath(nonFlake.sourceInfo.storePath); + + if (state.allowedPaths) + state.allowedPaths->insert(nonFlake.sourceInfo.storePath); + + return nonFlake; +} + +bool allowedToWrite(HandleLockFile handle) +{ + return handle == UpdateLockFile || handle == RecreateLockFile; +} + +bool recreateLockFile(HandleLockFile handle) +{ + return handle == RecreateLockFile || handle == UseNewLockFile; +} + +bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) +{ + if (handle == AllPure) return false; + else if (handle == TopRefUsesRegistries) return isTopRef; + else if (handle == UpdateLockFile) return true; + else if (handle == UseUpdatedLockFile) return true; + else if (handle == RecreateLockFile) return true; + else if (handle == UseNewLockFile) return true; + else assert(false); +} + +/* Given a flakeref and its subtree of the lockfile, return an updated + subtree of the lockfile. That is, if the 'flake.nix' of the + referenced flake has inputs that don't have a corresponding entry + in the lockfile, they're added to the lockfile; conversely, any + lockfile entries that don't have a corresponding entry in flake.nix + are removed. + + Note that this is lazy: we only recursively fetch inputs that are + not in the lockfile yet. */ +static std::pair<Flake, FlakeInput> updateLocks( + EvalState & state, + const Flake & flake, + HandleLockFile handleLockFile, + const FlakeInputs & oldEntry, + bool topRef) +{ + FlakeInput newEntry( + flake.id, + flake.sourceInfo.resolvedRef, + flake.sourceInfo.narHash); + + for (auto & input : flake.nonFlakeInputs) { + auto & id = input.first; + auto & ref = input.second; + auto i = oldEntry.nonFlakeInputs.find(id); + if (i != oldEntry.nonFlakeInputs.end()) { + newEntry.nonFlakeInputs.insert_or_assign(i->first, i->second); + } else { + if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) + throw Error("cannot update non-flake dependency '%s' in pure mode", id); + auto nonFlake = getNonFlake(state, ref, allowedToUseRegistries(handleLockFile, false)); + newEntry.nonFlakeInputs.insert_or_assign(id, + NonFlakeInput( + nonFlake.sourceInfo.resolvedRef, + nonFlake.sourceInfo.narHash)); + } + } + + for (auto & inputRef : flake.inputs) { + auto i = oldEntry.flakeInputs.find(inputRef); + if (i != oldEntry.flakeInputs.end()) { + newEntry.flakeInputs.insert_or_assign(inputRef, i->second); + } else { + if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) + throw Error("cannot update flake dependency '%s' in pure mode", inputRef); + newEntry.flakeInputs.insert_or_assign(inputRef, + updateLocks(state, + getFlake(state, inputRef, allowedToUseRegistries(handleLockFile, false)), + handleLockFile, {}, false).second); + } + } + + return {flake, newEntry}; +} + +/* Compute an in-memory lockfile for the specified top-level flake, + and optionally write it to file, it the flake is writable. */ +ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile) +{ + auto flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); + + LockFile oldLockFile; + + if (!recreateLockFile(handleLockFile)) { + // If recreateLockFile, start with an empty lockfile + // FIXME: symlink attack + oldLockFile = LockFile::read( + state.store->toRealPath(flake.sourceInfo.storePath) + + "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock"); + } + + LockFile lockFile(updateLocks( + state, flake, handleLockFile, oldLockFile, true).second); + + if (!(lockFile == oldLockFile)) { + if (allowedToWrite(handleLockFile)) { + if (auto refData = std::get_if<FlakeRef::IsPath>(&topRef.data)) { + lockFile.write(refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock"); + + // Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store. + runProgram("git", true, { "-C", refData->path, "add", + (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock" }); + } else + warn("cannot write lockfile of remote flake '%s'", topRef); + } else if (handleLockFile != AllPure && handleLockFile != TopRefUsesRegistries) + warn("using updated lockfile without writing it to file"); + } + + return ResolvedFlake(std::move(flake), std::move(lockFile)); +} + +void updateLockFile(EvalState & state, const FlakeRef & flakeRef, bool recreateLockFile) +{ + resolveFlake(state, flakeRef, recreateLockFile ? RecreateLockFile : UpdateLockFile); +} + +static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo, Value & vAttrs) +{ + auto & path = sourceInfo.storePath; + assert(state.store->isValidPath(path)); + mkString(*state.allocAttr(vAttrs, state.sOutPath), path, {path}); + + if (sourceInfo.resolvedRef.rev) { + mkString(*state.allocAttr(vAttrs, state.symbols.create("rev")), + sourceInfo.resolvedRef.rev->gitRev()); + mkString(*state.allocAttr(vAttrs, state.symbols.create("shortRev")), + sourceInfo.resolvedRef.rev->gitShortRev()); + } + + if (sourceInfo.revCount) + mkInt(*state.allocAttr(vAttrs, state.symbols.create("revCount")), *sourceInfo.revCount); + + if (sourceInfo.lastModified) + mkString(*state.allocAttr(vAttrs, state.symbols.create("lastModified")), + fmt("%s", + std::put_time(std::gmtime(&*sourceInfo.lastModified), "%Y%m%d%H%M%S"))); +} + +/* Helper primop to make callFlake (below) fetch/call its inputs + lazily. Note that this primop cannot be called by user code since + it doesn't appear in 'builtins'. */ +static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto lazyFlake = (FlakeInput *) args[0]->attrs; + auto flake = getFlake(state, lazyFlake->ref, false); + + if (flake.sourceInfo.narHash != lazyFlake->narHash) + throw Error("the content hash of flake '%s' doesn't match the hash recorded in the referring lockfile", flake.sourceInfo.resolvedRef); + + callFlake(state, flake, *lazyFlake, v); +} + +static void prim_callNonFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto lazyNonFlake = (NonFlakeInput *) args[0]->attrs; + + auto nonFlake = getNonFlake(state, lazyNonFlake->ref); + + if (nonFlake.sourceInfo.narHash != lazyNonFlake->narHash) + throw Error("the content hash of repository '%s' doesn't match the hash recorded in the referring lockfile", nonFlake.sourceInfo.resolvedRef); + + state.mkAttrs(v, 8); + + assert(state.store->isValidPath(nonFlake.sourceInfo.storePath)); + + mkString(*state.allocAttr(v, state.sOutPath), + nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath}); + + emitSourceInfoAttrs(state, nonFlake.sourceInfo, v); +} + +void callFlake(EvalState & state, + const Flake & flake, + const FlakeInputs & inputs, + Value & vRes) +{ + // Construct the resulting attrset '{outputs, ...}'. This attrset + // is passed lazily as an argument to the 'outputs' function. + + auto & v = *state.allocValue(); + + state.mkAttrs(v, + inputs.flakeInputs.size() + + inputs.nonFlakeInputs.size() + 8); + + for (auto & dep : inputs.flakeInputs) { + auto vFlake = state.allocAttr(v, dep.second.id); + auto vPrimOp = state.allocValue(); + static auto primOp = new PrimOp(prim_callFlake, 1, state.symbols.create("callFlake")); + vPrimOp->type = tPrimOp; + vPrimOp->primOp = primOp; + auto vArg = state.allocValue(); + vArg->type = tNull; + // FIXME: leak + vArg->attrs = (Bindings *) new FlakeInput(dep.second); // evil! also inefficient + mkApp(*vFlake, *vPrimOp, *vArg); + } + + for (auto & dep : inputs.nonFlakeInputs) { + auto vNonFlake = state.allocAttr(v, dep.first); + auto vPrimOp = state.allocValue(); + static auto primOp = new PrimOp(prim_callNonFlake, 1, state.symbols.create("callNonFlake")); + vPrimOp->type = tPrimOp; + vPrimOp->primOp = primOp; + auto vArg = state.allocValue(); + vArg->type = tNull; + // FIXME: leak + vArg->attrs = (Bindings *) new NonFlakeInput(dep.second); // evil! also inefficient + mkApp(*vNonFlake, *vPrimOp, *vArg); + } + + mkString(*state.allocAttr(v, state.sDescription), flake.description); + + emitSourceInfoAttrs(state, flake.sourceInfo, v); + + auto vOutputs = state.allocAttr(v, state.symbols.create("outputs")); + mkApp(*vOutputs, *flake.vOutputs, v); + + v.attrs->push_back(Attr(state.symbols.create("self"), &v)); + + v.attrs->sort(); + + /* For convenience, put the outputs directly in the result, so you + can refer to an output of an input as 'inputs.foo.bar' rather + than 'inputs.foo.outputs.bar'. */ + auto v2 = *state.allocValue(); + state.eval(state.parseExprFromString("res: res.outputs // res", "/"), v2); + + state.callFunction(v2, v, vRes, noPos); +} + +void callFlake(EvalState & state, + const ResolvedFlake & resFlake, + Value & v) +{ + callFlake(state, resFlake.flake, resFlake.lockFile, v); +} + +// This function is exposed to be used in nix files. +static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + callFlake(state, resolveFlake(state, state.forceStringNoCtx(*args[0], pos), + evalSettings.pureEval ? AllPure : UseUpdatedLockFile), v); +} + +static RegisterPrimOp r2("getFlake", 1, prim_getFlake); + +void gitCloneFlake(FlakeRef flakeRef, EvalState & state, Registries registries, const Path & destDir) +{ + flakeRef = lookupFlake(state, flakeRef, registries); + + std::string uri; + + Strings args = {"clone"}; + + if (auto refData = std::get_if<FlakeRef::IsGitHub>(&flakeRef.data)) { + uri = "git@github.com:" + refData->owner + "/" + refData->repo + ".git"; + args.push_back(uri); + if (flakeRef.ref) { + args.push_back("--branch"); + args.push_back(*flakeRef.ref); + } + } else if (auto refData = std::get_if<FlakeRef::IsGit>(&flakeRef.data)) { + args.push_back(refData->uri); + if (flakeRef.ref) { + args.push_back("--branch"); + args.push_back(*flakeRef.ref); + } + } + + if (destDir != "") + args.push_back(destDir); + + runProgram("git", true, args); +} + +} + +std::shared_ptr<flake::FlakeRegistry> EvalState::getGlobalFlakeRegistry() +{ + std::call_once(_globalFlakeRegistryInit, [&]() { + auto path = evalSettings.flakeRegistry; + + if (!hasPrefix(path, "/")) { + CachedDownloadRequest request(evalSettings.flakeRegistry); + request.name = "flake-registry.json"; + request.gcRoot = true; + path = getDownloader()->downloadCached(store, request).path; + } + + _globalFlakeRegistry = readRegistry(path); + }); + + return _globalFlakeRegistry; +} + +// This always returns a vector with flakeReg, userReg, globalReg. +// If one of them doesn't exist, the registry is left empty but does exist. +const Registries EvalState::getFlakeRegistries() +{ + Registries registries; + registries.push_back(getFlagRegistry(registryOverrides)); + registries.push_back(getUserRegistry()); + registries.push_back(getGlobalFlakeRegistry()); + return registries; +} + +} |