diff options
author | Eelco Dolstra <edolstra@gmail.com> | 2019-06-05 16:51:54 +0200 |
---|---|---|
committer | Eelco Dolstra <edolstra@gmail.com> | 2019-06-05 16:51:54 +0200 |
commit | 54aff8430c4e7739903f6dbed713cc088e38507f (patch) | |
tree | 4e8f84b5682b0480deb2c4943d244b9a5fe552e0 /src/libexpr/flake | |
parent | 1b057929885fd3f339d4c85b44ad9f10fef7d8a9 (diff) |
Move flake-related stuff to src/libexpr/flake
Diffstat (limited to 'src/libexpr/flake')
-rw-r--r-- | src/libexpr/flake/flake.cc | 604 | ||||
-rw-r--r-- | src/libexpr/flake/flake.hh | 111 | ||||
-rw-r--r-- | src/libexpr/flake/flakeref.cc | 252 | ||||
-rw-r--r-- | src/libexpr/flake/flakeref.hh | 188 | ||||
-rw-r--r-- | src/libexpr/flake/lockfile.cc | 102 | ||||
-rw-r--r-- | src/libexpr/flake/lockfile.hh | 112 |
6 files changed, 1369 insertions, 0 deletions
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc new file mode 100644 index 000000000..bb0543541 --- /dev/null +++ b/src/libexpr/flake/flake.cc @@ -0,0 +1,604 @@ +#include "flake.hh" +#include "lockfile.hh" +#include "primops.hh" +#include "eval-inline.hh" +#include "primops/fetchGit.hh" +#include "download.hh" +#include "args.hh" + +#include <iostream> +#include <queue> +#include <regex> +#include <ctime> +#include <iomanip> +#include <nlohmann/json.hpp> + +namespace nix { + +using namespace flake; + +namespace flake { + +/* Read a registry. */ +std::shared_ptr<FlakeRegistry> readRegistry(const Path & path) +{ + auto registry = std::make_shared<FlakeRegistry>(); + + if (!pathExists(path)) + return std::make_shared<FlakeRegistry>(); + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("flake registry '%s' has unsupported version %d", path, version); + + auto flakes = json["flakes"]; + for (auto i = flakes.begin(); i != flakes.end(); ++i) + registry->entries.emplace(i.key(), FlakeRef(i->value("uri", ""))); + + return registry; +} + +/* Write a registry to a file. */ +void writeRegistry(const FlakeRegistry & registry, const Path & path) +{ + nlohmann::json json; + json["version"] = 2; + for (auto elem : registry.entries) + json["flakes"][elem.first.to_string()] = { {"uri", elem.second.to_string()} }; + createDirs(dirOf(path)); + writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. +} + +Path getUserRegistryPath() +{ + return getHome() + "/.config/nix/registry.json"; +} + +std::shared_ptr<FlakeRegistry> getUserRegistry() +{ + return readRegistry(getUserRegistryPath()); +} + +std::shared_ptr<FlakeRegistry> getFlagRegistry(RegistryOverrides registryOverrides) +{ + auto flagRegistry = std::make_shared<FlakeRegistry>(); + for (auto const & x : registryOverrides) { + flagRegistry->entries.insert_or_assign(FlakeRef(x.first), FlakeRef(x.second)); + } + return flagRegistry; +} + +static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, + std::vector<FlakeRef> pastSearches = {}); + +FlakeRef updateFlakeRef(EvalState & state, const FlakeRef & newRef, const Registries & registries, std::vector<FlakeRef> pastSearches) +{ + std::string errorMsg = "found cycle in flake registries: "; + for (FlakeRef oldRef : pastSearches) { + errorMsg += oldRef.to_string(); + if (oldRef == newRef) + throw Error(errorMsg); + errorMsg += " - "; + } + pastSearches.push_back(newRef); + return lookupFlake(state, newRef, registries, pastSearches); +} + +static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, + std::vector<FlakeRef> pastSearches) +{ + if (registries.empty() && !flakeRef.isDirect()) + throw Error("indirect flake reference '%s' is not allowed", flakeRef); + + for (std::shared_ptr<FlakeRegistry> registry : registries) { + auto i = registry->entries.find(flakeRef); + if (i != registry->entries.end()) { + auto newRef = i->second; + return updateFlakeRef(state, newRef, registries, pastSearches); + } + + auto j = registry->entries.find(flakeRef.baseRef()); + if (j != registry->entries.end()) { + auto newRef = j->second; + newRef.ref = flakeRef.ref; + newRef.rev = flakeRef.rev; + return updateFlakeRef(state, newRef, registries, pastSearches); + } + } + + if (!flakeRef.isDirect()) + throw Error("could not resolve flake reference '%s'", flakeRef); + + return flakeRef; +} + +// Lookups happen here too +static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) +{ + FlakeRef resolvedRef = lookupFlake(state, flakeRef, + impureIsAllowed ? state.getFlakeRegistries() : std::vector<std::shared_ptr<FlakeRegistry>>()); + + if (evalSettings.pureEval && !impureIsAllowed && !resolvedRef.isImmutable()) + throw Error("requested to fetch mutable flake '%s' in pure mode", resolvedRef); + + auto doGit = [&](const GitInfo & gitInfo) { + FlakeRef ref(resolvedRef.baseRef()); + ref.ref = gitInfo.ref; + ref.rev = gitInfo.rev; + SourceInfo info(ref); + info.storePath = gitInfo.storePath; + info.revCount = gitInfo.revCount; + info.narHash = state.store->queryPathInfo(info.storePath)->narHash; + info.lastModified = gitInfo.lastModified; + return info; + }; + + // This only downloads only one revision of the repo, not the entire history. + if (auto refData = std::get_if<FlakeRef::IsGitHub>(&resolvedRef.data)) { + + // FIXME: use regular /archive URLs instead? api.github.com + // might have stricter rate limits. + + auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", + refData->owner, refData->repo, + resolvedRef.rev ? resolvedRef.rev->to_string(Base16, false) + : resolvedRef.ref ? *resolvedRef.ref : "master"); + + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken; + + CachedDownloadRequest request(url); + request.unpack = true; + request.name = "source"; + request.ttl = resolvedRef.rev ? 1000000000 : settings.tarballTtl; + request.getLastModified = true; + auto result = getDownloader()->downloadCached(state.store, request); + + if (!result.etag) + throw Error("did not receive an ETag header from '%s'", url); + + if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"') + throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url); + + FlakeRef ref(resolvedRef.baseRef()); + ref.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); + SourceInfo info(ref); + info.storePath = result.storePath; + info.narHash = state.store->queryPathInfo(info.storePath)->narHash; + info.lastModified = result.lastModified; + + return info; + } + + // This downloads the entire git history + else if (auto refData = std::get_if<FlakeRef::IsGit>(&resolvedRef.data)) { + return doGit(exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source")); + } + + else if (auto refData = std::get_if<FlakeRef::IsPath>(&resolvedRef.data)) { + if (!pathExists(refData->path + "/.git")) + throw Error("flake '%s' does not reference a Git repository", refData->path); + return doGit(exportGit(state.store, refData->path, {}, {}, "source")); + } + + else abort(); +} + +// This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within `fetchFlake`, which is used here. +Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) +{ + SourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); + debug("got flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); + + FlakeRef resolvedRef = sourceInfo.resolvedRef; + + state.store->assertStorePath(sourceInfo.storePath); + + if (state.allowedPaths) + state.allowedPaths->insert(state.store->toRealPath(sourceInfo.storePath)); + + // Guard against symlink attacks. + Path flakeFile = canonPath(sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix"); + Path realFlakeFile = state.store->toRealPath(flakeFile); + if (!isInDir(realFlakeFile, state.store->toRealPath(sourceInfo.storePath))) + throw Error("'flake.nix' file of flake '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath); + + Flake flake(flakeRef, sourceInfo); + + if (!pathExists(realFlakeFile)) + throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir); + + Value vInfo; + state.evalFile(realFlakeFile, vInfo); // FIXME: symlink attack + + state.forceAttrs(vInfo); + + auto sEpoch = state.symbols.create("epoch"); + + if (auto epoch = vInfo.attrs->get(sEpoch)) { + flake.epoch = state.forceInt(*(**epoch).value, *(**epoch).pos); + if (flake.epoch > 201906) + throw Error("flake '%s' requires unsupported epoch %d; please upgrade Nix", flakeRef, flake.epoch); + } else + throw Error("flake '%s' lacks attribute 'epoch'", flakeRef); + + if (auto name = vInfo.attrs->get(state.sName)) + flake.id = state.forceStringNoCtx(*(**name).value, *(**name).pos); + else + throw Error("flake '%s' lacks attribute 'name'", flakeRef); + + if (auto description = vInfo.attrs->get(state.sDescription)) + flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos); + + auto sInputs = state.symbols.create("inputs"); + + if (auto inputs = vInfo.attrs->get(sInputs)) { + state.forceList(*(**inputs).value, *(**inputs).pos); + for (unsigned int n = 0; n < (**inputs).value->listSize(); ++n) + flake.inputs.push_back(FlakeRef(state.forceStringNoCtx( + *(**inputs).value->listElems()[n], *(**inputs).pos))); + } + + auto sNonFlakeInputs = state.symbols.create("nonFlakeInputs"); + + if (std::optional<Attr *> nonFlakeInputs = vInfo.attrs->get(sNonFlakeInputs)) { + state.forceAttrs(*(**nonFlakeInputs).value, *(**nonFlakeInputs).pos); + for (Attr attr : *(*(**nonFlakeInputs).value).attrs) { + std::string myNonFlakeUri = state.forceStringNoCtx(*attr.value, *attr.pos); + FlakeRef nonFlakeRef = FlakeRef(myNonFlakeUri); + flake.nonFlakeInputs.insert_or_assign(attr.name, nonFlakeRef); + } + } + + auto sOutputs = state.symbols.create("outputs"); + + if (auto outputs = vInfo.attrs->get(sOutputs)) { + state.forceFunction(*(**outputs).value, *(**outputs).pos); + flake.vOutputs = (**outputs).value; + } else + throw Error("flake '%s' lacks attribute 'outputs'", flakeRef); + + for (auto & attr : *vInfo.attrs) { + if (attr.name != sEpoch && + attr.name != state.sName && + attr.name != state.sDescription && + attr.name != sInputs && + attr.name != sNonFlakeInputs && + attr.name != sOutputs) + throw Error("flake '%s' has an unsupported attribute '%s', at %s", + flakeRef, attr.name, *attr.pos); + } + + return flake; +} + +// Get the `NonFlake` corresponding to a `FlakeRef`. +NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) +{ + auto sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); + debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); + + FlakeRef resolvedRef = sourceInfo.resolvedRef; + + NonFlake nonFlake(flakeRef, sourceInfo); + + state.store->assertStorePath(nonFlake.sourceInfo.storePath); + + if (state.allowedPaths) + state.allowedPaths->insert(nonFlake.sourceInfo.storePath); + + return nonFlake; +} + +bool allowedToWrite(HandleLockFile handle) +{ + return handle == UpdateLockFile || handle == RecreateLockFile; +} + +bool recreateLockFile(HandleLockFile handle) +{ + return handle == RecreateLockFile || handle == UseNewLockFile; +} + +bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) +{ + if (handle == AllPure) return false; + else if (handle == TopRefUsesRegistries) return isTopRef; + else if (handle == UpdateLockFile) return true; + else if (handle == UseUpdatedLockFile) return true; + else if (handle == RecreateLockFile) return true; + else if (handle == UseNewLockFile) return true; + else assert(false); +} + +/* Given a flakeref and its subtree of the lockfile, return an updated + subtree of the lockfile. That is, if the 'flake.nix' of the + referenced flake has inputs that don't have a corresponding entry + in the lockfile, they're added to the lockfile; conversely, any + lockfile entries that don't have a corresponding entry in flake.nix + are removed. + + Note that this is lazy: we only recursively fetch inputs that are + not in the lockfile yet. */ +static std::pair<Flake, FlakeInput> updateLocks( + EvalState & state, + const Flake & flake, + HandleLockFile handleLockFile, + const FlakeInputs & oldEntry, + bool topRef) +{ + FlakeInput newEntry( + flake.id, + flake.sourceInfo.resolvedRef, + flake.sourceInfo.narHash); + + for (auto & input : flake.nonFlakeInputs) { + auto & id = input.first; + auto & ref = input.second; + auto i = oldEntry.nonFlakeInputs.find(id); + if (i != oldEntry.nonFlakeInputs.end()) { + newEntry.nonFlakeInputs.insert_or_assign(i->first, i->second); + } else { + if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) + throw Error("cannot update non-flake dependency '%s' in pure mode", id); + auto nonFlake = getNonFlake(state, ref, allowedToUseRegistries(handleLockFile, false)); + newEntry.nonFlakeInputs.insert_or_assign(id, + NonFlakeInput( + nonFlake.sourceInfo.resolvedRef, + nonFlake.sourceInfo.narHash)); + } + } + + for (auto & inputRef : flake.inputs) { + auto i = oldEntry.flakeInputs.find(inputRef); + if (i != oldEntry.flakeInputs.end()) { + newEntry.flakeInputs.insert_or_assign(inputRef, i->second); + } else { + if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) + throw Error("cannot update flake dependency '%s' in pure mode", inputRef); + newEntry.flakeInputs.insert_or_assign(inputRef, + updateLocks(state, + getFlake(state, inputRef, allowedToUseRegistries(handleLockFile, false)), + handleLockFile, {}, false).second); + } + } + + return {flake, newEntry}; +} + +/* Compute an in-memory lockfile for the specified top-level flake, + and optionally write it to file, it the flake is writable. */ +ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile) +{ + auto flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); + + LockFile oldLockFile; + + if (!recreateLockFile(handleLockFile)) { + // If recreateLockFile, start with an empty lockfile + // FIXME: symlink attack + oldLockFile = LockFile::read( + state.store->toRealPath(flake.sourceInfo.storePath) + + "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock"); + } + + LockFile lockFile(updateLocks( + state, flake, handleLockFile, oldLockFile, true).second); + + if (!(lockFile == oldLockFile)) { + if (allowedToWrite(handleLockFile)) { + if (auto refData = std::get_if<FlakeRef::IsPath>(&topRef.data)) { + lockFile.write(refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock"); + + // Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store. + runProgram("git", true, { "-C", refData->path, "add", + (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock" }); + } else + warn("cannot write lockfile of remote flake '%s'", topRef); + } else if (handleLockFile != AllPure && handleLockFile != TopRefUsesRegistries) + warn("using updated lockfile without writing it to file"); + } + + return ResolvedFlake(std::move(flake), std::move(lockFile)); +} + +void updateLockFile(EvalState & state, const FlakeRef & flakeRef, bool recreateLockFile) +{ + resolveFlake(state, flakeRef, recreateLockFile ? RecreateLockFile : UpdateLockFile); +} + +static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo, Value & vAttrs) +{ + auto & path = sourceInfo.storePath; + assert(state.store->isValidPath(path)); + mkString(*state.allocAttr(vAttrs, state.sOutPath), path, {path}); + + if (sourceInfo.resolvedRef.rev) { + mkString(*state.allocAttr(vAttrs, state.symbols.create("rev")), + sourceInfo.resolvedRef.rev->gitRev()); + mkString(*state.allocAttr(vAttrs, state.symbols.create("shortRev")), + sourceInfo.resolvedRef.rev->gitShortRev()); + } + + if (sourceInfo.revCount) + mkInt(*state.allocAttr(vAttrs, state.symbols.create("revCount")), *sourceInfo.revCount); + + if (sourceInfo.lastModified) + mkString(*state.allocAttr(vAttrs, state.symbols.create("lastModified")), + fmt("%s", + std::put_time(std::gmtime(&*sourceInfo.lastModified), "%Y%m%d%H%M%S"))); +} + +/* Helper primop to make callFlake (below) fetch/call its inputs + lazily. Note that this primop cannot be called by user code since + it doesn't appear in 'builtins'. */ +static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto lazyFlake = (FlakeInput *) args[0]->attrs; + auto flake = getFlake(state, lazyFlake->ref, false); + + if (flake.sourceInfo.narHash != lazyFlake->narHash) + throw Error("the content hash of flake '%s' doesn't match the hash recorded in the referring lockfile", flake.sourceInfo.resolvedRef); + + callFlake(state, flake, *lazyFlake, v); +} + +static void prim_callNonFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto lazyNonFlake = (NonFlakeInput *) args[0]->attrs; + + auto nonFlake = getNonFlake(state, lazyNonFlake->ref); + + if (nonFlake.sourceInfo.narHash != lazyNonFlake->narHash) + throw Error("the content hash of repository '%s' doesn't match the hash recorded in the referring lockfile", nonFlake.sourceInfo.resolvedRef); + + state.mkAttrs(v, 8); + + assert(state.store->isValidPath(nonFlake.sourceInfo.storePath)); + + mkString(*state.allocAttr(v, state.sOutPath), + nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath}); + + emitSourceInfoAttrs(state, nonFlake.sourceInfo, v); +} + +void callFlake(EvalState & state, + const Flake & flake, + const FlakeInputs & inputs, + Value & vRes) +{ + // Construct the resulting attrset '{outputs, ...}'. This attrset + // is passed lazily as an argument to the 'outputs' function. + + auto & v = *state.allocValue(); + + state.mkAttrs(v, + inputs.flakeInputs.size() + + inputs.nonFlakeInputs.size() + 8); + + for (auto & dep : inputs.flakeInputs) { + auto vFlake = state.allocAttr(v, dep.second.id); + auto vPrimOp = state.allocValue(); + static auto primOp = new PrimOp(prim_callFlake, 1, state.symbols.create("callFlake")); + vPrimOp->type = tPrimOp; + vPrimOp->primOp = primOp; + auto vArg = state.allocValue(); + vArg->type = tNull; + // FIXME: leak + vArg->attrs = (Bindings *) new FlakeInput(dep.second); // evil! also inefficient + mkApp(*vFlake, *vPrimOp, *vArg); + } + + for (auto & dep : inputs.nonFlakeInputs) { + auto vNonFlake = state.allocAttr(v, dep.first); + auto vPrimOp = state.allocValue(); + static auto primOp = new PrimOp(prim_callNonFlake, 1, state.symbols.create("callNonFlake")); + vPrimOp->type = tPrimOp; + vPrimOp->primOp = primOp; + auto vArg = state.allocValue(); + vArg->type = tNull; + // FIXME: leak + vArg->attrs = (Bindings *) new NonFlakeInput(dep.second); // evil! also inefficient + mkApp(*vNonFlake, *vPrimOp, *vArg); + } + + mkString(*state.allocAttr(v, state.sDescription), flake.description); + + emitSourceInfoAttrs(state, flake.sourceInfo, v); + + auto vOutputs = state.allocAttr(v, state.symbols.create("outputs")); + mkApp(*vOutputs, *flake.vOutputs, v); + + v.attrs->push_back(Attr(state.symbols.create("self"), &v)); + + v.attrs->sort(); + + /* For convenience, put the outputs directly in the result, so you + can refer to an output of an input as 'inputs.foo.bar' rather + than 'inputs.foo.outputs.bar'. */ + auto v2 = *state.allocValue(); + state.eval(state.parseExprFromString("res: res.outputs // res", "/"), v2); + + state.callFunction(v2, v, vRes, noPos); +} + +void callFlake(EvalState & state, + const ResolvedFlake & resFlake, + Value & v) +{ + callFlake(state, resFlake.flake, resFlake.lockFile, v); +} + +// This function is exposed to be used in nix files. +static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + callFlake(state, resolveFlake(state, state.forceStringNoCtx(*args[0], pos), + evalSettings.pureEval ? AllPure : UseUpdatedLockFile), v); +} + +static RegisterPrimOp r2("getFlake", 1, prim_getFlake); + +void gitCloneFlake(FlakeRef flakeRef, EvalState & state, Registries registries, const Path & destDir) +{ + flakeRef = lookupFlake(state, flakeRef, registries); + + std::string uri; + + Strings args = {"clone"}; + + if (auto refData = std::get_if<FlakeRef::IsGitHub>(&flakeRef.data)) { + uri = "git@github.com:" + refData->owner + "/" + refData->repo + ".git"; + args.push_back(uri); + if (flakeRef.ref) { + args.push_back("--branch"); + args.push_back(*flakeRef.ref); + } + } else if (auto refData = std::get_if<FlakeRef::IsGit>(&flakeRef.data)) { + args.push_back(refData->uri); + if (flakeRef.ref) { + args.push_back("--branch"); + args.push_back(*flakeRef.ref); + } + } + + if (destDir != "") + args.push_back(destDir); + + runProgram("git", true, args); +} + +} + +std::shared_ptr<flake::FlakeRegistry> EvalState::getGlobalFlakeRegistry() +{ + std::call_once(_globalFlakeRegistryInit, [&]() { + auto path = evalSettings.flakeRegistry; + + if (!hasPrefix(path, "/")) { + CachedDownloadRequest request(evalSettings.flakeRegistry); + request.name = "flake-registry.json"; + request.gcRoot = true; + path = getDownloader()->downloadCached(store, request).path; + } + + _globalFlakeRegistry = readRegistry(path); + }); + + return _globalFlakeRegistry; +} + +// This always returns a vector with flakeReg, userReg, globalReg. +// If one of them doesn't exist, the registry is left empty but does exist. +const Registries EvalState::getFlakeRegistries() +{ + Registries registries; + registries.push_back(getFlagRegistry(registryOverrides)); + registries.push_back(getUserRegistry()); + registries.push_back(getGlobalFlakeRegistry()); + return registries; +} + +} diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh new file mode 100644 index 000000000..b8d0da252 --- /dev/null +++ b/src/libexpr/flake/flake.hh @@ -0,0 +1,111 @@ +#pragma once + +#include "types.hh" +#include "flakeref.hh" +#include "lockfile.hh" + +namespace nix { + +struct Value; +class EvalState; + +namespace flake { + +static const size_t FLAG_REGISTRY = 0; +static const size_t USER_REGISTRY = 1; +static const size_t GLOBAL_REGISTRY = 2; + +struct FlakeRegistry +{ + std::map<FlakeRef, FlakeRef> entries; +}; + +typedef std::vector<std::shared_ptr<FlakeRegistry>> Registries; + +std::shared_ptr<FlakeRegistry> readRegistry(const Path &); + +void writeRegistry(const FlakeRegistry &, const Path &); + +Path getUserRegistryPath(); + +enum HandleLockFile : unsigned int + { AllPure // Everything is handled 100% purely + , TopRefUsesRegistries // The top FlakeRef uses the registries, apart from that, everything happens 100% purely + , UpdateLockFile // Update the existing lockfile and write it to file + , UseUpdatedLockFile // `UpdateLockFile` without writing to file + , RecreateLockFile // Recreate the lockfile from scratch and write it to file + , UseNewLockFile // `RecreateLockFile` without writing to file + }; + +struct SourceInfo +{ + // Immutable flakeref that this source tree was obtained from. + FlakeRef resolvedRef; + + Path storePath; + + // Number of ancestors of the most recent commit. + std::optional<uint64_t> revCount; + + // NAR hash of the store path. + Hash narHash; + + // A stable timestamp of this source tree. For Git and GitHub + // flakes, the commit date (not author date!) of the most recent + // commit. + std::optional<time_t> lastModified; + + SourceInfo(const FlakeRef & resolvRef) : resolvedRef(resolvRef) {}; +}; + +struct Flake +{ + FlakeId id; + FlakeRef originalRef; + std::string description; + SourceInfo sourceInfo; + std::vector<FlakeRef> inputs; + std::map<FlakeAlias, FlakeRef> nonFlakeInputs; + Value * vOutputs; // FIXME: gc + unsigned int epoch; + + Flake(const FlakeRef & origRef, const SourceInfo & sourceInfo) + : originalRef(origRef), sourceInfo(sourceInfo) {}; +}; + +struct NonFlake +{ + FlakeRef originalRef; + SourceInfo sourceInfo; + NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) + : originalRef(origRef), sourceInfo(sourceInfo) {}; +}; + +Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); + +struct ResolvedFlake +{ + Flake flake; + LockFile lockFile; + ResolvedFlake(Flake && flake, LockFile && lockFile) + : flake(flake), lockFile(lockFile) {} +}; + +ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile); + +void callFlake(EvalState & state, + const Flake & flake, + const FlakeInputs & inputs, + Value & v); + +void callFlake(EvalState & state, + const ResolvedFlake & resFlake, + Value & v); + +void updateLockFile(EvalState &, const FlakeRef & flakeRef, bool recreateLockFile); + +void gitCloneFlake(FlakeRef flakeRef, EvalState &, Registries, const Path & destDir); + +} + +} diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc new file mode 100644 index 000000000..24af09124 --- /dev/null +++ b/src/libexpr/flake/flakeref.cc @@ -0,0 +1,252 @@ +#include "flakeref.hh" +#include "store-api.hh" + +#include <regex> + +namespace nix { + +// A Git ref (i.e. branch or tag name). +const static std::string refRegex = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check + +// A Git revision (a SHA-1 commit hash). +const static std::string revRegexS = "[0-9a-fA-F]{40}"; +std::regex revRegex(revRegexS, std::regex::ECMAScript); + +// A Git ref or revision. +const static std::string revOrRefRegex = "(?:(" + revRegexS + ")|(" + refRegex + "))"; + +// A rev ("e72daba8250068216d79d2aeef40d4d95aff6666"), or a ref +// optionally followed by a rev (e.g. "master" or +// "master/e72daba8250068216d79d2aeef40d4d95aff6666"). +const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegex + ")(?:/(" + revRegexS + "))?))"; + +const static std::string flakeAlias = "[a-zA-Z][a-zA-Z0-9_-]*"; + +// GitHub references. +const static std::string ownerRegex = "[a-zA-Z][a-zA-Z0-9_-]*"; +const static std::string repoRegex = "[a-zA-Z][a-zA-Z0-9_-]*"; + +// URI stuff. +const static std::string schemeRegex = "(?:http|https|ssh|git|file)"; +const static std::string authorityRegex = "[a-zA-Z0-9._~-]*"; +const static std::string segmentRegex = "[a-zA-Z0-9._~-]+"; +const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*"; + +// 'dir' path elements cannot start with a '.'. We also reject +// potentially dangerous characters like ';'. +const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)"; +const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*"; + + +FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) +{ + // FIXME: could combine this into one regex. + + static std::regex flakeRegex( + "(?:flake:)?(" + flakeAlias + ")(?:/(?:" + refAndOrRevRegex + "))?", + std::regex::ECMAScript); + + static std::regex githubRegex( + "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?", + std::regex::ECMAScript); + + static std::regex uriRegex( + "((" + schemeRegex + "):" + + "(?://(" + authorityRegex + "))?" + + "(" + pathRegex + "))", + std::regex::ECMAScript); + + static std::regex refRegex2(refRegex, std::regex::ECMAScript); + + static std::regex subDirRegex2(subDirRegex, std::regex::ECMAScript); + + auto [uri2, params] = splitUriAndParams(uri_); + std::string uri(uri2); + + auto handleSubdir = [&](const std::string & name, const std::string & value) { + if (name == "dir") { + if (value != "" && !std::regex_match(value, subDirRegex2)) + throw BadFlakeRef("flake '%s' has invalid subdirectory '%s'", uri, value); + subdir = value; + return true; + } else + return false; + }; + + auto handleGitParams = [&](const std::string & name, const std::string & value) { + if (name == "rev") { + if (!std::regex_match(value, revRegex)) + throw BadFlakeRef("invalid Git revision '%s'", value); + rev = Hash(value, htSHA1); + } else if (name == "ref") { + if (!std::regex_match(value, refRegex2)) + throw BadFlakeRef("invalid Git ref '%s'", value); + ref = value; + } else if (handleSubdir(name, value)) + ; + else return false; + return true; + }; + + std::cmatch match; + if (std::regex_match(uri.c_str(), match, flakeRegex)) { + IsAlias d; + d.alias = match[1]; + if (match[2].matched) + rev = Hash(match[2], htSHA1); + else if (match[3].matched) { + ref = match[3]; + if (match[4].matched) + rev = Hash(match[4], htSHA1); + } + data = d; + } + + else if (std::regex_match(uri.c_str(), match, githubRegex)) { + IsGitHub d; + d.owner = match[1]; + d.repo = match[2]; + if (match[3].matched) + rev = Hash(match[3], htSHA1); + else if (match[4].matched) { + ref = match[4]; + } + for (auto & param : params) { + if (handleSubdir(param.first, param.second)) + ; + else + throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); + } + data = d; + } + + else if (std::regex_match(uri.c_str(), match, uriRegex) + && (match[2] == "file" || hasSuffix(match[4], ".git"))) + { + IsGit d; + d.uri = match[1]; + for (auto & param : params) { + if (handleGitParams(param.first, param.second)) + ; + else + // FIXME: should probably pass through unknown parameters + throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); + } + if (rev && !ref) + throw BadFlakeRef("flake URI '%s' lacks a Git ref", uri); + data = d; + } + + else if ((hasPrefix(uri, "/") || (allowRelative && (hasPrefix(uri, "./") || hasPrefix(uri, "../") || uri == "."))) + && uri.find(':') == std::string::npos) + { + IsPath d; + if (allowRelative) { + d.path = absPath(uri); + while (true) { + if (pathExists(d.path + "/.git")) break; + subdir = baseNameOf(d.path) + (subdir.empty() ? "" : "/" + subdir); + d.path = dirOf(d.path); + if (d.path == "/") + throw BadFlakeRef("path '%s' does not reference a Git repository", uri); + } + } else + d.path = canonPath(uri); + data = d; + for (auto & param : params) { + if (handleGitParams(param.first, param.second)) + ; + else + throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); + } + } + + else + throw BadFlakeRef("'%s' is not a valid flake reference", uri); +} + +std::string FlakeRef::to_string() const +{ + std::string string; + bool first = true; + + auto addParam = + [&](const std::string & name, std::string value) { + string += first ? '?' : '&'; + first = false; + string += name; + string += '='; + string += value; // FIXME: escaping + }; + + if (auto refData = std::get_if<FlakeRef::IsAlias>(&data)) { + string = refData->alias; + if (ref) string += '/' + *ref; + if (rev) string += '/' + rev->gitRev(); + } + + else if (auto refData = std::get_if<FlakeRef::IsPath>(&data)) { + string = refData->path; + if (ref) addParam("ref", *ref); + if (rev) addParam("rev", rev->gitRev()); + if (subdir != "") addParam("dir", subdir); + } + + else if (auto refData = std::get_if<FlakeRef::IsGitHub>(&data)) { + assert(!(ref && rev)); + string = "github:" + refData->owner + "/" + refData->repo; + if (ref) { string += '/'; string += *ref; } + if (rev) { string += '/'; string += rev->gitRev(); } + if (subdir != "") addParam("dir", subdir); + } + + else if (auto refData = std::get_if<FlakeRef::IsGit>(&data)) { + assert(!rev || ref); + string = refData->uri; + + if (ref) { + addParam("ref", *ref); + if (rev) + addParam("rev", rev->gitRev()); + } + + if (subdir != "") addParam("dir", subdir); + } + + else abort(); + + assert(FlakeRef(string) == *this); + + return string; +} + +std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) +{ + str << flakeRef.to_string(); + return str; +} + +bool FlakeRef::isImmutable() const +{ + return (bool) rev; +} + +FlakeRef FlakeRef::baseRef() const // Removes the ref and rev from a FlakeRef. +{ + FlakeRef result(*this); + result.ref = std::nullopt; + result.rev = std::nullopt; + return result; +} + +std::optional<FlakeRef> parseFlakeRef( + const std::string & uri, bool allowRelative) +{ + try { + return FlakeRef(uri, allowRelative); + } catch (BadFlakeRef & e) { + return {}; + } +} + +} diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh new file mode 100644 index 000000000..52bb82ddb --- /dev/null +++ b/src/libexpr/flake/flakeref.hh @@ -0,0 +1,188 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" + +#include <variant> + +namespace nix { + +/* Flake references are a URI-like syntax to specify a flake. + + Examples: + + * <flake-id>(/rev-or-ref(/rev)?)? + + Look up a flake by ID in the flake lock file or in the flake + registry. These must specify an actual location for the flake + using the formats listed below. Note that in pure evaluation + mode, the flake registry is empty. + + Optionally, the rev or ref from the dereferenced flake can be + overriden. For example, + + nixpkgs/19.09 + + uses the "19.09" branch of the nixpkgs' flake GitHub repository, + while + + nixpkgs/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f + + uses the specified revision. For Git (rather than GitHub) + repositories, both the rev and ref must be given, e.g. + + nixpkgs/19.09/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f + + * github:<owner>/<repo>(/<rev-or-ref>)? + + A repository on GitHub. These differ from Git references in that + they're downloaded in a efficient way (via the tarball mechanism) + and that they support downloading a specific revision without + specifying a branch. <rev-or-ref> is either a commit hash ("rev") + or a branch or tag name ("ref"). The default is: "master" if none + is specified. Note that in pure evaluation mode, a commit hash + must be used. + + Flakes fetched in this manner expose "rev" and "lastModified" + attributes, but not "revCount". + + Examples: + + github:edolstra/dwarffs + github:edolstra/dwarffs/unstable + github:edolstra/dwarffs/41c0c1bf292ea3ac3858ff393b49ca1123dbd553 + + * https://<server>/<path>.git(\?attr(&attr)*)? + ssh://<server>/<path>.git(\?attr(&attr)*)? + git://<server>/<path>.git(\?attr(&attr)*)? + file:///<path>(\?attr(&attr)*)? + + where 'attr' is one of: + rev=<rev> + ref=<ref> + + A Git repository fetched through https. Note that the path must + end in ".git". The default for "ref" is "master". + + Examples: + + https://example.org/my/repo.git + https://example.org/my/repo.git?ref=release-1.2.3 + https://example.org/my/repo.git?rev=e72daba8250068216d79d2aeef40d4d95aff6666 + git://github.com/edolstra/dwarffs.git?ref=flake&rev=2efca4bc9da70fb001b26c3dc858c6397d3c4817 + + * /path.git(\?attr(&attr)*)? + + Like file://path.git, but if no "ref" or "rev" is specified, the + (possibly dirty) working tree will be used. Using a working tree + is not allowed in pure evaluation mode. + + Examples: + + /path/to/my/repo + /path/to/my/repo?ref=develop + /path/to/my/repo?rev=e72daba8250068216d79d2aeef40d4d95aff6666 + + * https://<server>/<path>.tar.xz(?hash=<sri-hash>) + file:///<path>.tar.xz(?hash=<sri-hash>) + + A flake distributed as a tarball. In pure evaluation mode, an SRI + hash is mandatory. It exposes a "lastModified" attribute, being + the newest file inside the tarball. + + Example: + + https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz + https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz?hash=sha256-56bbc099995ea8581ead78f22832fee7dbcb0a0b6319293d8c2d0aef5379397c + + Note: currently, there can be only one flake per Git repository, and + it must be at top-level. In the future, we may want to add a field + (e.g. "dir=<dir>") to specify a subdirectory inside the repository. +*/ + +typedef std::string FlakeId; +typedef std::string FlakeAlias; +typedef std::string FlakeUri; + +struct FlakeRef +{ + struct IsAlias + { + FlakeAlias alias; + bool operator<(const IsAlias & b) const { return alias < b.alias; }; + bool operator==(const IsAlias & b) const { return alias == b.alias; }; + }; + + struct IsGitHub { + std::string owner, repo; + bool operator<(const IsGitHub & b) const { + return std::make_tuple(owner, repo) < std::make_tuple(b.owner, b.repo); + } + bool operator==(const IsGitHub & b) const { + return owner == b.owner && repo == b.repo; + } + }; + + // Git, Tarball + struct IsGit + { + std::string uri; + bool operator<(const IsGit & b) const { return uri < b.uri; } + bool operator==(const IsGit & b) const { return uri == b.uri; } + }; + + struct IsPath + { + Path path; + bool operator<(const IsPath & b) const { return path < b.path; } + bool operator==(const IsPath & b) const { return path == b.path; } + }; + + // Git, Tarball + + std::variant<IsAlias, IsGitHub, IsGit, IsPath> data; + + std::optional<std::string> ref; + std::optional<Hash> rev; + Path subdir = ""; // This is a relative path pointing at the flake.nix file's directory, relative to the git root. + + bool operator<(const FlakeRef & flakeRef) const + { + return std::make_tuple(data, ref, rev, subdir) < + std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev, subdir); + } + + bool operator==(const FlakeRef & flakeRef) const + { + return std::make_tuple(data, ref, rev, subdir) == + std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev, flakeRef.subdir); + } + + // Parse a flake URI. + FlakeRef(const std::string & uri, bool allowRelative = false); + + // FIXME: change to operator <<. + std::string to_string() const; + + /* Check whether this is a "direct" flake reference, that is, not + a flake ID, which requires a lookup in the flake registry. */ + bool isDirect() const + { + return !std::get_if<FlakeRef::IsAlias>(&data); + } + + /* Check whether this is an "immutable" flake reference, that is, + one that contains a commit hash or content hash. */ + bool isImmutable() const; + + FlakeRef baseRef() const; +}; + +std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); + +MakeError(BadFlakeRef, Error); + +std::optional<FlakeRef> parseFlakeRef( + const std::string & uri, bool allowRelative = false); + +} diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc new file mode 100644 index 000000000..97c748c66 --- /dev/null +++ b/src/libexpr/flake/lockfile.cc @@ -0,0 +1,102 @@ +#include "lockfile.hh" +#include "store-api.hh" + +namespace nix::flake { + +AbstractInput::AbstractInput(const nlohmann::json & json) + : ref(json["uri"]) + , narHash(Hash((std::string) json["narHash"])) +{ + if (!ref.isImmutable()) + throw Error("lockfile contains mutable flakeref '%s'", ref); +} + +nlohmann::json AbstractInput::toJson() const +{ + nlohmann::json json; + json["uri"] = ref.to_string(); + json["narHash"] = narHash.to_string(SRI); + return json; +} + +Path AbstractInput::computeStorePath(Store & store) const +{ + return store.makeFixedOutputPath(true, narHash, "source"); +} + +FlakeInput::FlakeInput(const nlohmann::json & json) + : FlakeInputs(json) + , AbstractInput(json) + , id(json["id"]) +{ +} + +nlohmann::json FlakeInput::toJson() const +{ + auto json = FlakeInputs::toJson(); + json.update(AbstractInput::toJson()); + json["id"] = id; + return json; +} + +FlakeInputs::FlakeInputs(const nlohmann::json & json) +{ + for (auto & i : json["nonFlakeInputs"].items()) + nonFlakeInputs.insert_or_assign(i.key(), NonFlakeInput(i.value())); + + for (auto & i : json["inputs"].items()) + flakeInputs.insert_or_assign(i.key(), FlakeInput(i.value())); +} + +nlohmann::json FlakeInputs::toJson() const +{ + nlohmann::json json; + { + auto j = nlohmann::json::object(); + for (auto & i : nonFlakeInputs) + j[i.first] = i.second.toJson(); + json["nonFlakeInputs"] = std::move(j); + } + { + auto j = nlohmann::json::object(); + for (auto & i : flakeInputs) + j[i.first.to_string()] = i.second.toJson(); + json["inputs"] = std::move(j); + } + return json; +} + +nlohmann::json LockFile::toJson() const +{ + auto json = FlakeInputs::toJson(); + json["version"] = 2; + return json; +} + +LockFile LockFile::read(const Path & path) +{ + if (pathExists(path)) { + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + if (version != 2) + throw Error("lock file '%s' has unsupported version %d", path, version); + + return LockFile(json); + } else + return LockFile(); +} + +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) +{ + stream << lockFile.toJson().dump(4); // '4' = indentation in json file + return stream; +} + +void LockFile::write(const Path & path) const +{ + createDirs(dirOf(path)); + writeFile(path, fmt("%s\n", *this)); +} + +} diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh new file mode 100644 index 000000000..b76124190 --- /dev/null +++ b/src/libexpr/flake/lockfile.hh @@ -0,0 +1,112 @@ +#pragma once + +#include "flakeref.hh" + +#include <nlohmann/json.hpp> + +namespace nix { +class Store; +} + +namespace nix::flake { + +/* Common lock file information about a flake input, namely the + immutable ref and the NAR hash. */ +struct AbstractInput +{ + FlakeRef ref; + Hash narHash; + + AbstractInput(const FlakeRef & flakeRef, const Hash & narHash) + : ref(flakeRef), narHash(narHash) + { + assert(ref.isImmutable()); + }; + + AbstractInput(const nlohmann::json & json); + + nlohmann::json toJson() const; + + Path computeStorePath(Store & store) const; +}; + +/* Lock file information about a non-flake input. */ +struct NonFlakeInput : AbstractInput +{ + using AbstractInput::AbstractInput; + + bool operator ==(const NonFlakeInput & other) const + { + return ref == other.ref && narHash == other.narHash; + } +}; + +struct FlakeInput; + +/* Lock file information about the dependencies of a flake. */ +struct FlakeInputs +{ + std::map<FlakeRef, FlakeInput> flakeInputs; + std::map<FlakeAlias, NonFlakeInput> nonFlakeInputs; + + FlakeInputs() {}; + FlakeInputs(const nlohmann::json & json); + + nlohmann::json toJson() const; +}; + +/* Lock file information about a flake input. */ +struct FlakeInput : FlakeInputs, AbstractInput +{ + FlakeId id; + + FlakeInput(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash) + : AbstractInput(flakeRef, narHash), id(id) {}; + + FlakeInput(const nlohmann::json & json); + + bool operator ==(const FlakeInput & other) const + { + return + id == other.id + && ref == other.ref + && narHash == other.narHash + && flakeInputs == other.flakeInputs + && nonFlakeInputs == other.nonFlakeInputs; + } + + nlohmann::json toJson() const; +}; + +/* An entire lock file. Note that this cannot be a FlakeInput for the + top-level flake, because then the lock file would need to contain + the hash of the top-level flake, but committing the lock file + would invalidate that hash. */ +struct LockFile : FlakeInputs +{ + bool operator ==(const LockFile & other) const + { + return + flakeInputs == other.flakeInputs + && nonFlakeInputs == other.nonFlakeInputs; + } + + LockFile() {} + LockFile(const nlohmann::json & json) : FlakeInputs(json) {} + LockFile(FlakeInput && dep) + { + flakeInputs = std::move(dep.flakeInputs); + nonFlakeInputs = std::move(dep.nonFlakeInputs); + } + + nlohmann::json toJson() const; + + static LockFile read(const Path & path); + + void write(const Path & path) const; +}; + +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile); + +} + |