From 54aff8430c4e7739903f6dbed713cc088e38507f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 5 Jun 2019 16:51:54 +0200 Subject: Move flake-related stuff to src/libexpr/flake --- src/libexpr/eval.cc | 2 +- src/libexpr/flake/flake.cc | 604 ++++++++++++++++++++++++++++++++++++++++ src/libexpr/flake/flake.hh | 111 ++++++++ src/libexpr/flake/flakeref.cc | 252 +++++++++++++++++ src/libexpr/flake/flakeref.hh | 188 +++++++++++++ src/libexpr/flake/lockfile.cc | 102 +++++++ src/libexpr/flake/lockfile.hh | 112 ++++++++ src/libexpr/local.mk | 7 +- src/libexpr/primops/flake.cc | 604 ---------------------------------------- src/libexpr/primops/flake.hh | 111 -------- src/libexpr/primops/flakeref.cc | 252 ----------------- src/libexpr/primops/flakeref.hh | 188 ------------- src/libexpr/primops/lockfile.cc | 102 ------- src/libexpr/primops/lockfile.hh | 112 -------- src/nix/flake.cc | 2 +- src/nix/installables.cc | 2 +- 16 files changed, 1378 insertions(+), 1373 deletions(-) create mode 100644 src/libexpr/flake/flake.cc create mode 100644 src/libexpr/flake/flake.hh create mode 100644 src/libexpr/flake/flakeref.cc create mode 100644 src/libexpr/flake/flakeref.hh create mode 100644 src/libexpr/flake/lockfile.cc create mode 100644 src/libexpr/flake/lockfile.hh delete mode 100644 src/libexpr/primops/flake.cc delete mode 100644 src/libexpr/primops/flake.hh delete mode 100644 src/libexpr/primops/flakeref.cc delete mode 100644 src/libexpr/primops/flakeref.hh delete mode 100644 src/libexpr/primops/lockfile.cc delete mode 100644 src/libexpr/primops/lockfile.hh diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 0f8a105b1..46c622ee8 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -7,7 +7,7 @@ #include "eval-inline.hh" #include "download.hh" #include "json.hh" -#include "primops/flake.hh" +#include "flake/flake.hh" #include #include diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc new file mode 100644 index 000000000..bb0543541 --- /dev/null +++ b/src/libexpr/flake/flake.cc @@ -0,0 +1,604 @@ +#include "flake.hh" +#include "lockfile.hh" +#include "primops.hh" +#include "eval-inline.hh" +#include "primops/fetchGit.hh" +#include "download.hh" +#include "args.hh" + +#include +#include +#include +#include +#include +#include + +namespace nix { + +using namespace flake; + +namespace flake { + +/* Read a registry. */ +std::shared_ptr readRegistry(const Path & path) +{ + auto registry = std::make_shared(); + + if (!pathExists(path)) + return std::make_shared(); + + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + if (version != 1) + throw Error("flake registry '%s' has unsupported version %d", path, version); + + auto flakes = json["flakes"]; + for (auto i = flakes.begin(); i != flakes.end(); ++i) + registry->entries.emplace(i.key(), FlakeRef(i->value("uri", ""))); + + return registry; +} + +/* Write a registry to a file. */ +void writeRegistry(const FlakeRegistry & registry, const Path & path) +{ + nlohmann::json json; + json["version"] = 2; + for (auto elem : registry.entries) + json["flakes"][elem.first.to_string()] = { {"uri", elem.second.to_string()} }; + createDirs(dirOf(path)); + writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. +} + +Path getUserRegistryPath() +{ + return getHome() + "/.config/nix/registry.json"; +} + +std::shared_ptr getUserRegistry() +{ + return readRegistry(getUserRegistryPath()); +} + +std::shared_ptr getFlagRegistry(RegistryOverrides registryOverrides) +{ + auto flagRegistry = std::make_shared(); + for (auto const & x : registryOverrides) { + flagRegistry->entries.insert_or_assign(FlakeRef(x.first), FlakeRef(x.second)); + } + return flagRegistry; +} + +static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, + std::vector pastSearches = {}); + +FlakeRef updateFlakeRef(EvalState & state, const FlakeRef & newRef, const Registries & registries, std::vector pastSearches) +{ + std::string errorMsg = "found cycle in flake registries: "; + for (FlakeRef oldRef : pastSearches) { + errorMsg += oldRef.to_string(); + if (oldRef == newRef) + throw Error(errorMsg); + errorMsg += " - "; + } + pastSearches.push_back(newRef); + return lookupFlake(state, newRef, registries, pastSearches); +} + +static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, + std::vector pastSearches) +{ + if (registries.empty() && !flakeRef.isDirect()) + throw Error("indirect flake reference '%s' is not allowed", flakeRef); + + for (std::shared_ptr registry : registries) { + auto i = registry->entries.find(flakeRef); + if (i != registry->entries.end()) { + auto newRef = i->second; + return updateFlakeRef(state, newRef, registries, pastSearches); + } + + auto j = registry->entries.find(flakeRef.baseRef()); + if (j != registry->entries.end()) { + auto newRef = j->second; + newRef.ref = flakeRef.ref; + newRef.rev = flakeRef.rev; + return updateFlakeRef(state, newRef, registries, pastSearches); + } + } + + if (!flakeRef.isDirect()) + throw Error("could not resolve flake reference '%s'", flakeRef); + + return flakeRef; +} + +// Lookups happen here too +static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) +{ + FlakeRef resolvedRef = lookupFlake(state, flakeRef, + impureIsAllowed ? state.getFlakeRegistries() : std::vector>()); + + if (evalSettings.pureEval && !impureIsAllowed && !resolvedRef.isImmutable()) + throw Error("requested to fetch mutable flake '%s' in pure mode", resolvedRef); + + auto doGit = [&](const GitInfo & gitInfo) { + FlakeRef ref(resolvedRef.baseRef()); + ref.ref = gitInfo.ref; + ref.rev = gitInfo.rev; + SourceInfo info(ref); + info.storePath = gitInfo.storePath; + info.revCount = gitInfo.revCount; + info.narHash = state.store->queryPathInfo(info.storePath)->narHash; + info.lastModified = gitInfo.lastModified; + return info; + }; + + // This only downloads only one revision of the repo, not the entire history. + if (auto refData = std::get_if(&resolvedRef.data)) { + + // FIXME: use regular /archive URLs instead? api.github.com + // might have stricter rate limits. + + auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", + refData->owner, refData->repo, + resolvedRef.rev ? resolvedRef.rev->to_string(Base16, false) + : resolvedRef.ref ? *resolvedRef.ref : "master"); + + std::string accessToken = settings.githubAccessToken.get(); + if (accessToken != "") + url += "?access_token=" + accessToken; + + CachedDownloadRequest request(url); + request.unpack = true; + request.name = "source"; + request.ttl = resolvedRef.rev ? 1000000000 : settings.tarballTtl; + request.getLastModified = true; + auto result = getDownloader()->downloadCached(state.store, request); + + if (!result.etag) + throw Error("did not receive an ETag header from '%s'", url); + + if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"') + throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url); + + FlakeRef ref(resolvedRef.baseRef()); + ref.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); + SourceInfo info(ref); + info.storePath = result.storePath; + info.narHash = state.store->queryPathInfo(info.storePath)->narHash; + info.lastModified = result.lastModified; + + return info; + } + + // This downloads the entire git history + else if (auto refData = std::get_if(&resolvedRef.data)) { + return doGit(exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source")); + } + + else if (auto refData = std::get_if(&resolvedRef.data)) { + if (!pathExists(refData->path + "/.git")) + throw Error("flake '%s' does not reference a Git repository", refData->path); + return doGit(exportGit(state.store, refData->path, {}, {}, "source")); + } + + else abort(); +} + +// This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within `fetchFlake`, which is used here. +Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) +{ + SourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); + debug("got flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); + + FlakeRef resolvedRef = sourceInfo.resolvedRef; + + state.store->assertStorePath(sourceInfo.storePath); + + if (state.allowedPaths) + state.allowedPaths->insert(state.store->toRealPath(sourceInfo.storePath)); + + // Guard against symlink attacks. + Path flakeFile = canonPath(sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix"); + Path realFlakeFile = state.store->toRealPath(flakeFile); + if (!isInDir(realFlakeFile, state.store->toRealPath(sourceInfo.storePath))) + throw Error("'flake.nix' file of flake '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath); + + Flake flake(flakeRef, sourceInfo); + + if (!pathExists(realFlakeFile)) + throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir); + + Value vInfo; + state.evalFile(realFlakeFile, vInfo); // FIXME: symlink attack + + state.forceAttrs(vInfo); + + auto sEpoch = state.symbols.create("epoch"); + + if (auto epoch = vInfo.attrs->get(sEpoch)) { + flake.epoch = state.forceInt(*(**epoch).value, *(**epoch).pos); + if (flake.epoch > 201906) + throw Error("flake '%s' requires unsupported epoch %d; please upgrade Nix", flakeRef, flake.epoch); + } else + throw Error("flake '%s' lacks attribute 'epoch'", flakeRef); + + if (auto name = vInfo.attrs->get(state.sName)) + flake.id = state.forceStringNoCtx(*(**name).value, *(**name).pos); + else + throw Error("flake '%s' lacks attribute 'name'", flakeRef); + + if (auto description = vInfo.attrs->get(state.sDescription)) + flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos); + + auto sInputs = state.symbols.create("inputs"); + + if (auto inputs = vInfo.attrs->get(sInputs)) { + state.forceList(*(**inputs).value, *(**inputs).pos); + for (unsigned int n = 0; n < (**inputs).value->listSize(); ++n) + flake.inputs.push_back(FlakeRef(state.forceStringNoCtx( + *(**inputs).value->listElems()[n], *(**inputs).pos))); + } + + auto sNonFlakeInputs = state.symbols.create("nonFlakeInputs"); + + if (std::optional nonFlakeInputs = vInfo.attrs->get(sNonFlakeInputs)) { + state.forceAttrs(*(**nonFlakeInputs).value, *(**nonFlakeInputs).pos); + for (Attr attr : *(*(**nonFlakeInputs).value).attrs) { + std::string myNonFlakeUri = state.forceStringNoCtx(*attr.value, *attr.pos); + FlakeRef nonFlakeRef = FlakeRef(myNonFlakeUri); + flake.nonFlakeInputs.insert_or_assign(attr.name, nonFlakeRef); + } + } + + auto sOutputs = state.symbols.create("outputs"); + + if (auto outputs = vInfo.attrs->get(sOutputs)) { + state.forceFunction(*(**outputs).value, *(**outputs).pos); + flake.vOutputs = (**outputs).value; + } else + throw Error("flake '%s' lacks attribute 'outputs'", flakeRef); + + for (auto & attr : *vInfo.attrs) { + if (attr.name != sEpoch && + attr.name != state.sName && + attr.name != state.sDescription && + attr.name != sInputs && + attr.name != sNonFlakeInputs && + attr.name != sOutputs) + throw Error("flake '%s' has an unsupported attribute '%s', at %s", + flakeRef, attr.name, *attr.pos); + } + + return flake; +} + +// Get the `NonFlake` corresponding to a `FlakeRef`. +NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) +{ + auto sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); + debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); + + FlakeRef resolvedRef = sourceInfo.resolvedRef; + + NonFlake nonFlake(flakeRef, sourceInfo); + + state.store->assertStorePath(nonFlake.sourceInfo.storePath); + + if (state.allowedPaths) + state.allowedPaths->insert(nonFlake.sourceInfo.storePath); + + return nonFlake; +} + +bool allowedToWrite(HandleLockFile handle) +{ + return handle == UpdateLockFile || handle == RecreateLockFile; +} + +bool recreateLockFile(HandleLockFile handle) +{ + return handle == RecreateLockFile || handle == UseNewLockFile; +} + +bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) +{ + if (handle == AllPure) return false; + else if (handle == TopRefUsesRegistries) return isTopRef; + else if (handle == UpdateLockFile) return true; + else if (handle == UseUpdatedLockFile) return true; + else if (handle == RecreateLockFile) return true; + else if (handle == UseNewLockFile) return true; + else assert(false); +} + +/* Given a flakeref and its subtree of the lockfile, return an updated + subtree of the lockfile. That is, if the 'flake.nix' of the + referenced flake has inputs that don't have a corresponding entry + in the lockfile, they're added to the lockfile; conversely, any + lockfile entries that don't have a corresponding entry in flake.nix + are removed. + + Note that this is lazy: we only recursively fetch inputs that are + not in the lockfile yet. */ +static std::pair updateLocks( + EvalState & state, + const Flake & flake, + HandleLockFile handleLockFile, + const FlakeInputs & oldEntry, + bool topRef) +{ + FlakeInput newEntry( + flake.id, + flake.sourceInfo.resolvedRef, + flake.sourceInfo.narHash); + + for (auto & input : flake.nonFlakeInputs) { + auto & id = input.first; + auto & ref = input.second; + auto i = oldEntry.nonFlakeInputs.find(id); + if (i != oldEntry.nonFlakeInputs.end()) { + newEntry.nonFlakeInputs.insert_or_assign(i->first, i->second); + } else { + if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) + throw Error("cannot update non-flake dependency '%s' in pure mode", id); + auto nonFlake = getNonFlake(state, ref, allowedToUseRegistries(handleLockFile, false)); + newEntry.nonFlakeInputs.insert_or_assign(id, + NonFlakeInput( + nonFlake.sourceInfo.resolvedRef, + nonFlake.sourceInfo.narHash)); + } + } + + for (auto & inputRef : flake.inputs) { + auto i = oldEntry.flakeInputs.find(inputRef); + if (i != oldEntry.flakeInputs.end()) { + newEntry.flakeInputs.insert_or_assign(inputRef, i->second); + } else { + if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) + throw Error("cannot update flake dependency '%s' in pure mode", inputRef); + newEntry.flakeInputs.insert_or_assign(inputRef, + updateLocks(state, + getFlake(state, inputRef, allowedToUseRegistries(handleLockFile, false)), + handleLockFile, {}, false).second); + } + } + + return {flake, newEntry}; +} + +/* Compute an in-memory lockfile for the specified top-level flake, + and optionally write it to file, it the flake is writable. */ +ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile) +{ + auto flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); + + LockFile oldLockFile; + + if (!recreateLockFile(handleLockFile)) { + // If recreateLockFile, start with an empty lockfile + // FIXME: symlink attack + oldLockFile = LockFile::read( + state.store->toRealPath(flake.sourceInfo.storePath) + + "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock"); + } + + LockFile lockFile(updateLocks( + state, flake, handleLockFile, oldLockFile, true).second); + + if (!(lockFile == oldLockFile)) { + if (allowedToWrite(handleLockFile)) { + if (auto refData = std::get_if(&topRef.data)) { + lockFile.write(refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock"); + + // Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store. + runProgram("git", true, { "-C", refData->path, "add", + (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock" }); + } else + warn("cannot write lockfile of remote flake '%s'", topRef); + } else if (handleLockFile != AllPure && handleLockFile != TopRefUsesRegistries) + warn("using updated lockfile without writing it to file"); + } + + return ResolvedFlake(std::move(flake), std::move(lockFile)); +} + +void updateLockFile(EvalState & state, const FlakeRef & flakeRef, bool recreateLockFile) +{ + resolveFlake(state, flakeRef, recreateLockFile ? RecreateLockFile : UpdateLockFile); +} + +static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo, Value & vAttrs) +{ + auto & path = sourceInfo.storePath; + assert(state.store->isValidPath(path)); + mkString(*state.allocAttr(vAttrs, state.sOutPath), path, {path}); + + if (sourceInfo.resolvedRef.rev) { + mkString(*state.allocAttr(vAttrs, state.symbols.create("rev")), + sourceInfo.resolvedRef.rev->gitRev()); + mkString(*state.allocAttr(vAttrs, state.symbols.create("shortRev")), + sourceInfo.resolvedRef.rev->gitShortRev()); + } + + if (sourceInfo.revCount) + mkInt(*state.allocAttr(vAttrs, state.symbols.create("revCount")), *sourceInfo.revCount); + + if (sourceInfo.lastModified) + mkString(*state.allocAttr(vAttrs, state.symbols.create("lastModified")), + fmt("%s", + std::put_time(std::gmtime(&*sourceInfo.lastModified), "%Y%m%d%H%M%S"))); +} + +/* Helper primop to make callFlake (below) fetch/call its inputs + lazily. Note that this primop cannot be called by user code since + it doesn't appear in 'builtins'. */ +static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto lazyFlake = (FlakeInput *) args[0]->attrs; + auto flake = getFlake(state, lazyFlake->ref, false); + + if (flake.sourceInfo.narHash != lazyFlake->narHash) + throw Error("the content hash of flake '%s' doesn't match the hash recorded in the referring lockfile", flake.sourceInfo.resolvedRef); + + callFlake(state, flake, *lazyFlake, v); +} + +static void prim_callNonFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + auto lazyNonFlake = (NonFlakeInput *) args[0]->attrs; + + auto nonFlake = getNonFlake(state, lazyNonFlake->ref); + + if (nonFlake.sourceInfo.narHash != lazyNonFlake->narHash) + throw Error("the content hash of repository '%s' doesn't match the hash recorded in the referring lockfile", nonFlake.sourceInfo.resolvedRef); + + state.mkAttrs(v, 8); + + assert(state.store->isValidPath(nonFlake.sourceInfo.storePath)); + + mkString(*state.allocAttr(v, state.sOutPath), + nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath}); + + emitSourceInfoAttrs(state, nonFlake.sourceInfo, v); +} + +void callFlake(EvalState & state, + const Flake & flake, + const FlakeInputs & inputs, + Value & vRes) +{ + // Construct the resulting attrset '{outputs, ...}'. This attrset + // is passed lazily as an argument to the 'outputs' function. + + auto & v = *state.allocValue(); + + state.mkAttrs(v, + inputs.flakeInputs.size() + + inputs.nonFlakeInputs.size() + 8); + + for (auto & dep : inputs.flakeInputs) { + auto vFlake = state.allocAttr(v, dep.second.id); + auto vPrimOp = state.allocValue(); + static auto primOp = new PrimOp(prim_callFlake, 1, state.symbols.create("callFlake")); + vPrimOp->type = tPrimOp; + vPrimOp->primOp = primOp; + auto vArg = state.allocValue(); + vArg->type = tNull; + // FIXME: leak + vArg->attrs = (Bindings *) new FlakeInput(dep.second); // evil! also inefficient + mkApp(*vFlake, *vPrimOp, *vArg); + } + + for (auto & dep : inputs.nonFlakeInputs) { + auto vNonFlake = state.allocAttr(v, dep.first); + auto vPrimOp = state.allocValue(); + static auto primOp = new PrimOp(prim_callNonFlake, 1, state.symbols.create("callNonFlake")); + vPrimOp->type = tPrimOp; + vPrimOp->primOp = primOp; + auto vArg = state.allocValue(); + vArg->type = tNull; + // FIXME: leak + vArg->attrs = (Bindings *) new NonFlakeInput(dep.second); // evil! also inefficient + mkApp(*vNonFlake, *vPrimOp, *vArg); + } + + mkString(*state.allocAttr(v, state.sDescription), flake.description); + + emitSourceInfoAttrs(state, flake.sourceInfo, v); + + auto vOutputs = state.allocAttr(v, state.symbols.create("outputs")); + mkApp(*vOutputs, *flake.vOutputs, v); + + v.attrs->push_back(Attr(state.symbols.create("self"), &v)); + + v.attrs->sort(); + + /* For convenience, put the outputs directly in the result, so you + can refer to an output of an input as 'inputs.foo.bar' rather + than 'inputs.foo.outputs.bar'. */ + auto v2 = *state.allocValue(); + state.eval(state.parseExprFromString("res: res.outputs // res", "/"), v2); + + state.callFunction(v2, v, vRes, noPos); +} + +void callFlake(EvalState & state, + const ResolvedFlake & resFlake, + Value & v) +{ + callFlake(state, resFlake.flake, resFlake.lockFile, v); +} + +// This function is exposed to be used in nix files. +static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) +{ + callFlake(state, resolveFlake(state, state.forceStringNoCtx(*args[0], pos), + evalSettings.pureEval ? AllPure : UseUpdatedLockFile), v); +} + +static RegisterPrimOp r2("getFlake", 1, prim_getFlake); + +void gitCloneFlake(FlakeRef flakeRef, EvalState & state, Registries registries, const Path & destDir) +{ + flakeRef = lookupFlake(state, flakeRef, registries); + + std::string uri; + + Strings args = {"clone"}; + + if (auto refData = std::get_if(&flakeRef.data)) { + uri = "git@github.com:" + refData->owner + "/" + refData->repo + ".git"; + args.push_back(uri); + if (flakeRef.ref) { + args.push_back("--branch"); + args.push_back(*flakeRef.ref); + } + } else if (auto refData = std::get_if(&flakeRef.data)) { + args.push_back(refData->uri); + if (flakeRef.ref) { + args.push_back("--branch"); + args.push_back(*flakeRef.ref); + } + } + + if (destDir != "") + args.push_back(destDir); + + runProgram("git", true, args); +} + +} + +std::shared_ptr EvalState::getGlobalFlakeRegistry() +{ + std::call_once(_globalFlakeRegistryInit, [&]() { + auto path = evalSettings.flakeRegistry; + + if (!hasPrefix(path, "/")) { + CachedDownloadRequest request(evalSettings.flakeRegistry); + request.name = "flake-registry.json"; + request.gcRoot = true; + path = getDownloader()->downloadCached(store, request).path; + } + + _globalFlakeRegistry = readRegistry(path); + }); + + return _globalFlakeRegistry; +} + +// This always returns a vector with flakeReg, userReg, globalReg. +// If one of them doesn't exist, the registry is left empty but does exist. +const Registries EvalState::getFlakeRegistries() +{ + Registries registries; + registries.push_back(getFlagRegistry(registryOverrides)); + registries.push_back(getUserRegistry()); + registries.push_back(getGlobalFlakeRegistry()); + return registries; +} + +} diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh new file mode 100644 index 000000000..b8d0da252 --- /dev/null +++ b/src/libexpr/flake/flake.hh @@ -0,0 +1,111 @@ +#pragma once + +#include "types.hh" +#include "flakeref.hh" +#include "lockfile.hh" + +namespace nix { + +struct Value; +class EvalState; + +namespace flake { + +static const size_t FLAG_REGISTRY = 0; +static const size_t USER_REGISTRY = 1; +static const size_t GLOBAL_REGISTRY = 2; + +struct FlakeRegistry +{ + std::map entries; +}; + +typedef std::vector> Registries; + +std::shared_ptr readRegistry(const Path &); + +void writeRegistry(const FlakeRegistry &, const Path &); + +Path getUserRegistryPath(); + +enum HandleLockFile : unsigned int + { AllPure // Everything is handled 100% purely + , TopRefUsesRegistries // The top FlakeRef uses the registries, apart from that, everything happens 100% purely + , UpdateLockFile // Update the existing lockfile and write it to file + , UseUpdatedLockFile // `UpdateLockFile` without writing to file + , RecreateLockFile // Recreate the lockfile from scratch and write it to file + , UseNewLockFile // `RecreateLockFile` without writing to file + }; + +struct SourceInfo +{ + // Immutable flakeref that this source tree was obtained from. + FlakeRef resolvedRef; + + Path storePath; + + // Number of ancestors of the most recent commit. + std::optional revCount; + + // NAR hash of the store path. + Hash narHash; + + // A stable timestamp of this source tree. For Git and GitHub + // flakes, the commit date (not author date!) of the most recent + // commit. + std::optional lastModified; + + SourceInfo(const FlakeRef & resolvRef) : resolvedRef(resolvRef) {}; +}; + +struct Flake +{ + FlakeId id; + FlakeRef originalRef; + std::string description; + SourceInfo sourceInfo; + std::vector inputs; + std::map nonFlakeInputs; + Value * vOutputs; // FIXME: gc + unsigned int epoch; + + Flake(const FlakeRef & origRef, const SourceInfo & sourceInfo) + : originalRef(origRef), sourceInfo(sourceInfo) {}; +}; + +struct NonFlake +{ + FlakeRef originalRef; + SourceInfo sourceInfo; + NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) + : originalRef(origRef), sourceInfo(sourceInfo) {}; +}; + +Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); + +struct ResolvedFlake +{ + Flake flake; + LockFile lockFile; + ResolvedFlake(Flake && flake, LockFile && lockFile) + : flake(flake), lockFile(lockFile) {} +}; + +ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile); + +void callFlake(EvalState & state, + const Flake & flake, + const FlakeInputs & inputs, + Value & v); + +void callFlake(EvalState & state, + const ResolvedFlake & resFlake, + Value & v); + +void updateLockFile(EvalState &, const FlakeRef & flakeRef, bool recreateLockFile); + +void gitCloneFlake(FlakeRef flakeRef, EvalState &, Registries, const Path & destDir); + +} + +} diff --git a/src/libexpr/flake/flakeref.cc b/src/libexpr/flake/flakeref.cc new file mode 100644 index 000000000..24af09124 --- /dev/null +++ b/src/libexpr/flake/flakeref.cc @@ -0,0 +1,252 @@ +#include "flakeref.hh" +#include "store-api.hh" + +#include + +namespace nix { + +// A Git ref (i.e. branch or tag name). +const static std::string refRegex = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check + +// A Git revision (a SHA-1 commit hash). +const static std::string revRegexS = "[0-9a-fA-F]{40}"; +std::regex revRegex(revRegexS, std::regex::ECMAScript); + +// A Git ref or revision. +const static std::string revOrRefRegex = "(?:(" + revRegexS + ")|(" + refRegex + "))"; + +// A rev ("e72daba8250068216d79d2aeef40d4d95aff6666"), or a ref +// optionally followed by a rev (e.g. "master" or +// "master/e72daba8250068216d79d2aeef40d4d95aff6666"). +const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegex + ")(?:/(" + revRegexS + "))?))"; + +const static std::string flakeAlias = "[a-zA-Z][a-zA-Z0-9_-]*"; + +// GitHub references. +const static std::string ownerRegex = "[a-zA-Z][a-zA-Z0-9_-]*"; +const static std::string repoRegex = "[a-zA-Z][a-zA-Z0-9_-]*"; + +// URI stuff. +const static std::string schemeRegex = "(?:http|https|ssh|git|file)"; +const static std::string authorityRegex = "[a-zA-Z0-9._~-]*"; +const static std::string segmentRegex = "[a-zA-Z0-9._~-]+"; +const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*"; + +// 'dir' path elements cannot start with a '.'. We also reject +// potentially dangerous characters like ';'. +const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)"; +const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*"; + + +FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) +{ + // FIXME: could combine this into one regex. + + static std::regex flakeRegex( + "(?:flake:)?(" + flakeAlias + ")(?:/(?:" + refAndOrRevRegex + "))?", + std::regex::ECMAScript); + + static std::regex githubRegex( + "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?", + std::regex::ECMAScript); + + static std::regex uriRegex( + "((" + schemeRegex + "):" + + "(?://(" + authorityRegex + "))?" + + "(" + pathRegex + "))", + std::regex::ECMAScript); + + static std::regex refRegex2(refRegex, std::regex::ECMAScript); + + static std::regex subDirRegex2(subDirRegex, std::regex::ECMAScript); + + auto [uri2, params] = splitUriAndParams(uri_); + std::string uri(uri2); + + auto handleSubdir = [&](const std::string & name, const std::string & value) { + if (name == "dir") { + if (value != "" && !std::regex_match(value, subDirRegex2)) + throw BadFlakeRef("flake '%s' has invalid subdirectory '%s'", uri, value); + subdir = value; + return true; + } else + return false; + }; + + auto handleGitParams = [&](const std::string & name, const std::string & value) { + if (name == "rev") { + if (!std::regex_match(value, revRegex)) + throw BadFlakeRef("invalid Git revision '%s'", value); + rev = Hash(value, htSHA1); + } else if (name == "ref") { + if (!std::regex_match(value, refRegex2)) + throw BadFlakeRef("invalid Git ref '%s'", value); + ref = value; + } else if (handleSubdir(name, value)) + ; + else return false; + return true; + }; + + std::cmatch match; + if (std::regex_match(uri.c_str(), match, flakeRegex)) { + IsAlias d; + d.alias = match[1]; + if (match[2].matched) + rev = Hash(match[2], htSHA1); + else if (match[3].matched) { + ref = match[3]; + if (match[4].matched) + rev = Hash(match[4], htSHA1); + } + data = d; + } + + else if (std::regex_match(uri.c_str(), match, githubRegex)) { + IsGitHub d; + d.owner = match[1]; + d.repo = match[2]; + if (match[3].matched) + rev = Hash(match[3], htSHA1); + else if (match[4].matched) { + ref = match[4]; + } + for (auto & param : params) { + if (handleSubdir(param.first, param.second)) + ; + else + throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); + } + data = d; + } + + else if (std::regex_match(uri.c_str(), match, uriRegex) + && (match[2] == "file" || hasSuffix(match[4], ".git"))) + { + IsGit d; + d.uri = match[1]; + for (auto & param : params) { + if (handleGitParams(param.first, param.second)) + ; + else + // FIXME: should probably pass through unknown parameters + throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); + } + if (rev && !ref) + throw BadFlakeRef("flake URI '%s' lacks a Git ref", uri); + data = d; + } + + else if ((hasPrefix(uri, "/") || (allowRelative && (hasPrefix(uri, "./") || hasPrefix(uri, "../") || uri == "."))) + && uri.find(':') == std::string::npos) + { + IsPath d; + if (allowRelative) { + d.path = absPath(uri); + while (true) { + if (pathExists(d.path + "/.git")) break; + subdir = baseNameOf(d.path) + (subdir.empty() ? "" : "/" + subdir); + d.path = dirOf(d.path); + if (d.path == "/") + throw BadFlakeRef("path '%s' does not reference a Git repository", uri); + } + } else + d.path = canonPath(uri); + data = d; + for (auto & param : params) { + if (handleGitParams(param.first, param.second)) + ; + else + throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); + } + } + + else + throw BadFlakeRef("'%s' is not a valid flake reference", uri); +} + +std::string FlakeRef::to_string() const +{ + std::string string; + bool first = true; + + auto addParam = + [&](const std::string & name, std::string value) { + string += first ? '?' : '&'; + first = false; + string += name; + string += '='; + string += value; // FIXME: escaping + }; + + if (auto refData = std::get_if(&data)) { + string = refData->alias; + if (ref) string += '/' + *ref; + if (rev) string += '/' + rev->gitRev(); + } + + else if (auto refData = std::get_if(&data)) { + string = refData->path; + if (ref) addParam("ref", *ref); + if (rev) addParam("rev", rev->gitRev()); + if (subdir != "") addParam("dir", subdir); + } + + else if (auto refData = std::get_if(&data)) { + assert(!(ref && rev)); + string = "github:" + refData->owner + "/" + refData->repo; + if (ref) { string += '/'; string += *ref; } + if (rev) { string += '/'; string += rev->gitRev(); } + if (subdir != "") addParam("dir", subdir); + } + + else if (auto refData = std::get_if(&data)) { + assert(!rev || ref); + string = refData->uri; + + if (ref) { + addParam("ref", *ref); + if (rev) + addParam("rev", rev->gitRev()); + } + + if (subdir != "") addParam("dir", subdir); + } + + else abort(); + + assert(FlakeRef(string) == *this); + + return string; +} + +std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) +{ + str << flakeRef.to_string(); + return str; +} + +bool FlakeRef::isImmutable() const +{ + return (bool) rev; +} + +FlakeRef FlakeRef::baseRef() const // Removes the ref and rev from a FlakeRef. +{ + FlakeRef result(*this); + result.ref = std::nullopt; + result.rev = std::nullopt; + return result; +} + +std::optional parseFlakeRef( + const std::string & uri, bool allowRelative) +{ + try { + return FlakeRef(uri, allowRelative); + } catch (BadFlakeRef & e) { + return {}; + } +} + +} diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh new file mode 100644 index 000000000..52bb82ddb --- /dev/null +++ b/src/libexpr/flake/flakeref.hh @@ -0,0 +1,188 @@ +#pragma once + +#include "types.hh" +#include "hash.hh" + +#include + +namespace nix { + +/* Flake references are a URI-like syntax to specify a flake. + + Examples: + + * (/rev-or-ref(/rev)?)? + + Look up a flake by ID in the flake lock file or in the flake + registry. These must specify an actual location for the flake + using the formats listed below. Note that in pure evaluation + mode, the flake registry is empty. + + Optionally, the rev or ref from the dereferenced flake can be + overriden. For example, + + nixpkgs/19.09 + + uses the "19.09" branch of the nixpkgs' flake GitHub repository, + while + + nixpkgs/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f + + uses the specified revision. For Git (rather than GitHub) + repositories, both the rev and ref must be given, e.g. + + nixpkgs/19.09/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f + + * github:/(/)? + + A repository on GitHub. These differ from Git references in that + they're downloaded in a efficient way (via the tarball mechanism) + and that they support downloading a specific revision without + specifying a branch. is either a commit hash ("rev") + or a branch or tag name ("ref"). The default is: "master" if none + is specified. Note that in pure evaluation mode, a commit hash + must be used. + + Flakes fetched in this manner expose "rev" and "lastModified" + attributes, but not "revCount". + + Examples: + + github:edolstra/dwarffs + github:edolstra/dwarffs/unstable + github:edolstra/dwarffs/41c0c1bf292ea3ac3858ff393b49ca1123dbd553 + + * https:///.git(\?attr(&attr)*)? + ssh:///.git(\?attr(&attr)*)? + git:///.git(\?attr(&attr)*)? + file:///(\?attr(&attr)*)? + + where 'attr' is one of: + rev= + ref= + + A Git repository fetched through https. Note that the path must + end in ".git". The default for "ref" is "master". + + Examples: + + https://example.org/my/repo.git + https://example.org/my/repo.git?ref=release-1.2.3 + https://example.org/my/repo.git?rev=e72daba8250068216d79d2aeef40d4d95aff6666 + git://github.com/edolstra/dwarffs.git?ref=flake&rev=2efca4bc9da70fb001b26c3dc858c6397d3c4817 + + * /path.git(\?attr(&attr)*)? + + Like file://path.git, but if no "ref" or "rev" is specified, the + (possibly dirty) working tree will be used. Using a working tree + is not allowed in pure evaluation mode. + + Examples: + + /path/to/my/repo + /path/to/my/repo?ref=develop + /path/to/my/repo?rev=e72daba8250068216d79d2aeef40d4d95aff6666 + + * https:///.tar.xz(?hash=) + file:///.tar.xz(?hash=) + + A flake distributed as a tarball. In pure evaluation mode, an SRI + hash is mandatory. It exposes a "lastModified" attribute, being + the newest file inside the tarball. + + Example: + + https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz + https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz?hash=sha256-56bbc099995ea8581ead78f22832fee7dbcb0a0b6319293d8c2d0aef5379397c + + Note: currently, there can be only one flake per Git repository, and + it must be at top-level. In the future, we may want to add a field + (e.g. "dir=") to specify a subdirectory inside the repository. +*/ + +typedef std::string FlakeId; +typedef std::string FlakeAlias; +typedef std::string FlakeUri; + +struct FlakeRef +{ + struct IsAlias + { + FlakeAlias alias; + bool operator<(const IsAlias & b) const { return alias < b.alias; }; + bool operator==(const IsAlias & b) const { return alias == b.alias; }; + }; + + struct IsGitHub { + std::string owner, repo; + bool operator<(const IsGitHub & b) const { + return std::make_tuple(owner, repo) < std::make_tuple(b.owner, b.repo); + } + bool operator==(const IsGitHub & b) const { + return owner == b.owner && repo == b.repo; + } + }; + + // Git, Tarball + struct IsGit + { + std::string uri; + bool operator<(const IsGit & b) const { return uri < b.uri; } + bool operator==(const IsGit & b) const { return uri == b.uri; } + }; + + struct IsPath + { + Path path; + bool operator<(const IsPath & b) const { return path < b.path; } + bool operator==(const IsPath & b) const { return path == b.path; } + }; + + // Git, Tarball + + std::variant data; + + std::optional ref; + std::optional rev; + Path subdir = ""; // This is a relative path pointing at the flake.nix file's directory, relative to the git root. + + bool operator<(const FlakeRef & flakeRef) const + { + return std::make_tuple(data, ref, rev, subdir) < + std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev, subdir); + } + + bool operator==(const FlakeRef & flakeRef) const + { + return std::make_tuple(data, ref, rev, subdir) == + std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev, flakeRef.subdir); + } + + // Parse a flake URI. + FlakeRef(const std::string & uri, bool allowRelative = false); + + // FIXME: change to operator <<. + std::string to_string() const; + + /* Check whether this is a "direct" flake reference, that is, not + a flake ID, which requires a lookup in the flake registry. */ + bool isDirect() const + { + return !std::get_if(&data); + } + + /* Check whether this is an "immutable" flake reference, that is, + one that contains a commit hash or content hash. */ + bool isImmutable() const; + + FlakeRef baseRef() const; +}; + +std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); + +MakeError(BadFlakeRef, Error); + +std::optional parseFlakeRef( + const std::string & uri, bool allowRelative = false); + +} diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc new file mode 100644 index 000000000..97c748c66 --- /dev/null +++ b/src/libexpr/flake/lockfile.cc @@ -0,0 +1,102 @@ +#include "lockfile.hh" +#include "store-api.hh" + +namespace nix::flake { + +AbstractInput::AbstractInput(const nlohmann::json & json) + : ref(json["uri"]) + , narHash(Hash((std::string) json["narHash"])) +{ + if (!ref.isImmutable()) + throw Error("lockfile contains mutable flakeref '%s'", ref); +} + +nlohmann::json AbstractInput::toJson() const +{ + nlohmann::json json; + json["uri"] = ref.to_string(); + json["narHash"] = narHash.to_string(SRI); + return json; +} + +Path AbstractInput::computeStorePath(Store & store) const +{ + return store.makeFixedOutputPath(true, narHash, "source"); +} + +FlakeInput::FlakeInput(const nlohmann::json & json) + : FlakeInputs(json) + , AbstractInput(json) + , id(json["id"]) +{ +} + +nlohmann::json FlakeInput::toJson() const +{ + auto json = FlakeInputs::toJson(); + json.update(AbstractInput::toJson()); + json["id"] = id; + return json; +} + +FlakeInputs::FlakeInputs(const nlohmann::json & json) +{ + for (auto & i : json["nonFlakeInputs"].items()) + nonFlakeInputs.insert_or_assign(i.key(), NonFlakeInput(i.value())); + + for (auto & i : json["inputs"].items()) + flakeInputs.insert_or_assign(i.key(), FlakeInput(i.value())); +} + +nlohmann::json FlakeInputs::toJson() const +{ + nlohmann::json json; + { + auto j = nlohmann::json::object(); + for (auto & i : nonFlakeInputs) + j[i.first] = i.second.toJson(); + json["nonFlakeInputs"] = std::move(j); + } + { + auto j = nlohmann::json::object(); + for (auto & i : flakeInputs) + j[i.first.to_string()] = i.second.toJson(); + json["inputs"] = std::move(j); + } + return json; +} + +nlohmann::json LockFile::toJson() const +{ + auto json = FlakeInputs::toJson(); + json["version"] = 2; + return json; +} + +LockFile LockFile::read(const Path & path) +{ + if (pathExists(path)) { + auto json = nlohmann::json::parse(readFile(path)); + + auto version = json.value("version", 0); + if (version != 2) + throw Error("lock file '%s' has unsupported version %d", path, version); + + return LockFile(json); + } else + return LockFile(); +} + +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) +{ + stream << lockFile.toJson().dump(4); // '4' = indentation in json file + return stream; +} + +void LockFile::write(const Path & path) const +{ + createDirs(dirOf(path)); + writeFile(path, fmt("%s\n", *this)); +} + +} diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh new file mode 100644 index 000000000..b76124190 --- /dev/null +++ b/src/libexpr/flake/lockfile.hh @@ -0,0 +1,112 @@ +#pragma once + +#include "flakeref.hh" + +#include + +namespace nix { +class Store; +} + +namespace nix::flake { + +/* Common lock file information about a flake input, namely the + immutable ref and the NAR hash. */ +struct AbstractInput +{ + FlakeRef ref; + Hash narHash; + + AbstractInput(const FlakeRef & flakeRef, const Hash & narHash) + : ref(flakeRef), narHash(narHash) + { + assert(ref.isImmutable()); + }; + + AbstractInput(const nlohmann::json & json); + + nlohmann::json toJson() const; + + Path computeStorePath(Store & store) const; +}; + +/* Lock file information about a non-flake input. */ +struct NonFlakeInput : AbstractInput +{ + using AbstractInput::AbstractInput; + + bool operator ==(const NonFlakeInput & other) const + { + return ref == other.ref && narHash == other.narHash; + } +}; + +struct FlakeInput; + +/* Lock file information about the dependencies of a flake. */ +struct FlakeInputs +{ + std::map flakeInputs; + std::map nonFlakeInputs; + + FlakeInputs() {}; + FlakeInputs(const nlohmann::json & json); + + nlohmann::json toJson() const; +}; + +/* Lock file information about a flake input. */ +struct FlakeInput : FlakeInputs, AbstractInput +{ + FlakeId id; + + FlakeInput(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash) + : AbstractInput(flakeRef, narHash), id(id) {}; + + FlakeInput(const nlohmann::json & json); + + bool operator ==(const FlakeInput & other) const + { + return + id == other.id + && ref == other.ref + && narHash == other.narHash + && flakeInputs == other.flakeInputs + && nonFlakeInputs == other.nonFlakeInputs; + } + + nlohmann::json toJson() const; +}; + +/* An entire lock file. Note that this cannot be a FlakeInput for the + top-level flake, because then the lock file would need to contain + the hash of the top-level flake, but committing the lock file + would invalidate that hash. */ +struct LockFile : FlakeInputs +{ + bool operator ==(const LockFile & other) const + { + return + flakeInputs == other.flakeInputs + && nonFlakeInputs == other.nonFlakeInputs; + } + + LockFile() {} + LockFile(const nlohmann::json & json) : FlakeInputs(json) {} + LockFile(FlakeInput && dep) + { + flakeInputs = std::move(dep.flakeInputs); + nonFlakeInputs = std::move(dep.nonFlakeInputs); + } + + nlohmann::json toJson() const; + + static LockFile read(const Path & path); + + void write(const Path & path) const; +}; + +std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile); + +} + diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index ccd5293e4..a9cb6b7b6 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -4,7 +4,12 @@ libexpr_NAME = libnixexpr libexpr_DIR := $(d) -libexpr_SOURCES := $(wildcard $(d)/*.cc) $(wildcard $(d)/primops/*.cc) $(d)/lexer-tab.cc $(d)/parser-tab.cc +libexpr_SOURCES := \ + $(wildcard $(d)/*.cc) \ + $(wildcard $(d)/primops/*.cc) \ + $(wildcard $(d)/flake/*.cc) \ + $(d)/lexer-tab.cc \ + $(d)/parser-tab.cc libexpr_LIBS = libutil libstore diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc deleted file mode 100644 index 793d6da35..000000000 --- a/src/libexpr/primops/flake.cc +++ /dev/null @@ -1,604 +0,0 @@ -#include "flake.hh" -#include "lockfile.hh" -#include "primops.hh" -#include "eval-inline.hh" -#include "fetchGit.hh" -#include "download.hh" -#include "args.hh" - -#include -#include -#include -#include -#include -#include - -namespace nix { - -using namespace flake; - -namespace flake { - -/* Read a registry. */ -std::shared_ptr readRegistry(const Path & path) -{ - auto registry = std::make_shared(); - - if (!pathExists(path)) - return std::make_shared(); - - auto json = nlohmann::json::parse(readFile(path)); - - auto version = json.value("version", 0); - if (version != 1) - throw Error("flake registry '%s' has unsupported version %d", path, version); - - auto flakes = json["flakes"]; - for (auto i = flakes.begin(); i != flakes.end(); ++i) - registry->entries.emplace(i.key(), FlakeRef(i->value("uri", ""))); - - return registry; -} - -/* Write a registry to a file. */ -void writeRegistry(const FlakeRegistry & registry, const Path & path) -{ - nlohmann::json json; - json["version"] = 2; - for (auto elem : registry.entries) - json["flakes"][elem.first.to_string()] = { {"uri", elem.second.to_string()} }; - createDirs(dirOf(path)); - writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. -} - -Path getUserRegistryPath() -{ - return getHome() + "/.config/nix/registry.json"; -} - -std::shared_ptr getUserRegistry() -{ - return readRegistry(getUserRegistryPath()); -} - -std::shared_ptr getFlagRegistry(RegistryOverrides registryOverrides) -{ - auto flagRegistry = std::make_shared(); - for (auto const & x : registryOverrides) { - flagRegistry->entries.insert_or_assign(FlakeRef(x.first), FlakeRef(x.second)); - } - return flagRegistry; -} - -static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, - std::vector pastSearches = {}); - -FlakeRef updateFlakeRef(EvalState & state, const FlakeRef & newRef, const Registries & registries, std::vector pastSearches) -{ - std::string errorMsg = "found cycle in flake registries: "; - for (FlakeRef oldRef : pastSearches) { - errorMsg += oldRef.to_string(); - if (oldRef == newRef) - throw Error(errorMsg); - errorMsg += " - "; - } - pastSearches.push_back(newRef); - return lookupFlake(state, newRef, registries, pastSearches); -} - -static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, - std::vector pastSearches) -{ - if (registries.empty() && !flakeRef.isDirect()) - throw Error("indirect flake reference '%s' is not allowed", flakeRef); - - for (std::shared_ptr registry : registries) { - auto i = registry->entries.find(flakeRef); - if (i != registry->entries.end()) { - auto newRef = i->second; - return updateFlakeRef(state, newRef, registries, pastSearches); - } - - auto j = registry->entries.find(flakeRef.baseRef()); - if (j != registry->entries.end()) { - auto newRef = j->second; - newRef.ref = flakeRef.ref; - newRef.rev = flakeRef.rev; - return updateFlakeRef(state, newRef, registries, pastSearches); - } - } - - if (!flakeRef.isDirect()) - throw Error("could not resolve flake reference '%s'", flakeRef); - - return flakeRef; -} - -// Lookups happen here too -static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) -{ - FlakeRef resolvedRef = lookupFlake(state, flakeRef, - impureIsAllowed ? state.getFlakeRegistries() : std::vector>()); - - if (evalSettings.pureEval && !impureIsAllowed && !resolvedRef.isImmutable()) - throw Error("requested to fetch mutable flake '%s' in pure mode", resolvedRef); - - auto doGit = [&](const GitInfo & gitInfo) { - FlakeRef ref(resolvedRef.baseRef()); - ref.ref = gitInfo.ref; - ref.rev = gitInfo.rev; - SourceInfo info(ref); - info.storePath = gitInfo.storePath; - info.revCount = gitInfo.revCount; - info.narHash = state.store->queryPathInfo(info.storePath)->narHash; - info.lastModified = gitInfo.lastModified; - return info; - }; - - // This only downloads only one revision of the repo, not the entire history. - if (auto refData = std::get_if(&resolvedRef.data)) { - - // FIXME: use regular /archive URLs instead? api.github.com - // might have stricter rate limits. - - auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", - refData->owner, refData->repo, - resolvedRef.rev ? resolvedRef.rev->to_string(Base16, false) - : resolvedRef.ref ? *resolvedRef.ref : "master"); - - std::string accessToken = settings.githubAccessToken.get(); - if (accessToken != "") - url += "?access_token=" + accessToken; - - CachedDownloadRequest request(url); - request.unpack = true; - request.name = "source"; - request.ttl = resolvedRef.rev ? 1000000000 : settings.tarballTtl; - request.getLastModified = true; - auto result = getDownloader()->downloadCached(state.store, request); - - if (!result.etag) - throw Error("did not receive an ETag header from '%s'", url); - - if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"') - throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url); - - FlakeRef ref(resolvedRef.baseRef()); - ref.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); - SourceInfo info(ref); - info.storePath = result.storePath; - info.narHash = state.store->queryPathInfo(info.storePath)->narHash; - info.lastModified = result.lastModified; - - return info; - } - - // This downloads the entire git history - else if (auto refData = std::get_if(&resolvedRef.data)) { - return doGit(exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source")); - } - - else if (auto refData = std::get_if(&resolvedRef.data)) { - if (!pathExists(refData->path + "/.git")) - throw Error("flake '%s' does not reference a Git repository", refData->path); - return doGit(exportGit(state.store, refData->path, {}, {}, "source")); - } - - else abort(); -} - -// This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within `fetchFlake`, which is used here. -Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) -{ - SourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); - debug("got flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); - - FlakeRef resolvedRef = sourceInfo.resolvedRef; - - state.store->assertStorePath(sourceInfo.storePath); - - if (state.allowedPaths) - state.allowedPaths->insert(state.store->toRealPath(sourceInfo.storePath)); - - // Guard against symlink attacks. - Path flakeFile = canonPath(sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix"); - Path realFlakeFile = state.store->toRealPath(flakeFile); - if (!isInDir(realFlakeFile, state.store->toRealPath(sourceInfo.storePath))) - throw Error("'flake.nix' file of flake '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath); - - Flake flake(flakeRef, sourceInfo); - - if (!pathExists(realFlakeFile)) - throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir); - - Value vInfo; - state.evalFile(realFlakeFile, vInfo); // FIXME: symlink attack - - state.forceAttrs(vInfo); - - auto sEpoch = state.symbols.create("epoch"); - - if (auto epoch = vInfo.attrs->get(sEpoch)) { - flake.epoch = state.forceInt(*(**epoch).value, *(**epoch).pos); - if (flake.epoch > 201906) - throw Error("flake '%s' requires unsupported epoch %d; please upgrade Nix", flakeRef, flake.epoch); - } else - throw Error("flake '%s' lacks attribute 'epoch'", flakeRef); - - if (auto name = vInfo.attrs->get(state.sName)) - flake.id = state.forceStringNoCtx(*(**name).value, *(**name).pos); - else - throw Error("flake '%s' lacks attribute 'name'", flakeRef); - - if (auto description = vInfo.attrs->get(state.sDescription)) - flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos); - - auto sInputs = state.symbols.create("inputs"); - - if (auto inputs = vInfo.attrs->get(sInputs)) { - state.forceList(*(**inputs).value, *(**inputs).pos); - for (unsigned int n = 0; n < (**inputs).value->listSize(); ++n) - flake.inputs.push_back(FlakeRef(state.forceStringNoCtx( - *(**inputs).value->listElems()[n], *(**inputs).pos))); - } - - auto sNonFlakeInputs = state.symbols.create("nonFlakeInputs"); - - if (std::optional nonFlakeInputs = vInfo.attrs->get(sNonFlakeInputs)) { - state.forceAttrs(*(**nonFlakeInputs).value, *(**nonFlakeInputs).pos); - for (Attr attr : *(*(**nonFlakeInputs).value).attrs) { - std::string myNonFlakeUri = state.forceStringNoCtx(*attr.value, *attr.pos); - FlakeRef nonFlakeRef = FlakeRef(myNonFlakeUri); - flake.nonFlakeInputs.insert_or_assign(attr.name, nonFlakeRef); - } - } - - auto sOutputs = state.symbols.create("outputs"); - - if (auto outputs = vInfo.attrs->get(sOutputs)) { - state.forceFunction(*(**outputs).value, *(**outputs).pos); - flake.vOutputs = (**outputs).value; - } else - throw Error("flake '%s' lacks attribute 'outputs'", flakeRef); - - for (auto & attr : *vInfo.attrs) { - if (attr.name != sEpoch && - attr.name != state.sName && - attr.name != state.sDescription && - attr.name != sInputs && - attr.name != sNonFlakeInputs && - attr.name != sOutputs) - throw Error("flake '%s' has an unsupported attribute '%s', at %s", - flakeRef, attr.name, *attr.pos); - } - - return flake; -} - -// Get the `NonFlake` corresponding to a `FlakeRef`. -NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) -{ - auto sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); - debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); - - FlakeRef resolvedRef = sourceInfo.resolvedRef; - - NonFlake nonFlake(flakeRef, sourceInfo); - - state.store->assertStorePath(nonFlake.sourceInfo.storePath); - - if (state.allowedPaths) - state.allowedPaths->insert(nonFlake.sourceInfo.storePath); - - return nonFlake; -} - -bool allowedToWrite(HandleLockFile handle) -{ - return handle == UpdateLockFile || handle == RecreateLockFile; -} - -bool recreateLockFile(HandleLockFile handle) -{ - return handle == RecreateLockFile || handle == UseNewLockFile; -} - -bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) -{ - if (handle == AllPure) return false; - else if (handle == TopRefUsesRegistries) return isTopRef; - else if (handle == UpdateLockFile) return true; - else if (handle == UseUpdatedLockFile) return true; - else if (handle == RecreateLockFile) return true; - else if (handle == UseNewLockFile) return true; - else assert(false); -} - -/* Given a flakeref and its subtree of the lockfile, return an updated - subtree of the lockfile. That is, if the 'flake.nix' of the - referenced flake has inputs that don't have a corresponding entry - in the lockfile, they're added to the lockfile; conversely, any - lockfile entries that don't have a corresponding entry in flake.nix - are removed. - - Note that this is lazy: we only recursively fetch inputs that are - not in the lockfile yet. */ -static std::pair updateLocks( - EvalState & state, - const Flake & flake, - HandleLockFile handleLockFile, - const FlakeInputs & oldEntry, - bool topRef) -{ - FlakeInput newEntry( - flake.id, - flake.sourceInfo.resolvedRef, - flake.sourceInfo.narHash); - - for (auto & input : flake.nonFlakeInputs) { - auto & id = input.first; - auto & ref = input.second; - auto i = oldEntry.nonFlakeInputs.find(id); - if (i != oldEntry.nonFlakeInputs.end()) { - newEntry.nonFlakeInputs.insert_or_assign(i->first, i->second); - } else { - if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) - throw Error("cannot update non-flake dependency '%s' in pure mode", id); - auto nonFlake = getNonFlake(state, ref, allowedToUseRegistries(handleLockFile, false)); - newEntry.nonFlakeInputs.insert_or_assign(id, - NonFlakeInput( - nonFlake.sourceInfo.resolvedRef, - nonFlake.sourceInfo.narHash)); - } - } - - for (auto & inputRef : flake.inputs) { - auto i = oldEntry.flakeInputs.find(inputRef); - if (i != oldEntry.flakeInputs.end()) { - newEntry.flakeInputs.insert_or_assign(inputRef, i->second); - } else { - if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) - throw Error("cannot update flake dependency '%s' in pure mode", inputRef); - newEntry.flakeInputs.insert_or_assign(inputRef, - updateLocks(state, - getFlake(state, inputRef, allowedToUseRegistries(handleLockFile, false)), - handleLockFile, {}, false).second); - } - } - - return {flake, newEntry}; -} - -/* Compute an in-memory lockfile for the specified top-level flake, - and optionally write it to file, it the flake is writable. */ -ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile) -{ - auto flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); - - LockFile oldLockFile; - - if (!recreateLockFile(handleLockFile)) { - // If recreateLockFile, start with an empty lockfile - // FIXME: symlink attack - oldLockFile = LockFile::read( - state.store->toRealPath(flake.sourceInfo.storePath) - + "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock"); - } - - LockFile lockFile(updateLocks( - state, flake, handleLockFile, oldLockFile, true).second); - - if (!(lockFile == oldLockFile)) { - if (allowedToWrite(handleLockFile)) { - if (auto refData = std::get_if(&topRef.data)) { - lockFile.write(refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock"); - - // Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store. - runProgram("git", true, { "-C", refData->path, "add", - (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock" }); - } else - warn("cannot write lockfile of remote flake '%s'", topRef); - } else if (handleLockFile != AllPure && handleLockFile != TopRefUsesRegistries) - warn("using updated lockfile without writing it to file"); - } - - return ResolvedFlake(std::move(flake), std::move(lockFile)); -} - -void updateLockFile(EvalState & state, const FlakeRef & flakeRef, bool recreateLockFile) -{ - resolveFlake(state, flakeRef, recreateLockFile ? RecreateLockFile : UpdateLockFile); -} - -static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo, Value & vAttrs) -{ - auto & path = sourceInfo.storePath; - assert(state.store->isValidPath(path)); - mkString(*state.allocAttr(vAttrs, state.sOutPath), path, {path}); - - if (sourceInfo.resolvedRef.rev) { - mkString(*state.allocAttr(vAttrs, state.symbols.create("rev")), - sourceInfo.resolvedRef.rev->gitRev()); - mkString(*state.allocAttr(vAttrs, state.symbols.create("shortRev")), - sourceInfo.resolvedRef.rev->gitShortRev()); - } - - if (sourceInfo.revCount) - mkInt(*state.allocAttr(vAttrs, state.symbols.create("revCount")), *sourceInfo.revCount); - - if (sourceInfo.lastModified) - mkString(*state.allocAttr(vAttrs, state.symbols.create("lastModified")), - fmt("%s", - std::put_time(std::gmtime(&*sourceInfo.lastModified), "%Y%m%d%H%M%S"))); -} - -/* Helper primop to make callFlake (below) fetch/call its inputs - lazily. Note that this primop cannot be called by user code since - it doesn't appear in 'builtins'. */ -static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - auto lazyFlake = (FlakeInput *) args[0]->attrs; - auto flake = getFlake(state, lazyFlake->ref, false); - - if (flake.sourceInfo.narHash != lazyFlake->narHash) - throw Error("the content hash of flake '%s' doesn't match the hash recorded in the referring lockfile", flake.sourceInfo.resolvedRef); - - callFlake(state, flake, *lazyFlake, v); -} - -static void prim_callNonFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - auto lazyNonFlake = (NonFlakeInput *) args[0]->attrs; - - auto nonFlake = getNonFlake(state, lazyNonFlake->ref); - - if (nonFlake.sourceInfo.narHash != lazyNonFlake->narHash) - throw Error("the content hash of repository '%s' doesn't match the hash recorded in the referring lockfile", nonFlake.sourceInfo.resolvedRef); - - state.mkAttrs(v, 8); - - assert(state.store->isValidPath(nonFlake.sourceInfo.storePath)); - - mkString(*state.allocAttr(v, state.sOutPath), - nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath}); - - emitSourceInfoAttrs(state, nonFlake.sourceInfo, v); -} - -void callFlake(EvalState & state, - const Flake & flake, - const FlakeInputs & inputs, - Value & vRes) -{ - // Construct the resulting attrset '{outputs, ...}'. This attrset - // is passed lazily as an argument to the 'outputs' function. - - auto & v = *state.allocValue(); - - state.mkAttrs(v, - inputs.flakeInputs.size() + - inputs.nonFlakeInputs.size() + 8); - - for (auto & dep : inputs.flakeInputs) { - auto vFlake = state.allocAttr(v, dep.second.id); - auto vPrimOp = state.allocValue(); - static auto primOp = new PrimOp(prim_callFlake, 1, state.symbols.create("callFlake")); - vPrimOp->type = tPrimOp; - vPrimOp->primOp = primOp; - auto vArg = state.allocValue(); - vArg->type = tNull; - // FIXME: leak - vArg->attrs = (Bindings *) new FlakeInput(dep.second); // evil! also inefficient - mkApp(*vFlake, *vPrimOp, *vArg); - } - - for (auto & dep : inputs.nonFlakeInputs) { - auto vNonFlake = state.allocAttr(v, dep.first); - auto vPrimOp = state.allocValue(); - static auto primOp = new PrimOp(prim_callNonFlake, 1, state.symbols.create("callNonFlake")); - vPrimOp->type = tPrimOp; - vPrimOp->primOp = primOp; - auto vArg = state.allocValue(); - vArg->type = tNull; - // FIXME: leak - vArg->attrs = (Bindings *) new NonFlakeInput(dep.second); // evil! also inefficient - mkApp(*vNonFlake, *vPrimOp, *vArg); - } - - mkString(*state.allocAttr(v, state.sDescription), flake.description); - - emitSourceInfoAttrs(state, flake.sourceInfo, v); - - auto vOutputs = state.allocAttr(v, state.symbols.create("outputs")); - mkApp(*vOutputs, *flake.vOutputs, v); - - v.attrs->push_back(Attr(state.symbols.create("self"), &v)); - - v.attrs->sort(); - - /* For convenience, put the outputs directly in the result, so you - can refer to an output of an input as 'inputs.foo.bar' rather - than 'inputs.foo.outputs.bar'. */ - auto v2 = *state.allocValue(); - state.eval(state.parseExprFromString("res: res.outputs // res", "/"), v2); - - state.callFunction(v2, v, vRes, noPos); -} - -void callFlake(EvalState & state, - const ResolvedFlake & resFlake, - Value & v) -{ - callFlake(state, resFlake.flake, resFlake.lockFile, v); -} - -// This function is exposed to be used in nix files. -static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - callFlake(state, resolveFlake(state, state.forceStringNoCtx(*args[0], pos), - evalSettings.pureEval ? AllPure : UseUpdatedLockFile), v); -} - -static RegisterPrimOp r2("getFlake", 1, prim_getFlake); - -void gitCloneFlake(FlakeRef flakeRef, EvalState & state, Registries registries, const Path & destDir) -{ - flakeRef = lookupFlake(state, flakeRef, registries); - - std::string uri; - - Strings args = {"clone"}; - - if (auto refData = std::get_if(&flakeRef.data)) { - uri = "git@github.com:" + refData->owner + "/" + refData->repo + ".git"; - args.push_back(uri); - if (flakeRef.ref) { - args.push_back("--branch"); - args.push_back(*flakeRef.ref); - } - } else if (auto refData = std::get_if(&flakeRef.data)) { - args.push_back(refData->uri); - if (flakeRef.ref) { - args.push_back("--branch"); - args.push_back(*flakeRef.ref); - } - } - - if (destDir != "") - args.push_back(destDir); - - runProgram("git", true, args); -} - -} - -std::shared_ptr EvalState::getGlobalFlakeRegistry() -{ - std::call_once(_globalFlakeRegistryInit, [&]() { - auto path = evalSettings.flakeRegistry; - - if (!hasPrefix(path, "/")) { - CachedDownloadRequest request(evalSettings.flakeRegistry); - request.name = "flake-registry.json"; - request.gcRoot = true; - path = getDownloader()->downloadCached(store, request).path; - } - - _globalFlakeRegistry = readRegistry(path); - }); - - return _globalFlakeRegistry; -} - -// This always returns a vector with flakeReg, userReg, globalReg. -// If one of them doesn't exist, the registry is left empty but does exist. -const Registries EvalState::getFlakeRegistries() -{ - Registries registries; - registries.push_back(getFlagRegistry(registryOverrides)); - registries.push_back(getUserRegistry()); - registries.push_back(getGlobalFlakeRegistry()); - return registries; -} - -} diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh deleted file mode 100644 index b8d0da252..000000000 --- a/src/libexpr/primops/flake.hh +++ /dev/null @@ -1,111 +0,0 @@ -#pragma once - -#include "types.hh" -#include "flakeref.hh" -#include "lockfile.hh" - -namespace nix { - -struct Value; -class EvalState; - -namespace flake { - -static const size_t FLAG_REGISTRY = 0; -static const size_t USER_REGISTRY = 1; -static const size_t GLOBAL_REGISTRY = 2; - -struct FlakeRegistry -{ - std::map entries; -}; - -typedef std::vector> Registries; - -std::shared_ptr readRegistry(const Path &); - -void writeRegistry(const FlakeRegistry &, const Path &); - -Path getUserRegistryPath(); - -enum HandleLockFile : unsigned int - { AllPure // Everything is handled 100% purely - , TopRefUsesRegistries // The top FlakeRef uses the registries, apart from that, everything happens 100% purely - , UpdateLockFile // Update the existing lockfile and write it to file - , UseUpdatedLockFile // `UpdateLockFile` without writing to file - , RecreateLockFile // Recreate the lockfile from scratch and write it to file - , UseNewLockFile // `RecreateLockFile` without writing to file - }; - -struct SourceInfo -{ - // Immutable flakeref that this source tree was obtained from. - FlakeRef resolvedRef; - - Path storePath; - - // Number of ancestors of the most recent commit. - std::optional revCount; - - // NAR hash of the store path. - Hash narHash; - - // A stable timestamp of this source tree. For Git and GitHub - // flakes, the commit date (not author date!) of the most recent - // commit. - std::optional lastModified; - - SourceInfo(const FlakeRef & resolvRef) : resolvedRef(resolvRef) {}; -}; - -struct Flake -{ - FlakeId id; - FlakeRef originalRef; - std::string description; - SourceInfo sourceInfo; - std::vector inputs; - std::map nonFlakeInputs; - Value * vOutputs; // FIXME: gc - unsigned int epoch; - - Flake(const FlakeRef & origRef, const SourceInfo & sourceInfo) - : originalRef(origRef), sourceInfo(sourceInfo) {}; -}; - -struct NonFlake -{ - FlakeRef originalRef; - SourceInfo sourceInfo; - NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) - : originalRef(origRef), sourceInfo(sourceInfo) {}; -}; - -Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); - -struct ResolvedFlake -{ - Flake flake; - LockFile lockFile; - ResolvedFlake(Flake && flake, LockFile && lockFile) - : flake(flake), lockFile(lockFile) {} -}; - -ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile); - -void callFlake(EvalState & state, - const Flake & flake, - const FlakeInputs & inputs, - Value & v); - -void callFlake(EvalState & state, - const ResolvedFlake & resFlake, - Value & v); - -void updateLockFile(EvalState &, const FlakeRef & flakeRef, bool recreateLockFile); - -void gitCloneFlake(FlakeRef flakeRef, EvalState &, Registries, const Path & destDir); - -} - -} diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc deleted file mode 100644 index 24af09124..000000000 --- a/src/libexpr/primops/flakeref.cc +++ /dev/null @@ -1,252 +0,0 @@ -#include "flakeref.hh" -#include "store-api.hh" - -#include - -namespace nix { - -// A Git ref (i.e. branch or tag name). -const static std::string refRegex = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check - -// A Git revision (a SHA-1 commit hash). -const static std::string revRegexS = "[0-9a-fA-F]{40}"; -std::regex revRegex(revRegexS, std::regex::ECMAScript); - -// A Git ref or revision. -const static std::string revOrRefRegex = "(?:(" + revRegexS + ")|(" + refRegex + "))"; - -// A rev ("e72daba8250068216d79d2aeef40d4d95aff6666"), or a ref -// optionally followed by a rev (e.g. "master" or -// "master/e72daba8250068216d79d2aeef40d4d95aff6666"). -const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegex + ")(?:/(" + revRegexS + "))?))"; - -const static std::string flakeAlias = "[a-zA-Z][a-zA-Z0-9_-]*"; - -// GitHub references. -const static std::string ownerRegex = "[a-zA-Z][a-zA-Z0-9_-]*"; -const static std::string repoRegex = "[a-zA-Z][a-zA-Z0-9_-]*"; - -// URI stuff. -const static std::string schemeRegex = "(?:http|https|ssh|git|file)"; -const static std::string authorityRegex = "[a-zA-Z0-9._~-]*"; -const static std::string segmentRegex = "[a-zA-Z0-9._~-]+"; -const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*"; - -// 'dir' path elements cannot start with a '.'. We also reject -// potentially dangerous characters like ';'. -const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)"; -const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*"; - - -FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) -{ - // FIXME: could combine this into one regex. - - static std::regex flakeRegex( - "(?:flake:)?(" + flakeAlias + ")(?:/(?:" + refAndOrRevRegex + "))?", - std::regex::ECMAScript); - - static std::regex githubRegex( - "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?", - std::regex::ECMAScript); - - static std::regex uriRegex( - "((" + schemeRegex + "):" + - "(?://(" + authorityRegex + "))?" + - "(" + pathRegex + "))", - std::regex::ECMAScript); - - static std::regex refRegex2(refRegex, std::regex::ECMAScript); - - static std::regex subDirRegex2(subDirRegex, std::regex::ECMAScript); - - auto [uri2, params] = splitUriAndParams(uri_); - std::string uri(uri2); - - auto handleSubdir = [&](const std::string & name, const std::string & value) { - if (name == "dir") { - if (value != "" && !std::regex_match(value, subDirRegex2)) - throw BadFlakeRef("flake '%s' has invalid subdirectory '%s'", uri, value); - subdir = value; - return true; - } else - return false; - }; - - auto handleGitParams = [&](const std::string & name, const std::string & value) { - if (name == "rev") { - if (!std::regex_match(value, revRegex)) - throw BadFlakeRef("invalid Git revision '%s'", value); - rev = Hash(value, htSHA1); - } else if (name == "ref") { - if (!std::regex_match(value, refRegex2)) - throw BadFlakeRef("invalid Git ref '%s'", value); - ref = value; - } else if (handleSubdir(name, value)) - ; - else return false; - return true; - }; - - std::cmatch match; - if (std::regex_match(uri.c_str(), match, flakeRegex)) { - IsAlias d; - d.alias = match[1]; - if (match[2].matched) - rev = Hash(match[2], htSHA1); - else if (match[3].matched) { - ref = match[3]; - if (match[4].matched) - rev = Hash(match[4], htSHA1); - } - data = d; - } - - else if (std::regex_match(uri.c_str(), match, githubRegex)) { - IsGitHub d; - d.owner = match[1]; - d.repo = match[2]; - if (match[3].matched) - rev = Hash(match[3], htSHA1); - else if (match[4].matched) { - ref = match[4]; - } - for (auto & param : params) { - if (handleSubdir(param.first, param.second)) - ; - else - throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); - } - data = d; - } - - else if (std::regex_match(uri.c_str(), match, uriRegex) - && (match[2] == "file" || hasSuffix(match[4], ".git"))) - { - IsGit d; - d.uri = match[1]; - for (auto & param : params) { - if (handleGitParams(param.first, param.second)) - ; - else - // FIXME: should probably pass through unknown parameters - throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); - } - if (rev && !ref) - throw BadFlakeRef("flake URI '%s' lacks a Git ref", uri); - data = d; - } - - else if ((hasPrefix(uri, "/") || (allowRelative && (hasPrefix(uri, "./") || hasPrefix(uri, "../") || uri == "."))) - && uri.find(':') == std::string::npos) - { - IsPath d; - if (allowRelative) { - d.path = absPath(uri); - while (true) { - if (pathExists(d.path + "/.git")) break; - subdir = baseNameOf(d.path) + (subdir.empty() ? "" : "/" + subdir); - d.path = dirOf(d.path); - if (d.path == "/") - throw BadFlakeRef("path '%s' does not reference a Git repository", uri); - } - } else - d.path = canonPath(uri); - data = d; - for (auto & param : params) { - if (handleGitParams(param.first, param.second)) - ; - else - throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); - } - } - - else - throw BadFlakeRef("'%s' is not a valid flake reference", uri); -} - -std::string FlakeRef::to_string() const -{ - std::string string; - bool first = true; - - auto addParam = - [&](const std::string & name, std::string value) { - string += first ? '?' : '&'; - first = false; - string += name; - string += '='; - string += value; // FIXME: escaping - }; - - if (auto refData = std::get_if(&data)) { - string = refData->alias; - if (ref) string += '/' + *ref; - if (rev) string += '/' + rev->gitRev(); - } - - else if (auto refData = std::get_if(&data)) { - string = refData->path; - if (ref) addParam("ref", *ref); - if (rev) addParam("rev", rev->gitRev()); - if (subdir != "") addParam("dir", subdir); - } - - else if (auto refData = std::get_if(&data)) { - assert(!(ref && rev)); - string = "github:" + refData->owner + "/" + refData->repo; - if (ref) { string += '/'; string += *ref; } - if (rev) { string += '/'; string += rev->gitRev(); } - if (subdir != "") addParam("dir", subdir); - } - - else if (auto refData = std::get_if(&data)) { - assert(!rev || ref); - string = refData->uri; - - if (ref) { - addParam("ref", *ref); - if (rev) - addParam("rev", rev->gitRev()); - } - - if (subdir != "") addParam("dir", subdir); - } - - else abort(); - - assert(FlakeRef(string) == *this); - - return string; -} - -std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) -{ - str << flakeRef.to_string(); - return str; -} - -bool FlakeRef::isImmutable() const -{ - return (bool) rev; -} - -FlakeRef FlakeRef::baseRef() const // Removes the ref and rev from a FlakeRef. -{ - FlakeRef result(*this); - result.ref = std::nullopt; - result.rev = std::nullopt; - return result; -} - -std::optional parseFlakeRef( - const std::string & uri, bool allowRelative) -{ - try { - return FlakeRef(uri, allowRelative); - } catch (BadFlakeRef & e) { - return {}; - } -} - -} diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh deleted file mode 100644 index 52bb82ddb..000000000 --- a/src/libexpr/primops/flakeref.hh +++ /dev/null @@ -1,188 +0,0 @@ -#pragma once - -#include "types.hh" -#include "hash.hh" - -#include - -namespace nix { - -/* Flake references are a URI-like syntax to specify a flake. - - Examples: - - * (/rev-or-ref(/rev)?)? - - Look up a flake by ID in the flake lock file or in the flake - registry. These must specify an actual location for the flake - using the formats listed below. Note that in pure evaluation - mode, the flake registry is empty. - - Optionally, the rev or ref from the dereferenced flake can be - overriden. For example, - - nixpkgs/19.09 - - uses the "19.09" branch of the nixpkgs' flake GitHub repository, - while - - nixpkgs/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f - - uses the specified revision. For Git (rather than GitHub) - repositories, both the rev and ref must be given, e.g. - - nixpkgs/19.09/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f - - * github:/(/)? - - A repository on GitHub. These differ from Git references in that - they're downloaded in a efficient way (via the tarball mechanism) - and that they support downloading a specific revision without - specifying a branch. is either a commit hash ("rev") - or a branch or tag name ("ref"). The default is: "master" if none - is specified. Note that in pure evaluation mode, a commit hash - must be used. - - Flakes fetched in this manner expose "rev" and "lastModified" - attributes, but not "revCount". - - Examples: - - github:edolstra/dwarffs - github:edolstra/dwarffs/unstable - github:edolstra/dwarffs/41c0c1bf292ea3ac3858ff393b49ca1123dbd553 - - * https:///.git(\?attr(&attr)*)? - ssh:///.git(\?attr(&attr)*)? - git:///.git(\?attr(&attr)*)? - file:///(\?attr(&attr)*)? - - where 'attr' is one of: - rev= - ref= - - A Git repository fetched through https. Note that the path must - end in ".git". The default for "ref" is "master". - - Examples: - - https://example.org/my/repo.git - https://example.org/my/repo.git?ref=release-1.2.3 - https://example.org/my/repo.git?rev=e72daba8250068216d79d2aeef40d4d95aff6666 - git://github.com/edolstra/dwarffs.git?ref=flake&rev=2efca4bc9da70fb001b26c3dc858c6397d3c4817 - - * /path.git(\?attr(&attr)*)? - - Like file://path.git, but if no "ref" or "rev" is specified, the - (possibly dirty) working tree will be used. Using a working tree - is not allowed in pure evaluation mode. - - Examples: - - /path/to/my/repo - /path/to/my/repo?ref=develop - /path/to/my/repo?rev=e72daba8250068216d79d2aeef40d4d95aff6666 - - * https:///.tar.xz(?hash=) - file:///.tar.xz(?hash=) - - A flake distributed as a tarball. In pure evaluation mode, an SRI - hash is mandatory. It exposes a "lastModified" attribute, being - the newest file inside the tarball. - - Example: - - https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz - https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz?hash=sha256-56bbc099995ea8581ead78f22832fee7dbcb0a0b6319293d8c2d0aef5379397c - - Note: currently, there can be only one flake per Git repository, and - it must be at top-level. In the future, we may want to add a field - (e.g. "dir=") to specify a subdirectory inside the repository. -*/ - -typedef std::string FlakeId; -typedef std::string FlakeAlias; -typedef std::string FlakeUri; - -struct FlakeRef -{ - struct IsAlias - { - FlakeAlias alias; - bool operator<(const IsAlias & b) const { return alias < b.alias; }; - bool operator==(const IsAlias & b) const { return alias == b.alias; }; - }; - - struct IsGitHub { - std::string owner, repo; - bool operator<(const IsGitHub & b) const { - return std::make_tuple(owner, repo) < std::make_tuple(b.owner, b.repo); - } - bool operator==(const IsGitHub & b) const { - return owner == b.owner && repo == b.repo; - } - }; - - // Git, Tarball - struct IsGit - { - std::string uri; - bool operator<(const IsGit & b) const { return uri < b.uri; } - bool operator==(const IsGit & b) const { return uri == b.uri; } - }; - - struct IsPath - { - Path path; - bool operator<(const IsPath & b) const { return path < b.path; } - bool operator==(const IsPath & b) const { return path == b.path; } - }; - - // Git, Tarball - - std::variant data; - - std::optional ref; - std::optional rev; - Path subdir = ""; // This is a relative path pointing at the flake.nix file's directory, relative to the git root. - - bool operator<(const FlakeRef & flakeRef) const - { - return std::make_tuple(data, ref, rev, subdir) < - std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev, subdir); - } - - bool operator==(const FlakeRef & flakeRef) const - { - return std::make_tuple(data, ref, rev, subdir) == - std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev, flakeRef.subdir); - } - - // Parse a flake URI. - FlakeRef(const std::string & uri, bool allowRelative = false); - - // FIXME: change to operator <<. - std::string to_string() const; - - /* Check whether this is a "direct" flake reference, that is, not - a flake ID, which requires a lookup in the flake registry. */ - bool isDirect() const - { - return !std::get_if(&data); - } - - /* Check whether this is an "immutable" flake reference, that is, - one that contains a commit hash or content hash. */ - bool isImmutable() const; - - FlakeRef baseRef() const; -}; - -std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); - -MakeError(BadFlakeRef, Error); - -std::optional parseFlakeRef( - const std::string & uri, bool allowRelative = false); - -} diff --git a/src/libexpr/primops/lockfile.cc b/src/libexpr/primops/lockfile.cc deleted file mode 100644 index 97c748c66..000000000 --- a/src/libexpr/primops/lockfile.cc +++ /dev/null @@ -1,102 +0,0 @@ -#include "lockfile.hh" -#include "store-api.hh" - -namespace nix::flake { - -AbstractInput::AbstractInput(const nlohmann::json & json) - : ref(json["uri"]) - , narHash(Hash((std::string) json["narHash"])) -{ - if (!ref.isImmutable()) - throw Error("lockfile contains mutable flakeref '%s'", ref); -} - -nlohmann::json AbstractInput::toJson() const -{ - nlohmann::json json; - json["uri"] = ref.to_string(); - json["narHash"] = narHash.to_string(SRI); - return json; -} - -Path AbstractInput::computeStorePath(Store & store) const -{ - return store.makeFixedOutputPath(true, narHash, "source"); -} - -FlakeInput::FlakeInput(const nlohmann::json & json) - : FlakeInputs(json) - , AbstractInput(json) - , id(json["id"]) -{ -} - -nlohmann::json FlakeInput::toJson() const -{ - auto json = FlakeInputs::toJson(); - json.update(AbstractInput::toJson()); - json["id"] = id; - return json; -} - -FlakeInputs::FlakeInputs(const nlohmann::json & json) -{ - for (auto & i : json["nonFlakeInputs"].items()) - nonFlakeInputs.insert_or_assign(i.key(), NonFlakeInput(i.value())); - - for (auto & i : json["inputs"].items()) - flakeInputs.insert_or_assign(i.key(), FlakeInput(i.value())); -} - -nlohmann::json FlakeInputs::toJson() const -{ - nlohmann::json json; - { - auto j = nlohmann::json::object(); - for (auto & i : nonFlakeInputs) - j[i.first] = i.second.toJson(); - json["nonFlakeInputs"] = std::move(j); - } - { - auto j = nlohmann::json::object(); - for (auto & i : flakeInputs) - j[i.first.to_string()] = i.second.toJson(); - json["inputs"] = std::move(j); - } - return json; -} - -nlohmann::json LockFile::toJson() const -{ - auto json = FlakeInputs::toJson(); - json["version"] = 2; - return json; -} - -LockFile LockFile::read(const Path & path) -{ - if (pathExists(path)) { - auto json = nlohmann::json::parse(readFile(path)); - - auto version = json.value("version", 0); - if (version != 2) - throw Error("lock file '%s' has unsupported version %d", path, version); - - return LockFile(json); - } else - return LockFile(); -} - -std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) -{ - stream << lockFile.toJson().dump(4); // '4' = indentation in json file - return stream; -} - -void LockFile::write(const Path & path) const -{ - createDirs(dirOf(path)); - writeFile(path, fmt("%s\n", *this)); -} - -} diff --git a/src/libexpr/primops/lockfile.hh b/src/libexpr/primops/lockfile.hh deleted file mode 100644 index b76124190..000000000 --- a/src/libexpr/primops/lockfile.hh +++ /dev/null @@ -1,112 +0,0 @@ -#pragma once - -#include "flakeref.hh" - -#include - -namespace nix { -class Store; -} - -namespace nix::flake { - -/* Common lock file information about a flake input, namely the - immutable ref and the NAR hash. */ -struct AbstractInput -{ - FlakeRef ref; - Hash narHash; - - AbstractInput(const FlakeRef & flakeRef, const Hash & narHash) - : ref(flakeRef), narHash(narHash) - { - assert(ref.isImmutable()); - }; - - AbstractInput(const nlohmann::json & json); - - nlohmann::json toJson() const; - - Path computeStorePath(Store & store) const; -}; - -/* Lock file information about a non-flake input. */ -struct NonFlakeInput : AbstractInput -{ - using AbstractInput::AbstractInput; - - bool operator ==(const NonFlakeInput & other) const - { - return ref == other.ref && narHash == other.narHash; - } -}; - -struct FlakeInput; - -/* Lock file information about the dependencies of a flake. */ -struct FlakeInputs -{ - std::map flakeInputs; - std::map nonFlakeInputs; - - FlakeInputs() {}; - FlakeInputs(const nlohmann::json & json); - - nlohmann::json toJson() const; -}; - -/* Lock file information about a flake input. */ -struct FlakeInput : FlakeInputs, AbstractInput -{ - FlakeId id; - - FlakeInput(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash) - : AbstractInput(flakeRef, narHash), id(id) {}; - - FlakeInput(const nlohmann::json & json); - - bool operator ==(const FlakeInput & other) const - { - return - id == other.id - && ref == other.ref - && narHash == other.narHash - && flakeInputs == other.flakeInputs - && nonFlakeInputs == other.nonFlakeInputs; - } - - nlohmann::json toJson() const; -}; - -/* An entire lock file. Note that this cannot be a FlakeInput for the - top-level flake, because then the lock file would need to contain - the hash of the top-level flake, but committing the lock file - would invalidate that hash. */ -struct LockFile : FlakeInputs -{ - bool operator ==(const LockFile & other) const - { - return - flakeInputs == other.flakeInputs - && nonFlakeInputs == other.nonFlakeInputs; - } - - LockFile() {} - LockFile(const nlohmann::json & json) : FlakeInputs(json) {} - LockFile(FlakeInput && dep) - { - flakeInputs = std::move(dep.flakeInputs); - nonFlakeInputs = std::move(dep.nonFlakeInputs); - } - - nlohmann::json toJson() const; - - static LockFile read(const Path & path); - - void write(const Path & path) const; -}; - -std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile); - -} - diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 653154aaa..af1a361b3 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -4,7 +4,7 @@ #include "progress-bar.hh" #include "eval.hh" #include "eval-inline.hh" -#include "primops/flake.hh" +#include "flake/flake.hh" #include "get-drvs.hh" #include "store-api.hh" diff --git a/src/nix/installables.cc b/src/nix/installables.cc index ca88ec0da..a85295a09 100644 --- a/src/nix/installables.cc +++ b/src/nix/installables.cc @@ -7,7 +7,7 @@ #include "get-drvs.hh" #include "store-api.hh" #include "shared.hh" -#include "primops/flake.hh" +#include "flake/flake.hh" #include #include -- cgit v1.2.3