diff options
author | Eelco Dolstra <edolstra@gmail.com> | 2019-06-05 16:51:54 +0200 |
---|---|---|
committer | Eelco Dolstra <edolstra@gmail.com> | 2019-06-05 16:51:54 +0200 |
commit | 54aff8430c4e7739903f6dbed713cc088e38507f (patch) | |
tree | 4e8f84b5682b0480deb2c4943d244b9a5fe552e0 /src/libexpr/primops | |
parent | 1b057929885fd3f339d4c85b44ad9f10fef7d8a9 (diff) |
Move flake-related stuff to src/libexpr/flake
Diffstat (limited to 'src/libexpr/primops')
-rw-r--r-- | src/libexpr/primops/flake.cc | 604 | ||||
-rw-r--r-- | src/libexpr/primops/flake.hh | 111 | ||||
-rw-r--r-- | src/libexpr/primops/flakeref.cc | 252 | ||||
-rw-r--r-- | src/libexpr/primops/flakeref.hh | 188 | ||||
-rw-r--r-- | src/libexpr/primops/lockfile.cc | 102 | ||||
-rw-r--r-- | src/libexpr/primops/lockfile.hh | 112 |
6 files changed, 0 insertions, 1369 deletions
diff --git a/src/libexpr/primops/flake.cc b/src/libexpr/primops/flake.cc deleted file mode 100644 index 793d6da35..000000000 --- a/src/libexpr/primops/flake.cc +++ /dev/null @@ -1,604 +0,0 @@ -#include "flake.hh" -#include "lockfile.hh" -#include "primops.hh" -#include "eval-inline.hh" -#include "fetchGit.hh" -#include "download.hh" -#include "args.hh" - -#include <iostream> -#include <queue> -#include <regex> -#include <ctime> -#include <iomanip> -#include <nlohmann/json.hpp> - -namespace nix { - -using namespace flake; - -namespace flake { - -/* Read a registry. */ -std::shared_ptr<FlakeRegistry> readRegistry(const Path & path) -{ - auto registry = std::make_shared<FlakeRegistry>(); - - if (!pathExists(path)) - return std::make_shared<FlakeRegistry>(); - - auto json = nlohmann::json::parse(readFile(path)); - - auto version = json.value("version", 0); - if (version != 1) - throw Error("flake registry '%s' has unsupported version %d", path, version); - - auto flakes = json["flakes"]; - for (auto i = flakes.begin(); i != flakes.end(); ++i) - registry->entries.emplace(i.key(), FlakeRef(i->value("uri", ""))); - - return registry; -} - -/* Write a registry to a file. */ -void writeRegistry(const FlakeRegistry & registry, const Path & path) -{ - nlohmann::json json; - json["version"] = 2; - for (auto elem : registry.entries) - json["flakes"][elem.first.to_string()] = { {"uri", elem.second.to_string()} }; - createDirs(dirOf(path)); - writeFile(path, json.dump(4)); // The '4' is the number of spaces used in the indentation in the json file. -} - -Path getUserRegistryPath() -{ - return getHome() + "/.config/nix/registry.json"; -} - -std::shared_ptr<FlakeRegistry> getUserRegistry() -{ - return readRegistry(getUserRegistryPath()); -} - -std::shared_ptr<FlakeRegistry> getFlagRegistry(RegistryOverrides registryOverrides) -{ - auto flagRegistry = std::make_shared<FlakeRegistry>(); - for (auto const & x : registryOverrides) { - flagRegistry->entries.insert_or_assign(FlakeRef(x.first), FlakeRef(x.second)); - } - return flagRegistry; -} - -static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, - std::vector<FlakeRef> pastSearches = {}); - -FlakeRef updateFlakeRef(EvalState & state, const FlakeRef & newRef, const Registries & registries, std::vector<FlakeRef> pastSearches) -{ - std::string errorMsg = "found cycle in flake registries: "; - for (FlakeRef oldRef : pastSearches) { - errorMsg += oldRef.to_string(); - if (oldRef == newRef) - throw Error(errorMsg); - errorMsg += " - "; - } - pastSearches.push_back(newRef); - return lookupFlake(state, newRef, registries, pastSearches); -} - -static FlakeRef lookupFlake(EvalState & state, const FlakeRef & flakeRef, const Registries & registries, - std::vector<FlakeRef> pastSearches) -{ - if (registries.empty() && !flakeRef.isDirect()) - throw Error("indirect flake reference '%s' is not allowed", flakeRef); - - for (std::shared_ptr<FlakeRegistry> registry : registries) { - auto i = registry->entries.find(flakeRef); - if (i != registry->entries.end()) { - auto newRef = i->second; - return updateFlakeRef(state, newRef, registries, pastSearches); - } - - auto j = registry->entries.find(flakeRef.baseRef()); - if (j != registry->entries.end()) { - auto newRef = j->second; - newRef.ref = flakeRef.ref; - newRef.rev = flakeRef.rev; - return updateFlakeRef(state, newRef, registries, pastSearches); - } - } - - if (!flakeRef.isDirect()) - throw Error("could not resolve flake reference '%s'", flakeRef); - - return flakeRef; -} - -// Lookups happen here too -static SourceInfo fetchFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) -{ - FlakeRef resolvedRef = lookupFlake(state, flakeRef, - impureIsAllowed ? state.getFlakeRegistries() : std::vector<std::shared_ptr<FlakeRegistry>>()); - - if (evalSettings.pureEval && !impureIsAllowed && !resolvedRef.isImmutable()) - throw Error("requested to fetch mutable flake '%s' in pure mode", resolvedRef); - - auto doGit = [&](const GitInfo & gitInfo) { - FlakeRef ref(resolvedRef.baseRef()); - ref.ref = gitInfo.ref; - ref.rev = gitInfo.rev; - SourceInfo info(ref); - info.storePath = gitInfo.storePath; - info.revCount = gitInfo.revCount; - info.narHash = state.store->queryPathInfo(info.storePath)->narHash; - info.lastModified = gitInfo.lastModified; - return info; - }; - - // This only downloads only one revision of the repo, not the entire history. - if (auto refData = std::get_if<FlakeRef::IsGitHub>(&resolvedRef.data)) { - - // FIXME: use regular /archive URLs instead? api.github.com - // might have stricter rate limits. - - auto url = fmt("https://api.github.com/repos/%s/%s/tarball/%s", - refData->owner, refData->repo, - resolvedRef.rev ? resolvedRef.rev->to_string(Base16, false) - : resolvedRef.ref ? *resolvedRef.ref : "master"); - - std::string accessToken = settings.githubAccessToken.get(); - if (accessToken != "") - url += "?access_token=" + accessToken; - - CachedDownloadRequest request(url); - request.unpack = true; - request.name = "source"; - request.ttl = resolvedRef.rev ? 1000000000 : settings.tarballTtl; - request.getLastModified = true; - auto result = getDownloader()->downloadCached(state.store, request); - - if (!result.etag) - throw Error("did not receive an ETag header from '%s'", url); - - if (result.etag->size() != 42 || (*result.etag)[0] != '"' || (*result.etag)[41] != '"') - throw Error("ETag header '%s' from '%s' is not a Git revision", *result.etag, url); - - FlakeRef ref(resolvedRef.baseRef()); - ref.rev = Hash(std::string(*result.etag, 1, result.etag->size() - 2), htSHA1); - SourceInfo info(ref); - info.storePath = result.storePath; - info.narHash = state.store->queryPathInfo(info.storePath)->narHash; - info.lastModified = result.lastModified; - - return info; - } - - // This downloads the entire git history - else if (auto refData = std::get_if<FlakeRef::IsGit>(&resolvedRef.data)) { - return doGit(exportGit(state.store, refData->uri, resolvedRef.ref, resolvedRef.rev, "source")); - } - - else if (auto refData = std::get_if<FlakeRef::IsPath>(&resolvedRef.data)) { - if (!pathExists(refData->path + "/.git")) - throw Error("flake '%s' does not reference a Git repository", refData->path); - return doGit(exportGit(state.store, refData->path, {}, {}, "source")); - } - - else abort(); -} - -// This will return the flake which corresponds to a given FlakeRef. The lookupFlake is done within `fetchFlake`, which is used here. -Flake getFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) -{ - SourceInfo sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); - debug("got flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); - - FlakeRef resolvedRef = sourceInfo.resolvedRef; - - state.store->assertStorePath(sourceInfo.storePath); - - if (state.allowedPaths) - state.allowedPaths->insert(state.store->toRealPath(sourceInfo.storePath)); - - // Guard against symlink attacks. - Path flakeFile = canonPath(sourceInfo.storePath + "/" + resolvedRef.subdir + "/flake.nix"); - Path realFlakeFile = state.store->toRealPath(flakeFile); - if (!isInDir(realFlakeFile, state.store->toRealPath(sourceInfo.storePath))) - throw Error("'flake.nix' file of flake '%s' escapes from '%s'", resolvedRef, sourceInfo.storePath); - - Flake flake(flakeRef, sourceInfo); - - if (!pathExists(realFlakeFile)) - throw Error("source tree referenced by '%s' does not contain a '%s/flake.nix' file", resolvedRef, resolvedRef.subdir); - - Value vInfo; - state.evalFile(realFlakeFile, vInfo); // FIXME: symlink attack - - state.forceAttrs(vInfo); - - auto sEpoch = state.symbols.create("epoch"); - - if (auto epoch = vInfo.attrs->get(sEpoch)) { - flake.epoch = state.forceInt(*(**epoch).value, *(**epoch).pos); - if (flake.epoch > 201906) - throw Error("flake '%s' requires unsupported epoch %d; please upgrade Nix", flakeRef, flake.epoch); - } else - throw Error("flake '%s' lacks attribute 'epoch'", flakeRef); - - if (auto name = vInfo.attrs->get(state.sName)) - flake.id = state.forceStringNoCtx(*(**name).value, *(**name).pos); - else - throw Error("flake '%s' lacks attribute 'name'", flakeRef); - - if (auto description = vInfo.attrs->get(state.sDescription)) - flake.description = state.forceStringNoCtx(*(**description).value, *(**description).pos); - - auto sInputs = state.symbols.create("inputs"); - - if (auto inputs = vInfo.attrs->get(sInputs)) { - state.forceList(*(**inputs).value, *(**inputs).pos); - for (unsigned int n = 0; n < (**inputs).value->listSize(); ++n) - flake.inputs.push_back(FlakeRef(state.forceStringNoCtx( - *(**inputs).value->listElems()[n], *(**inputs).pos))); - } - - auto sNonFlakeInputs = state.symbols.create("nonFlakeInputs"); - - if (std::optional<Attr *> nonFlakeInputs = vInfo.attrs->get(sNonFlakeInputs)) { - state.forceAttrs(*(**nonFlakeInputs).value, *(**nonFlakeInputs).pos); - for (Attr attr : *(*(**nonFlakeInputs).value).attrs) { - std::string myNonFlakeUri = state.forceStringNoCtx(*attr.value, *attr.pos); - FlakeRef nonFlakeRef = FlakeRef(myNonFlakeUri); - flake.nonFlakeInputs.insert_or_assign(attr.name, nonFlakeRef); - } - } - - auto sOutputs = state.symbols.create("outputs"); - - if (auto outputs = vInfo.attrs->get(sOutputs)) { - state.forceFunction(*(**outputs).value, *(**outputs).pos); - flake.vOutputs = (**outputs).value; - } else - throw Error("flake '%s' lacks attribute 'outputs'", flakeRef); - - for (auto & attr : *vInfo.attrs) { - if (attr.name != sEpoch && - attr.name != state.sName && - attr.name != state.sDescription && - attr.name != sInputs && - attr.name != sNonFlakeInputs && - attr.name != sOutputs) - throw Error("flake '%s' has an unsupported attribute '%s', at %s", - flakeRef, attr.name, *attr.pos); - } - - return flake; -} - -// Get the `NonFlake` corresponding to a `FlakeRef`. -NonFlake getNonFlake(EvalState & state, const FlakeRef & flakeRef, bool impureIsAllowed = false) -{ - auto sourceInfo = fetchFlake(state, flakeRef, impureIsAllowed); - debug("got non-flake source '%s' with flakeref %s", sourceInfo.storePath, sourceInfo.resolvedRef.to_string()); - - FlakeRef resolvedRef = sourceInfo.resolvedRef; - - NonFlake nonFlake(flakeRef, sourceInfo); - - state.store->assertStorePath(nonFlake.sourceInfo.storePath); - - if (state.allowedPaths) - state.allowedPaths->insert(nonFlake.sourceInfo.storePath); - - return nonFlake; -} - -bool allowedToWrite(HandleLockFile handle) -{ - return handle == UpdateLockFile || handle == RecreateLockFile; -} - -bool recreateLockFile(HandleLockFile handle) -{ - return handle == RecreateLockFile || handle == UseNewLockFile; -} - -bool allowedToUseRegistries(HandleLockFile handle, bool isTopRef) -{ - if (handle == AllPure) return false; - else if (handle == TopRefUsesRegistries) return isTopRef; - else if (handle == UpdateLockFile) return true; - else if (handle == UseUpdatedLockFile) return true; - else if (handle == RecreateLockFile) return true; - else if (handle == UseNewLockFile) return true; - else assert(false); -} - -/* Given a flakeref and its subtree of the lockfile, return an updated - subtree of the lockfile. That is, if the 'flake.nix' of the - referenced flake has inputs that don't have a corresponding entry - in the lockfile, they're added to the lockfile; conversely, any - lockfile entries that don't have a corresponding entry in flake.nix - are removed. - - Note that this is lazy: we only recursively fetch inputs that are - not in the lockfile yet. */ -static std::pair<Flake, FlakeInput> updateLocks( - EvalState & state, - const Flake & flake, - HandleLockFile handleLockFile, - const FlakeInputs & oldEntry, - bool topRef) -{ - FlakeInput newEntry( - flake.id, - flake.sourceInfo.resolvedRef, - flake.sourceInfo.narHash); - - for (auto & input : flake.nonFlakeInputs) { - auto & id = input.first; - auto & ref = input.second; - auto i = oldEntry.nonFlakeInputs.find(id); - if (i != oldEntry.nonFlakeInputs.end()) { - newEntry.nonFlakeInputs.insert_or_assign(i->first, i->second); - } else { - if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) - throw Error("cannot update non-flake dependency '%s' in pure mode", id); - auto nonFlake = getNonFlake(state, ref, allowedToUseRegistries(handleLockFile, false)); - newEntry.nonFlakeInputs.insert_or_assign(id, - NonFlakeInput( - nonFlake.sourceInfo.resolvedRef, - nonFlake.sourceInfo.narHash)); - } - } - - for (auto & inputRef : flake.inputs) { - auto i = oldEntry.flakeInputs.find(inputRef); - if (i != oldEntry.flakeInputs.end()) { - newEntry.flakeInputs.insert_or_assign(inputRef, i->second); - } else { - if (handleLockFile == AllPure || handleLockFile == TopRefUsesRegistries) - throw Error("cannot update flake dependency '%s' in pure mode", inputRef); - newEntry.flakeInputs.insert_or_assign(inputRef, - updateLocks(state, - getFlake(state, inputRef, allowedToUseRegistries(handleLockFile, false)), - handleLockFile, {}, false).second); - } - } - - return {flake, newEntry}; -} - -/* Compute an in-memory lockfile for the specified top-level flake, - and optionally write it to file, it the flake is writable. */ -ResolvedFlake resolveFlake(EvalState & state, const FlakeRef & topRef, HandleLockFile handleLockFile) -{ - auto flake = getFlake(state, topRef, allowedToUseRegistries(handleLockFile, true)); - - LockFile oldLockFile; - - if (!recreateLockFile(handleLockFile)) { - // If recreateLockFile, start with an empty lockfile - // FIXME: symlink attack - oldLockFile = LockFile::read( - state.store->toRealPath(flake.sourceInfo.storePath) - + "/" + flake.sourceInfo.resolvedRef.subdir + "/flake.lock"); - } - - LockFile lockFile(updateLocks( - state, flake, handleLockFile, oldLockFile, true).second); - - if (!(lockFile == oldLockFile)) { - if (allowedToWrite(handleLockFile)) { - if (auto refData = std::get_if<FlakeRef::IsPath>(&topRef.data)) { - lockFile.write(refData->path + (topRef.subdir == "" ? "" : "/" + topRef.subdir) + "/flake.lock"); - - // Hack: Make sure that flake.lock is visible to Git, so it ends up in the Nix store. - runProgram("git", true, { "-C", refData->path, "add", - (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock" }); - } else - warn("cannot write lockfile of remote flake '%s'", topRef); - } else if (handleLockFile != AllPure && handleLockFile != TopRefUsesRegistries) - warn("using updated lockfile without writing it to file"); - } - - return ResolvedFlake(std::move(flake), std::move(lockFile)); -} - -void updateLockFile(EvalState & state, const FlakeRef & flakeRef, bool recreateLockFile) -{ - resolveFlake(state, flakeRef, recreateLockFile ? RecreateLockFile : UpdateLockFile); -} - -static void emitSourceInfoAttrs(EvalState & state, const SourceInfo & sourceInfo, Value & vAttrs) -{ - auto & path = sourceInfo.storePath; - assert(state.store->isValidPath(path)); - mkString(*state.allocAttr(vAttrs, state.sOutPath), path, {path}); - - if (sourceInfo.resolvedRef.rev) { - mkString(*state.allocAttr(vAttrs, state.symbols.create("rev")), - sourceInfo.resolvedRef.rev->gitRev()); - mkString(*state.allocAttr(vAttrs, state.symbols.create("shortRev")), - sourceInfo.resolvedRef.rev->gitShortRev()); - } - - if (sourceInfo.revCount) - mkInt(*state.allocAttr(vAttrs, state.symbols.create("revCount")), *sourceInfo.revCount); - - if (sourceInfo.lastModified) - mkString(*state.allocAttr(vAttrs, state.symbols.create("lastModified")), - fmt("%s", - std::put_time(std::gmtime(&*sourceInfo.lastModified), "%Y%m%d%H%M%S"))); -} - -/* Helper primop to make callFlake (below) fetch/call its inputs - lazily. Note that this primop cannot be called by user code since - it doesn't appear in 'builtins'. */ -static void prim_callFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - auto lazyFlake = (FlakeInput *) args[0]->attrs; - auto flake = getFlake(state, lazyFlake->ref, false); - - if (flake.sourceInfo.narHash != lazyFlake->narHash) - throw Error("the content hash of flake '%s' doesn't match the hash recorded in the referring lockfile", flake.sourceInfo.resolvedRef); - - callFlake(state, flake, *lazyFlake, v); -} - -static void prim_callNonFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - auto lazyNonFlake = (NonFlakeInput *) args[0]->attrs; - - auto nonFlake = getNonFlake(state, lazyNonFlake->ref); - - if (nonFlake.sourceInfo.narHash != lazyNonFlake->narHash) - throw Error("the content hash of repository '%s' doesn't match the hash recorded in the referring lockfile", nonFlake.sourceInfo.resolvedRef); - - state.mkAttrs(v, 8); - - assert(state.store->isValidPath(nonFlake.sourceInfo.storePath)); - - mkString(*state.allocAttr(v, state.sOutPath), - nonFlake.sourceInfo.storePath, {nonFlake.sourceInfo.storePath}); - - emitSourceInfoAttrs(state, nonFlake.sourceInfo, v); -} - -void callFlake(EvalState & state, - const Flake & flake, - const FlakeInputs & inputs, - Value & vRes) -{ - // Construct the resulting attrset '{outputs, ...}'. This attrset - // is passed lazily as an argument to the 'outputs' function. - - auto & v = *state.allocValue(); - - state.mkAttrs(v, - inputs.flakeInputs.size() + - inputs.nonFlakeInputs.size() + 8); - - for (auto & dep : inputs.flakeInputs) { - auto vFlake = state.allocAttr(v, dep.second.id); - auto vPrimOp = state.allocValue(); - static auto primOp = new PrimOp(prim_callFlake, 1, state.symbols.create("callFlake")); - vPrimOp->type = tPrimOp; - vPrimOp->primOp = primOp; - auto vArg = state.allocValue(); - vArg->type = tNull; - // FIXME: leak - vArg->attrs = (Bindings *) new FlakeInput(dep.second); // evil! also inefficient - mkApp(*vFlake, *vPrimOp, *vArg); - } - - for (auto & dep : inputs.nonFlakeInputs) { - auto vNonFlake = state.allocAttr(v, dep.first); - auto vPrimOp = state.allocValue(); - static auto primOp = new PrimOp(prim_callNonFlake, 1, state.symbols.create("callNonFlake")); - vPrimOp->type = tPrimOp; - vPrimOp->primOp = primOp; - auto vArg = state.allocValue(); - vArg->type = tNull; - // FIXME: leak - vArg->attrs = (Bindings *) new NonFlakeInput(dep.second); // evil! also inefficient - mkApp(*vNonFlake, *vPrimOp, *vArg); - } - - mkString(*state.allocAttr(v, state.sDescription), flake.description); - - emitSourceInfoAttrs(state, flake.sourceInfo, v); - - auto vOutputs = state.allocAttr(v, state.symbols.create("outputs")); - mkApp(*vOutputs, *flake.vOutputs, v); - - v.attrs->push_back(Attr(state.symbols.create("self"), &v)); - - v.attrs->sort(); - - /* For convenience, put the outputs directly in the result, so you - can refer to an output of an input as 'inputs.foo.bar' rather - than 'inputs.foo.outputs.bar'. */ - auto v2 = *state.allocValue(); - state.eval(state.parseExprFromString("res: res.outputs // res", "/"), v2); - - state.callFunction(v2, v, vRes, noPos); -} - -void callFlake(EvalState & state, - const ResolvedFlake & resFlake, - Value & v) -{ - callFlake(state, resFlake.flake, resFlake.lockFile, v); -} - -// This function is exposed to be used in nix files. -static void prim_getFlake(EvalState & state, const Pos & pos, Value * * args, Value & v) -{ - callFlake(state, resolveFlake(state, state.forceStringNoCtx(*args[0], pos), - evalSettings.pureEval ? AllPure : UseUpdatedLockFile), v); -} - -static RegisterPrimOp r2("getFlake", 1, prim_getFlake); - -void gitCloneFlake(FlakeRef flakeRef, EvalState & state, Registries registries, const Path & destDir) -{ - flakeRef = lookupFlake(state, flakeRef, registries); - - std::string uri; - - Strings args = {"clone"}; - - if (auto refData = std::get_if<FlakeRef::IsGitHub>(&flakeRef.data)) { - uri = "git@github.com:" + refData->owner + "/" + refData->repo + ".git"; - args.push_back(uri); - if (flakeRef.ref) { - args.push_back("--branch"); - args.push_back(*flakeRef.ref); - } - } else if (auto refData = std::get_if<FlakeRef::IsGit>(&flakeRef.data)) { - args.push_back(refData->uri); - if (flakeRef.ref) { - args.push_back("--branch"); - args.push_back(*flakeRef.ref); - } - } - - if (destDir != "") - args.push_back(destDir); - - runProgram("git", true, args); -} - -} - -std::shared_ptr<flake::FlakeRegistry> EvalState::getGlobalFlakeRegistry() -{ - std::call_once(_globalFlakeRegistryInit, [&]() { - auto path = evalSettings.flakeRegistry; - - if (!hasPrefix(path, "/")) { - CachedDownloadRequest request(evalSettings.flakeRegistry); - request.name = "flake-registry.json"; - request.gcRoot = true; - path = getDownloader()->downloadCached(store, request).path; - } - - _globalFlakeRegistry = readRegistry(path); - }); - - return _globalFlakeRegistry; -} - -// This always returns a vector with flakeReg, userReg, globalReg. -// If one of them doesn't exist, the registry is left empty but does exist. -const Registries EvalState::getFlakeRegistries() -{ - Registries registries; - registries.push_back(getFlagRegistry(registryOverrides)); - registries.push_back(getUserRegistry()); - registries.push_back(getGlobalFlakeRegistry()); - return registries; -} - -} diff --git a/src/libexpr/primops/flake.hh b/src/libexpr/primops/flake.hh deleted file mode 100644 index b8d0da252..000000000 --- a/src/libexpr/primops/flake.hh +++ /dev/null @@ -1,111 +0,0 @@ -#pragma once - -#include "types.hh" -#include "flakeref.hh" -#include "lockfile.hh" - -namespace nix { - -struct Value; -class EvalState; - -namespace flake { - -static const size_t FLAG_REGISTRY = 0; -static const size_t USER_REGISTRY = 1; -static const size_t GLOBAL_REGISTRY = 2; - -struct FlakeRegistry -{ - std::map<FlakeRef, FlakeRef> entries; -}; - -typedef std::vector<std::shared_ptr<FlakeRegistry>> Registries; - -std::shared_ptr<FlakeRegistry> readRegistry(const Path &); - -void writeRegistry(const FlakeRegistry &, const Path &); - -Path getUserRegistryPath(); - -enum HandleLockFile : unsigned int - { AllPure // Everything is handled 100% purely - , TopRefUsesRegistries // The top FlakeRef uses the registries, apart from that, everything happens 100% purely - , UpdateLockFile // Update the existing lockfile and write it to file - , UseUpdatedLockFile // `UpdateLockFile` without writing to file - , RecreateLockFile // Recreate the lockfile from scratch and write it to file - , UseNewLockFile // `RecreateLockFile` without writing to file - }; - -struct SourceInfo -{ - // Immutable flakeref that this source tree was obtained from. - FlakeRef resolvedRef; - - Path storePath; - - // Number of ancestors of the most recent commit. - std::optional<uint64_t> revCount; - - // NAR hash of the store path. - Hash narHash; - - // A stable timestamp of this source tree. For Git and GitHub - // flakes, the commit date (not author date!) of the most recent - // commit. - std::optional<time_t> lastModified; - - SourceInfo(const FlakeRef & resolvRef) : resolvedRef(resolvRef) {}; -}; - -struct Flake -{ - FlakeId id; - FlakeRef originalRef; - std::string description; - SourceInfo sourceInfo; - std::vector<FlakeRef> inputs; - std::map<FlakeAlias, FlakeRef> nonFlakeInputs; - Value * vOutputs; // FIXME: gc - unsigned int epoch; - - Flake(const FlakeRef & origRef, const SourceInfo & sourceInfo) - : originalRef(origRef), sourceInfo(sourceInfo) {}; -}; - -struct NonFlake -{ - FlakeRef originalRef; - SourceInfo sourceInfo; - NonFlake(const FlakeRef & origRef, const SourceInfo & sourceInfo) - : originalRef(origRef), sourceInfo(sourceInfo) {}; -}; - -Flake getFlake(EvalState &, const FlakeRef &, bool impureIsAllowed); - -struct ResolvedFlake -{ - Flake flake; - LockFile lockFile; - ResolvedFlake(Flake && flake, LockFile && lockFile) - : flake(flake), lockFile(lockFile) {} -}; - -ResolvedFlake resolveFlake(EvalState &, const FlakeRef &, HandleLockFile); - -void callFlake(EvalState & state, - const Flake & flake, - const FlakeInputs & inputs, - Value & v); - -void callFlake(EvalState & state, - const ResolvedFlake & resFlake, - Value & v); - -void updateLockFile(EvalState &, const FlakeRef & flakeRef, bool recreateLockFile); - -void gitCloneFlake(FlakeRef flakeRef, EvalState &, Registries, const Path & destDir); - -} - -} diff --git a/src/libexpr/primops/flakeref.cc b/src/libexpr/primops/flakeref.cc deleted file mode 100644 index 24af09124..000000000 --- a/src/libexpr/primops/flakeref.cc +++ /dev/null @@ -1,252 +0,0 @@ -#include "flakeref.hh" -#include "store-api.hh" - -#include <regex> - -namespace nix { - -// A Git ref (i.e. branch or tag name). -const static std::string refRegex = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check - -// A Git revision (a SHA-1 commit hash). -const static std::string revRegexS = "[0-9a-fA-F]{40}"; -std::regex revRegex(revRegexS, std::regex::ECMAScript); - -// A Git ref or revision. -const static std::string revOrRefRegex = "(?:(" + revRegexS + ")|(" + refRegex + "))"; - -// A rev ("e72daba8250068216d79d2aeef40d4d95aff6666"), or a ref -// optionally followed by a rev (e.g. "master" or -// "master/e72daba8250068216d79d2aeef40d4d95aff6666"). -const static std::string refAndOrRevRegex = "(?:(" + revRegexS + ")|(?:(" + refRegex + ")(?:/(" + revRegexS + "))?))"; - -const static std::string flakeAlias = "[a-zA-Z][a-zA-Z0-9_-]*"; - -// GitHub references. -const static std::string ownerRegex = "[a-zA-Z][a-zA-Z0-9_-]*"; -const static std::string repoRegex = "[a-zA-Z][a-zA-Z0-9_-]*"; - -// URI stuff. -const static std::string schemeRegex = "(?:http|https|ssh|git|file)"; -const static std::string authorityRegex = "[a-zA-Z0-9._~-]*"; -const static std::string segmentRegex = "[a-zA-Z0-9._~-]+"; -const static std::string pathRegex = "/?" + segmentRegex + "(?:/" + segmentRegex + ")*"; - -// 'dir' path elements cannot start with a '.'. We also reject -// potentially dangerous characters like ';'. -const static std::string subDirElemRegex = "(?:[a-zA-Z0-9_-]+[a-zA-Z0-9._-]*)"; -const static std::string subDirRegex = subDirElemRegex + "(?:/" + subDirElemRegex + ")*"; - - -FlakeRef::FlakeRef(const std::string & uri_, bool allowRelative) -{ - // FIXME: could combine this into one regex. - - static std::regex flakeRegex( - "(?:flake:)?(" + flakeAlias + ")(?:/(?:" + refAndOrRevRegex + "))?", - std::regex::ECMAScript); - - static std::regex githubRegex( - "github:(" + ownerRegex + ")/(" + repoRegex + ")(?:/" + revOrRefRegex + ")?", - std::regex::ECMAScript); - - static std::regex uriRegex( - "((" + schemeRegex + "):" + - "(?://(" + authorityRegex + "))?" + - "(" + pathRegex + "))", - std::regex::ECMAScript); - - static std::regex refRegex2(refRegex, std::regex::ECMAScript); - - static std::regex subDirRegex2(subDirRegex, std::regex::ECMAScript); - - auto [uri2, params] = splitUriAndParams(uri_); - std::string uri(uri2); - - auto handleSubdir = [&](const std::string & name, const std::string & value) { - if (name == "dir") { - if (value != "" && !std::regex_match(value, subDirRegex2)) - throw BadFlakeRef("flake '%s' has invalid subdirectory '%s'", uri, value); - subdir = value; - return true; - } else - return false; - }; - - auto handleGitParams = [&](const std::string & name, const std::string & value) { - if (name == "rev") { - if (!std::regex_match(value, revRegex)) - throw BadFlakeRef("invalid Git revision '%s'", value); - rev = Hash(value, htSHA1); - } else if (name == "ref") { - if (!std::regex_match(value, refRegex2)) - throw BadFlakeRef("invalid Git ref '%s'", value); - ref = value; - } else if (handleSubdir(name, value)) - ; - else return false; - return true; - }; - - std::cmatch match; - if (std::regex_match(uri.c_str(), match, flakeRegex)) { - IsAlias d; - d.alias = match[1]; - if (match[2].matched) - rev = Hash(match[2], htSHA1); - else if (match[3].matched) { - ref = match[3]; - if (match[4].matched) - rev = Hash(match[4], htSHA1); - } - data = d; - } - - else if (std::regex_match(uri.c_str(), match, githubRegex)) { - IsGitHub d; - d.owner = match[1]; - d.repo = match[2]; - if (match[3].matched) - rev = Hash(match[3], htSHA1); - else if (match[4].matched) { - ref = match[4]; - } - for (auto & param : params) { - if (handleSubdir(param.first, param.second)) - ; - else - throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); - } - data = d; - } - - else if (std::regex_match(uri.c_str(), match, uriRegex) - && (match[2] == "file" || hasSuffix(match[4], ".git"))) - { - IsGit d; - d.uri = match[1]; - for (auto & param : params) { - if (handleGitParams(param.first, param.second)) - ; - else - // FIXME: should probably pass through unknown parameters - throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); - } - if (rev && !ref) - throw BadFlakeRef("flake URI '%s' lacks a Git ref", uri); - data = d; - } - - else if ((hasPrefix(uri, "/") || (allowRelative && (hasPrefix(uri, "./") || hasPrefix(uri, "../") || uri == "."))) - && uri.find(':') == std::string::npos) - { - IsPath d; - if (allowRelative) { - d.path = absPath(uri); - while (true) { - if (pathExists(d.path + "/.git")) break; - subdir = baseNameOf(d.path) + (subdir.empty() ? "" : "/" + subdir); - d.path = dirOf(d.path); - if (d.path == "/") - throw BadFlakeRef("path '%s' does not reference a Git repository", uri); - } - } else - d.path = canonPath(uri); - data = d; - for (auto & param : params) { - if (handleGitParams(param.first, param.second)) - ; - else - throw BadFlakeRef("invalid Git flakeref parameter '%s', in '%s'", param.first, uri); - } - } - - else - throw BadFlakeRef("'%s' is not a valid flake reference", uri); -} - -std::string FlakeRef::to_string() const -{ - std::string string; - bool first = true; - - auto addParam = - [&](const std::string & name, std::string value) { - string += first ? '?' : '&'; - first = false; - string += name; - string += '='; - string += value; // FIXME: escaping - }; - - if (auto refData = std::get_if<FlakeRef::IsAlias>(&data)) { - string = refData->alias; - if (ref) string += '/' + *ref; - if (rev) string += '/' + rev->gitRev(); - } - - else if (auto refData = std::get_if<FlakeRef::IsPath>(&data)) { - string = refData->path; - if (ref) addParam("ref", *ref); - if (rev) addParam("rev", rev->gitRev()); - if (subdir != "") addParam("dir", subdir); - } - - else if (auto refData = std::get_if<FlakeRef::IsGitHub>(&data)) { - assert(!(ref && rev)); - string = "github:" + refData->owner + "/" + refData->repo; - if (ref) { string += '/'; string += *ref; } - if (rev) { string += '/'; string += rev->gitRev(); } - if (subdir != "") addParam("dir", subdir); - } - - else if (auto refData = std::get_if<FlakeRef::IsGit>(&data)) { - assert(!rev || ref); - string = refData->uri; - - if (ref) { - addParam("ref", *ref); - if (rev) - addParam("rev", rev->gitRev()); - } - - if (subdir != "") addParam("dir", subdir); - } - - else abort(); - - assert(FlakeRef(string) == *this); - - return string; -} - -std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef) -{ - str << flakeRef.to_string(); - return str; -} - -bool FlakeRef::isImmutable() const -{ - return (bool) rev; -} - -FlakeRef FlakeRef::baseRef() const // Removes the ref and rev from a FlakeRef. -{ - FlakeRef result(*this); - result.ref = std::nullopt; - result.rev = std::nullopt; - return result; -} - -std::optional<FlakeRef> parseFlakeRef( - const std::string & uri, bool allowRelative) -{ - try { - return FlakeRef(uri, allowRelative); - } catch (BadFlakeRef & e) { - return {}; - } -} - -} diff --git a/src/libexpr/primops/flakeref.hh b/src/libexpr/primops/flakeref.hh deleted file mode 100644 index 52bb82ddb..000000000 --- a/src/libexpr/primops/flakeref.hh +++ /dev/null @@ -1,188 +0,0 @@ -#pragma once - -#include "types.hh" -#include "hash.hh" - -#include <variant> - -namespace nix { - -/* Flake references are a URI-like syntax to specify a flake. - - Examples: - - * <flake-id>(/rev-or-ref(/rev)?)? - - Look up a flake by ID in the flake lock file or in the flake - registry. These must specify an actual location for the flake - using the formats listed below. Note that in pure evaluation - mode, the flake registry is empty. - - Optionally, the rev or ref from the dereferenced flake can be - overriden. For example, - - nixpkgs/19.09 - - uses the "19.09" branch of the nixpkgs' flake GitHub repository, - while - - nixpkgs/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f - - uses the specified revision. For Git (rather than GitHub) - repositories, both the rev and ref must be given, e.g. - - nixpkgs/19.09/98a2a5b5370c1e2092d09cb38b9dcff6d98a109f - - * github:<owner>/<repo>(/<rev-or-ref>)? - - A repository on GitHub. These differ from Git references in that - they're downloaded in a efficient way (via the tarball mechanism) - and that they support downloading a specific revision without - specifying a branch. <rev-or-ref> is either a commit hash ("rev") - or a branch or tag name ("ref"). The default is: "master" if none - is specified. Note that in pure evaluation mode, a commit hash - must be used. - - Flakes fetched in this manner expose "rev" and "lastModified" - attributes, but not "revCount". - - Examples: - - github:edolstra/dwarffs - github:edolstra/dwarffs/unstable - github:edolstra/dwarffs/41c0c1bf292ea3ac3858ff393b49ca1123dbd553 - - * https://<server>/<path>.git(\?attr(&attr)*)? - ssh://<server>/<path>.git(\?attr(&attr)*)? - git://<server>/<path>.git(\?attr(&attr)*)? - file:///<path>(\?attr(&attr)*)? - - where 'attr' is one of: - rev=<rev> - ref=<ref> - - A Git repository fetched through https. Note that the path must - end in ".git". The default for "ref" is "master". - - Examples: - - https://example.org/my/repo.git - https://example.org/my/repo.git?ref=release-1.2.3 - https://example.org/my/repo.git?rev=e72daba8250068216d79d2aeef40d4d95aff6666 - git://github.com/edolstra/dwarffs.git?ref=flake&rev=2efca4bc9da70fb001b26c3dc858c6397d3c4817 - - * /path.git(\?attr(&attr)*)? - - Like file://path.git, but if no "ref" or "rev" is specified, the - (possibly dirty) working tree will be used. Using a working tree - is not allowed in pure evaluation mode. - - Examples: - - /path/to/my/repo - /path/to/my/repo?ref=develop - /path/to/my/repo?rev=e72daba8250068216d79d2aeef40d4d95aff6666 - - * https://<server>/<path>.tar.xz(?hash=<sri-hash>) - file:///<path>.tar.xz(?hash=<sri-hash>) - - A flake distributed as a tarball. In pure evaluation mode, an SRI - hash is mandatory. It exposes a "lastModified" attribute, being - the newest file inside the tarball. - - Example: - - https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz - https://releases.nixos.org/nixos/unstable/nixos-19.03pre167858.f2a1a4e93be/nixexprs.tar.xz?hash=sha256-56bbc099995ea8581ead78f22832fee7dbcb0a0b6319293d8c2d0aef5379397c - - Note: currently, there can be only one flake per Git repository, and - it must be at top-level. In the future, we may want to add a field - (e.g. "dir=<dir>") to specify a subdirectory inside the repository. -*/ - -typedef std::string FlakeId; -typedef std::string FlakeAlias; -typedef std::string FlakeUri; - -struct FlakeRef -{ - struct IsAlias - { - FlakeAlias alias; - bool operator<(const IsAlias & b) const { return alias < b.alias; }; - bool operator==(const IsAlias & b) const { return alias == b.alias; }; - }; - - struct IsGitHub { - std::string owner, repo; - bool operator<(const IsGitHub & b) const { - return std::make_tuple(owner, repo) < std::make_tuple(b.owner, b.repo); - } - bool operator==(const IsGitHub & b) const { - return owner == b.owner && repo == b.repo; - } - }; - - // Git, Tarball - struct IsGit - { - std::string uri; - bool operator<(const IsGit & b) const { return uri < b.uri; } - bool operator==(const IsGit & b) const { return uri == b.uri; } - }; - - struct IsPath - { - Path path; - bool operator<(const IsPath & b) const { return path < b.path; } - bool operator==(const IsPath & b) const { return path == b.path; } - }; - - // Git, Tarball - - std::variant<IsAlias, IsGitHub, IsGit, IsPath> data; - - std::optional<std::string> ref; - std::optional<Hash> rev; - Path subdir = ""; // This is a relative path pointing at the flake.nix file's directory, relative to the git root. - - bool operator<(const FlakeRef & flakeRef) const - { - return std::make_tuple(data, ref, rev, subdir) < - std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev, subdir); - } - - bool operator==(const FlakeRef & flakeRef) const - { - return std::make_tuple(data, ref, rev, subdir) == - std::make_tuple(flakeRef.data, flakeRef.ref, flakeRef.rev, flakeRef.subdir); - } - - // Parse a flake URI. - FlakeRef(const std::string & uri, bool allowRelative = false); - - // FIXME: change to operator <<. - std::string to_string() const; - - /* Check whether this is a "direct" flake reference, that is, not - a flake ID, which requires a lookup in the flake registry. */ - bool isDirect() const - { - return !std::get_if<FlakeRef::IsAlias>(&data); - } - - /* Check whether this is an "immutable" flake reference, that is, - one that contains a commit hash or content hash. */ - bool isImmutable() const; - - FlakeRef baseRef() const; -}; - -std::ostream & operator << (std::ostream & str, const FlakeRef & flakeRef); - -MakeError(BadFlakeRef, Error); - -std::optional<FlakeRef> parseFlakeRef( - const std::string & uri, bool allowRelative = false); - -} diff --git a/src/libexpr/primops/lockfile.cc b/src/libexpr/primops/lockfile.cc deleted file mode 100644 index 97c748c66..000000000 --- a/src/libexpr/primops/lockfile.cc +++ /dev/null @@ -1,102 +0,0 @@ -#include "lockfile.hh" -#include "store-api.hh" - -namespace nix::flake { - -AbstractInput::AbstractInput(const nlohmann::json & json) - : ref(json["uri"]) - , narHash(Hash((std::string) json["narHash"])) -{ - if (!ref.isImmutable()) - throw Error("lockfile contains mutable flakeref '%s'", ref); -} - -nlohmann::json AbstractInput::toJson() const -{ - nlohmann::json json; - json["uri"] = ref.to_string(); - json["narHash"] = narHash.to_string(SRI); - return json; -} - -Path AbstractInput::computeStorePath(Store & store) const -{ - return store.makeFixedOutputPath(true, narHash, "source"); -} - -FlakeInput::FlakeInput(const nlohmann::json & json) - : FlakeInputs(json) - , AbstractInput(json) - , id(json["id"]) -{ -} - -nlohmann::json FlakeInput::toJson() const -{ - auto json = FlakeInputs::toJson(); - json.update(AbstractInput::toJson()); - json["id"] = id; - return json; -} - -FlakeInputs::FlakeInputs(const nlohmann::json & json) -{ - for (auto & i : json["nonFlakeInputs"].items()) - nonFlakeInputs.insert_or_assign(i.key(), NonFlakeInput(i.value())); - - for (auto & i : json["inputs"].items()) - flakeInputs.insert_or_assign(i.key(), FlakeInput(i.value())); -} - -nlohmann::json FlakeInputs::toJson() const -{ - nlohmann::json json; - { - auto j = nlohmann::json::object(); - for (auto & i : nonFlakeInputs) - j[i.first] = i.second.toJson(); - json["nonFlakeInputs"] = std::move(j); - } - { - auto j = nlohmann::json::object(); - for (auto & i : flakeInputs) - j[i.first.to_string()] = i.second.toJson(); - json["inputs"] = std::move(j); - } - return json; -} - -nlohmann::json LockFile::toJson() const -{ - auto json = FlakeInputs::toJson(); - json["version"] = 2; - return json; -} - -LockFile LockFile::read(const Path & path) -{ - if (pathExists(path)) { - auto json = nlohmann::json::parse(readFile(path)); - - auto version = json.value("version", 0); - if (version != 2) - throw Error("lock file '%s' has unsupported version %d", path, version); - - return LockFile(json); - } else - return LockFile(); -} - -std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile) -{ - stream << lockFile.toJson().dump(4); // '4' = indentation in json file - return stream; -} - -void LockFile::write(const Path & path) const -{ - createDirs(dirOf(path)); - writeFile(path, fmt("%s\n", *this)); -} - -} diff --git a/src/libexpr/primops/lockfile.hh b/src/libexpr/primops/lockfile.hh deleted file mode 100644 index b76124190..000000000 --- a/src/libexpr/primops/lockfile.hh +++ /dev/null @@ -1,112 +0,0 @@ -#pragma once - -#include "flakeref.hh" - -#include <nlohmann/json.hpp> - -namespace nix { -class Store; -} - -namespace nix::flake { - -/* Common lock file information about a flake input, namely the - immutable ref and the NAR hash. */ -struct AbstractInput -{ - FlakeRef ref; - Hash narHash; - - AbstractInput(const FlakeRef & flakeRef, const Hash & narHash) - : ref(flakeRef), narHash(narHash) - { - assert(ref.isImmutable()); - }; - - AbstractInput(const nlohmann::json & json); - - nlohmann::json toJson() const; - - Path computeStorePath(Store & store) const; -}; - -/* Lock file information about a non-flake input. */ -struct NonFlakeInput : AbstractInput -{ - using AbstractInput::AbstractInput; - - bool operator ==(const NonFlakeInput & other) const - { - return ref == other.ref && narHash == other.narHash; - } -}; - -struct FlakeInput; - -/* Lock file information about the dependencies of a flake. */ -struct FlakeInputs -{ - std::map<FlakeRef, FlakeInput> flakeInputs; - std::map<FlakeAlias, NonFlakeInput> nonFlakeInputs; - - FlakeInputs() {}; - FlakeInputs(const nlohmann::json & json); - - nlohmann::json toJson() const; -}; - -/* Lock file information about a flake input. */ -struct FlakeInput : FlakeInputs, AbstractInput -{ - FlakeId id; - - FlakeInput(const FlakeId & id, const FlakeRef & flakeRef, const Hash & narHash) - : AbstractInput(flakeRef, narHash), id(id) {}; - - FlakeInput(const nlohmann::json & json); - - bool operator ==(const FlakeInput & other) const - { - return - id == other.id - && ref == other.ref - && narHash == other.narHash - && flakeInputs == other.flakeInputs - && nonFlakeInputs == other.nonFlakeInputs; - } - - nlohmann::json toJson() const; -}; - -/* An entire lock file. Note that this cannot be a FlakeInput for the - top-level flake, because then the lock file would need to contain - the hash of the top-level flake, but committing the lock file - would invalidate that hash. */ -struct LockFile : FlakeInputs -{ - bool operator ==(const LockFile & other) const - { - return - flakeInputs == other.flakeInputs - && nonFlakeInputs == other.nonFlakeInputs; - } - - LockFile() {} - LockFile(const nlohmann::json & json) : FlakeInputs(json) {} - LockFile(FlakeInput && dep) - { - flakeInputs = std::move(dep.flakeInputs); - nonFlakeInputs = std::move(dep.nonFlakeInputs); - } - - nlohmann::json toJson() const; - - static LockFile read(const Path & path); - - void write(const Path & path) const; -}; - -std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile); - -} - |