diff options
Diffstat (limited to 'src')
51 files changed, 1073 insertions, 283 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc index 57f2cd32d..0904f2ce9 100644 --- a/src/build-remote/build-remote.cc +++ b/src/build-remote/build-remote.cc @@ -277,7 +277,16 @@ connected: auto drv = store->readDerivation(*drvPath); auto outputHashes = staticOutputHashes(*store, drv); - drv.inputSrcs = store->parseStorePathSet(inputs); + + // Hijack the inputs paths of the derivation to include all the paths + // that come from the `inputDrvs` set. + // We don’t do that for the derivations whose `inputDrvs` is empty + // because + // 1. It’s not needed + // 2. Changing the `inputSrcs` set changes the associated output ids, + // which break CA derivations + if (!drv.inputDrvs.empty()) + drv.inputSrcs = store->parseStorePathSet(inputs); auto result = sshStore->buildDerivation(*drvPath, drv); diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 569c4b9e4..6777b23be 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -188,7 +188,7 @@ void MixProfile::updateProfile(const BuiltPaths & buildables) } if (result.size() != 1) - throw Error("'--profile' requires that the arguments produce a single store path, but there are %d", result.size()); + throw UsageError("'--profile' requires that the arguments produce a single store path, but there are %d", result.size()); updateProfile(result[0]); } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index fe52912cf..658b415f3 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -171,14 +171,50 @@ Strings SourceExprCommand::getDefaultFlakeAttrPathPrefixes() void SourceExprCommand::completeInstallable(std::string_view prefix) { - if (file) return; // FIXME + if (file) { + evalSettings.pureEval = false; + auto state = getEvalState(); + Expr *e = state->parseExprFromFile( + resolveExprPath(state->checkSourcePath(lookupFileArg(*state, *file))) + ); + + Value root; + state->eval(e, root); + + auto autoArgs = getAutoArgs(*state); + + std::string prefix_ = std::string(prefix); + auto sep = prefix_.rfind('.'); + std::string searchWord; + if (sep != std::string::npos) { + searchWord = prefix_.substr(sep, std::string::npos); + prefix_ = prefix_.substr(0, sep); + } else { + searchWord = prefix_; + prefix_ = ""; + } - completeFlakeRefWithFragment( - getEvalState(), - lockFlags, - getDefaultFlakeAttrPathPrefixes(), - getDefaultFlakeAttrPaths(), - prefix); + Value &v1(*findAlongAttrPath(*state, prefix_, *autoArgs, root).first); + state->forceValue(v1); + Value v2; + state->autoCallFunction(*autoArgs, v1, v2); + + if (v2.type() == nAttrs) { + for (auto & i : *v2.attrs) { + std::string name = i.name; + if (name.find(searchWord) == 0) { + completions->add(i.name); + } + } + } + } else { + completeFlakeRefWithFragment( + getEvalState(), + lockFlags, + getDefaultFlakeAttrPathPrefixes(), + getDefaultFlakeAttrPaths(), + prefix); + } } void completeFlakeRefWithFragment( @@ -573,10 +609,10 @@ InstallableFlake::getCursors(EvalState & state) std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const { + flake::LockFlags lockFlagsApplyConfig = lockFlags; + lockFlagsApplyConfig.applyNixConfig = true; if (!_lockedFlake) { - _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlags)); - _lockedFlake->flake.config.apply(); - // FIXME: send new config to the daemon. + _lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlagsApplyConfig)); } return _lockedFlake; } diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index 9dd557205..867eb13a5 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -19,7 +19,7 @@ static Strings parseAttrPath(std::string_view s) ++i; while (1) { if (i == s.end()) - throw Error("missing closing quote in selection path '%1%'", s); + throw ParseError("missing closing quote in selection path '%1%'", s); if (*i == '"') break; cur.push_back(*i++); } @@ -116,14 +116,14 @@ Pos findDerivationFilename(EvalState & state, Value & v, std::string what) auto colon = pos.rfind(':'); if (colon == std::string::npos) - throw Error("cannot parse meta.position attribute '%s'", pos); + throw ParseError("cannot parse meta.position attribute '%s'", pos); std::string filename(pos, 0, colon); unsigned int lineno; try { lineno = std::stoi(std::string(pos, colon + 1)); } catch (std::invalid_argument & e) { - throw Error("cannot parse line number '%s'", pos); + throw ParseError("cannot parse line number '%s'", pos); } Symbol file = state.symbols.create(filename); diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index ef9f8efca..c3206a577 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -64,7 +64,11 @@ static char * dupStringWithLen(const char * s, size_t size) RootValue allocRootValue(Value * v) { +#if HAVE_BOEHMGC return std::allocate_shared<Value *>(traceable_allocator<Value *>(), v); +#else + return std::make_shared<Value *>(v); +#endif } @@ -233,22 +237,34 @@ static void * oomHandler(size_t requested) } class BoehmGCStackAllocator : public StackAllocator { - boost::coroutines2::protected_fixedsize_stack stack { - // We allocate 8 MB, the default max stack size on NixOS. - // A smaller stack might be quicker to allocate but reduces the stack - // depth available for source filter expressions etc. - std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024)) + boost::coroutines2::protected_fixedsize_stack stack { + // We allocate 8 MB, the default max stack size on NixOS. + // A smaller stack might be quicker to allocate but reduces the stack + // depth available for source filter expressions etc. + std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024)) }; + // This is specific to boost::coroutines2::protected_fixedsize_stack. + // The stack protection page is included in sctx.size, so we have to + // subtract one page size from the stack size. + std::size_t pfss_usable_stack_size(boost::context::stack_context &sctx) { + return sctx.size - boost::context::stack_traits::page_size(); + } + public: boost::context::stack_context allocate() override { auto sctx = stack.allocate(); - GC_add_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp); + + // Stacks generally start at a high address and grow to lower addresses. + // Architectures that do the opposite are rare; in fact so rare that + // boost_routine does not implement it. + // So we subtract the stack size. + GC_add_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); return sctx; } void deallocate(boost::context::stack_context sctx) override { - GC_remove_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp); + GC_remove_roots(static_cast<char *>(sctx.sp) - pfss_usable_stack_size(sctx), sctx.sp); stack.deallocate(sctx); } @@ -908,7 +924,7 @@ void EvalState::evalFile(const Path & path_, Value & v, bool mustBeTrivial) // computation. if (mustBeTrivial && !(dynamic_cast<ExprAttrs *>(e))) - throw Error("file '%s' must be an attribute set", path); + throw EvalError("file '%s' must be an attribute set", path); eval(e, v); } catch (Error & e) { addErrorTrace(e, "while evaluating the file '%1%':", path2); diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc index 2e94490d4..e266bc36d 100644 --- a/src/libexpr/flake/flake.cc +++ b/src/libexpr/flake/flake.cc @@ -298,6 +298,11 @@ LockedFlake lockFlake( auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache); + if (lockFlags.applyNixConfig) { + flake.config.apply(); + // FIXME: send new config to the daemon. + } + try { // FIXME: symlink attack @@ -359,7 +364,12 @@ LockedFlake lockFlake( ancestors? */ auto i = overrides.find(inputPath); bool hasOverride = i != overrides.end(); - if (hasOverride) overridesUsed.insert(inputPath); + if (hasOverride) { + overridesUsed.insert(inputPath); + // Respect the “flakeness” of the input even if we + // override it + i->second.isFlake = input2.isFlake; + } auto & input = hasOverride ? i->second : input2; /* Resolve 'follows' later (since it may refer to an input diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh index d17d5e183..4479e95db 100644 --- a/src/libexpr/flake/flake.hh +++ b/src/libexpr/flake/flake.hh @@ -104,6 +104,10 @@ struct LockFlags references like 'nixpkgs'. */ bool useRegistries = true; + /* Whether to apply flake's nixConfig attribute to the configuration */ + + bool applyNixConfig = false; + /* Whether mutable flake references (i.e. those without a Git revision or similar) without a corresponding lock are allowed. Mutable flake references with a lock are always diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk index c40abfb78..1aed8e152 100644 --- a/src/libexpr/local.mk +++ b/src/libexpr/local.mk @@ -16,7 +16,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib libexpr_LIBS = libutil libstore libfetchers libexpr_LDFLAGS = -lboost_context -ifeq ($(OS), Linux) +ifdef HOST_LINUX libexpr_LDFLAGS += -ldl endif diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 4830ebec3..3f88ccb91 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -62,6 +62,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar fetchers::Attrs attrs; attrs.insert_or_assign("type", "hg"); attrs.insert_or_assign("url", url.find("://") != std::string::npos ? url : "file://" + url); + attrs.insert_or_assign("name", name); if (ref) attrs.insert_or_assign("ref", *ref); if (rev) attrs.insert_or_assign("rev", rev->gitRev()); auto input = fetchers::Input::fromAttrs(std::move(attrs)); diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index b8b99d4fa..730db84ed 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -7,6 +7,7 @@ #include <ctime> #include <iomanip> +#include <regex> namespace nix { @@ -60,10 +61,19 @@ void emitTreeAttrs( v.attrs->sort(); } -std::string fixURI(std::string uri, EvalState &state) +std::string fixURI(std::string uri, EvalState &state, const std::string & defaultScheme = "file") { state.checkURI(uri); - return uri.find("://") != std::string::npos ? uri : "file://" + uri; + return uri.find("://") != std::string::npos ? uri : defaultScheme + "://" + uri; +} + +std::string fixURIForGit(std::string uri, EvalState & state) +{ + static std::regex scp_uri("([^/].*)@(.*):(.*)"); + if (uri[0] != '/' && std::regex_match(uri, scp_uri)) + return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh"); + else + return fixURI(uri, state); } void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v) @@ -72,13 +82,18 @@ void addURI(EvalState &state, fetchers::Attrs &attrs, Symbol name, std::string v attrs.emplace(name, n == "url" ? fixURI(v, state) : v); } +struct FetchTreeParams { + bool emptyRevFallback = false; + bool allowNameArgument = false; +}; + static void fetchTree( EvalState &state, const Pos &pos, Value **args, Value &v, const std::optional<std::string> type, - bool emptyRevFallback = false + const FetchTreeParams & params = FetchTreeParams{} ) { fetchers::Input input; PathSet context; @@ -119,17 +134,25 @@ static void fetchTree( .errPos = pos }); + if (!params.allowNameArgument) + if (auto nameIter = attrs.find("name"); nameIter != attrs.end()) + throw Error({ + .msg = hintfmt("attribute 'name' isn’t supported in call to 'fetchTree'"), + .errPos = pos + }); + + input = fetchers::Input::fromAttrs(std::move(attrs)); } else { - auto url = fixURI(state.coerceToString(pos, *args[0], context, false, false), state); + auto url = state.coerceToString(pos, *args[0], context, false, false); if (type == "git") { fetchers::Attrs attrs; attrs.emplace("type", "git"); - attrs.emplace("url", url); + attrs.emplace("url", fixURIForGit(url, state)); input = fetchers::Input::fromAttrs(std::move(attrs)); } else { - input = fetchers::Input::fromURL(url); + input = fetchers::Input::fromURL(fixURI(url, state)); } } @@ -144,13 +167,13 @@ static void fetchTree( if (state.allowedPaths) state.allowedPaths->insert(tree.actualPath); - emitTreeAttrs(state, tree, input2, v, emptyRevFallback); + emitTreeAttrs(state, tree, input2, v, params.emptyRevFallback); } static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, Value & v) { settings.requireExperimentalFeature("flakes"); - fetchTree(state, pos, args, v, std::nullopt); + fetchTree(state, pos, args, v, std::nullopt, FetchTreeParams { .allowNameArgument = false }); } // FIXME: document @@ -292,7 +315,7 @@ static RegisterPrimOp primop_fetchTarball({ static void prim_fetchGit(EvalState &state, const Pos &pos, Value **args, Value &v) { - fetchTree(state, pos, args, v, "git", true); + fetchTree(state, pos, args, v, "git", FetchTreeParams { .emptyRevFallback = true, .allowNameArgument = true }); } static RegisterPrimOp primop_fetchGit({ diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 916e0a8e8..e158d914b 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -200,12 +200,17 @@ void Input::markChangedFile( return scheme->markChangedFile(*this, file, commitMsg); } +std::string Input::getName() const +{ + return maybeGetStrAttr(attrs, "name").value_or("source"); +} + StorePath Input::computeStorePath(Store & store) const { auto narHash = getNarHash(); if (!narHash) throw Error("cannot compute store path for mutable input '%s'", to_string()); - return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, "source"); + return store.makeFixedOutputPath(FileIngestionMethod::Recursive, *narHash, getName()); } std::string Input::getType() const diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh index a72cfafa4..c839cf23b 100644 --- a/src/libfetchers/fetchers.hh +++ b/src/libfetchers/fetchers.hh @@ -81,6 +81,8 @@ public: std::string_view file, std::optional<std::string> commitMsg) const; + std::string getName() const; + StorePath computeStorePath(Store & store) const; // Convenience functions for common attributes. diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index d8e0dbe0a..bc1930138 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -60,7 +60,7 @@ struct GitInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "git") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs") + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs" && name != "name") throw Error("unsupported Git input attribute '%s'", name); parseURL(getStrAttr(attrs, "url")); @@ -167,10 +167,10 @@ struct GitInputScheme : InputScheme std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override { - auto name = "source"; - Input input(_input); + std::string name = input.getName(); + bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false); bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false); bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false); @@ -270,7 +270,7 @@ struct GitInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); // FIXME: maybe we should use the timestamp of the last // modified dirty file? diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 8352ef02d..298c05f9a 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -207,7 +207,7 @@ struct GitArchiveInputScheme : InputScheme auto url = getDownloadUrl(input); - auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers); + auto [tree, lastModified] = downloadTarball(store, url.url, input.getName(), true, url.headers); input.attrs.insert_or_assign("lastModified", uint64_t(lastModified)); diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 0eb401e10..efb4ee8db 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -74,7 +74,7 @@ struct MercurialInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "hg") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash") + if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "revCount" && name != "narHash" && name != "name") throw Error("unsupported Mercurial input attribute '%s'", name); parseURL(getStrAttr(attrs, "url")); @@ -147,10 +147,10 @@ struct MercurialInputScheme : InputScheme std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override { - auto name = "source"; - Input input(_input); + auto name = input.getName(); + auto [isLocal, actualUrl_] = getActualUrl(input); auto actualUrl = actualUrl_; // work around clang bug @@ -193,7 +193,7 @@ struct MercurialInputScheme : InputScheme return files.count(file); }; - auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); + auto storePath = store->addToStore(input.getName(), actualUrl, FileIngestionMethod::Recursive, htSHA256, filter); return { Tree(store->toRealPath(storePath), std::move(storePath)), diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc index 74376adc0..f35359d4b 100644 --- a/src/libfetchers/registry.cc +++ b/src/libfetchers/registry.cc @@ -124,6 +124,13 @@ std::shared_ptr<Registry> getUserRegistry() return userRegistry; } +std::shared_ptr<Registry> getCustomRegistry(const Path & p) +{ + static auto customRegistry = + Registry::read(p, Registry::Custom); + return customRegistry; +} + static std::shared_ptr<Registry> flagRegistry = std::make_shared<Registry>(Registry::Flag); diff --git a/src/libfetchers/registry.hh b/src/libfetchers/registry.hh index 1077af020..260a2c460 100644 --- a/src/libfetchers/registry.hh +++ b/src/libfetchers/registry.hh @@ -14,6 +14,7 @@ struct Registry User = 1, System = 2, Global = 3, + Custom = 4, }; RegistryType type; @@ -48,6 +49,8 @@ typedef std::vector<std::shared_ptr<Registry>> Registries; std::shared_ptr<Registry> getUserRegistry(); +std::shared_ptr<Registry> getCustomRegistry(const Path & p); + Path getUserRegistryPath(); Registries getRegistries(ref<Store> store); diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 257465bae..031ccc5f7 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -196,7 +196,7 @@ struct TarballInputScheme : InputScheme if (maybeGetStrAttr(attrs, "type") != "tarball") return {}; for (auto & [name, value] : attrs) - if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash") + if (name != "type" && name != "url" && /* name != "hash" && */ name != "narHash" && name != "name") throw Error("unsupported tarball input attribute '%s'", name); Input input; @@ -226,7 +226,7 @@ struct TarballInputScheme : InputScheme std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override { - auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first; + auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), input.getName(), false).first; return {std::move(tree), input}; } }; diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index 15354549a..b2a6e2a82 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -484,7 +484,7 @@ Logger * makeProgressBar(bool printBuildLogs) { return new ProgressBar( printBuildLogs, - isatty(STDERR_FILENO) && getEnv("TERM").value_or("dumb") != "dumb" + shouldANSI() ); } diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 4c49f3cab..3cbcf5199 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -738,6 +738,63 @@ void DerivationGoal::cleanupPostOutputsRegisteredModeNonCheck() { } +void runPostBuildHook( + Store & store, + Logger & logger, + const StorePath & drvPath, + StorePathSet outputPaths +) +{ + auto hook = settings.postBuildHook; + if (hook == "") + return; + + Activity act(logger, lvlInfo, actPostBuildHook, + fmt("running post-build-hook '%s'", settings.postBuildHook), + Logger::Fields{store.printStorePath(drvPath)}); + PushActivity pact(act.id); + std::map<std::string, std::string> hookEnvironment = getEnv(); + + hookEnvironment.emplace("DRV_PATH", store.printStorePath(drvPath)); + hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", store.printStorePathSet(outputPaths)))); + + RunOptions opts(settings.postBuildHook, {}); + opts.environment = hookEnvironment; + + struct LogSink : Sink { + Activity & act; + std::string currentLine; + + LogSink(Activity & act) : act(act) { } + + void operator() (std::string_view data) override { + for (auto c : data) { + if (c == '\n') { + flushLine(); + } else { + currentLine += c; + } + } + } + + void flushLine() { + act.result(resPostBuildLogLine, currentLine); + currentLine.clear(); + } + + ~LogSink() { + if (currentLine != "") { + currentLine += '\n'; + flushLine(); + } + } + }; + LogSink sink(act); + + opts.standardOut = &sink; + opts.mergeStderrToStdout = true; + runProgram2(opts); +} void DerivationGoal::buildDone() { @@ -803,57 +860,15 @@ void DerivationGoal::buildDone() being valid. */ registerOutputs(); - if (settings.postBuildHook != "") { - Activity act(*logger, lvlInfo, actPostBuildHook, - fmt("running post-build-hook '%s'", settings.postBuildHook), - Logger::Fields{worker.store.printStorePath(drvPath)}); - PushActivity pact(act.id); - StorePathSet outputPaths; - for (auto i : drv->outputs) { - outputPaths.insert(finalOutputs.at(i.first)); - } - std::map<std::string, std::string> hookEnvironment = getEnv(); - - hookEnvironment.emplace("DRV_PATH", worker.store.printStorePath(drvPath)); - hookEnvironment.emplace("OUT_PATHS", chomp(concatStringsSep(" ", worker.store.printStorePathSet(outputPaths)))); - - RunOptions opts(settings.postBuildHook, {}); - opts.environment = hookEnvironment; - - struct LogSink : Sink { - Activity & act; - std::string currentLine; - - LogSink(Activity & act) : act(act) { } - - void operator() (std::string_view data) override { - for (auto c : data) { - if (c == '\n') { - flushLine(); - } else { - currentLine += c; - } - } - } - - void flushLine() { - act.result(resPostBuildLogLine, currentLine); - currentLine.clear(); - } - - ~LogSink() { - if (currentLine != "") { - currentLine += '\n'; - flushLine(); - } - } - }; - LogSink sink(act); - - opts.standardOut = &sink; - opts.mergeStderrToStdout = true; - runProgram2(opts); - } + StorePathSet outputPaths; + for (auto & [_, path] : finalOutputs) + outputPaths.insert(path); + runPostBuildHook( + worker.store, + *logger, + drvPath, + outputPaths + ); if (buildMode == bmCheck) { cleanupPostOutputsRegisteredModeCheck(); @@ -909,6 +924,8 @@ void DerivationGoal::resolvedFinished() { auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv); + StorePathSet outputPaths; + // `wantedOutputs` might be empty, which means “all the outputs” auto realWantedOutputs = wantedOutputs; if (realWantedOutputs.empty()) @@ -926,8 +943,10 @@ void DerivationGoal::resolvedFinished() { auto newRealisation = *realisation; newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput}; newRealisation.signatures.clear(); + newRealisation.dependentRealisations = drvOutputReferences(worker.store, *drv, realisation->outPath); signRealisation(newRealisation); worker.store.registerDrvOutput(newRealisation); + outputPaths.insert(realisation->outPath); } else { // If we don't have a realisation, then it must mean that something // failed when building the resolved drv @@ -935,6 +954,13 @@ void DerivationGoal::resolvedFinished() { } } + runPostBuildHook( + worker.store, + *logger, + drvPath, + outputPaths + ); + // This is potentially a bit fishy in terms of error reporting. Not sure // how to do it in a cleaner way amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex); diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index a5ac4c49d..be270d079 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -17,6 +17,13 @@ DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker void DrvOutputSubstitutionGoal::init() { trace("init"); + + /* If the derivation already exists, we’re done */ + if (worker.store.queryRealisation(id)) { + amDone(ecSuccess); + return; + } + subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>(); tryNext(); } @@ -53,6 +60,26 @@ void DrvOutputSubstitutionGoal::tryNext() return; } + for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { + if (depId != id) { + if (auto localOutputInfo = worker.store.queryRealisation(depId); + localOutputInfo && localOutputInfo->outPath != depPath) { + warn( + "substituter '%s' has an incompatible realisation for '%s', ignoring.\n" + "Local: %s\n" + "Remote: %s", + sub->getUri(), + depId.to_string(), + worker.store.printStorePath(localOutputInfo->outPath), + worker.store.printStorePath(depPath) + ); + tryNext(); + return; + } + addWaitee(worker.makeDrvOutputSubstitutionGoal(depId)); + } + } + addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath)); if (waitees.empty()) outPathValid(); diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 259134eba..02f902666 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1248,13 +1248,18 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo std::optional<const Realisation> queryRealisation(const DrvOutput & id) override // XXX: This should probably be allowed if the realisation corresponds to // an allowed derivation - { throw Error("queryRealisation"); } + { + if (!goal.isAllowed(id)) + throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string()); + return next->queryRealisation(id); + } void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override { if (buildMode != bmNormal) throw Error("unsupported build mode"); StorePathSet newPaths; + std::set<Realisation> newRealisations; for (auto & req : paths) { if (!goal.isAllowed(req)) @@ -1267,16 +1272,28 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo auto p = std::get_if<DerivedPath::Built>(&path); if (!p) continue; auto & bfd = *p; + auto drv = readDerivation(bfd.drvPath); + auto drvHashes = staticOutputHashes(*this, drv); auto outputs = next->queryDerivationOutputMap(bfd.drvPath); for (auto & [outputName, outputPath] : outputs) - if (wantOutput(outputName, bfd.outputs)) + if (wantOutput(outputName, bfd.outputs)) { newPaths.insert(outputPath); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + auto thisRealisation = next->queryRealisation( + DrvOutput{drvHashes.at(outputName), outputName} + ); + assert(thisRealisation); + newRealisations.insert(*thisRealisation); + } + } } StorePathSet closure; next->computeFSClosure(newPaths, closure); for (auto & path : closure) goal.addDependency(path); + for (auto & real : Realisation::closure(*next, newRealisations)) + goal.addedDrvOutputs.insert(real.id); } BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, @@ -2379,6 +2396,7 @@ void LocalDerivationGoal::registerOutputs() floating CA derivations and hash-mismatching fixed-output derivations. */ PathLocks dynamicOutputLock; + dynamicOutputLock.setDeletion(true); auto optFixedPath = output.path(worker.store, drv->name, outputName); if (!optFixedPath || worker.store.printStorePath(*optFixedPath) != finalDestPath) diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh index d30be2351..088a57209 100644 --- a/src/libstore/build/local-derivation-goal.hh +++ b/src/libstore/build/local-derivation-goal.hh @@ -108,6 +108,9 @@ struct LocalDerivationGoal : public DerivationGoal /* Paths that were added via recursive Nix calls. */ StorePathSet addedPaths; + /* Realisations that were added via recursive Nix calls. */ + std::set<DrvOutput> addedDrvOutputs; + /* Recursive Nix calls are only allowed to build or realize paths in the original input closure or added via a recursive Nix call (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where @@ -116,6 +119,11 @@ struct LocalDerivationGoal : public DerivationGoal { return inputPaths.count(path) || addedPaths.count(path); } + bool isAllowed(const DrvOutput & id) + { + return addedDrvOutputs.count(id); + } + bool isAllowed(const DerivedPath & req); friend struct RestrictedStore; diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql index 20ee046a1..08af0cc1f 100644 --- a/src/libstore/ca-specific-schema.sql +++ b/src/libstore/ca-specific-schema.sql @@ -3,10 +3,19 @@ -- is enabled create table if not exists Realisations ( + id integer primary key autoincrement not null, drvPath text not null, outputName text not null, -- symbolic output id, usually "out" outputPath integer not null, signatures text, -- space-separated list - primary key (drvPath, outputName), foreign key (outputPath) references ValidPaths(id) on delete cascade ); + +create index if not exists IndexRealisations on Realisations(drvPath, outputName); + +create table if not exists RealisationsRefs ( + referrer integer not null, + realisationReference integer, + foreign key (referrer) references Realisations(id) on delete cascade, + foreign key (realisationReference) references Realisations(id) on delete restrict +); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 28c175a51..d7c7f8e1d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -53,12 +53,15 @@ struct LocalStore::State::Stmts { SQLiteStmt InvalidatePath; SQLiteStmt AddDerivationOutput; SQLiteStmt RegisterRealisedOutput; + SQLiteStmt UpdateRealisedOutput; SQLiteStmt QueryValidDerivers; SQLiteStmt QueryDerivationOutputs; SQLiteStmt QueryRealisedOutput; SQLiteStmt QueryAllRealisedOutputs; SQLiteStmt QueryPathFromHashPart; SQLiteStmt QueryValidPaths; + SQLiteStmt QueryRealisationReferences; + SQLiteStmt AddRealisationReference; }; int getSchema(Path schemaPath) @@ -76,7 +79,7 @@ int getSchema(Path schemaPath) void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) { - const int nixCASchemaVersion = 1; + const int nixCASchemaVersion = 2; int curCASchema = getSchema(schemaPath); if (curCASchema != nixCASchemaVersion) { if (curCASchema > nixCASchemaVersion) { @@ -94,7 +97,39 @@ void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd) #include "ca-specific-schema.sql.gen.hh" ; db.exec(schema); + curCASchema = nixCASchemaVersion; } + + if (curCASchema < 2) { + SQLiteTxn txn(db); + // Ugly little sql dance to add a new `id` column and make it the primary key + db.exec(R"( + create table Realisations2 ( + id integer primary key autoincrement not null, + drvPath text not null, + outputName text not null, -- symbolic output id, usually "out" + outputPath integer not null, + signatures text, -- space-separated list + foreign key (outputPath) references ValidPaths(id) on delete cascade + ); + insert into Realisations2 (drvPath, outputName, outputPath, signatures) + select drvPath, outputName, outputPath, signatures from Realisations; + drop table Realisations; + alter table Realisations2 rename to Realisations; + )"); + db.exec(R"( + create index if not exists IndexRealisations on Realisations(drvPath, outputName); + + create table if not exists RealisationsRefs ( + referrer integer not null, + realisationReference integer, + foreign key (referrer) references Realisations(id) on delete cascade, + foreign key (realisationReference) references Realisations(id) on delete restrict + ); + )"); + txn.commit(); + } + writeFile(schemaPath, fmt("%d", nixCASchemaVersion)); lockFile(lockFd.get(), ltRead, true); } @@ -311,9 +346,18 @@ LocalStore::LocalStore(const Params & params) values (?, ?, (select id from ValidPaths where path = ?), ?) ; )"); + state->stmts->UpdateRealisedOutput.create(state->db, + R"( + update Realisations + set signatures = ? + where + drvPath = ? and + outputName = ? + ; + )"); state->stmts->QueryRealisedOutput.create(state->db, R"( - select Output.path, Realisations.signatures from Realisations + select Realisations.id, Output.path, Realisations.signatures from Realisations inner join ValidPaths as Output on Output.id = Realisations.outputPath where drvPath = ? and outputName = ? ; @@ -325,6 +369,19 @@ LocalStore::LocalStore(const Params & params) where drvPath = ? ; )"); + state->stmts->QueryRealisationReferences.create(state->db, + R"( + select drvPath, outputName from Realisations + join RealisationsRefs on realisationReference = Realisations.id + where referrer = ?; + )"); + state->stmts->AddRealisationReference.create(state->db, + R"( + insert or replace into RealisationsRefs (referrer, realisationReference) + values ( + ?, + (select id from Realisations where drvPath = ? and outputName = ?)); + )"); } } @@ -661,14 +718,54 @@ void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag check void LocalStore::registerDrvOutput(const Realisation & info) { settings.requireExperimentalFeature("ca-derivations"); - auto state(_state.lock()); retrySQLite<void>([&]() { - state->stmts->RegisterRealisedOutput.use() - (info.id.strHash()) - (info.id.outputName) - (printStorePath(info.outPath)) - (concatStringsSep(" ", info.signatures)) - .exec(); + auto state(_state.lock()); + if (auto oldR = queryRealisation_(*state, info.id)) { + if (info.isCompatibleWith(*oldR)) { + auto combinedSignatures = oldR->signatures; + combinedSignatures.insert(info.signatures.begin(), + info.signatures.end()); + state->stmts->UpdateRealisedOutput.use() + (concatStringsSep(" ", combinedSignatures)) + (info.id.strHash()) + (info.id.outputName) + .exec(); + } else { + throw Error("Trying to register a realisation of '%s', but we already " + "have another one locally.\n" + "Local: %s\n" + "Remote: %s", + info.id.to_string(), + printStorePath(oldR->outPath), + printStorePath(info.outPath) + ); + } + } else { + state->stmts->RegisterRealisedOutput.use() + (info.id.strHash()) + (info.id.outputName) + (printStorePath(info.outPath)) + (concatStringsSep(" ", info.signatures)) + .exec(); + } + uint64_t myId = state->db.getLastInsertedRowId(); + for (auto & [outputId, depPath] : info.dependentRealisations) { + auto localRealisation = queryRealisationCore_(*state, outputId); + if (!localRealisation) + throw Error("unable to register the derivation '%s' as it " + "depends on the non existent '%s'", + info.id.to_string(), outputId.to_string()); + if (localRealisation->second.outPath != depPath) + throw Error("unable to register the derivation '%s' as it " + "depends on a realisation of '%s' that doesn’t" + "match what we have locally", + info.id.to_string(), outputId.to_string()); + state->stmts->AddRealisationReference.use() + (myId) + (outputId.strHash()) + (outputId.outputName) + .exec(); + } }); } @@ -1679,23 +1776,69 @@ void LocalStore::createUser(const std::string & userName, uid_t userId) } } -std::optional<const Realisation> LocalStore::queryRealisation( - const DrvOutput& id) { - typedef std::optional<const Realisation> Ret; - return retrySQLite<Ret>([&]() -> Ret { +std::optional<std::pair<int64_t, Realisation>> LocalStore::queryRealisationCore_( + LocalStore::State & state, + const DrvOutput & id) +{ + auto useQueryRealisedOutput( + state.stmts->QueryRealisedOutput.use() + (id.strHash()) + (id.outputName)); + if (!useQueryRealisedOutput.next()) + return std::nullopt; + auto realisationDbId = useQueryRealisedOutput.getInt(0); + auto outputPath = parseStorePath(useQueryRealisedOutput.getStr(1)); + auto signatures = + tokenizeString<StringSet>(useQueryRealisedOutput.getStr(2)); + + return {{ + realisationDbId, + Realisation{ + .id = id, + .outPath = outputPath, + .signatures = signatures, + } + }}; +} + +std::optional<const Realisation> LocalStore::queryRealisation_( + LocalStore::State & state, + const DrvOutput & id) +{ + auto maybeCore = queryRealisationCore_(state, id); + if (!maybeCore) + return std::nullopt; + auto [realisationDbId, res] = *maybeCore; + + std::map<DrvOutput, StorePath> dependentRealisations; + auto useRealisationRefs( + state.stmts->QueryRealisationReferences.use() + (realisationDbId)); + while (useRealisationRefs.next()) { + auto depId = DrvOutput { + Hash::parseAnyPrefixed(useRealisationRefs.getStr(0)), + useRealisationRefs.getStr(1), + }; + auto dependentRealisation = queryRealisationCore_(state, depId); + assert(dependentRealisation); // Enforced by the db schema + auto outputPath = dependentRealisation->second.outPath; + dependentRealisations.insert({depId, outputPath}); + } + + res.dependentRealisations = dependentRealisations; + + return { res }; +} + +std::optional<const Realisation> +LocalStore::queryRealisation(const DrvOutput & id) +{ + return retrySQLite<std::optional<const Realisation>>([&]() { auto state(_state.lock()); - auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())( - id.outputName)); - if (!use.next()) - return std::nullopt; - auto outputPath = parseStorePath(use.getStr(0)); - auto signatures = tokenizeString<StringSet>(use.getStr(1)); - return Ret{Realisation{ - .id = id, .outPath = outputPath, .signatures = signatures}}; + return queryRealisation_(*state, id); }); } - FixedOutputHash LocalStore::hashCAPath( const FileIngestionMethod & method, const HashType & hashType, const StorePath & path) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 15c7fc306..a01d48c4b 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -203,6 +203,8 @@ public: void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override; void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output); + std::optional<const Realisation> queryRealisation_(State & state, const DrvOutput & id); + std::optional<std::pair<int64_t, Realisation>> queryRealisationCore_(State & state, const DrvOutput & id); std::optional<const Realisation> queryRealisation(const DrvOutput&) override; private: diff --git a/src/libstore/local.mk b/src/libstore/local.mk index b6652984c..2fc334a82 100644 --- a/src/libstore/local.mk +++ b/src/libstore/local.mk @@ -9,11 +9,11 @@ libstore_SOURCES := $(wildcard $(d)/*.cc $(d)/builtins/*.cc $(d)/build/*.cc) libstore_LIBS = libutil libstore_LDFLAGS = $(SQLITE3_LIBS) -lbz2 $(LIBCURL_LIBS) $(SODIUM_LIBS) -pthread -ifeq ($(OS), Linux) +ifdef HOST_LINUX libstore_LDFLAGS += -ldl endif -ifeq ($(OS), Darwin) +ifdef HOST_DARWIN libstore_FILES = sandbox-defaults.sb sandbox-minimal.sb sandbox-network.sb endif @@ -23,7 +23,7 @@ ifeq ($(ENABLE_S3), 1) libstore_LDFLAGS += -laws-cpp-sdk-transfer -laws-cpp-sdk-s3 -laws-cpp-sdk-core endif -ifeq ($(OS), SunOS) +ifdef HOST_SOLARIS libstore_LDFLAGS += -lsocket endif diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc index b42e5e434..9843ccf04 100644 --- a/src/libstore/machines.cc +++ b/src/libstore/machines.cc @@ -16,13 +16,18 @@ Machine::Machine(decltype(storeUri) storeUri, decltype(mandatoryFeatures) mandatoryFeatures, decltype(sshPublicHostKey) sshPublicHostKey) : storeUri( - // Backwards compatibility: if the URI is a hostname, - // prepend ssh://. + // Backwards compatibility: if the URI is schemeless, is not a path, + // and is not one of the special store connection words, prepend + // ssh://. storeUri.find("://") != std::string::npos - || hasPrefix(storeUri, "local") - || hasPrefix(storeUri, "remote") - || hasPrefix(storeUri, "auto") - || hasPrefix(storeUri, "/") + || storeUri.find("/") != std::string::npos + || storeUri == "auto" + || storeUri == "daemon" + || storeUri == "local" + || hasPrefix(storeUri, "auto?") + || hasPrefix(storeUri, "daemon?") + || hasPrefix(storeUri, "local?") + || hasPrefix(storeUri, "?") ? storeUri : "ssh://" + storeUri), systemTypes(systemTypes), diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index bc5fd968c..b4929b445 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -29,9 +29,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths, res.insert(i); if (includeDerivers && path.isDerivation()) - for (auto& i : queryDerivationOutputs(path)) - if (isValidPath(i) && queryPathInfo(i)->deriver == path) - res.insert(i); + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + if (maybeOutPath && isValidPath(*maybeOutPath)) + res.insert(*maybeOutPath); return res; }; else @@ -44,9 +44,9 @@ void Store::computeFSClosure(const StorePathSet & startPaths, res.insert(ref); if (includeOutputs && path.isDerivation()) - for (auto& i : queryDerivationOutputs(path)) - if (isValidPath(i)) - res.insert(i); + for (auto& [_, maybeOutPath] : queryPartialDerivationOutputMap(path)) + if (maybeOutPath && isValidPath(*maybeOutPath)) + res.insert(*maybeOutPath); if (includeDerivers && info->deriver && isValidPath(*info->deriver)) res.insert(*info->deriver); @@ -254,5 +254,44 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) }}); } +std::map<DrvOutput, StorePath> drvOutputReferences( + const std::set<Realisation> & inputRealisations, + const StorePathSet & pathReferences) +{ + std::map<DrvOutput, StorePath> res; + + for (const auto & input : inputRealisations) { + if (pathReferences.count(input.outPath)) { + res.insert({input.id, input.outPath}); + } + } + + return res; +} +std::map<DrvOutput, StorePath> drvOutputReferences( + Store & store, + const Derivation & drv, + const StorePath & outputPath) +{ + std::set<Realisation> inputRealisations; + + for (const auto& [inputDrv, outputNames] : drv.inputDrvs) { + auto outputHashes = + staticOutputHashes(store, store.readDerivation(inputDrv)); + for (const auto& outputName : outputNames) { + auto thisRealisation = store.queryRealisation( + DrvOutput{outputHashes.at(outputName), outputName}); + if (!thisRealisation) + throw Error( + "output '%s' of derivation '%s' isn’t built", outputName, + store.printStorePath(inputDrv)); + inputRealisations.insert(*thisRealisation); + } + } + + auto info = store.queryPathInfo(outputPath); + + return drvOutputReferences(Realisation::closure(store, inputRealisations), info->references); +} } diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc index ee6bbb045..345235d66 100644 --- a/src/libstore/parsed-derivations.cc +++ b/src/libstore/parsed-derivations.cc @@ -93,6 +93,8 @@ StringSet ParsedDerivation::getRequiredSystemFeatures() const StringSet res; for (auto & i : getStringsAttr("requiredSystemFeatures").value_or(Strings())) res.insert(i); + if (!derivationHasKnownOutputPaths(drv.type())) + res.insert("ca-derivations"); return res; } diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 638065547..eadec594c 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -1,5 +1,6 @@ #include "realisation.hh" #include "store-api.hh" +#include "closure.hh" #include <nlohmann/json.hpp> namespace nix { @@ -21,11 +22,52 @@ std::string DrvOutput::to_string() const { return strHash() + "!" + outputName; } +std::set<Realisation> Realisation::closure(Store & store, const std::set<Realisation> & startOutputs) +{ + std::set<Realisation> res; + Realisation::closure(store, startOutputs, res); + return res; +} + +void Realisation::closure(Store & store, const std::set<Realisation> & startOutputs, std::set<Realisation> & res) +{ + auto getDeps = [&](const Realisation& current) -> std::set<Realisation> { + std::set<Realisation> res; + for (auto& [currentDep, _] : current.dependentRealisations) { + if (auto currentRealisation = store.queryRealisation(currentDep)) + res.insert(*currentRealisation); + else + throw Error( + "Unrealised derivation '%s'", currentDep.to_string()); + } + return res; + }; + + computeClosure<Realisation>( + startOutputs, res, + [&](const Realisation& current, + std::function<void(std::promise<std::set<Realisation>>&)> + processEdges) { + std::promise<std::set<Realisation>> promise; + try { + auto res = getDeps(current); + promise.set_value(res); + } catch (...) { + promise.set_exception(std::current_exception()); + } + return processEdges(promise); + }); +} + nlohmann::json Realisation::toJSON() const { + auto jsonDependentRealisations = nlohmann::json::object(); + for (auto & [depId, depOutPath] : dependentRealisations) + jsonDependentRealisations.emplace(depId.to_string(), depOutPath.to_string()); return nlohmann::json{ {"id", id.to_string()}, {"outPath", outPath.to_string()}, {"signatures", signatures}, + {"dependentRealisations", jsonDependentRealisations}, }; } @@ -51,10 +93,16 @@ Realisation Realisation::fromJSON( if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end()) signatures.insert(signaturesIterator->begin(), signaturesIterator->end()); + std::map <DrvOutput, StorePath> dependentRealisations; + if (auto jsonDependencies = json.find("dependentRealisations"); jsonDependencies != json.end()) + for (auto & [jsonDepId, jsonDepOutPath] : jsonDependencies->get<std::map<std::string, std::string>>()) + dependentRealisations.insert({DrvOutput::parse(jsonDepId), StorePath(jsonDepOutPath)}); + return Realisation{ .id = DrvOutput::parse(getField("id")), .outPath = StorePath(getField("outPath")), .signatures = signatures, + .dependentRealisations = dependentRealisations, }; } @@ -92,6 +140,16 @@ StorePath RealisedPath::path() const { return std::visit([](auto && arg) { return arg.getPath(); }, raw); } +bool Realisation::isCompatibleWith(const Realisation & other) const +{ + assert (id == other.id); + if (outPath == other.outPath) { + assert(dependentRealisations == other.dependentRealisations); + return true; + } + return false; +} + void RealisedPath::closure( Store& store, const RealisedPath::Set& startPaths, diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh index f5049c9e9..05d2bc44f 100644 --- a/src/libstore/realisation.hh +++ b/src/libstore/realisation.hh @@ -28,6 +28,14 @@ struct Realisation { StringSet signatures; + /** + * The realisations that are required for the current one to be valid. + * + * When importing this realisation, the store will first check that all its + * dependencies exist, and map to the correct output path + */ + std::map<DrvOutput, StorePath> dependentRealisations; + nlohmann::json toJSON() const; static Realisation fromJSON(const nlohmann::json& json, const std::string& whence); @@ -36,6 +44,11 @@ struct Realisation { bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; size_t checkSignatures(const PublicKeys & publicKeys) const; + static std::set<Realisation> closure(Store &, const std::set<Realisation> &); + static void closure(Store &, const std::set<Realisation> &, std::set<Realisation>& res); + + bool isCompatibleWith(const Realisation & other) const; + StorePath getPath() const { return outPath; } GENERATE_CMP(Realisation, me->id, me->outPath); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index cfdfb5269..18b9f3256 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -337,6 +337,13 @@ ValidPathInfo Store::addToStoreSlow(std::string_view name, const Path & srcPath, return info; } +StringSet StoreConfig::getDefaultSystemFeatures() +{ + auto res = settings.systemFeatures.get(); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) + res.insert("ca-derivations"); + return res; +} Store::Store(const Params & params) : StoreConfig(params) @@ -816,20 +823,39 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute) { StorePathSet storePaths; - std::set<Realisation> realisations; + std::set<Realisation> toplevelRealisations; for (auto & path : paths) { storePaths.insert(path.path()); if (auto realisation = std::get_if<Realisation>(&path.raw)) { settings.requireExperimentalFeature("ca-derivations"); - realisations.insert(*realisation); + toplevelRealisations.insert(*realisation); } } auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute); + + ThreadPool pool; + try { - for (auto & realisation : realisations) { - dstStore->registerDrvOutput(realisation, checkSigs); - } - } catch (MissingExperimentalFeature & e) { + // Copy the realisation closure + processGraph<Realisation>( + pool, Realisation::closure(*srcStore, toplevelRealisations), + [&](const Realisation& current) -> std::set<Realisation> { + std::set<Realisation> children; + for (const auto& [drvOutput, _] : current.dependentRealisations) { + auto currentChild = srcStore->queryRealisation(drvOutput); + if (!currentChild) + throw Error( + "Incomplete realisation closure: '%s' is a " + "dependency of '%s' but isn’t registered", + drvOutput.to_string(), current.id.to_string()); + children.insert(*currentChild); + } + return children; + }, + [&](const Realisation& current) -> void { + dstStore->registerDrvOutput(current, checkSigs); + }); + } catch (MissingExperimentalFeature& e) { // Don't fail if the remote doesn't support CA derivations is it might // not be within our control to change that, and we might still want // to at least copy the output paths. diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 7415cb54c..1b7645488 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -180,6 +180,8 @@ struct StoreConfig : public Config StoreConfig() = delete; + StringSet getDefaultSystemFeatures(); + virtual ~StoreConfig() { } virtual const std::string name() = 0; @@ -196,7 +198,7 @@ struct StoreConfig : public Config Setting<bool> wantMassQuery{this, false, "want-mass-query", "whether this substituter can be queried efficiently for path validity"}; - Setting<StringSet> systemFeatures{this, settings.systemFeatures, + Setting<StringSet> systemFeatures{this, getDefaultSystemFeatures(), "system-features", "Optional features that the system this store builds on implements (like \"kvm\")."}; @@ -869,4 +871,9 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri) std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv); +std::map<DrvOutput, StorePath> drvOutputReferences( + Store & store, + const Derivation & drv, + const StorePath & outputPath); + } diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh index 0315dc506..eecd5b819 100644 --- a/src/libutil/comparator.hh +++ b/src/libutil/comparator.hh @@ -25,6 +25,8 @@ } #define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args) #define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args) +#define GENERATE_NEQ(args...) GENERATE_ONE_CMP(!=, args) #define GENERATE_CMP(args...) \ GENERATE_EQUAL(args) \ - GENERATE_LEQ(args) + GENERATE_LEQ(args) \ + GENERATE_NEQ(args) diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index d2e801175..6b9b850ca 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -46,7 +46,7 @@ public: : printBuildLogs(printBuildLogs) { systemd = getEnv("IN_SYSTEMD") == "1"; - tty = isatty(STDERR_FILENO); + tty = shouldANSI(); } bool isVerbose() override { diff --git a/src/libutil/url.cc b/src/libutil/url.cc index c1bab866c..f6232d255 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -32,7 +32,7 @@ ParsedURL parseURL(const std::string & url) auto isFile = scheme.find("file") != std::string::npos; if (authority && *authority != "" && isFile) - throw Error("file:// URL '%s' has unexpected authority '%s'", + throw BadURL("file:// URL '%s' has unexpected authority '%s'", url, *authority); if (isFile && path.empty()) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 7e57fd7ca..ee9f17228 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -1372,6 +1372,12 @@ void ignoreException() } } +bool shouldANSI() +{ + return isatty(STDERR_FILENO) + && getEnv("TERM").value_or("dumb") != "dumb" + && !getEnv("NO_COLOR").has_value(); +} std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned int width) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index f84d0fb31..a8dd4bd47 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -482,6 +482,9 @@ constexpr char treeLast[] = "└───"; constexpr char treeLine[] = "│ "; constexpr char treeNull[] = " "; +/* Determine whether ANSI escape sequences are appropriate for the + present output. */ +bool shouldANSI(); /* Truncate a string to 'width' printable characters. If 'filterAll' is true, all ANSI escape sequences are filtered out. Otherwise, diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc index 519cbd5bb..29be46cf5 100755 --- a/src/nix-build/nix-build.cc +++ b/src/nix-build/nix-build.cc @@ -392,6 +392,12 @@ static void main_nix_build(int argc, char * * argv) if (dryRun) return; + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + auto resolvedDrv = drv.tryResolve(*store); + assert(resolvedDrv && "Successfully resolved the derivation"); + drv = *resolvedDrv; + } + // Set the environment. auto env = getEnv(); diff --git a/src/nix/develop.cc b/src/nix/develop.cc index ee782c4ec..26f53db09 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -8,7 +8,7 @@ #include "affinity.hh" #include "progress-bar.hh" -#include <regex> +#include <nlohmann/json.hpp> using namespace nix; @@ -25,94 +25,142 @@ static DevelopSettings developSettings; static GlobalConfig::Register rDevelopSettings(&developSettings); -struct Var -{ - bool exported = true; - bool associative = false; - std::string quoted; // quoted string or array -}; - struct BuildEnvironment { - std::map<std::string, Var> env; - std::string bashFunctions; -}; + struct String + { + bool exported; + std::string value; -BuildEnvironment readEnvironment(const Path & path) -{ - BuildEnvironment res; + bool operator == (const String & other) const + { + return exported == other.exported && value == other.value; + } + }; - std::set<std::string> exported; + using Array = std::vector<std::string>; - debug("reading environment file '%s'", path); + using Associative = std::map<std::string, std::string>; - auto file = readFile(path); + using Value = std::variant<String, Array, Associative>; - auto pos = file.cbegin(); + std::map<std::string, Value> vars; + std::map<std::string, std::string> bashFunctions; - static std::string varNameRegex = - R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re"; + static BuildEnvironment fromJSON(std::string_view in) + { + BuildEnvironment res; - static std::string simpleStringRegex = - R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re"; + std::set<std::string> exported; - static std::string dquotedStringRegex = - R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re"; + auto json = nlohmann::json::parse(in); - static std::string squotedStringRegex = - R"re((?:\$?(?:'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'|\\')+))re"; + for (auto & [name, info] : json["variables"].items()) { + std::string type = info["type"]; + if (type == "var" || type == "exported") + res.vars.insert({name, BuildEnvironment::String { .exported = type == "exported", .value = info["value"] }}); + else if (type == "array") + res.vars.insert({name, (Array) info["value"]}); + else if (type == "associative") + res.vars.insert({name, (Associative) info["value"]}); + } - static std::string indexedArrayRegex = - R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re"; + for (auto & [name, def] : json["bashFunctions"].items()) { + res.bashFunctions.insert({name, def}); + } - static std::regex declareRegex( - "^declare -a?x (" + varNameRegex + ")(=(" + - dquotedStringRegex + "|" + indexedArrayRegex + "))?\n"); + return res; + } - static std::regex varRegex( - "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + squotedStringRegex + "|" + indexedArrayRegex + ")\n"); + std::string toJSON() const + { + auto res = nlohmann::json::object(); + + auto vars2 = nlohmann::json::object(); + for (auto & [name, value] : vars) { + auto info = nlohmann::json::object(); + if (auto str = std::get_if<String>(&value)) { + info["type"] = str->exported ? "exported" : "var"; + info["value"] = str->value; + } + else if (auto arr = std::get_if<Array>(&value)) { + info["type"] = "array"; + info["value"] = *arr; + } + else if (auto arr = std::get_if<Associative>(&value)) { + info["type"] = "associative"; + info["value"] = *arr; + } + vars2[name] = std::move(info); + } + res["variables"] = std::move(vars2); - /* Note: we distinguish between an indexed and associative array - using the space before the closing parenthesis. Will - undoubtedly regret this some day. */ - static std::regex assocArrayRegex( - "^(" + varNameRegex + ")=" + R"re((?:\(( *\[[^\]]+\]="(?:[^"\\]|\\.)*")* *\)))re" + "\n"); + res["bashFunctions"] = bashFunctions; - static std::regex functionRegex( - "^" + varNameRegex + " \\(\\) *\n"); + auto json = res.dump(); - while (pos != file.end()) { + assert(BuildEnvironment::fromJSON(json) == *this); - std::smatch match; + return json; + } - if (std::regex_search(pos, file.cend(), match, declareRegex, std::regex_constants::match_continuous)) { - pos = match[0].second; - exported.insert(match[1]); + void toBash(std::ostream & out, const std::set<std::string> & ignoreVars) const + { + for (auto & [name, value] : vars) { + if (!ignoreVars.count(name)) { + if (auto str = std::get_if<String>(&value)) { + out << fmt("%s=%s\n", name, shellEscape(str->value)); + if (str->exported) + out << fmt("export %s\n", name); + } + else if (auto arr = std::get_if<Array>(&value)) { + out << "declare -a " << name << "=("; + for (auto & s : *arr) + out << shellEscape(s) << " "; + out << ")\n"; + } + else if (auto arr = std::get_if<Associative>(&value)) { + out << "declare -A " << name << "=("; + for (auto & [n, v] : *arr) + out << "[" << shellEscape(n) << "]=" << shellEscape(v) << " "; + out << ")\n"; + } + } } - else if (std::regex_search(pos, file.cend(), match, varRegex, std::regex_constants::match_continuous)) { - pos = match[0].second; - res.env.insert({match[1], Var { .exported = exported.count(match[1]) > 0, .quoted = match[2] }}); + for (auto & [name, def] : bashFunctions) { + out << name << " ()\n{\n" << def << "}\n"; } + } - else if (std::regex_search(pos, file.cend(), match, assocArrayRegex, std::regex_constants::match_continuous)) { - pos = match[0].second; - res.env.insert({match[1], Var { .associative = true, .quoted = match[2] }}); - } + static std::string getString(const Value & value) + { + if (auto str = std::get_if<String>(&value)) + return str->value; + else + throw Error("bash variable is not a string"); + } - else if (std::regex_search(pos, file.cend(), match, functionRegex, std::regex_constants::match_continuous)) { - res.bashFunctions = std::string(pos, file.cend()); - break; + static Array getStrings(const Value & value) + { + if (auto str = std::get_if<String>(&value)) + return tokenizeString<Array>(str->value); + else if (auto arr = std::get_if<Array>(&value)) { + return *arr; + } else if (auto assoc = std::get_if<Associative>(&value)) { + Array assocKeys; + std::for_each(assoc->begin(), assoc->end(), [&](auto & n) { assocKeys.push_back(n.first); }); + return assocKeys; } - - else throw Error("shell environment '%s' has unexpected line '%s'", - path, file.substr(pos - file.cbegin(), 60)); + else + throw Error("bash variable is not a string or array"); } - res.env.erase("__output"); - - return res; -} + bool operator == (const BuildEnvironment & other) const + { + return vars == other.vars && bashFunctions == other.bashFunctions; + } +}; const static std::string getEnvSh = #include "get-env.sh.gen.hh" @@ -144,17 +192,26 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath) /* Rehash and write the derivation. FIXME: would be nice to use 'buildDerivation', but that's privileged. */ drv.name += "-env"; - for (auto & output : drv.outputs) { - output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; - drv.env[output.first] = ""; - } drv.inputSrcs.insert(std::move(getEnvShPath)); - Hash h = std::get<0>(hashDerivationModulo(*store, drv, true)); + if (settings.isExperimentalFeatureEnabled("ca-derivations")) { + for (auto & output : drv.outputs) { + output.second = { + .output = DerivationOutputDeferred{}, + }; + drv.env[output.first] = hashPlaceholder(output.first); + } + } else { + for (auto & output : drv.outputs) { + output.second = { .output = DerivationOutputInputAddressed { .path = StorePath::dummy } }; + drv.env[output.first] = ""; + } + Hash h = std::get<0>(hashDerivationModulo(*store, drv, true)); - for (auto & output : drv.outputs) { - auto outPath = store->makeOutputPath(output.first, h, drv.name); - output.second = { .output = DerivationOutputInputAddressed { .path = outPath } }; - drv.env[output.first] = store->printStorePath(outPath); + for (auto & output : drv.outputs) { + auto outPath = store->makeOutputPath(output.first, h, drv.name); + output.second = { .output = DerivationOutputInputAddressed { .path = outPath } }; + drv.env[output.first] = store->printStorePath(outPath); + } } auto shellDrvPath = writeDerivation(*store, drv); @@ -162,8 +219,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath) /* Build the derivation. */ store->buildPaths({DerivedPath::Built{shellDrvPath}}); - for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) { - auto & [_1, optPath] = outputAndOptPath; + for (auto & [_0, optPath] : store->queryPartialDerivationOutputMap(shellDrvPath)) { assert(optPath); auto & outPath = *optPath; assert(store->isValidPath(outPath)); @@ -177,19 +233,15 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath) struct Common : InstallableCommand, MixProfile { - std::set<string> ignoreVars{ + std::set<std::string> ignoreVars{ "BASHOPTS", - "EUID", "HOME", // FIXME: don't ignore in pure mode? - "HOSTNAME", "NIX_BUILD_TOP", "NIX_ENFORCE_PURITY", "NIX_LOG_FD", "NIX_REMOTE", "PPID", - "PWD", "SHELLOPTS", - "SHLVL", "SSL_CERT_FILE", // FIXME: only want to ignore /no-cert-file.crt "TEMP", "TEMPDIR", @@ -225,22 +277,10 @@ struct Common : InstallableCommand, MixProfile out << "nix_saved_PATH=\"$PATH\"\n"; - for (auto & i : buildEnvironment.env) { - if (!ignoreVars.count(i.first) && !hasPrefix(i.first, "BASH_")) { - if (i.second.associative) - out << fmt("declare -A %s=(%s)\n", i.first, i.second.quoted); - else { - out << fmt("%s=%s\n", i.first, i.second.quoted); - if (i.second.exported) - out << fmt("export %s\n", i.first); - } - } - } + buildEnvironment.toBash(out, ignoreVars); out << "PATH=\"$PATH:$nix_saved_PATH\"\n"; - out << buildEnvironment.bashFunctions << "\n"; - out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n"; for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"}) out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i); @@ -250,24 +290,16 @@ struct Common : InstallableCommand, MixProfile auto script = out.str(); /* Substitute occurrences of output paths. */ - auto outputs = buildEnvironment.env.find("outputs"); - assert(outputs != buildEnvironment.env.end()); + auto outputs = buildEnvironment.vars.find("outputs"); + assert(outputs != buildEnvironment.vars.end()); // FIXME: properly unquote 'outputs'. StringMap rewrites; - for (auto & outputName : tokenizeString<std::vector<std::string>>(replaceStrings(outputs->second.quoted, "'", ""))) { - // Hacky way to obtain the key of an associate array. This is needed for strctured attrs where - // `outputs` is an associative array. If the regex isn't matched, the non-structured-attrs behavior will - // be used. - std::regex ptrn(R"re(\[([A-z0-9]+)\]=.*)re"); - std::smatch match; - if (std::regex_match(outputName, match, ptrn)) { - outputName = match[1]; - } - auto from = buildEnvironment.env.find(outputName); - assert(from != buildEnvironment.env.end()); + for (auto & outputName : BuildEnvironment::getStrings(outputs->second)) { + auto from = buildEnvironment.vars.find(outputName); + assert(from != buildEnvironment.vars.end()); // FIXME: unquote - rewrites.insert({from->second.quoted, outputsDir + "/" + outputName}); + rewrites.insert({BuildEnvironment::getString(from->second), outputsDir + "/" + outputName}); } /* Substitute redirects. */ @@ -321,7 +353,9 @@ struct Common : InstallableCommand, MixProfile updateProfile(shellOutPath); - return {readEnvironment(strPath), strPath}; + debug("reading environment file '%s'", strPath); + + return {BuildEnvironment::fromJSON(readFile(strPath)), strPath}; } }; @@ -443,13 +477,17 @@ struct CmdDevelop : Common, MixEnvironment try { auto state = getEvalState(); + auto nixpkgsLockFlags = lockFlags; + nixpkgsLockFlags.inputOverrides = {}; + nixpkgsLockFlags.inputUpdates = {}; + auto bashInstallable = std::make_shared<InstallableFlake>( this, state, installable->nixpkgsFlakeRef(), Strings{"bashInteractive"}, Strings{"legacyPackages." + settings.thisSystem.get() + "."}, - lockFlags); + nixpkgsLockFlags); shell = state->store->printStorePath( toStorePath(state->store, Realise::Outputs, OperateOn::Output, bashInstallable)) + "/bin/bash"; @@ -470,7 +508,7 @@ struct CmdDevelop : Common, MixEnvironment } }; -struct CmdPrintDevEnv : Common +struct CmdPrintDevEnv : Common, MixJSON { std::string description() override { @@ -492,7 +530,10 @@ struct CmdPrintDevEnv : Common stopProgressBar(); - std::cout << makeRcScript(store, buildEnvironment); + logger->writeToStdout( + json + ? buildEnvironment.toJSON() + : makeRcScript(store, buildEnvironment)); } }; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index 64fcfc000..9055b07eb 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -84,6 +84,7 @@ struct CmdFlakeUpdate : FlakeCommand lockFlags.recreateLockFile = true; lockFlags.writeLockFile = true; + lockFlags.applyNixConfig = true; lockFlake(); } @@ -114,6 +115,7 @@ struct CmdFlakeLock : FlakeCommand settings.tarballTtl = 0; lockFlags.writeLockFile = true; + lockFlags.applyNixConfig = true; lockFlake(); } @@ -270,6 +272,8 @@ struct CmdFlakeCheck : FlakeCommand settings.readOnlyMode = !build; auto state = getEvalState(); + + lockFlags.applyNixConfig = true; auto flake = lockFlake(); bool hasErrors = false; diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index 431440516..42c806450 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -8,6 +8,109 @@ if [[ -n $stdenv ]]; then source $stdenv/setup fi +# Better to use compgen, but stdenv bash doesn't have it. +__vars="$(declare -p)" +__functions="$(declare -F)" + +__dumpEnv() { + printf '{\n' + + printf ' "bashFunctions": {\n' + local __first=1 + while read __line; do + if ! [[ $__line =~ ^declare\ -f\ (.*) ]]; then continue; fi + __fun_name="${BASH_REMATCH[1]}" + __fun_body="$(type $__fun_name)" + if [[ $__fun_body =~ \{(.*)\} ]]; then + if [[ -z $__first ]]; then printf ',\n'; else __first=; fi + __fun_body="${BASH_REMATCH[1]}" + printf " " + __escapeString "$__fun_name" + printf ':' + __escapeString "$__fun_body" + else + printf "Cannot parse definition of function '%s'.\n" "$__fun_name" >&2 + return 1 + fi + done < <(printf "%s\n" "$__functions") + printf '\n },\n' + + printf ' "variables": {\n' + local __first=1 + while read __line; do + if ! [[ $__line =~ ^declare\ (-[^ ])\ ([^=]*) ]]; then continue; fi + local type="${BASH_REMATCH[1]}" + local __var_name="${BASH_REMATCH[2]}" + + if [[ $__var_name =~ ^BASH_ || \ + $__var_name = _ || \ + $__var_name = DIRSTACK || \ + $__var_name = EUID || \ + $__var_name = FUNCNAME || \ + $__var_name = HISTCMD || \ + $__var_name = HOSTNAME || \ + $__var_name = GROUPS || \ + $__var_name = PIPESTATUS || \ + $__var_name = PWD || \ + $__var_name = RANDOM || \ + $__var_name = SHLVL || \ + $__var_name = SECONDS \ + ]]; then continue; fi + + if [[ -z $__first ]]; then printf ',\n'; else __first=; fi + + printf " " + __escapeString "$__var_name" + printf ': {' + + # FIXME: handle -i, -r, -n. + if [[ $type == -x ]]; then + printf '"type": "exported", "value": ' + __escapeString "${!__var_name}" + elif [[ $type == -- ]]; then + printf '"type": "var", "value": ' + __escapeString "${!__var_name}" + elif [[ $type == -a ]]; then + printf '"type": "array", "value": [' + local __first2=1 + __var_name="$__var_name[@]" + for __i in "${!__var_name}"; do + if [[ -z $__first2 ]]; then printf ', '; else __first2=; fi + __escapeString "$__i" + printf ' ' + done + printf ']' + elif [[ $type == -A ]]; then + printf '"type": "associative", "value": {\n' + local __first2=1 + declare -n __var_name2="$__var_name" + for __i in "${!__var_name2[@]}"; do + if [[ -z $__first2 ]]; then printf ',\n'; else __first2=; fi + printf " " + __escapeString "$__i" + printf ": " + __escapeString "${__var_name2[$__i]}" + done + printf '\n }' + else + printf '"type": "unknown"' + fi + + printf "}" + done < <(printf "%s\n" "$__vars") + printf '\n }\n}' +} + +__escapeString() { + local __s="$1" + __s="${__s//\\/\\\\}" + __s="${__s//\"/\\\"}" + __s="${__s//$'\n'/\\n}" + __s="${__s//$'\r'/\\r}" + __s="${__s//$'\t'/\\t}" + printf '"%s"' "$__s" +} + # In case of `__structuredAttrs = true;` the list of outputs is an associative # array with a format like `outname => /nix/store/hash-drvname-outname`, so `__olist` # must contain the array's keys (hence `${!...[@]}`) in this case. @@ -19,8 +122,7 @@ fi for __output in $__olist; do if [[ -z $__done ]]; then - export > "${!__output}" - set >> "${!__output}" + __dumpEnv > ${!__output} __done=1 else echo -n >> "${!__output}" diff --git a/src/nix/print-dev-env.md b/src/nix/print-dev-env.md index b80252acf..2aad491de 100644 --- a/src/nix/print-dev-env.md +++ b/src/nix/print-dev-env.md @@ -8,12 +8,43 @@ R""( # . <(nix print-dev-env nixpkgs#hello) ``` +* Get the build environment in JSON format: + + ```console + # nix print-dev-env nixpkgs#hello --json + ``` + + The output will look like this: + + ```json + { + "bashFunctions": { + "buildPhase": " \n runHook preBuild;\n...", + ... + }, + "variables": { + "src": { + "type": "exported", + "value": "/nix/store/3x7dwzq014bblazs7kq20p9hyzz0qh8g-hello-2.10.tar.gz" + }, + "postUnpackHooks": { + "type": "array", + "value": ["_updateSourceDateEpochFromSourceRoot"] + }, + ... + } + } + ``` + # Description -This command prints a shell script that can be sourced by `b`ash and -that sets the environment variables and shell functions defined by the -build process of *installable*. This allows you to get a similar build +This command prints a shell script that can be sourced by `bash` and +that sets the variables and shell functions defined by the build +process of *installable*. This allows you to get a similar build environment in your current shell rather than in a subshell (as with `nix develop`). +With `--json`, the output is a JSON serialisation of the variables and +functions defined by the build process. + )"" diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md index dcf825da9..ba85441d8 100644 --- a/src/nix/profile-remove.md +++ b/src/nix/profile-remove.md @@ -15,6 +15,7 @@ R""( ``` * Remove all packages: + ```console # nix profile remove '.*' ``` diff --git a/src/nix/registry-add.md b/src/nix/registry-add.md index 80a31996a..a947fa0b3 100644 --- a/src/nix/registry-add.md +++ b/src/nix/registry-add.md @@ -21,6 +21,13 @@ R""( # nix registry add nixpkgs/nixos-20.03 ~/Dev/nixpkgs ``` +* Add `nixpkgs` pointing to `github:nixos/nixpkgs` to your custom flake + registry: + + ```console + nix registry add --registry ./custom-flake-registry.json nixpkgs github:nixos/nixpkgs + ``` + # Description This command adds an entry to the user registry that maps flake diff --git a/src/nix/registry-pin.md b/src/nix/registry-pin.md index 6e97e003e..ebc0e3eff 100644 --- a/src/nix/registry-pin.md +++ b/src/nix/registry-pin.md @@ -24,6 +24,13 @@ R""( … ``` +* Pin `nixpkgs` in a custom registry to its most recent Git revision: + + ```console + # nix registry pin --registry ./custom-flake-registry.json nixpkgs + ``` + + # Description This command adds an entry to the user registry that maps flake diff --git a/src/nix/registry-remove.md b/src/nix/registry-remove.md index 4c0eb4947..eecd4c6e7 100644 --- a/src/nix/registry-remove.md +++ b/src/nix/registry-remove.md @@ -8,6 +8,12 @@ R""( # nix registry remove nixpkgs ``` +* Remove the entry `nixpkgs` from a custom registry: + + ```console + # nix registry remove --registry ./custom-flake-registry.json nixpkgs + ``` + # Description This command removes from the user registry any entry for flake diff --git a/src/nix/registry.cc b/src/nix/registry.cc index f9719600f..6a92576c7 100644 --- a/src/nix/registry.cc +++ b/src/nix/registry.cc @@ -10,6 +10,46 @@ using namespace nix; using namespace nix::flake; + +class RegistryCommand : virtual Args +{ + std::string registry_path; + + std::shared_ptr<fetchers::Registry> registry; + +public: + + RegistryCommand() + { + addFlag({ + .longName = "registry", + .description = "The registry to operate on.", + .labels = {"registry"}, + .handler = {®istry_path}, + }); + } + + std::shared_ptr<fetchers::Registry> getRegistry() + { + if (registry) return registry; + if (registry_path.empty()) { + registry = fetchers::getUserRegistry(); + } else { + registry = fetchers::getCustomRegistry(registry_path); + } + return registry; + } + + Path getRegistryPath() + { + if (registry_path.empty()) { + return fetchers::getUserRegistryPath(); + } else { + return registry_path; + } + } +}; + struct CmdRegistryList : StoreCommand { std::string description() override @@ -45,7 +85,7 @@ struct CmdRegistryList : StoreCommand } }; -struct CmdRegistryAdd : MixEvalArgs, Command +struct CmdRegistryAdd : MixEvalArgs, Command, RegistryCommand { std::string fromUrl, toUrl; @@ -71,16 +111,16 @@ struct CmdRegistryAdd : MixEvalArgs, Command { auto fromRef = parseFlakeRef(fromUrl); auto toRef = parseFlakeRef(toUrl); + auto registry = getRegistry(); fetchers::Attrs extraAttrs; if (toRef.subdir != "") extraAttrs["dir"] = toRef.subdir; - auto userRegistry = fetchers::getUserRegistry(); - userRegistry->remove(fromRef.input); - userRegistry->add(fromRef.input, toRef.input, extraAttrs); - userRegistry->write(fetchers::getUserRegistryPath()); + registry->remove(fromRef.input); + registry->add(fromRef.input, toRef.input, extraAttrs); + registry->write(getRegistryPath()); } }; -struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command +struct CmdRegistryRemove : RegistryCommand, Command { std::string url; @@ -103,19 +143,21 @@ struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command void run() override { - auto userRegistry = fetchers::getUserRegistry(); - userRegistry->remove(parseFlakeRef(url).input); - userRegistry->write(fetchers::getUserRegistryPath()); + auto registry = getRegistry(); + registry->remove(parseFlakeRef(url).input); + registry->write(getRegistryPath()); } }; -struct CmdRegistryPin : virtual Args, EvalCommand +struct CmdRegistryPin : RegistryCommand, EvalCommand { std::string url; + std::string locked; + std::string description() override { - return "pin a flake to its current version in user flake registry"; + return "pin a flake to its current version or to the current version of a flake URL"; } std::string doc() override @@ -128,18 +170,31 @@ struct CmdRegistryPin : virtual Args, EvalCommand CmdRegistryPin() { expectArg("url", &url); + + expectArgs({ + .label = "locked", + .optional = true, + .handler = {&locked}, + .completer = {[&](size_t, std::string_view prefix) { + completeFlakeRef(getStore(), prefix); + }} + }); } void run(nix::ref<nix::Store> store) override { + if (locked.empty()) { + locked = url; + } + auto registry = getRegistry(); auto ref = parseFlakeRef(url); - auto userRegistry = fetchers::getUserRegistry(); - userRegistry->remove(ref.input); - auto [tree, resolved] = ref.resolve(store).input.fetch(store); + auto locked_ref = parseFlakeRef(locked); + registry->remove(ref.input); + auto [tree, resolved] = locked_ref.resolve(store).input.fetch(store); fetchers::Attrs extraAttrs; if (ref.subdir != "") extraAttrs["dir"] = ref.subdir; - userRegistry->add(ref.input, resolved, extraAttrs); - userRegistry->write(fetchers::getUserRegistryPath()); + registry->add(ref.input, resolved, extraAttrs); + registry->write(getRegistryPath()); } }; diff --git a/src/nix/repl.cc b/src/nix/repl.cc index eed79c332..0275feae7 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -104,6 +104,26 @@ NixRepl::~NixRepl() write_history(historyFile.c_str()); } +string runNix(Path program, const Strings & args, + const std::optional<std::string> & input = {}) +{ + auto experimentalFeatures = concatStringsSep(" ", settings.experimentalFeatures.get()); + auto nixConf = getEnv("NIX_CONFIG").value_or(""); + nixConf.append("\nexperimental-features = " + experimentalFeatures); + auto subprocessEnv = getEnv(); + subprocessEnv["NIX_CONFIG"] = nixConf; + RunOptions opts(settings.nixBinDir+ "/" + program, args); + opts.input = input; + opts.environment = subprocessEnv; + + auto res = runProgram(opts); + + if (!statusOk(res.first)) + throw ExecError(res.first, fmt("program '%1%' %2%", program, statusToString(res.first))); + + return res.second; +} + static NixRepl * curRepl; // ugly static char * completionCallback(char * s, int *match) { @@ -463,7 +483,7 @@ bool NixRepl::processLine(string line) state->callFunction(f, v, result, Pos()); StorePath drvPath = getDerivationPath(result); - runProgram(settings.nixBinDir + "/nix-shell", true, {state->store->printStorePath(drvPath)}); + runNix("nix-shell", {state->store->printStorePath(drvPath)}); } else if (command == ":b" || command == ":i" || command == ":s") { @@ -477,7 +497,7 @@ bool NixRepl::processLine(string line) but doing it in a child makes it easier to recover from problems / SIGINT. */ try { - runProgram(settings.nixBinDir + "/nix", true, {"build", "--no-link", drvPathRaw}); + runNix("nix", {"build", "--no-link", drvPathRaw}); auto drv = state->store->readDerivation(drvPath); std::cout << std::endl << "this derivation produced the following outputs:" << std::endl; for (auto & i : drv.outputsAndOptPaths(*state->store)) @@ -485,9 +505,9 @@ bool NixRepl::processLine(string line) } catch (ExecError &) { } } else if (command == ":i") { - runProgram(settings.nixBinDir + "/nix-env", true, {"-i", drvPathRaw}); + runNix("nix-env", {"-i", drvPathRaw}); } else { - runProgram(settings.nixBinDir + "/nix-shell", true, {drvPathRaw}); + runNix("nix-shell", {drvPathRaw}); } } diff --git a/src/resolve-system-dependencies/local.mk b/src/resolve-system-dependencies/local.mk index 054ae01cb..fc48a8417 100644 --- a/src/resolve-system-dependencies/local.mk +++ b/src/resolve-system-dependencies/local.mk @@ -1,4 +1,4 @@ -ifeq ($(OS), Darwin) +ifdef HOST_DARWIN programs += resolve-system-dependencies endif |