diff options
29 files changed, 575 insertions, 113 deletions
diff --git a/.github/workflows/backport.yml b/.github/workflows/backport.yml index af12190e2..ec7ab4516 100644 --- a/.github/workflows/backport.yml +++ b/.github/workflows/backport.yml @@ -15,7 +15,7 @@ jobs: fetch-depth: 0 - name: Create backport PRs # should be kept in sync with `version` - uses: zeebe-io/backport-action@v0.0.5 + uses: zeebe-io/backport-action@v0.0.7 with: # Config README: https://github.com/zeebe-io/backport-action#backport-action github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/doc/manual/src/SUMMARY.md.in b/doc/manual/src/SUMMARY.md.in index 3869f4791..8d9b061ba 100644 --- a/doc/manual/src/SUMMARY.md.in +++ b/doc/manual/src/SUMMARY.md.in @@ -9,6 +9,7 @@ - [Prerequisites](installation/prerequisites-source.md) - [Obtaining a Source Distribution](installation/obtaining-source.md) - [Building Nix from Source](installation/building-source.md) + - [Using Nix within Docker](installation/installing-docker.md) - [Security](installation/nix-security.md) - [Single-User Mode](installation/single-user.md) - [Multi-User Mode](installation/multi-user.md) diff --git a/doc/manual/src/command-ref/conf-file-prefix.md b/doc/manual/src/command-ref/conf-file-prefix.md index d660db502..44b7ba86d 100644 --- a/doc/manual/src/command-ref/conf-file-prefix.md +++ b/doc/manual/src/command-ref/conf-file-prefix.md @@ -16,8 +16,9 @@ By default Nix reads settings from the following places: will be loaded in reverse order. Otherwise it will look for `nix/nix.conf` files in `XDG_CONFIG_DIRS` - and `XDG_CONFIG_HOME`. If these are unset, it will look in - `$HOME/.config/nix/nix.conf`. + and `XDG_CONFIG_HOME`. If unset, `XDG_CONFIG_DIRS` defaults to + `/etc/xdg`, and `XDG_CONFIG_HOME` defaults to `$HOME/.config` + as per [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). - If `NIX_CONFIG` is set, its contents is treated as the contents of a configuration file. diff --git a/doc/manual/src/installation/installing-docker.md b/doc/manual/src/installation/installing-docker.md new file mode 100644 index 000000000..3d2255b7a --- /dev/null +++ b/doc/manual/src/installation/installing-docker.md @@ -0,0 +1,59 @@ +# Using Nix within Docker + +To run the latest stable release of Nix with Docker run the following command: + +```console +$ docker -ti run nixos/nix +Unable to find image 'nixos/nix:latest' locally +latest: Pulling from nixos/nix +5843afab3874: Pull complete +b52bf13f109c: Pull complete +1e2415612aa3: Pull complete +Digest: sha256:27f6e7f60227e959ee7ece361f75d4844a40e1cc6878b6868fe30140420031ff +Status: Downloaded newer image for nixos/nix:latest +35ca4ada6e96:/# nix --version +nix (Nix) 2.3.12 +35ca4ada6e96:/# exit +``` + +# What is included in Nix' Docker image? + +The official Docker image is created using `pkgs.dockerTools.buildLayeredImage` +(and not with `Dockerfile` as it is usual with Docker images). You can still +base your custom Docker image on it as you would do with any other Docker +image. + +The Docker image is also not based on any other image and includes minimal set +of runtime dependencies that are required to use Nix: + + - pkgs.nix + - pkgs.bashInteractive + - pkgs.coreutils-full + - pkgs.gnutar + - pkgs.gzip + - pkgs.gnugrep + - pkgs.which + - pkgs.curl + - pkgs.less + - pkgs.wget + - pkgs.man + - pkgs.cacert.out + - pkgs.findutils + +# Docker image with the latest development version of Nix + +To get the latest image that was built by [Hydra](https://hydra.nixos.org) run +the following command: + +```console +$ curl -L https://hydra.nixos.org/job/nix/master/dockerImage.x86_64-linux/latest/download/1 | docker load +$ docker run -ti nix:2.5pre20211105 +``` + +You can also build a Docker image from source yourself: + +```console +$ nix build ./\#hydraJobs.dockerImage.x86_64-linux +$ docker load -i ./result +$ docker run -ti nix:2.5pre20211105 +``` diff --git a/docker.nix b/docker.nix new file mode 100644 index 000000000..2a13c23fb --- /dev/null +++ b/docker.nix @@ -0,0 +1,251 @@ +{ pkgs ? import <nixpkgs> { } +, lib ? pkgs.lib +, name ? "nix" +, tag ? "latest" +, channelName ? "nixpkgs" +, channelURL ? "https://nixos.org/channels/nixpkgs-unstable" +}: +let + defaultPkgs = with pkgs; [ + nix + bashInteractive + coreutils-full + gnutar + gzip + gnugrep + which + curl + less + wget + man + cacert.out + findutils + ]; + + users = { + + root = { + uid = 0; + shell = "/bin/bash"; + home = "/root"; + gid = 0; + }; + + } // lib.listToAttrs ( + map + ( + n: { + name = "nixbld${toString n}"; + value = { + uid = 30000 + n; + gid = 30000; + groups = [ "nixbld" ]; + description = "Nix build user ${toString n}"; + }; + } + ) + (lib.lists.range 1 32) + ); + + groups = { + root.gid = 0; + nixbld.gid = 30000; + }; + + userToPasswd = ( + k: + { uid + , gid ? 65534 + , home ? "/var/empty" + , description ? "" + , shell ? "/bin/false" + , groups ? [ ] + }: "${k}:x:${toString uid}:${toString gid}:${description}:${home}:${shell}" + ); + passwdContents = ( + lib.concatStringsSep "\n" + (lib.attrValues (lib.mapAttrs userToPasswd users)) + ); + + userToShadow = k: { ... }: "${k}:!:1::::::"; + shadowContents = ( + lib.concatStringsSep "\n" + (lib.attrValues (lib.mapAttrs userToShadow users)) + ); + + # Map groups to members + # { + # group = [ "user1" "user2" ]; + # } + groupMemberMap = ( + let + # Create a flat list of user/group mappings + mappings = ( + builtins.foldl' + ( + acc: user: + let + groups = users.${user}.groups or [ ]; + in + acc ++ map + (group: { + inherit user group; + }) + groups + ) + [ ] + (lib.attrNames users) + ); + in + ( + builtins.foldl' + ( + acc: v: acc // { + ${v.group} = acc.${v.group} or [ ] ++ [ v.user ]; + } + ) + { } + mappings) + ); + + groupToGroup = k: { gid }: + let + members = groupMemberMap.${k} or [ ]; + in + "${k}:x:${toString gid}:${lib.concatStringsSep "," members}"; + groupContents = ( + lib.concatStringsSep "\n" + (lib.attrValues (lib.mapAttrs groupToGroup groups)) + ); + + nixConf = { + sandbox = "false"; + build-users-group = "nixbld"; + trusted-public-keys = "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="; + }; + nixConfContents = (lib.concatStringsSep "\n" (lib.mapAttrsFlatten (n: v: "${n} = ${v}") nixConf)) + "\n"; + + baseSystem = + let + nixpkgs = pkgs.path; + channel = pkgs.runCommand "channel-nixos" { } '' + mkdir $out + ln -s ${nixpkgs} $out/nixpkgs + echo "[]" > $out/manifest.nix + ''; + rootEnv = pkgs.buildPackages.buildEnv { + name = "root-profile-env"; + paths = defaultPkgs; + }; + profile = pkgs.buildPackages.runCommand "user-environment" { } '' + mkdir $out + cp -a ${rootEnv}/* $out/ + + cat > $out/manifest.nix <<EOF + [ + ${lib.concatStringsSep "\n" (builtins.map (drv: let + outputs = drv.outputsToInstall or [ "out" ]; + in '' + { + ${lib.concatStringsSep "\n" (builtins.map (output: '' + ${output} = { outPath = "${lib.getOutput output drv}"; }; + '') outputs)} + outputs = [ ${lib.concatStringsSep " " (builtins.map (x: "\"${x}\"") outputs)} ]; + name = "${drv.name}"; + outPath = "${drv}"; + system = "${drv.system}"; + type = "derivation"; + meta = { }; + } + '') defaultPkgs)} + ] + EOF + ''; + in + pkgs.runCommand "base-system" + { + inherit passwdContents groupContents shadowContents nixConfContents; + passAsFile = [ + "passwdContents" + "groupContents" + "shadowContents" + "nixConfContents" + ]; + allowSubstitutes = false; + preferLocalBuild = true; + } '' + env + set -x + mkdir -p $out/etc + + cat $passwdContentsPath > $out/etc/passwd + echo "" >> $out/etc/passwd + + cat $groupContentsPath > $out/etc/group + echo "" >> $out/etc/group + + cat $shadowContentsPath > $out/etc/shadow + echo "" >> $out/etc/shadow + + mkdir -p $out/usr + ln -s /nix/var/nix/profiles/share $out/usr/ + + mkdir -p $out/nix/var/nix/gcroots + + mkdir $out/tmp + + mkdir -p $out/etc/nix + cat $nixConfContentsPath > $out/etc/nix/nix.conf + + mkdir -p $out/root + mkdir -p $out/nix/var/nix/profiles/per-user/root + + ln -s ${profile} $out/nix/var/nix/profiles/default-1-link + ln -s $out/nix/var/nix/profiles/default-1-link $out/nix/var/nix/profiles/default + ln -s /nix/var/nix/profiles/default $out/root/.nix-profile + + ln -s ${channel} $out/nix/var/nix/profiles/per-user/root/channels-1-link + ln -s $out/nix/var/nix/profiles/per-user/root/channels-1-link $out/nix/var/nix/profiles/per-user/root/channels + + mkdir -p $out/root/.nix-defexpr + ln -s $out/nix/var/nix/profiles/per-user/root/channels $out/root/.nix-defexpr/channels + echo "${channelURL} ${channelName}" > $out/root/.nix-channels + + mkdir -p $out/bin $out/usr/bin + ln -s ${pkgs.coreutils}/bin/env $out/usr/bin/env + ln -s ${pkgs.bashInteractive}/bin/bash $out/bin/sh + ''; + +in +pkgs.dockerTools.buildLayeredImageWithNixDb { + + inherit name tag; + + contents = [ baseSystem ]; + + extraCommands = '' + rm -rf nix-support + ln -s /nix/var/nix/profiles nix/var/nix/gcroots/profiles + ''; + + config = { + Cmd = [ "/root/.nix-profile/bin/bash" ]; + Env = [ + "USER=root" + "PATH=${lib.concatStringsSep ":" [ + "/root/.nix-profile/bin" + "/nix/var/nix/profiles/default/bin" + "/nix/var/nix/profiles/default/sbin" + ]}" + "MANPATH=${lib.concatStringsSep ":" [ + "/root/.nix-profile/share/man" + "/nix/var/nix/profiles/default/share/man" + ]}" + "SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "GIT_SSL_CAINFO=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "NIX_SSL_CERT_FILE=/nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt" + "NIX_PATH=/nix/var/nix/profiles/per-user/root/channels:/root/.nix-defexpr/channels" + ]; + }; + +} @@ -405,6 +405,13 @@ installerScript = installScriptFor [ "x86_64-linux" "i686-linux" "aarch64-linux" "x86_64-darwin" "aarch64-darwin" "armv6l-linux" "armv7l-linux" ]; installerScriptForGHA = installScriptFor [ "x86_64-linux" "x86_64-darwin" "armv6l-linux" "armv7l-linux"]; + # docker image with Nix inside + dockerImage = nixpkgs.lib.genAttrs linux64BitSystems (system: + import ./docker.nix { + pkgs = nixpkgsFor.${system}; + tag = version; + }); + # Line coverage analysis. coverage = with nixpkgsFor.x86_64-linux; diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index 402de78ad..f1ff3a6e0 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -1275,6 +1275,8 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & } }; + Attr * functor; + while (nrArgs > 0) { if (vCur.isLambda()) { @@ -1403,16 +1405,16 @@ void EvalState::callFunction(Value & fun, size_t nrArgs, Value * * args, Value & } } - else if (vCur.type() == nAttrs) { - if (auto functor = vCur.attrs->get(sFunctor)) { - /* 'vCur" may be allocated on the stack of the calling - function, but for functors we may keep a reference, - so heap-allocate a copy and use that instead. */ - Value * args2[] = {allocValue()}; - *args2[0] = vCur; - /* !!! Should we use the attr pos here? */ - callFunction(*functor->value, 1, args2, vCur, pos); - } + else if (vCur.type() == nAttrs && (functor = vCur.attrs->get(sFunctor))) { + /* 'vCur' may be allocated on the stack of the calling + function, but for functors we may keep a reference, so + heap-allocate a copy and use that instead. */ + Value * args2[] = {allocValue(), args[0]}; + *args2[0] = vCur; + /* !!! Should we use the attr pos here? */ + callFunction(*functor->value, 2, args2, vCur, pos); + nrArgs--; + args++; } else diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 923997bf6..c1f4e72e0 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -39,12 +39,6 @@ namespace nix { { }; }; - // Helper to prevent an expensive dynamic_cast call in expr_app. - struct App - { - Expr * e; - bool isCall; - }; } #define YY_DECL int yylex \ @@ -284,12 +278,10 @@ void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * err char * uri; std::vector<nix::AttrName> * attrNames; std::vector<nix::Expr *> * string_parts; - nix::App app; // bool == whether this is an ExprCall } %type <e> start expr expr_function expr_if expr_op -%type <e> expr_select expr_simple -%type <app> expr_app +%type <e> expr_select expr_simple expr_app %type <list> expr_list %type <attrs> binds %type <formals> formals @@ -377,20 +369,18 @@ expr_op | expr_op '*' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__mul")), {$1, $3}); } | expr_op '/' expr_op { $$ = new ExprCall(CUR_POS, new ExprVar(data->symbols.create("__div")), {$1, $3}); } | expr_op CONCAT expr_op { $$ = new ExprOpConcatLists(CUR_POS, $1, $3); } - | expr_app { $$ = $1.e; } + | expr_app ; expr_app : expr_app expr_select { - if ($1.isCall) { - ((ExprCall *) $1.e)->args.push_back($2); + if (auto e2 = dynamic_cast<ExprCall *>($1)) { + e2->args.push_back($2); $$ = $1; - } else { - $$.e = new ExprCall(CUR_POS, $1.e, {$2}); - $$.isCall = true; - } + } else + $$ = new ExprCall(CUR_POS, $1, {$2}); } - | expr_select { $$.e = $1; $$.isCall = false; } + | expr_select ; expr_select diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 1d59ab9cf..5bd4e5545 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -988,8 +988,9 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * * } if (i->name == state.sContentAddressed) { - settings.requireExperimentalFeature(Xp::CaDerivations); contentAddressed = state.forceBool(*i->value, pos); + if (contentAddressed) + settings.requireExperimentalFeature(Xp::CaDerivations); } /* The `args' attribute is special: it supplies the diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index e6becdafc..079513873 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -74,7 +74,10 @@ std::string fixURI(std::string uri, EvalState & state, const std::string & defau std::string fixURIForGit(std::string uri, EvalState & state) { - static std::regex scp_uri("([^/].*)@(.*):(.*)"); + /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes + * them by removing the `:` and assuming a scheme of `ssh://` + * */ + static std::regex scp_uri("([^/]*)@(.*):(.*)"); if (uri[0] != '/' && std::regex_match(uri, scp_uri)) return fixURI(std::regex_replace(uri, scp_uri, "$1@$2/$3"), state, "ssh"); else diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index a3f4e42a3..544d2ffbf 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -324,17 +324,13 @@ struct GitInputScheme : InputScheme Path cacheDir = getCacheDir() + "/nix/gitv3/" + hashString(htSHA256, actualUrl).to_string(Base32, false); repoDir = cacheDir; - Path cacheDirLock = cacheDir + ".lock"; createDirs(dirOf(cacheDir)); - AutoCloseFD lock = openLockFile(cacheDirLock, true); - lockFile(lock.get(), ltWrite, true); + PathLocks cacheDirLock({cacheDir + ".lock"}); if (!pathExists(cacheDir)) { runProgram("git", true, { "-c", "init.defaultBranch=" + gitInitialBranch, "init", "--bare", repoDir }); } - deleteLockFile(cacheDirLock, lock.get()); - Path localRefFile = input.getRef()->compare(0, 5, "refs/") == 0 ? cacheDir + "/" + *input.getRef() @@ -399,6 +395,8 @@ struct GitInputScheme : InputScheme if (!input.getRev()) input.attrs.insert_or_assign("rev", Hash::parseAny(chomp(readFile(localRefFile)), htSHA1).gitRev()); + + // cache dir lock is removed at scope end; we will only use read-only operations on specific revisions in the remainder } bool isShallow = chomp(runProgram("git", true, { "-C", repoDir, "rev-parse", "--is-shallow-repository" })) == "true"; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index ffc44e9e2..1c539b80e 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -300,7 +300,7 @@ struct GitLabInputScheme : GitArchiveInputScheme if ("PAT" == token.substr(0, fldsplit)) return std::make_pair("Private-token", token.substr(fldsplit+1)); warn("Unrecognized GitLab token type %s", token.substr(0, fldsplit)); - return std::nullopt; + return std::make_pair(token.substr(0,fldsplit), token.substr(fldsplit+1)); } Hash getRevFromRef(nix::ref<Store> store, const Input & input) const override diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 5ca14a372..13c086a46 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -439,40 +439,29 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s })->path; } -std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id) +void BinaryCacheStore::queryRealisationUncached(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept { - if (diskCache) { - auto [cacheOutcome, maybeCachedRealisation] = - diskCache->lookupRealisation(getUri(), id); - switch (cacheOutcome) { - case NarInfoDiskCache::oValid: - debug("Returning a cached realisation for %s", id.to_string()); - return *maybeCachedRealisation; - case NarInfoDiskCache::oInvalid: - debug("Returning a cached missing realisation for %s", id.to_string()); - return {}; - case NarInfoDiskCache::oUnknown: - break; - } - } - auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi"; - auto rawOutputInfo = getFile(outputInfoFilePath); - if (rawOutputInfo) { - auto realisation = Realisation::fromJSON( - nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath); + auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback)); + + Callback<std::shared_ptr<std::string>> newCallback = { + [=](std::future<std::shared_ptr<std::string>> fut) { + try { + auto data = fut.get(); + if (!data) return (*callbackPtr)(nullptr); - if (diskCache) - diskCache->upsertRealisation( - getUri(), realisation); + auto realisation = Realisation::fromJSON( + nlohmann::json::parse(*data), outputInfoFilePath); + return (*callbackPtr)(std::make_shared<const Realisation>(realisation)); + } catch (...) { + callbackPtr->rethrow(); + } + } + }; - return {realisation}; - } else { - if (diskCache) - diskCache->upsertAbsentRealisation(getUri(), id); - return std::nullopt; - } + getFile(outputInfoFilePath, std::move(newCallback)); } void BinaryCacheStore::registerDrvOutput(const Realisation& info) { diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh index a703994d3..9815af591 100644 --- a/src/libstore/binary-cache-store.hh +++ b/src/libstore/binary-cache-store.hh @@ -108,7 +108,8 @@ public: void registerDrvOutput(const Realisation & info) override; - std::optional<const Realisation> queryRealisation(const DrvOutput &) override; + void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override; void narFromPath(const StorePath & path, Sink & sink) override; diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc index be270d079..b9602e696 100644 --- a/src/libstore/build/drv-output-substitution-goal.cc +++ b/src/libstore/build/drv-output-substitution-goal.cc @@ -1,6 +1,8 @@ #include "drv-output-substitution-goal.hh" +#include "finally.hh" #include "worker.hh" #include "substitution-goal.hh" +#include "callback.hh" namespace nix { @@ -50,14 +52,42 @@ void DrvOutputSubstitutionGoal::tryNext() return; } - auto sub = subs.front(); + sub = subs.front(); subs.pop_front(); // FIXME: Make async - outputInfo = sub->queryRealisation(id); + // outputInfo = sub->queryRealisation(id); + outPipe.create(); + promise = decltype(promise)(); + + sub->queryRealisation( + id, { [&](std::future<std::shared_ptr<const Realisation>> res) { + try { + Finally updateStats([this]() { outPipe.writeSide.close(); }); + promise.set_value(res.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + } }); + + worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false); + + state = &DrvOutputSubstitutionGoal::realisationFetched; +} + +void DrvOutputSubstitutionGoal::realisationFetched() +{ + worker.childTerminated(this); + + try { + outputInfo = promise.get_future().get(); + } catch (std::exception & e) { + printError(e.what()); + substituterFailed = true; + } + if (!outputInfo) { - tryNext(); - return; + return tryNext(); } for (const auto & [depId, depPath] : outputInfo->dependentRealisations) { @@ -119,4 +149,10 @@ void DrvOutputSubstitutionGoal::work() (this->*state)(); } +void DrvOutputSubstitutionGoal::handleEOF(int fd) +{ + if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this()); +} + + } diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh index 63ab53d89..67ae2624a 100644 --- a/src/libstore/build/drv-output-substitution-goal.hh +++ b/src/libstore/build/drv-output-substitution-goal.hh @@ -3,6 +3,8 @@ #include "store-api.hh" #include "goal.hh" #include "realisation.hh" +#include <thread> +#include <future> namespace nix { @@ -20,11 +22,18 @@ private: // The realisation corresponding to the given output id. // Will be filled once we can get it. - std::optional<Realisation> outputInfo; + std::shared_ptr<const Realisation> outputInfo; /* The remaining substituters. */ std::list<ref<Store>> subs; + /* The current substituter. */ + std::shared_ptr<Store> sub; + + Pipe outPipe; + std::thread thr; + std::promise<std::shared_ptr<const Realisation>> promise; + /* Whether a substituter failed. */ bool substituterFailed = false; @@ -36,6 +45,7 @@ public: void init(); void tryNext(); + void realisationFetched(); void outPathValid(); void finished(); @@ -44,7 +54,7 @@ public: string key() override; void work() override; - + void handleEOF(int fd) override; }; } diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc index 589b449d0..3c7bd695e 100644 --- a/src/libstore/build/local-derivation-goal.cc +++ b/src/libstore/build/local-derivation-goal.cc @@ -1226,13 +1226,14 @@ struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual Lo // corresponds to an allowed derivation { throw Error("registerDrvOutput"); } - std::optional<const Realisation> queryRealisation(const DrvOutput & id) override + void queryRealisationUncached(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override // XXX: This should probably be allowed if the realisation corresponds to // an allowed derivation { if (!goal.isAllowed(id)) - throw InvalidPath("cannot query an unknown output id '%s' in recursive Nix", id.to_string()); - return next->queryRealisation(id); + callback(nullptr); + next->queryRealisation(id, std::move(callback)); } void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index 36c6e725c..62dc21c59 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -50,8 +50,9 @@ struct DummyStore : public virtual DummyStoreConfig, public virtual Store void narFromPath(const StorePath & path, Sink & sink) override { unsupported("narFromPath"); } - std::optional<const Realisation> queryRealisation(const DrvOutput&) override - { unsupported("queryRealisation"); } + void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override + { callback(nullptr); } }; static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore; diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index c4fb91364..4861d185e 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -367,7 +367,8 @@ public: return conn->remoteVersion; } - std::optional<const Realisation> queryRealisation(const DrvOutput&) override + void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override // TODO: Implement { unsupported("queryRealisation"); } }; diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index eecd407f5..64019314f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1836,13 +1836,24 @@ std::optional<const Realisation> LocalStore::queryRealisation_( return { res }; } -std::optional<const Realisation> -LocalStore::queryRealisation(const DrvOutput & id) +void LocalStore::queryRealisationUncached(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept { - return retrySQLite<std::optional<const Realisation>>([&]() { - auto state(_state.lock()); - return queryRealisation_(*state, id); - }); + try { + auto maybeRealisation + = retrySQLite<std::optional<const Realisation>>([&]() { + auto state(_state.lock()); + return queryRealisation_(*state, id); + }); + if (maybeRealisation) + callback( + std::make_shared<const Realisation>(maybeRealisation.value())); + else + callback(nullptr); + + } catch (...) { + callback.rethrow(); + } } FixedOutputHash LocalStore::hashCAPath( diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 7ddb1490f..115ea046a 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -207,7 +207,8 @@ public: std::optional<const Realisation> queryRealisation_(State & state, const DrvOutput & id); std::optional<std::pair<int64_t, Realisation>> queryRealisationCore_(State & state, const DrvOutput & id); - std::optional<const Realisation> queryRealisation(const DrvOutput&) override; + void queryRealisationUncached(const DrvOutput&, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override; private: diff --git a/src/libstore/references.cc b/src/libstore/references.cc index c369b14ac..91b3fc142 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -54,12 +54,12 @@ void RefScanSink::operator () (std::string_view data) fragment, so search in the concatenation of the tail of the previous fragment and the start of the current fragment. */ auto s = tail; - s.append(data.data(), refLength); + auto tailLen = std::min(data.size(), refLength); + s.append(data.data(), tailLen); search(s, hashes, seen); search(data, hashes, seen); - auto tailLen = std::min(data.size(), refLength); auto rest = refLength - tailLen; if (rest < tail.size()) tail = tail.substr(tail.size() - rest); diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 0e3ef6be4..a627e9cf1 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -680,23 +680,33 @@ void RemoteStore::registerDrvOutput(const Realisation & info) conn.processStderr(); } -std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput & id) +void RemoteStore::queryRealisationUncached(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept { auto conn(getConnection()); conn->to << wopQueryRealisation; conn->to << id.to_string(); conn.processStderr(); - if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { - auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{}); - if (outPaths.empty()) - return std::nullopt; - return {Realisation{.id = id, .outPath = *outPaths.begin()}}; - } else { - auto realisations = worker_proto::read(*this, conn->from, Phantom<std::set<Realisation>>{}); - if (realisations.empty()) - return std::nullopt; - return *realisations.begin(); - } + + auto real = [&]() -> std::shared_ptr<const Realisation> { + if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 31) { + auto outPaths = worker_proto::read( + *this, conn->from, Phantom<std::set<StorePath>> {}); + if (outPaths.empty()) + return nullptr; + return std::make_shared<const Realisation>(Realisation { .id = id, .outPath = *outPaths.begin() }); + } else { + auto realisations = worker_proto::read( + *this, conn->from, Phantom<std::set<Realisation>> {}); + if (realisations.empty()) + return nullptr; + return std::make_shared<const Realisation>(*realisations.begin()); + } + }(); + + try { + callback(std::shared_ptr<const Realisation>(real)); + } catch (...) { return callback.rethrow(); } } static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs) diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 6c0496b28..0fd67f371 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -88,7 +88,8 @@ public: void registerDrvOutput(const Realisation & info) override; - std::optional<const Realisation> queryRealisation(const DrvOutput &) override; + void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept override; void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode, std::shared_ptr<Store> evalStore) override; diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index ba13026f3..c88dfe179 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -542,6 +542,74 @@ void Store::queryPathInfo(const StorePath & storePath, }}); } +void Store::queryRealisation(const DrvOutput & id, + Callback<std::shared_ptr<const Realisation>> callback) noexcept +{ + + try { + if (diskCache) { + auto [cacheOutcome, maybeCachedRealisation] + = diskCache->lookupRealisation(getUri(), id); + switch (cacheOutcome) { + case NarInfoDiskCache::oValid: + debug("Returning a cached realisation for %s", id.to_string()); + callback(maybeCachedRealisation); + return; + case NarInfoDiskCache::oInvalid: + debug( + "Returning a cached missing realisation for %s", + id.to_string()); + callback(nullptr); + return; + case NarInfoDiskCache::oUnknown: + break; + } + } + } catch (...) { + return callback.rethrow(); + } + + auto callbackPtr + = std::make_shared<decltype(callback)>(std::move(callback)); + + queryRealisationUncached( + id, + { [this, id, callbackPtr]( + std::future<std::shared_ptr<const Realisation>> fut) { + try { + auto info = fut.get(); + + if (diskCache) { + if (info) + diskCache->upsertRealisation(getUri(), *info); + else + diskCache->upsertAbsentRealisation(getUri(), id); + } + + (*callbackPtr)(std::shared_ptr<const Realisation>(info)); + + } catch (...) { + callbackPtr->rethrow(); + } + } }); +} + +std::shared_ptr<const Realisation> Store::queryRealisation(const DrvOutput & id) +{ + using RealPtr = std::shared_ptr<const Realisation>; + std::promise<RealPtr> promise; + + queryRealisation(id, + {[&](std::future<RealPtr> result) { + try { + promise.set_value(result.get()); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); + + return promise.get_future().get(); +} void Store::substitutePaths(const StorePathSet & paths) { diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b7899dd44..aa44651d4 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -369,6 +369,14 @@ public: void queryPathInfo(const StorePath & path, Callback<ref<const ValidPathInfo>> callback) noexcept; + /* Query the information about a realisation. */ + std::shared_ptr<const Realisation> queryRealisation(const DrvOutput &); + + /* Asynchronous version of queryRealisation(). */ + void queryRealisation(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept; + + /* Check whether the given valid path info is sufficiently attested, by either being signed by a trusted public key or content-addressed, in order to be included in the given store. @@ -393,11 +401,11 @@ protected: virtual void queryPathInfoUncached(const StorePath & path, Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept = 0; + virtual void queryRealisationUncached(const DrvOutput &, + Callback<std::shared_ptr<const Realisation>> callback) noexcept = 0; public: - virtual std::optional<const Realisation> queryRealisation(const DrvOutput &) = 0; - /* Queries the set of incoming FS references for a store path. The result is not cleared. */ virtual void queryReferrers(const StorePath & path, StorePathSet & referrers) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index a6552ebca..5468d1ed1 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -562,7 +562,7 @@ Path getConfigDir() std::vector<Path> getConfigDirs() { Path configHome = getConfigDir(); - string configDirs = getEnv("XDG_CONFIG_DIRS").value_or(""); + string configDirs = getEnv("XDG_CONFIG_DIRS").value_or("/etc/xdg"); std::vector<Path> result = tokenizeString<std::vector<string>>(configDirs, ":"); result.insert(result.begin(), configHome); return result; @@ -1631,6 +1631,7 @@ void setStackSize(size_t stackSize) } #endif } + static AutoCloseFD fdSavedMountNamespace; void saveMountNamespace() @@ -1638,9 +1639,10 @@ void saveMountNamespace() #if __linux__ static std::once_flag done; std::call_once(done, []() { - fdSavedMountNamespace = open("/proc/self/ns/mnt", O_RDONLY); - if (!fdSavedMountNamespace) + AutoCloseFD fd = open("/proc/self/ns/mnt", O_RDONLY); + if (!fd) throw SysError("saving parent mount namespace"); + fdSavedMountNamespace = std::move(fd); }); #endif } @@ -1648,8 +1650,12 @@ void saveMountNamespace() void restoreMountNamespace() { #if __linux__ - if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) - throw SysError("restoring parent mount namespace"); + try { + if (fdSavedMountNamespace && setns(fdSavedMountNamespace.get(), CLONE_NEWNS) == -1) + throw SysError("restoring parent mount namespace"); + } catch (Error & e) { + debug(e.msg()); + } #endif } diff --git a/src/nix/main.cc b/src/nix/main.cc index 01889a71f..60b0aa410 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -257,9 +257,11 @@ void mainWrapped(int argc, char * * argv) #if __linux__ if (getuid() == 0) { - saveMountNamespace(); - if (unshare(CLONE_NEWNS) == -1) - throw SysError("setting up a private mount namespace"); + try { + saveMountNamespace(); + if (unshare(CLONE_NEWNS) == -1) + throw SysError("setting up a private mount namespace"); + } catch (Error & e) { } } #endif diff --git a/src/nix/repl.cc b/src/nix/repl.cc index 4f13ee05d..fd86174f2 100644 --- a/src/nix/repl.cc +++ b/src/nix/repl.cc @@ -471,7 +471,10 @@ bool NixRepl::processLine(string line) auto args = editorFor(pos); auto editor = args.front(); args.pop_front(); - runProgram(editor, true, args); + + // runProgram redirects stdout to a StringSink, + // using runProgram2 to allow editors to display their UI + runProgram2(RunOptions { .program = editor, .searchPath = true, .args = args }); // Reload right after exiting the editor state->resetFileCache(); |