aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorJohn Ericson <John.Ericson@Obsidian.Systems>2021-04-27 19:06:58 +0000
committerJohn Ericson <John.Ericson@Obsidian.Systems>2021-04-27 19:06:58 +0000
commite023c985d58094041e74ff59a51757bc75687ca7 (patch)
tree8865872040ac8752c8349b73fa71b82e80dc2584 /src
parentd3cfc14e3a370116e5715d5de5f64ed34dd2f912 (diff)
parent906adadacd2d1c98346a2f42c0b42a32d2806d94 (diff)
Merge remote-tracking branch 'upstream/master' into auto-uid-allocation
Diffstat (limited to 'src')
-rw-r--r--src/build-remote/build-remote.cc111
-rw-r--r--src/libcmd/command.cc (renamed from src/nix/command.cc)90
-rw-r--r--src/libcmd/command.hh (renamed from src/nix/command.hh)56
-rw-r--r--src/libcmd/installables.cc (renamed from src/nix/installables.cc)181
-rw-r--r--src/libcmd/installables.hh (renamed from src/nix/installables.hh)36
-rw-r--r--src/libcmd/legacy.cc (renamed from src/nix/legacy.cc)0
-rw-r--r--src/libcmd/legacy.hh (renamed from src/nix/legacy.hh)0
-rw-r--r--src/libcmd/local.mk15
-rw-r--r--src/libcmd/markdown.cc (renamed from src/nix/markdown.cc)6
-rw-r--r--src/libcmd/markdown.hh (renamed from src/nix/markdown.hh)0
-rw-r--r--src/libcmd/nix-cmd.pc.in9
-rw-r--r--src/libexpr/attr-path.cc16
-rw-r--r--src/libexpr/attr-set.cc4
-rw-r--r--src/libexpr/attr-set.hh2
-rw-r--r--src/libexpr/common-eval-args.cc17
-rw-r--r--src/libexpr/eval-cache.cc41
-rw-r--r--src/libexpr/eval-inline.hh20
-rw-r--r--src/libexpr/eval.cc322
-rw-r--r--src/libexpr/eval.hh2
-rw-r--r--src/libexpr/fetchurl.nix41
-rw-r--r--src/libexpr/flake/config.cc81
-rw-r--r--src/libexpr/flake/flake.cc637
-rw-r--r--src/libexpr/flake/flake.hh42
-rw-r--r--src/libexpr/flake/flakeref.hh23
-rw-r--r--src/libexpr/flake/lockfile.cc26
-rw-r--r--src/libexpr/flake/lockfile.hh2
-rw-r--r--src/libexpr/get-drvs.cc41
-rw-r--r--src/libexpr/lexer.l4
-rw-r--r--src/libexpr/local.mk6
-rw-r--r--src/libexpr/nixexpr.cc2
-rw-r--r--src/libexpr/nixexpr.hh5
-rw-r--r--src/libexpr/parser.y34
-rw-r--r--src/libexpr/primops.cc277
-rw-r--r--src/libexpr/primops/context.cc6
-rw-r--r--src/libexpr/primops/fetchMercurial.cc6
-rw-r--r--src/libexpr/primops/fetchTree.cc35
-rw-r--r--src/libexpr/primops/fromTOML.cc2
-rw-r--r--src/libexpr/value-to-json.cc25
-rw-r--r--src/libexpr/value-to-xml.cc33
-rw-r--r--src/libexpr/value.hh244
-rw-r--r--src/libfetchers/attrs.cc8
-rw-r--r--src/libfetchers/attrs.hh14
-rw-r--r--src/libfetchers/cache.cc20
-rw-r--r--src/libfetchers/fetchers.cc15
-rw-r--r--src/libfetchers/fetchers.hh26
-rw-r--r--src/libfetchers/git.cc63
-rw-r--r--src/libfetchers/github.cc26
-rw-r--r--src/libfetchers/mercurial.cc58
-rw-r--r--src/libfetchers/path.cc6
-rw-r--r--src/libfetchers/registry.cc8
-rw-r--r--src/libfetchers/tarball.cc22
-rw-r--r--src/libmain/common-args.cc28
-rw-r--r--src/libmain/common-args.hh23
-rw-r--r--src/libmain/loggers.cc4
-rw-r--r--src/libmain/loggers.hh2
-rw-r--r--src/libmain/progress-bar.cc12
-rw-r--r--src/libmain/shared.cc57
-rw-r--r--src/libmain/shared.hh21
-rw-r--r--src/libstore/binary-cache-store.cc25
-rw-r--r--src/libstore/binary-cache-store.hh14
-rw-r--r--src/libstore/build/derivation-goal.cc2926
-rw-r--r--src/libstore/build/derivation-goal.hh214
-rw-r--r--src/libstore/build/drv-output-substitution-goal.cc95
-rw-r--r--src/libstore/build/drv-output-substitution-goal.hh50
-rw-r--r--src/libstore/build/entry-points.cc (renamed from src/libstore/build/local-store-build.cc)67
-rw-r--r--src/libstore/build/goal.cc2
-rw-r--r--src/libstore/build/goal.hh6
-rw-r--r--src/libstore/build/local-derivation-goal.cc2906
-rw-r--r--src/libstore/build/local-derivation-goal.hh209
-rw-r--r--src/libstore/build/substitution-goal.cc74
-rw-r--r--src/libstore/build/substitution-goal.hh15
-rw-r--r--src/libstore/build/worker.cc98
-rw-r--r--src/libstore/build/worker.hh34
-rw-r--r--src/libstore/builtins/buildenv.cc10
-rw-r--r--src/libstore/ca-specific-schema.sql12
-rw-r--r--src/libstore/cgroup.cc5
-rw-r--r--src/libstore/crypto.cc54
-rw-r--r--src/libstore/crypto.hh24
-rw-r--r--src/libstore/daemon.cc70
-rw-r--r--src/libstore/derivations.cc140
-rw-r--r--src/libstore/derivations.hh40
-rw-r--r--src/libstore/derived-path.cc77
-rw-r--r--src/libstore/derived-path.hh129
-rw-r--r--src/libstore/dummy-store.cc11
-rw-r--r--src/libstore/filetransfer.cc40
-rw-r--r--src/libstore/filetransfer.hh4
-rw-r--r--src/libstore/fs-accessor.hh9
-rw-r--r--src/libstore/globals.cc72
-rw-r--r--src/libstore/globals.hh65
-rw-r--r--src/libstore/http-binary-cache-store.cc5
-rw-r--r--src/libstore/legacy-ssh-store.cc29
-rw-r--r--src/libstore/local-binary-cache-store.cc11
-rw-r--r--src/libstore/local-fs-store.cc8
-rw-r--r--src/libstore/local-fs-store.hh2
-rw-r--r--src/libstore/local-store.cc377
-rw-r--r--src/libstore/local-store.hh59
-rw-r--r--src/libstore/local.mk6
-rw-r--r--src/libstore/machines.cc6
-rw-r--r--src/libstore/misc.cc82
-rw-r--r--src/libstore/names.cc12
-rw-r--r--src/libstore/nar-accessor.cc6
-rw-r--r--src/libstore/nar-info-disk-cache.cc6
-rw-r--r--src/libstore/nar-info.cc8
-rw-r--r--src/libstore/optimise-store.cc17
-rw-r--r--src/libstore/parsed-derivations.cc4
-rw-r--r--src/libstore/path-info.hh2
-rw-r--r--src/libstore/path-with-outputs.cc71
-rw-r--r--src/libstore/path-with-outputs.hh35
-rw-r--r--src/libstore/path.cc15
-rw-r--r--src/libstore/path.hh10
-rw-r--r--src/libstore/profiles.cc11
-rw-r--r--src/libstore/realisation.cc122
-rw-r--r--src/libstore/realisation.hh83
-rw-r--r--src/libstore/references.cc53
-rw-r--r--src/libstore/references.hh4
-rw-r--r--src/libstore/remote-fs-accessor.cc10
-rw-r--r--src/libstore/remote-fs-accessor.hh4
-rw-r--r--src/libstore/remote-store.cc140
-rw-r--r--src/libstore/remote-store.hh10
-rw-r--r--src/libstore/s3-binary-cache-store.cc20
-rw-r--r--src/libstore/s3-binary-cache-store.hh6
-rw-r--r--src/libstore/serve-protocol.hh2
-rw-r--r--src/libstore/sqlite.cc6
-rw-r--r--src/libstore/ssh-store.cc6
-rw-r--r--src/libstore/ssh.cc20
-rw-r--r--src/libstore/ssh.hh3
-rw-r--r--src/libstore/store-api.cc171
-rw-r--r--src/libstore/store-api.hh102
-rw-r--r--src/libstore/uds-remote-store.cc3
-rw-r--r--src/libstore/uds-remote-store.hh7
-rw-r--r--src/libstore/worker-protocol.hh31
-rw-r--r--src/libutil/archive.cc24
-rw-r--r--src/libutil/archive.hh10
-rw-r--r--src/libutil/args.cc157
-rw-r--r--src/libutil/args.hh123
-rw-r--r--src/libutil/comparator.hh30
-rw-r--r--src/libutil/compression.cc113
-rw-r--r--src/libutil/compute-levels.cc80
-rw-r--r--src/libutil/compute-levels.hh7
-rw-r--r--src/libutil/config.cc106
-rw-r--r--src/libutil/config.hh33
-rw-r--r--src/libutil/error.cc240
-rw-r--r--src/libutil/error.hh22
-rw-r--r--src/libutil/hash.cc20
-rw-r--r--src/libutil/hash.hh2
-rw-r--r--src/libutil/local.mk4
-rw-r--r--src/libutil/logging.cc7
-rw-r--r--src/libutil/logging.hh15
-rw-r--r--src/libutil/serialise.cc135
-rw-r--r--src/libutil/serialise.hh108
-rw-r--r--src/libutil/tarfile.cc2
-rw-r--r--src/libutil/tests/config.cc36
-rw-r--r--src/libutil/tests/logging.cc41
-rw-r--r--src/libutil/tests/tests.cc37
-rw-r--r--src/libutil/tests/url.cc18
-rw-r--r--src/libutil/types.hh13
-rw-r--r--src/libutil/url-parts.hh5
-rw-r--r--src/libutil/util.cc127
-rw-r--r--src/libutil/util.hh69
-rwxr-xr-xsrc/nix-build/nix-build.cc50
-rwxr-xr-xsrc/nix-channel/nix-channel.cc4
-rw-r--r--src/nix-collect-garbage/nix-collect-garbage.cc4
-rwxr-xr-xsrc/nix-copy-closure/nix-copy-closure.cc10
-rw-r--r--src/nix-env/nix-env.cc89
-rw-r--r--src/nix-env/user-env.cc17
-rw-r--r--src/nix-instantiate/nix-instantiate.cc4
-rw-r--r--src/nix-prefetch-url/nix-prefetch-url.cc232
-rw-r--r--src/nix-store/nix-store.cc83
-rw-r--r--src/nix/add-file.md28
-rw-r--r--src/nix/add-path.md29
-rw-r--r--src/nix/add-to-store.cc78
-rw-r--r--src/nix/app.cc19
-rw-r--r--src/nix/build.cc41
-rw-r--r--src/nix/build.md92
-rw-r--r--src/nix/bundle.cc28
-rw-r--r--src/nix/bundle.md36
-rw-r--r--src/nix/cat.cc18
-rw-r--r--src/nix/copy.cc50
-rw-r--r--src/nix/copy.md58
-rw-r--r--src/nix/daemon.cc (renamed from src/nix-daemon/nix-daemon.cc)122
-rw-r--r--src/nix/daemon.md21
-rw-r--r--src/nix/develop.cc146
-rw-r--r--src/nix/develop.md94
-rw-r--r--src/nix/diff-closures.cc15
-rw-r--r--src/nix/diff-closures.md51
-rw-r--r--src/nix/dump-path.cc53
-rw-r--r--src/nix/edit.cc14
-rw-r--r--src/nix/edit.md31
-rw-r--r--src/nix/eval.cc92
-rw-r--r--src/nix/eval.md74
-rw-r--r--src/nix/flake-archive.md29
-rw-r--r--src/nix/flake-check.md68
-rw-r--r--src/nix/flake-clone.md18
-rw-r--r--src/nix/flake-init.md54
-rw-r--r--src/nix/flake-lock.md38
-rw-r--r--src/nix/flake-metadata.md110
-rw-r--r--src/nix/flake-new.md34
-rw-r--r--src/nix/flake-prefetch.md28
-rw-r--r--src/nix/flake-show.md38
-rw-r--r--src/nix/flake-update.md34
-rw-r--r--src/nix/flake.cc349
-rw-r--r--src/nix/flake.md566
-rw-r--r--src/nix/hash.cc101
-rw-r--r--src/nix/help.md17
-rw-r--r--src/nix/key-convert-secret-to-public.md19
-rw-r--r--src/nix/key-generate-secret.md48
-rw-r--r--src/nix/local.mk5
-rw-r--r--src/nix/log.cc27
-rw-r--r--src/nix/log.md40
-rw-r--r--src/nix/ls.cc63
-rw-r--r--src/nix/main.cc180
-rw-r--r--src/nix/make-content-addressable.cc27
-rw-r--r--src/nix/make-content-addressable.md59
-rw-r--r--src/nix/nar-cat.md19
-rw-r--r--src/nix/nar-dump-path.md17
-rw-r--r--src/nix/nar-ls.md24
-rw-r--r--src/nix/nar.cc33
-rw-r--r--src/nix/nar.md13
-rw-r--r--src/nix/nix.md119
-rw-r--r--src/nix/optimise-store.cc15
-rw-r--r--src/nix/optimise-store.md23
-rw-r--r--src/nix/path-info.cc65
-rw-r--r--src/nix/path-info.md94
-rw-r--r--src/nix/ping-store.cc17
-rw-r--r--src/nix/ping-store.md33
-rw-r--r--src/nix/prefetch.cc317
-rw-r--r--src/nix/print-dev-env.md19
-rw-r--r--src/nix/profile-diff-closures.md28
-rw-r--r--src/nix/profile-history.md26
-rw-r--r--src/nix/profile-install.md27
-rw-r--r--src/nix/profile-list.md31
-rw-r--r--src/nix/profile-remove.md32
-rw-r--r--src/nix/profile-upgrade.md41
-rw-r--r--src/nix/profile.cc262
-rw-r--r--src/nix/profile.md107
-rw-r--r--src/nix/realisation.cc78
-rw-r--r--src/nix/realisation/info.md15
-rw-r--r--src/nix/registry-add.md33
-rw-r--r--src/nix/registry-list.md29
-rw-r--r--src/nix/registry-pin.md38
-rw-r--r--src/nix/registry-remove.md16
-rw-r--r--src/nix/registry.cc37
-rw-r--r--src/nix/registry.md98
-rw-r--r--src/nix/repl.cc101
-rw-r--r--src/nix/repl.md57
-rw-r--r--src/nix/run.cc72
-rw-r--r--src/nix/run.md88
-rw-r--r--src/nix/search.cc42
-rw-r--r--src/nix/search.md72
-rw-r--r--src/nix/shell.md48
-rw-r--r--src/nix/show-config.cc4
-rw-r--r--src/nix/show-derivation.cc21
-rw-r--r--src/nix/show-derivation.md103
-rw-r--r--src/nix/sigs.cc105
-rw-r--r--src/nix/store-cat.md19
-rw-r--r--src/nix/store-delete.cc44
-rw-r--r--src/nix/store-delete.md24
-rw-r--r--src/nix/store-dump-path.md23
-rw-r--r--src/nix/store-gc.cc43
-rw-r--r--src/nix/store-gc.md21
-rw-r--r--src/nix/store-ls.md27
-rw-r--r--src/nix/store-prefetch-file.md32
-rw-r--r--src/nix/store-repair.cc27
-rw-r--r--src/nix/store-repair.md32
-rw-r--r--src/nix/store.cc26
-rw-r--r--src/nix/upgrade-nix.cc24
-rw-r--r--src/nix/upgrade-nix.md28
-rw-r--r--src/nix/verify.cc63
-rw-r--r--src/nix/verify.md49
-rw-r--r--src/nix/why-depends.cc23
-rw-r--r--src/nix/why-depends.md80
-rw-r--r--src/resolve-system-dependencies/resolve-system-dependencies.cc20
272 files changed, 13459 insertions, 6779 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 9a0e9f08d..57f2cd32d 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -17,7 +17,7 @@
#include "store-api.hh"
#include "derivations.hh"
#include "local-store.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
using namespace nix;
using std::cin;
@@ -53,6 +53,9 @@ static int main_build_remote(int argc, char * * argv)
unsetenv("DISPLAY");
unsetenv("SSH_ASKPASS");
+ /* If we ever use the common args framework, make sure to
+ remove initPlugins below and initialize settings first.
+ */
if (argc != 2)
throw UsageError("called without required arguments");
@@ -71,11 +74,15 @@ static int main_build_remote(int argc, char * * argv)
initPlugins();
- auto store = openStore().cast<LocalStore>();
+ auto store = openStore();
/* It would be more appropriate to use $XDG_RUNTIME_DIR, since
that gets cleared on reboot, but it wouldn't work on macOS. */
- currentLoad = store->stateDir + "/current-load";
+ auto currentLoadName = "/current-load";
+ if (auto localStore = store.dynamic_pointer_cast<LocalFSStore>())
+ currentLoad = std::string { localStore->stateDir } + currentLoadName;
+ else
+ currentLoad = settings.nixStateDir + currentLoadName;
std::shared_ptr<Store> sshStore;
AutoCloseFD bestSlotLock;
@@ -172,13 +179,14 @@ static int main_build_remote(int argc, char * * argv)
else
{
// build the hint template.
- string hintstring = "derivation: %s\nrequired (system, features): (%s, %s)";
- hintstring += "\n%s available machines:";
- hintstring += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)";
+ string errorText =
+ "Failed to find a machine for remote build!\n"
+ "derivation: %s\nrequired (system, features): (%s, %s)";
+ errorText += "\n%s available machines:";
+ errorText += "\n(systems, maxjobs, supportedFeatures, mandatoryFeatures)";
- for (unsigned int i = 0; i < machines.size(); ++i) {
- hintstring += "\n(%s, %s, %s, %s)";
- }
+ for (unsigned int i = 0; i < machines.size(); ++i)
+ errorText += "\n(%s, %s, %s, %s)";
// add the template values.
string drvstr;
@@ -187,25 +195,21 @@ static int main_build_remote(int argc, char * * argv)
else
drvstr = "<unknown>";
- auto hint = hintformat(hintstring);
- hint
- % drvstr
- % neededSystem
- % concatStringsSep<StringSet>(", ", requiredFeatures)
- % machines.size();
-
- for (auto & m : machines) {
- hint % concatStringsSep<vector<string>>(", ", m.systemTypes)
- % m.maxJobs
- % concatStringsSep<StringSet>(", ", m.supportedFeatures)
- % concatStringsSep<StringSet>(", ", m.mandatoryFeatures);
- }
+ auto error = hintformat(errorText);
+ error
+ % drvstr
+ % neededSystem
+ % concatStringsSep<StringSet>(", ", requiredFeatures)
+ % machines.size();
- logErrorInfo(lvlInfo, {
- .name = "Remote build",
- .description = "Failed to find a machine for remote build!",
- .hint = hint
- });
+ for (auto & m : machines)
+ error
+ % concatStringsSep<vector<string>>(", ", m.systemTypes)
+ % m.maxJobs
+ % concatStringsSep<StringSet>(", ", m.supportedFeatures)
+ % concatStringsSep<StringSet>(", ", m.mandatoryFeatures);
+
+ printMsg(canBuildLocally ? lvlChatty : lvlWarn, error);
std::cerr << "# decline\n";
}
@@ -230,12 +234,9 @@ static int main_build_remote(int argc, char * * argv)
} catch (std::exception & e) {
auto msg = chomp(drainFD(5, false));
- logError({
- .name = "Remote build",
- .hint = hintfmt("cannot build on '%s': %s%s",
- bestMachine->storeUri, e.what(),
- (msg.empty() ? "" : ": " + msg))
- });
+ printError("cannot build on '%s': %s%s",
+ bestMachine->storeUri, e.what(),
+ msg.empty() ? "" : ": " + msg);
bestMachine->enabled = false;
continue;
}
@@ -250,7 +251,7 @@ connected:
std::cerr << "# accept\n" << storeUri << "\n";
auto inputs = readStrings<PathSet>(source);
- auto outputs = readStrings<PathSet>(source);
+ auto wantedOutputs = readStrings<StringSet>(source);
AutoCloseFD uploadLock = openLockFile(currentLoad + "/" + escapeUri(storeUri) + ".upload-lock", true);
@@ -275,6 +276,7 @@ connected:
uploadLock = -1;
auto drv = store->readDerivation(*drvPath);
+ auto outputHashes = staticOutputHashes(*store, drv);
drv.inputSrcs = store->parseStorePathSet(inputs);
auto result = sshStore->buildDerivation(*drvPath, drv);
@@ -282,15 +284,42 @@ connected:
if (!result.success())
throw Error("build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg);
- StorePathSet missing;
- for (auto & path : outputs)
- if (!store->isValidPath(store->parseStorePath(path))) missing.insert(store->parseStorePath(path));
+ std::set<Realisation> missingRealisations;
+ StorePathSet missingPaths;
+ if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
+ for (auto & outputName : wantedOutputs) {
+ auto thisOutputHash = outputHashes.at(outputName);
+ auto thisOutputId = DrvOutput{ thisOutputHash, outputName };
+ if (!store->queryRealisation(thisOutputId)) {
+ debug("missing output %s", outputName);
+ assert(result.builtOutputs.count(thisOutputId));
+ auto newRealisation = result.builtOutputs.at(thisOutputId);
+ missingRealisations.insert(newRealisation);
+ missingPaths.insert(newRealisation.outPath);
+ }
+ }
+ } else {
+ auto outputPaths = drv.outputsAndOptPaths(*store);
+ for (auto & [outputName, hopefullyOutputPath] : outputPaths) {
+ assert(hopefullyOutputPath.second);
+ if (!store->isValidPath(*hopefullyOutputPath.second))
+ missingPaths.insert(*hopefullyOutputPath.second);
+ }
+ }
- if (!missing.empty()) {
+ if (!missingPaths.empty()) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
- for (auto & i : missing)
- store->locksHeld.insert(store->printStorePath(i)); /* FIXME: ugly */
- copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, NoSubstitute);
+ if (auto localStore = store.dynamic_pointer_cast<LocalStore>())
+ for (auto & path : missingPaths)
+ localStore->locksHeld.insert(store->printStorePath(path)); /* FIXME: ugly */
+ copyPaths(ref<Store>(sshStore), store, missingPaths, NoRepair, NoCheckSigs, NoSubstitute);
+ }
+ // XXX: Should be done as part of `copyPaths`
+ for (auto & realisation : missingRealisations) {
+ // Should hold, because if the feature isn't enabled the set
+ // of missing realisations should be empty
+ settings.requireExperimentalFeature("ca-derivations");
+ store->registerDrvOutput(realisation);
}
return 0;
diff --git a/src/nix/command.cc b/src/libcmd/command.cc
index 9a38c77f1..9da470c15 100644
--- a/src/nix/command.cc
+++ b/src/libcmd/command.cc
@@ -11,11 +11,20 @@ extern char * * environ __attribute__((weak));
namespace nix {
-Commands * RegisterCommand::commands = nullptr;
+RegisterCommand::Commands * RegisterCommand::commands = nullptr;
-void NixMultiCommand::printHelp(const string & programName, std::ostream & out)
+nix::Commands RegisterCommand::getCommandsFor(const std::vector<std::string> & prefix)
{
- MultiCommand::printHelp(programName, out);
+ nix::Commands res;
+ for (auto & [name, command] : *RegisterCommand::commands)
+ if (name.size() == prefix.size() + 1) {
+ bool equal = true;
+ for (size_t i = 0; i < prefix.size(); ++i)
+ if (name[i] != prefix[i]) equal = false;
+ if (equal)
+ res.insert_or_assign(name[prefix.size()], command);
+ }
+ return res;
}
nlohmann::json NixMultiCommand::toJSON()
@@ -45,57 +54,72 @@ void StoreCommand::run()
run(getStore());
}
-StorePathsCommand::StorePathsCommand(bool recursive)
+RealisedPathsCommand::RealisedPathsCommand(bool recursive)
: recursive(recursive)
{
if (recursive)
addFlag({
.longName = "no-recursive",
- .description = "apply operation to specified paths only",
+ .description = "Apply operation to specified paths only.",
+ .category = installablesCategory,
.handler = {&this->recursive, false},
});
else
addFlag({
.longName = "recursive",
.shortName = 'r',
- .description = "apply operation to closure of the specified paths",
+ .description = "Apply operation to closure of the specified paths.",
+ .category = installablesCategory,
.handler = {&this->recursive, true},
});
- mkFlag(0, "all", "apply operation to the entire store", &all);
+ addFlag({
+ .longName = "all",
+ .description = "Apply the operation to every store path.",
+ .category = installablesCategory,
+ .handler = {&all, true},
+ });
}
-void StorePathsCommand::run(ref<Store> store)
+void RealisedPathsCommand::run(ref<Store> store)
{
- StorePaths storePaths;
-
+ std::vector<RealisedPath> paths;
if (all) {
if (installables.size())
throw UsageError("'--all' does not expect arguments");
+ // XXX: Only uses opaque paths, ignores all the realisations
for (auto & p : store->queryAllValidPaths())
- storePaths.push_back(p);
- }
-
- else {
- for (auto & p : toStorePaths(store, realiseMode, operateOn, installables))
- storePaths.push_back(p);
-
+ paths.push_back(p);
+ } else {
+ auto pathSet = toRealisedPaths(store, realiseMode, operateOn, installables);
if (recursive) {
- StorePathSet closure;
- store->computeFSClosure(StorePathSet(storePaths.begin(), storePaths.end()), closure, false, false);
- storePaths.clear();
- for (auto & p : closure)
- storePaths.push_back(p);
+ auto roots = std::move(pathSet);
+ pathSet = {};
+ RealisedPath::closure(*store, roots, pathSet);
}
+ for (auto & path : pathSet)
+ paths.push_back(path);
}
- run(store, std::move(storePaths));
+ run(store, std::move(paths));
+}
+
+StorePathsCommand::StorePathsCommand(bool recursive)
+ : RealisedPathsCommand(recursive)
+{
}
-void StorePathCommand::run(ref<Store> store)
+void StorePathsCommand::run(ref<Store> store, std::vector<RealisedPath> paths)
{
- auto storePaths = toStorePaths(store, Realise::Nothing, operateOn, installables);
+ StorePaths storePaths;
+ for (auto & p : paths)
+ storePaths.push_back(p.path());
+ run(store, std::move(storePaths));
+}
+
+void StorePathCommand::run(ref<Store> store, std::vector<StorePath> storePaths)
+{
if (storePaths.size() != 1)
throw UsageError("this command requires exactly one store path");
@@ -119,7 +143,7 @@ MixProfile::MixProfile()
{
addFlag({
.longName = "profile",
- .description = "profile to update",
+ .description = "The profile to update.",
.labels = {"path"},
.handler = {&profile},
.completer = completePath
@@ -138,7 +162,7 @@ void MixProfile::updateProfile(const StorePath & storePath)
profile2, storePath));
}
-void MixProfile::updateProfile(const Buildables & buildables)
+void MixProfile::updateProfile(const DerivedPathsWithHints & buildables)
{
if (!profile) return;
@@ -146,10 +170,10 @@ void MixProfile::updateProfile(const Buildables & buildables)
for (auto & buildable : buildables) {
std::visit(overloaded {
- [&](BuildableOpaque bo) {
+ [&](DerivedPathWithHints::Opaque bo) {
result.push_back(bo.path);
},
- [&](BuildableFromDrv bfd) {
+ [&](DerivedPathWithHints::Built bfd) {
for (auto & output : bfd.outputs) {
/* Output path should be known because we just tried to
build it. */
@@ -157,7 +181,7 @@ void MixProfile::updateProfile(const Buildables & buildables)
result.push_back(*output.second);
}
},
- }, buildable);
+ }, buildable.raw());
}
if (result.size() != 1)
@@ -176,14 +200,14 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false)
addFlag({
.longName = "ignore-environment",
.shortName = 'i',
- .description = "clear the entire environment (except those specified with --keep)",
+ .description = "Clear the entire environment (except those specified with `--keep`).",
.handler = {&ignoreEnvironment, true},
});
addFlag({
.longName = "keep",
.shortName = 'k',
- .description = "keep specified environment variable",
+ .description = "Keep the environment variable *name*.",
.labels = {"name"},
.handler = {[&](std::string s) { keep.insert(s); }},
});
@@ -191,7 +215,7 @@ MixEnvironment::MixEnvironment() : ignoreEnvironment(false)
addFlag({
.longName = "unset",
.shortName = 'u',
- .description = "unset specified environment variable",
+ .description = "Unset the environment variable *name*.",
.labels = {"name"},
.handler = {[&](std::string s) { unset.insert(s); }},
});
diff --git a/src/nix/command.hh b/src/libcmd/command.hh
index d60c8aeb6..9e18c6e51 100644
--- a/src/nix/command.hh
+++ b/src/libcmd/command.hh
@@ -13,6 +13,8 @@ namespace nix {
extern std::string programPath;
+extern char * * savedArgv;
+
class EvalState;
struct Pos;
class Store;
@@ -21,10 +23,10 @@ static constexpr Command::Category catSecondary = 100;
static constexpr Command::Category catUtility = 101;
static constexpr Command::Category catNixInstallation = 102;
+static constexpr auto installablesCategory = "Options that change the interpretation of installables";
+
struct NixMultiCommand : virtual MultiCommand, virtual Command
{
- void printHelp(const string & programName, std::ostream & out) override;
-
nlohmann::json toJSON() override;
};
@@ -46,6 +48,8 @@ struct EvalCommand : virtual StoreCommand, MixEvalArgs
ref<EvalState> getEvalState();
std::shared_ptr<EvalState> evalState;
+
+ ~EvalCommand();
};
struct MixFlakeOptions : virtual Args, EvalCommand
@@ -139,7 +143,7 @@ private:
};
/* A command that operates on zero or more store paths. */
-struct StorePathsCommand : public InstallablesCommand
+struct RealisedPathsCommand : public InstallablesCommand
{
private:
@@ -152,47 +156,67 @@ protected:
public:
- StorePathsCommand(bool recursive = false);
+ RealisedPathsCommand(bool recursive = false);
using StoreCommand::run;
- virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0;
+ virtual void run(ref<Store> store, std::vector<RealisedPath> paths) = 0;
void run(ref<Store> store) override;
bool useDefaultInstallables() override { return !all; }
};
+struct StorePathsCommand : public RealisedPathsCommand
+{
+ StorePathsCommand(bool recursive = false);
+
+ using RealisedPathsCommand::run;
+
+ virtual void run(ref<Store> store, std::vector<StorePath> storePaths) = 0;
+
+ void run(ref<Store> store, std::vector<RealisedPath> paths) override;
+};
+
/* A command that operates on exactly one store path. */
-struct StorePathCommand : public InstallablesCommand
+struct StorePathCommand : public StorePathsCommand
{
- using StoreCommand::run;
+ using StorePathsCommand::run;
virtual void run(ref<Store> store, const StorePath & storePath) = 0;
- void run(ref<Store> store) override;
+ void run(ref<Store> store, std::vector<StorePath> storePaths) override;
};
/* A helper class for registering commands globally. */
struct RegisterCommand
{
+ typedef std::map<std::vector<std::string>, std::function<ref<Command>()>> Commands;
static Commands * commands;
- RegisterCommand(const std::string & name,
+ RegisterCommand(std::vector<std::string> && name,
std::function<ref<Command>()> command)
{
if (!commands) commands = new Commands;
commands->emplace(name, command);
}
+
+ static nix::Commands getCommandsFor(const std::vector<std::string> & prefix);
};
template<class T>
static RegisterCommand registerCommand(const std::string & name)
{
- return RegisterCommand(name, [](){ return make_ref<T>(); });
+ return RegisterCommand({name}, [](){ return make_ref<T>(); });
+}
+
+template<class T>
+static RegisterCommand registerCommand2(std::vector<std::string> && name)
+{
+ return RegisterCommand(std::move(name), [](){ return make_ref<T>(); });
}
-Buildables build(ref<Store> store, Realise mode,
+DerivedPathsWithHints build(ref<Store> store, Realise mode,
std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode = bmNormal);
std::set<StorePath> toStorePaths(ref<Store> store,
@@ -207,6 +231,12 @@ std::set<StorePath> toDerivations(ref<Store> store,
std::vector<std::shared_ptr<Installable>> installables,
bool useDeriver = false);
+std::set<RealisedPath> toRealisedPaths(
+ ref<Store> store,
+ Realise mode,
+ OperateOn operateOn,
+ std::vector<std::shared_ptr<Installable>> installables);
+
/* Helper function to generate args that invoke $EDITOR on
filename:lineno. */
Strings editorFor(const Pos & pos);
@@ -222,7 +252,7 @@ struct MixProfile : virtual StoreCommand
/* If 'profile' is set, make it point at the store path produced
by 'buildables'. */
- void updateProfile(const Buildables & buildables);
+ void updateProfile(const DerivedPathsWithHints & buildables);
};
struct MixDefaultProfile : MixProfile
@@ -252,6 +282,8 @@ void completeFlakeRefWithFragment(
const Strings & defaultFlakeAttrPaths,
std::string_view prefix);
+std::string showVersions(const std::set<std::string> & versions);
+
void printClosureDiff(
ref<Store> store,
const StorePath & beforePath,
diff --git a/src/nix/installables.cc b/src/libcmd/installables.cc
index 7473c9758..5d3026c1a 100644
--- a/src/nix/installables.cc
+++ b/src/libcmd/installables.cc
@@ -16,6 +16,8 @@
#include <regex>
#include <queue>
+#include <nlohmann/json.hpp>
+
namespace nix {
void completeFlakeInputPath(
@@ -31,39 +33,47 @@ void completeFlakeInputPath(
MixFlakeOptions::MixFlakeOptions()
{
+ auto category = "Common flake-related options";
+
addFlag({
.longName = "recreate-lock-file",
- .description = "recreate lock file from scratch",
+ .description = "Recreate the flake's lock file from scratch.",
+ .category = category,
.handler = {&lockFlags.recreateLockFile, true}
});
addFlag({
.longName = "no-update-lock-file",
- .description = "do not allow any updates to the lock file",
+ .description = "Do not allow any updates to the flake's lock file.",
+ .category = category,
.handler = {&lockFlags.updateLockFile, false}
});
addFlag({
.longName = "no-write-lock-file",
- .description = "do not write the newly generated lock file",
+ .description = "Do not write the flake's newly generated lock file.",
+ .category = category,
.handler = {&lockFlags.writeLockFile, false}
});
addFlag({
.longName = "no-registries",
- .description = "don't use flake registries",
+ .description = "Don't allow lookups in the flake registries.",
+ .category = category,
.handler = {&lockFlags.useRegistries, false}
});
addFlag({
.longName = "commit-lock-file",
- .description = "commit changes to the lock file",
+ .description = "Commit changes to the flake's lock file.",
+ .category = category,
.handler = {&lockFlags.commitLockFile, true}
});
addFlag({
.longName = "update-input",
- .description = "update a specific flake input",
+ .description = "Update a specific flake input (ignoring its previous entry in the lock file).",
+ .category = category,
.labels = {"input-path"},
.handler = {[&](std::string s) {
lockFlags.inputUpdates.insert(flake::parseInputPath(s));
@@ -76,9 +86,11 @@ MixFlakeOptions::MixFlakeOptions()
addFlag({
.longName = "override-input",
- .description = "override a specific flake input (e.g. `dwarffs/nixpkgs`)",
+ .description = "Override a specific flake input (e.g. `dwarffs/nixpkgs`). This implies `--no-write-lock-file`.",
+ .category = category,
.labels = {"input-path", "flake-url"},
.handler = {[&](std::string inputPath, std::string flakeRef) {
+ lockFlags.writeLockFile = false;
lockFlags.inputOverrides.insert_or_assign(
flake::parseInputPath(inputPath),
parseFlakeRef(flakeRef, absPath(".")));
@@ -87,7 +99,8 @@ MixFlakeOptions::MixFlakeOptions()
addFlag({
.longName = "inputs-from",
- .description = "use the inputs of the specified flake as registry entries",
+ .description = "Use the inputs of the specified flake as registry entries.",
+ .category = category,
.labels = {"flake-url"},
.handler = {[&](std::string flakeRef) {
auto evalState = getEvalState();
@@ -116,22 +129,25 @@ SourceExprCommand::SourceExprCommand()
addFlag({
.longName = "file",
.shortName = 'f',
- .description = "evaluate *file* rather than the default",
+ .description = "Interpret installables as attribute paths relative to the Nix expression stored in *file*.",
+ .category = installablesCategory,
.labels = {"file"},
.handler = {&file},
.completer = completePath
});
addFlag({
- .longName ="expr",
- .description = "evaluate attributes from *expr*",
+ .longName = "expr",
+ .description = "Interpret installables as attribute paths relative to the Nix expression *expr*.",
+ .category = installablesCategory,
.labels = {"expr"},
.handler = {&expr}
});
addFlag({
- .longName ="derivation",
- .description = "operate on the store derivation rather than its outputs",
+ .longName = "derivation",
+ .description = "Operate on the store derivation rather than its outputs.",
+ .category = installablesCategory,
.handler = {&operateOn, OperateOn::Derivation},
});
}
@@ -240,6 +256,12 @@ ref<EvalState> EvalCommand::getEvalState()
return ref<EvalState>(evalState);
}
+EvalCommand::~EvalCommand()
+{
+ if (evalState)
+ evalState->printStats();
+}
+
void completeFlakeRef(ref<Store> store, std::string_view prefix)
{
if (prefix == "")
@@ -263,9 +285,9 @@ void completeFlakeRef(ref<Store> store, std::string_view prefix)
}
}
-Buildable Installable::toBuildable()
+DerivedPathWithHints Installable::toDerivedPathWithHints()
{
- auto buildables = toBuildables();
+ auto buildables = toDerivedPathsWithHints();
if (buildables.size() != 1)
throw Error("installable '%s' evaluates to %d derivations, where only one is expected", what(), buildables.size());
return std::move(buildables[0]);
@@ -299,7 +321,7 @@ struct InstallableStorePath : Installable
std::string what() override { return store->printStorePath(storePath); }
- Buildables toBuildables() override
+ DerivedPathsWithHints toDerivedPathsWithHints() override
{
if (storePath.isDerivation()) {
std::map<std::string, std::optional<StorePath>> outputs;
@@ -307,14 +329,14 @@ struct InstallableStorePath : Installable
for (auto & [name, output] : drv.outputsAndOptPaths(*store))
outputs.emplace(name, output.second);
return {
- BuildableFromDrv {
+ DerivedPathWithHints::Built {
.drvPath = storePath,
.outputs = std::move(outputs)
}
};
} else {
return {
- BuildableOpaque {
+ DerivedPathWithHints::Opaque {
.path = storePath,
}
};
@@ -327,9 +349,9 @@ struct InstallableStorePath : Installable
}
};
-Buildables InstallableValue::toBuildables()
+DerivedPathsWithHints InstallableValue::toDerivedPathsWithHints()
{
- Buildables res;
+ DerivedPathsWithHints res;
std::map<StorePath, std::map<std::string, std::optional<StorePath>>> drvsToOutputs;
@@ -342,7 +364,7 @@ Buildables InstallableValue::toBuildables()
}
for (auto & i : drvsToOutputs)
- res.push_back(BuildableFromDrv { i.first, i.second });
+ res.push_back(DerivedPathWithHints::Built { i.first, i.second });
return res;
}
@@ -382,7 +404,7 @@ std::vector<InstallableValue::DerivationInfo> InstallableAttrPath::toDerivations
for (auto & drvInfo : drvInfos) {
res.push_back({
state->store->parseStorePath(drvInfo.queryDrvPath()),
- state->store->parseStorePath(drvInfo.queryOutPath()),
+ state->store->maybeParseStorePath(drvInfo.queryOutPath()),
drvInfo.queryOutputName()
});
}
@@ -456,6 +478,23 @@ static std::string showAttrPaths(const std::vector<std::string> & paths)
return s;
}
+InstallableFlake::InstallableFlake(
+ SourceExprCommand * cmd,
+ ref<EvalState> state,
+ FlakeRef && flakeRef,
+ Strings && attrPaths,
+ Strings && prefixes,
+ const flake::LockFlags & lockFlags)
+ : InstallableValue(state),
+ flakeRef(flakeRef),
+ attrPaths(attrPaths),
+ prefixes(prefixes),
+ lockFlags(lockFlags)
+{
+ if (cmd && cmd->getAutoArgs(*state)->size())
+ throw UsageError("'--arg' and '--argstr' are incompatible with flakes");
+}
+
std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableFlake::toDerivation()
{
auto lockedFlake = getLockedFlake();
@@ -474,7 +513,7 @@ std::tuple<std::string, FlakeRef, InstallableValue::DerivationInfo> InstallableF
auto drvInfo = DerivationInfo{
std::move(drvPath),
- state->store->parseStorePath(attr->getAttr(state->sOutPath)->getString()),
+ state->store->maybeParseStorePath(attr->getAttr(state->sOutPath)->getString()),
attr->getAttr(state->sOutputName)->getString()
};
@@ -533,8 +572,11 @@ InstallableFlake::getCursors(EvalState & state)
std::shared_ptr<flake::LockedFlake> InstallableFlake::getLockedFlake() const
{
- if (!_lockedFlake)
+ if (!_lockedFlake) {
_lockedFlake = std::make_shared<flake::LockedFlake>(lockFlake(*state, flakeRef, lockFlags));
+ _lockedFlake->flake.config.apply();
+ // FIXME: send new config to the daemon.
+ }
return _lockedFlake;
}
@@ -585,9 +627,12 @@ std::vector<std::shared_ptr<Installable>> SourceExprCommand::parseInstallables(
try {
auto [flakeRef, fragment] = parseFlakeRefWithFragment(s, absPath("."));
result.push_back(std::make_shared<InstallableFlake>(
- getEvalState(), std::move(flakeRef),
+ this,
+ getEvalState(),
+ std::move(flakeRef),
fragment == "" ? getDefaultFlakeAttrPaths() : Strings{fragment},
- getDefaultFlakeAttrPathPrefixes(), lockFlags));
+ getDefaultFlakeAttrPathPrefixes(),
+ lockFlags));
continue;
} catch (...) {
ex = std::current_exception();
@@ -626,29 +671,30 @@ std::shared_ptr<Installable> SourceExprCommand::parseInstallable(
return installables.front();
}
-Buildables build(ref<Store> store, Realise mode,
+DerivedPathsWithHints build(ref<Store> store, Realise mode,
std::vector<std::shared_ptr<Installable>> installables, BuildMode bMode)
{
if (mode == Realise::Nothing)
settings.readOnlyMode = true;
- Buildables buildables;
+ DerivedPathsWithHints buildables;
- std::vector<StorePathWithOutputs> pathsToBuild;
+ std::vector<DerivedPath> pathsToBuild;
for (auto & i : installables) {
- for (auto & b : i->toBuildables()) {
+ for (auto & b : i->toDerivedPathsWithHints()) {
std::visit(overloaded {
- [&](BuildableOpaque bo) {
- pathsToBuild.push_back({bo.path});
+ [&](DerivedPathWithHints::Opaque bo) {
+ pathsToBuild.push_back(bo);
},
- [&](BuildableFromDrv bfd) {
+ [&](DerivedPathWithHints::Built bfd) {
StringSet outputNames;
for (auto & output : bfd.outputs)
outputNames.insert(output.first);
- pathsToBuild.push_back({bfd.drvPath, outputNames});
+ pathsToBuild.push_back(
+ DerivedPath::Built{bfd.drvPath, outputNames});
},
- }, b);
+ }, b.raw());
buildables.push_back(std::move(b));
}
}
@@ -661,36 +707,65 @@ Buildables build(ref<Store> store, Realise mode,
return buildables;
}
-StorePathSet toStorePaths(ref<Store> store,
- Realise mode, OperateOn operateOn,
+std::set<RealisedPath> toRealisedPaths(
+ ref<Store> store,
+ Realise mode,
+ OperateOn operateOn,
std::vector<std::shared_ptr<Installable>> installables)
{
- StorePathSet outPaths;
-
+ std::set<RealisedPath> res;
if (operateOn == OperateOn::Output) {
for (auto & b : build(store, mode, installables))
std::visit(overloaded {
- [&](BuildableOpaque bo) {
- outPaths.insert(bo.path);
+ [&](DerivedPathWithHints::Opaque bo) {
+ res.insert(bo.path);
},
- [&](BuildableFromDrv bfd) {
+ [&](DerivedPathWithHints::Built bfd) {
+ auto drv = store->readDerivation(bfd.drvPath);
+ auto outputHashes = staticOutputHashes(*store, drv);
for (auto & output : bfd.outputs) {
- if (!output.second)
- throw Error("Cannot operate on output of unbuilt CA drv");
- outPaths.insert(*output.second);
+ if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+ if (!outputHashes.count(output.first))
+ throw Error(
+ "the derivation '%s' doesn't have an output named '%s'",
+ store->printStorePath(bfd.drvPath),
+ output.first);
+ auto outputId = DrvOutput{outputHashes.at(output.first), output.first};
+ auto realisation = store->queryRealisation(outputId);
+ if (!realisation)
+ throw Error("cannot operate on an output of unbuilt content-addresed derivation '%s'", outputId.to_string());
+ res.insert(RealisedPath{*realisation});
+ }
+ else {
+ // If ca-derivations isn't enabled, behave as if
+ // all the paths are opaque to keep the default
+ // behavior
+ assert(output.second);
+ res.insert(*output.second);
+ }
}
},
- }, b);
+ }, b.raw());
} else {
if (mode == Realise::Nothing)
settings.readOnlyMode = true;
for (auto & i : installables)
- for (auto & b : i->toBuildables())
- if (auto bfd = std::get_if<BuildableFromDrv>(&b))
- outPaths.insert(bfd->drvPath);
+ for (auto & b : i->toDerivedPathsWithHints())
+ if (auto bfd = std::get_if<DerivedPathWithHints::Built>(&b))
+ res.insert(bfd->drvPath);
}
+ return res;
+}
+
+StorePathSet toStorePaths(ref<Store> store,
+ Realise mode, OperateOn operateOn,
+ std::vector<std::shared_ptr<Installable>> installables)
+{
+ StorePathSet outPaths;
+ for (auto & path : toRealisedPaths(store, mode, operateOn, installables))
+ outPaths.insert(path.path());
return outPaths;
}
@@ -712,9 +787,9 @@ StorePathSet toDerivations(ref<Store> store,
StorePathSet drvPaths;
for (auto & i : installables)
- for (auto & b : i->toBuildables())
+ for (auto & b : i->toDerivedPathsWithHints())
std::visit(overloaded {
- [&](BuildableOpaque bo) {
+ [&](DerivedPathWithHints::Opaque bo) {
if (!useDeriver)
throw Error("argument '%s' did not evaluate to a derivation", i->what());
auto derivers = store->queryValidDerivers(bo.path);
@@ -723,10 +798,10 @@ StorePathSet toDerivations(ref<Store> store,
// FIXME: use all derivers?
drvPaths.insert(*derivers.begin());
},
- [&](BuildableFromDrv bfd) {
+ [&](DerivedPathWithHints::Built bfd) {
drvPaths.insert(bfd.drvPath);
},
- }, b);
+ }, b.raw());
return drvPaths;
}
diff --git a/src/nix/installables.hh b/src/libcmd/installables.hh
index c7c2f8981..403403c07 100644
--- a/src/nix/installables.hh
+++ b/src/libcmd/installables.hh
@@ -2,6 +2,8 @@
#include "util.hh"
#include "path.hh"
+#include "path-with-outputs.hh"
+#include "derived-path.hh"
#include "eval.hh"
#include "flake/flake.hh"
@@ -14,22 +16,6 @@ struct SourceExprCommand;
namespace eval_cache { class EvalCache; class AttrCursor; }
-struct BuildableOpaque {
- StorePath path;
-};
-
-struct BuildableFromDrv {
- StorePath drvPath;
- std::map<std::string, std::optional<StorePath>> outputs;
-};
-
-typedef std::variant<
- BuildableOpaque,
- BuildableFromDrv
-> Buildable;
-
-typedef std::vector<Buildable> Buildables;
-
struct App
{
std::vector<StorePathWithOutputs> context;
@@ -43,9 +29,9 @@ struct Installable
virtual std::string what() = 0;
- virtual Buildables toBuildables() = 0;
+ virtual DerivedPathsWithHints toDerivedPathsWithHints() = 0;
- Buildable toBuildable();
+ DerivedPathWithHints toDerivedPathWithHints();
App toApp(EvalState & state);
@@ -88,7 +74,7 @@ struct InstallableValue : Installable
virtual std::vector<DerivationInfo> toDerivations() = 0;
- Buildables toBuildables() override;
+ DerivedPathsWithHints toDerivedPathsWithHints() override;
};
struct InstallableFlake : InstallableValue
@@ -99,11 +85,13 @@ struct InstallableFlake : InstallableValue
const flake::LockFlags & lockFlags;
mutable std::shared_ptr<flake::LockedFlake> _lockedFlake;
- InstallableFlake(ref<EvalState> state, FlakeRef && flakeRef,
- Strings && attrPaths, Strings && prefixes, const flake::LockFlags & lockFlags)
- : InstallableValue(state), flakeRef(flakeRef), attrPaths(attrPaths),
- prefixes(prefixes), lockFlags(lockFlags)
- { }
+ InstallableFlake(
+ SourceExprCommand * cmd,
+ ref<EvalState> state,
+ FlakeRef && flakeRef,
+ Strings && attrPaths,
+ Strings && prefixes,
+ const flake::LockFlags & lockFlags);
std::string what() override { return flakeRef.to_string() + "#" + *attrPaths.begin(); }
diff --git a/src/nix/legacy.cc b/src/libcmd/legacy.cc
index 6df09ee37..6df09ee37 100644
--- a/src/nix/legacy.cc
+++ b/src/libcmd/legacy.cc
diff --git a/src/nix/legacy.hh b/src/libcmd/legacy.hh
index f503b0da3..f503b0da3 100644
--- a/src/nix/legacy.hh
+++ b/src/libcmd/legacy.hh
diff --git a/src/libcmd/local.mk b/src/libcmd/local.mk
new file mode 100644
index 000000000..ab0e0e43d
--- /dev/null
+++ b/src/libcmd/local.mk
@@ -0,0 +1,15 @@
+libraries += libcmd
+
+libcmd_NAME = libnixcmd
+
+libcmd_DIR := $(d)
+
+libcmd_SOURCES := $(wildcard $(d)/*.cc)
+
+libcmd_CXXFLAGS += -I src/libutil -I src/libstore -I src/libexpr -I src/libmain -I src/libfetchers
+
+libcmd_LDFLAGS = -llowdown
+
+libcmd_LIBS = libstore libutil libexpr libmain libfetchers
+
+$(eval $(call install-file-in, $(d)/nix-cmd.pc, $(prefix)/lib/pkgconfig, 0644))
diff --git a/src/nix/markdown.cc b/src/libcmd/markdown.cc
index 40788a42f..d25113d93 100644
--- a/src/nix/markdown.cc
+++ b/src/libcmd/markdown.cc
@@ -3,9 +3,7 @@
#include "finally.hh"
#include <sys/queue.h>
-extern "C" {
#include <lowdown.h>
-}
namespace nix {
@@ -42,7 +40,9 @@ std::string renderMarkdownToTerminal(std::string_view markdown)
throw Error("cannot allocate Markdown output buffer");
Finally freeBuffer([&]() { lowdown_buf_free(buf); });
- lowdown_term_rndr(buf, nullptr, renderer, node);
+ int rndr_res = lowdown_term_rndr(buf, nullptr, renderer, node);
+ if (!rndr_res)
+ throw Error("allocation error while rendering Markdown");
return std::string(buf->data, buf->size);
}
diff --git a/src/nix/markdown.hh b/src/libcmd/markdown.hh
index 78320fcf5..78320fcf5 100644
--- a/src/nix/markdown.hh
+++ b/src/libcmd/markdown.hh
diff --git a/src/libcmd/nix-cmd.pc.in b/src/libcmd/nix-cmd.pc.in
new file mode 100644
index 000000000..1761a9f41
--- /dev/null
+++ b/src/libcmd/nix-cmd.pc.in
@@ -0,0 +1,9 @@
+prefix=@prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: Nix
+Description: Nix Package Manager
+Version: @PACKAGE_VERSION@
+Libs: -L${libdir} -lnixcmd
+Cflags: -I${includedir}/nix -std=c++17
diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc
index 83854df49..9dd557205 100644
--- a/src/libexpr/attr-path.cc
+++ b/src/libexpr/attr-path.cc
@@ -52,9 +52,7 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
for (auto & attr : tokens) {
/* Is i an index (integer) or a normal attribute name? */
- enum { apAttr, apIndex } apType = apAttr;
- unsigned int attrIndex;
- if (string2Int(attr, attrIndex)) apType = apIndex;
+ auto attrIndex = string2Int<unsigned int>(attr);
/* Evaluate the expression. */
Value * vNew = state.allocValue();
@@ -65,9 +63,9 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
/* It should evaluate to either a set or an expression,
according to what is specified in the attrPath. */
- if (apType == apAttr) {
+ if (!attrIndex) {
- if (v->type != tAttrs)
+ if (v->type() != nAttrs)
throw TypeError(
"the expression selected by the selection path '%1%' should be a set but is %2%",
attrPath,
@@ -82,17 +80,17 @@ std::pair<Value *, Pos> findAlongAttrPath(EvalState & state, const string & attr
pos = *a->pos;
}
- else if (apType == apIndex) {
+ else {
if (!v->isList())
throw TypeError(
"the expression selected by the selection path '%1%' should be a list but is %2%",
attrPath,
showType(*v));
- if (attrIndex >= v->listSize())
- throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", attrIndex, attrPath);
+ if (*attrIndex >= v->listSize())
+ throw AttrPathNotFound("list index %1% in selection path '%2%' is out of range", *attrIndex, attrPath);
- v = v->listElems()[attrIndex];
+ v = v->listElems()[*attrIndex];
pos = noPos;
}
diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc
index b1d61a285..b6091c955 100644
--- a/src/libexpr/attr-set.cc
+++ b/src/libexpr/attr-set.cc
@@ -24,9 +24,7 @@ void EvalState::mkAttrs(Value & v, size_t capacity)
v = vEmptySet;
return;
}
- clearValue(v);
- v.type = tAttrs;
- v.attrs = allocBindings(capacity);
+ v.mkAttrs(allocBindings(capacity));
nrAttrsets++;
nrAttrsInAttrsets += capacity;
}
diff --git a/src/libexpr/attr-set.hh b/src/libexpr/attr-set.hh
index 7eaa16c59..6d68e5df3 100644
--- a/src/libexpr/attr-set.hh
+++ b/src/libexpr/attr-set.hh
@@ -77,7 +77,7 @@ public:
auto a = get(name);
if (!a)
throw Error({
- .hint = hintfmt("attribute '%s' missing", name),
+ .msg = hintfmt("attribute '%s' missing", name),
.errPos = pos
});
diff --git a/src/libexpr/common-eval-args.cc b/src/libexpr/common-eval-args.cc
index 10c1a6975..aa14bf79b 100644
--- a/src/libexpr/common-eval-args.cc
+++ b/src/libexpr/common-eval-args.cc
@@ -12,16 +12,20 @@ namespace nix {
MixEvalArgs::MixEvalArgs()
{
+ auto category = "Common evaluation options";
+
addFlag({
.longName = "arg",
- .description = "argument to be passed to Nix functions",
+ .description = "Pass the value *expr* as the argument *name* to Nix functions.",
+ .category = category,
.labels = {"name", "expr"},
.handler = {[&](std::string name, std::string expr) { autoArgs[name] = 'E' + expr; }}
});
addFlag({
.longName = "argstr",
- .description = "string-valued argument to be passed to Nix functions",
+ .description = "Pass the string *string* as the argument *name* to Nix functions.",
+ .category = category,
.labels = {"name", "string"},
.handler = {[&](std::string name, std::string s) { autoArgs[name] = 'S' + s; }},
});
@@ -29,14 +33,16 @@ MixEvalArgs::MixEvalArgs()
addFlag({
.longName = "include",
.shortName = 'I',
- .description = "add a path to the list of locations used to look up `<...>` file names",
+ .description = "Add *path* to the list of locations used to look up `<...>` file names.",
+ .category = category,
.labels = {"path"},
.handler = {[&](std::string s) { searchPath.push_back(s); }}
});
addFlag({
.longName = "impure",
- .description = "allow access to mutable paths and repositories",
+ .description = "Allow access to mutable paths and repositories.",
+ .category = category,
.handler = {[&]() {
evalSettings.pureEval = false;
}},
@@ -44,7 +50,8 @@ MixEvalArgs::MixEvalArgs()
addFlag({
.longName = "override-flake",
- .description = "override a flake registry value",
+ .description = "Override the flake registries, redirecting *original-ref* to *resolved-ref*.",
+ .category = category,
.labels = {"original-ref", "resolved-ref"},
.handler = {[&](std::string _from, std::string _to) {
auto from = parseFlakeRef(_from, absPath("."));
diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc
index 381344b40..98d91c905 100644
--- a/src/libexpr/eval-cache.cc
+++ b/src/libexpr/eval-cache.cc
@@ -390,14 +390,14 @@ Value & AttrCursor::forceValue()
}
if (root->db && (!cachedValue || std::get_if<placeholder_t>(&cachedValue->second))) {
- if (v.type == tString)
+ if (v.type() == nString)
cachedValue = {root->db->setString(getKey(), v.string.s, v.string.context),
string_t{v.string.s, {}}};
- else if (v.type == tPath)
- cachedValue = {root->db->setString(getKey(), v.path), v.path};
- else if (v.type == tBool)
+ else if (v.type() == nPath)
+ cachedValue = {root->db->setString(getKey(), v.path), string_t{v.path, {}}};
+ else if (v.type() == nBool)
cachedValue = {root->db->setBool(getKey(), v.boolean), v.boolean};
- else if (v.type == tAttrs)
+ else if (v.type() == nAttrs)
; // FIXME: do something?
else
cachedValue = {root->db->setMisc(getKey()), misc_t()};
@@ -442,7 +442,7 @@ std::shared_ptr<AttrCursor> AttrCursor::maybeGetAttr(Symbol name, bool forceErro
auto & v = forceValue();
- if (v.type != tAttrs)
+ if (v.type() != nAttrs)
return nullptr;
//throw TypeError("'%s' is not an attribute set", getAttrPathStr());
@@ -512,10 +512,10 @@ std::string AttrCursor::getString()
auto & v = forceValue();
- if (v.type != tString && v.type != tPath)
- throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
+ if (v.type() != nString && v.type() != nPath)
+ throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()));
- return v.type == tString ? v.string.s : v.path;
+ return v.type() == nString ? v.string.s : v.path;
}
string_t AttrCursor::getStringWithContext()
@@ -525,8 +525,17 @@ string_t AttrCursor::getStringWithContext()
cachedValue = root->db->getAttr(getKey(), root->state.symbols);
if (cachedValue && !std::get_if<placeholder_t>(&cachedValue->second)) {
if (auto s = std::get_if<string_t>(&cachedValue->second)) {
- debug("using cached string attribute '%s'", getAttrPathStr());
- return *s;
+ bool valid = true;
+ for (auto & c : s->second) {
+ if (!root->state.store->isValidPath(root->state.store->parseStorePath(c.first))) {
+ valid = false;
+ break;
+ }
+ }
+ if (valid) {
+ debug("using cached string attribute '%s'", getAttrPathStr());
+ return *s;
+ }
} else
throw TypeError("'%s' is not a string", getAttrPathStr());
}
@@ -534,12 +543,12 @@ string_t AttrCursor::getStringWithContext()
auto & v = forceValue();
- if (v.type == tString)
+ if (v.type() == nString)
return {v.string.s, v.getContext()};
- else if (v.type == tPath)
+ else if (v.type() == nPath)
return {v.path, {}};
else
- throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type));
+ throw TypeError("'%s' is not a string but %s", getAttrPathStr(), showType(v.type()));
}
bool AttrCursor::getBool()
@@ -558,7 +567,7 @@ bool AttrCursor::getBool()
auto & v = forceValue();
- if (v.type != tBool)
+ if (v.type() != nBool)
throw TypeError("'%s' is not a Boolean", getAttrPathStr());
return v.boolean;
@@ -580,7 +589,7 @@ std::vector<Symbol> AttrCursor::getAttrs()
auto & v = forceValue();
- if (v.type != tAttrs)
+ if (v.type() != nAttrs)
throw TypeError("'%s' is not an attribute set", getAttrPathStr());
std::vector<Symbol> attrs;
diff --git a/src/libexpr/eval-inline.hh b/src/libexpr/eval-inline.hh
index 30f6ec7db..655408cd3 100644
--- a/src/libexpr/eval-inline.hh
+++ b/src/libexpr/eval-inline.hh
@@ -10,7 +10,7 @@ namespace nix {
LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s))
{
throw EvalError({
- .hint = hintfmt(s),
+ .msg = hintfmt(s),
.errPos = pos
});
}
@@ -24,7 +24,7 @@ LocalNoInlineNoReturn(void throwTypeError(const char * s, const Value & v))
LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const Value & v))
{
throw TypeError({
- .hint = hintfmt(s, showType(v)),
+ .msg = hintfmt(s, showType(v)),
.errPos = pos
});
}
@@ -32,23 +32,21 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
void EvalState::forceValue(Value & v, const Pos & pos)
{
- if (v.type == tThunk) {
+ if (v.isThunk()) {
Env * env = v.thunk.env;
Expr * expr = v.thunk.expr;
try {
- v.type = tBlackhole;
+ v.mkBlackhole();
//checkInterrupt();
expr->eval(*this, *env, v);
} catch (...) {
- v.type = tThunk;
- v.thunk.env = env;
- v.thunk.expr = expr;
+ v.mkThunk(env, expr);
throw;
}
}
- else if (v.type == tApp)
+ else if (v.isApp())
callFunction(*v.app.left, *v.app.right, v, noPos);
- else if (v.type == tBlackhole)
+ else if (v.isBlackhole())
throwEvalError(pos, "infinite recursion encountered");
}
@@ -56,7 +54,7 @@ void EvalState::forceValue(Value & v, const Pos & pos)
inline void EvalState::forceAttrs(Value & v)
{
forceValue(v);
- if (v.type != tAttrs)
+ if (v.type() != nAttrs)
throwTypeError("value is %1% while a set was expected", v);
}
@@ -64,7 +62,7 @@ inline void EvalState::forceAttrs(Value & v)
inline void EvalState::forceAttrs(Value & v, const Pos & pos)
{
forceValue(v, pos);
- if (v.type != tAttrs)
+ if (v.type() != nAttrs)
throwTypeError(pos, "value is %1% while a set was expected", v);
}
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index d6366050c..3afe2e47b 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -27,6 +27,10 @@
#include <gc/gc.h>
#include <gc/gc_cpp.h>
+#include <boost/coroutine2/coroutine.hpp>
+#include <boost/coroutine2/protected_fixedsize_stack.hpp>
+#include <boost/context/stack_context.hpp>
+
#endif
namespace nix {
@@ -64,7 +68,7 @@ RootValue allocRootValue(Value * v)
}
-static void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
+void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v)
{
checkInterrupt();
@@ -73,7 +77,7 @@ static void printValue(std::ostream & str, std::set<const Value *> & active, con
return;
}
- switch (v.type) {
+ switch (v.internalType) {
case tInt:
str << v.integer;
break;
@@ -154,32 +158,27 @@ std::ostream & operator << (std::ostream & str, const Value & v)
const Value *getPrimOp(const Value &v) {
const Value * primOp = &v;
- while (primOp->type == tPrimOpApp) {
+ while (primOp->isPrimOpApp()) {
primOp = primOp->primOpApp.left;
}
- assert(primOp->type == tPrimOp);
+ assert(primOp->isPrimOp());
return primOp;
}
-
string showType(ValueType type)
{
switch (type) {
- case tInt: return "an integer";
- case tBool: return "a Boolean";
- case tString: return "a string";
- case tPath: return "a path";
- case tNull: return "null";
- case tAttrs: return "a set";
- case tList1: case tList2: case tListN: return "a list";
- case tThunk: return "a thunk";
- case tApp: return "a function application";
- case tLambda: return "a function";
- case tBlackhole: return "a black hole";
- case tPrimOp: return "a built-in function";
- case tPrimOpApp: return "a partially applied built-in function";
- case tExternal: return "an external value";
- case tFloat: return "a float";
+ case nInt: return "an integer";
+ case nBool: return "a Boolean";
+ case nString: return "a string";
+ case nPath: return "a path";
+ case nNull: return "null";
+ case nAttrs: return "a set";
+ case nList: return "a list";
+ case nFunction: return "a function";
+ case nExternal: return "an external value";
+ case nFloat: return "a float";
+ case nThunk: return "a thunk";
}
abort();
}
@@ -187,15 +186,18 @@ string showType(ValueType type)
string showType(const Value & v)
{
- switch (v.type) {
+ switch (v.internalType) {
case tString: return v.string.context ? "a string with context" : "a string";
case tPrimOp:
return fmt("the built-in function '%s'", string(v.primOp->name));
case tPrimOpApp:
return fmt("the partially applied built-in function '%s'", string(getPrimOp(v)->primOp->name));
case tExternal: return v.external->showType();
+ case tThunk: return "a thunk";
+ case tApp: return "a function application";
+ case tBlackhole: return "a black hole";
default:
- return showType(v.type);
+ return showType(v.type());
}
}
@@ -203,12 +205,13 @@ string showType(const Value & v)
bool Value::isTrivial() const
{
return
- type != tApp
- && type != tPrimOpApp
- && (type != tThunk
+ internalType != tApp
+ && internalType != tPrimOpApp
+ && (internalType != tThunk
|| (dynamic_cast<ExprAttrs *>(thunk.expr)
&& ((ExprAttrs *) thunk.expr)->dynamicAttrs.empty())
- || dynamic_cast<ExprLambda *>(thunk.expr));
+ || dynamic_cast<ExprLambda *>(thunk.expr)
+ || dynamic_cast<ExprList *>(thunk.expr));
}
@@ -219,6 +222,31 @@ static void * oomHandler(size_t requested)
/* Convert this to a proper C++ exception. */
throw std::bad_alloc();
}
+
+class BoehmGCStackAllocator : public StackAllocator {
+ boost::coroutines2::protected_fixedsize_stack stack {
+ // We allocate 8 MB, the default max stack size on NixOS.
+ // A smaller stack might be quicker to allocate but reduces the stack
+ // depth available for source filter expressions etc.
+ std::max(boost::context::stack_traits::default_size(), static_cast<std::size_t>(8 * 1024 * 1024))
+ };
+
+ public:
+ boost::context::stack_context allocate() override {
+ auto sctx = stack.allocate();
+ GC_add_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp);
+ return sctx;
+ }
+
+ void deallocate(boost::context::stack_context sctx) override {
+ GC_remove_roots(static_cast<char *>(sctx.sp) - sctx.size, sctx.sp);
+ stack.deallocate(sctx);
+ }
+
+};
+
+static BoehmGCStackAllocator boehmGCStackAllocator;
+
#endif
@@ -256,6 +284,8 @@ void initGC()
GC_set_oom_fn(oomHandler);
+ StackAllocator::defaultAllocator = &boehmGCStackAllocator;
+
/* Set the initial heap size to something fairly big (25% of
physical RAM, up to a maximum of 384 MiB) so that in most cases
we don't need to garbage collect at all. (Collection has a
@@ -372,11 +402,6 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
for (auto & i : evalSettings.nixPath.get()) addToSearchPath(i);
}
- try {
- addToSearchPath("nix=" + canonPath(settings.nixDataDir + "/nix/corepkgs", true));
- } catch (Error &) {
- }
-
if (evalSettings.restrictEval || evalSettings.pureEval) {
allowedPaths = PathSet();
@@ -400,9 +425,7 @@ EvalState::EvalState(const Strings & _searchPath, ref<Store> store)
}
}
- clearValue(vEmptySet);
- vEmptySet.type = tAttrs;
- vEmptySet.attrs = allocBindings(0);
+ vEmptySet.mkAttrs(allocBindings(0));
createBaseEnv();
}
@@ -429,6 +452,8 @@ Path EvalState::checkSourcePath(const Path & path_)
*/
Path abspath = canonPath(path_);
+ if (hasPrefix(abspath, corepkgsPrefix)) return abspath;
+
for (auto & i : *allowedPaths) {
if (isDirOrInDir(abspath, i)) {
found = true;
@@ -518,16 +543,14 @@ Value * EvalState::addPrimOp(const string & name,
the primop to a dummy value. */
if (arity == 0) {
auto vPrimOp = allocValue();
- vPrimOp->type = tPrimOp;
- vPrimOp->primOp = new PrimOp { .fun = primOp, .arity = 1, .name = sym };
+ vPrimOp->mkPrimOp(new PrimOp { .fun = primOp, .arity = 1, .name = sym });
Value v;
mkApp(v, *vPrimOp, *vPrimOp);
return addConstant(name, v);
}
Value * v = allocValue();
- v->type = tPrimOp;
- v->primOp = new PrimOp { .fun = primOp, .arity = arity, .name = sym };
+ v->mkPrimOp(new PrimOp { .fun = primOp, .arity = arity, .name = sym });
staticBaseEnv.vars[symbols.create(name)] = baseEnvDispl;
baseEnv.values[baseEnvDispl++] = v;
baseEnv.values[0]->attrs->push_back(Attr(sym, v));
@@ -542,8 +565,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
if (primOp.arity == 0) {
primOp.arity = 1;
auto vPrimOp = allocValue();
- vPrimOp->type = tPrimOp;
- vPrimOp->primOp = new PrimOp(std::move(primOp));
+ vPrimOp->mkPrimOp(new PrimOp(std::move(primOp)));
Value v;
mkApp(v, *vPrimOp, *vPrimOp);
return addConstant(primOp.name, v);
@@ -554,8 +576,7 @@ Value * EvalState::addPrimOp(PrimOp && primOp)
primOp.name = symbols.create(std::string(primOp.name, 2));
Value * v = allocValue();
- v->type = tPrimOp;
- v->primOp = new PrimOp(std::move(primOp));
+ v->mkPrimOp(new PrimOp(std::move(primOp)));
staticBaseEnv.vars[envName] = baseEnvDispl;
baseEnv.values[baseEnvDispl++] = v;
baseEnv.values[0]->attrs->push_back(Attr(primOp.name, v));
@@ -571,10 +592,8 @@ Value & EvalState::getBuiltin(const string & name)
std::optional<EvalState::Doc> EvalState::getDoc(Value & v)
{
- if (v.type == tPrimOp || v.type == tPrimOpApp) {
+ if (v.isPrimOp()) {
auto v2 = &v;
- while (v2->type == tPrimOpApp)
- v2 = v2->primOpApp.left;
if (v2->primOp->doc)
return Doc {
.pos = noPos,
@@ -601,7 +620,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2))
LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2))
{
throw EvalError({
- .hint = hintfmt(s, s2),
+ .msg = hintfmt(s, s2),
.errPos = pos
});
}
@@ -614,7 +633,7 @@ LocalNoInlineNoReturn(void throwEvalError(const char * s, const string & s2, con
LocalNoInlineNoReturn(void throwEvalError(const Pos & pos, const char * s, const string & s2, const string & s3))
{
throw EvalError({
- .hint = hintfmt(s, s2, s3),
+ .msg = hintfmt(s, s2, s3),
.errPos = pos
});
}
@@ -623,7 +642,7 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const
{
// p1 is where the error occurred; p2 is a position mentioned in the message.
throw EvalError({
- .hint = hintfmt(s, sym, p2),
+ .msg = hintfmt(s, sym, p2),
.errPos = p1
});
}
@@ -631,20 +650,15 @@ LocalNoInlineNoReturn(void throwEvalError(const Pos & p1, const char * s, const
LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s))
{
throw TypeError({
- .hint = hintfmt(s),
+ .msg = hintfmt(s),
.errPos = pos
});
}
-LocalNoInlineNoReturn(void throwTypeError(const char * s, const string & s1))
-{
- throw TypeError(s, s1);
-}
-
LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const ExprLambda & fun, const Symbol & s2))
{
throw TypeError({
- .hint = hintfmt(s, fun.showNamePos(), s2),
+ .msg = hintfmt(s, fun.showNamePos(), s2),
.errPos = pos
});
}
@@ -652,7 +666,7 @@ LocalNoInlineNoReturn(void throwTypeError(const Pos & pos, const char * s, const
LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s, const string & s1))
{
throw AssertionError({
- .hint = hintfmt(s, s1),
+ .msg = hintfmt(s, s1),
.errPos = pos
});
}
@@ -660,7 +674,15 @@ LocalNoInlineNoReturn(void throwAssertionError(const Pos & pos, const char * s,
LocalNoInlineNoReturn(void throwUndefinedVarError(const Pos & pos, const char * s, const string & s1))
{
throw UndefinedVarError({
- .hint = hintfmt(s, s1),
+ .msg = hintfmt(s, s1),
+ .errPos = pos
+ });
+}
+
+LocalNoInlineNoReturn(void throwMissingArgumentError(const Pos & pos, const char * s, const string & s1))
+{
+ throw MissingArgumentError({
+ .msg = hintfmt(s, s1),
.errPos = pos
});
}
@@ -678,15 +700,13 @@ LocalNoInline(void addErrorTrace(Error & e, const Pos & pos, const char * s, con
void mkString(Value & v, const char * s)
{
- mkStringNoCopy(v, dupString(s));
+ v.mkString(dupString(s));
}
Value & mkString(Value & v, std::string_view s, const PathSet & context)
{
- v.type = tString;
- v.string.s = dupStringWithLen(s.data(), s.size());
- v.string.context = 0;
+ v.mkString(dupStringWithLen(s.data(), s.size()));
if (!context.empty()) {
size_t n = 0;
v.string.context = (const char * *)
@@ -701,7 +721,7 @@ Value & mkString(Value & v, std::string_view s, const PathSet & context)
void mkPath(Value & v, const char * s)
{
- mkPathNoCopy(v, dupString(s));
+ v.mkPath(dupString(s));
}
@@ -762,16 +782,9 @@ Env & EvalState::allocEnv(size_t size)
void EvalState::mkList(Value & v, size_t size)
{
- clearValue(v);
- if (size == 1)
- v.type = tList1;
- else if (size == 2)
- v.type = tList2;
- else {
- v.type = tListN;
- v.bigList.size = size;
- v.bigList.elems = size ? (Value * *) allocBytes(size * sizeof(Value *)) : 0;
- }
+ v.mkList(size);
+ if (size > 2)
+ v.bigList.elems = (Value * *) allocBytes(size * sizeof(Value *));
nrListElems += size;
}
@@ -780,9 +793,7 @@ unsigned long nrThunks = 0;
static inline void mkThunk(Value & v, Env & env, Expr * expr)
{
- v.type = tThunk;
- v.thunk.env = &env;
- v.thunk.expr = expr;
+ v.mkThunk(&env, expr);
nrThunks++;
}
@@ -917,7 +928,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e)
{
Value v;
e->eval(*this, env, v);
- if (v.type != tBool)
+ if (v.type() != nBool)
throwTypeError("value is %1% while a Boolean was expected", v);
return v.boolean;
}
@@ -927,7 +938,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
{
Value v;
e->eval(*this, env, v);
- if (v.type != tBool)
+ if (v.type() != nBool)
throwTypeError(pos, "value is %1% while a Boolean was expected", v);
return v.boolean;
}
@@ -936,7 +947,7 @@ inline bool EvalState::evalBool(Env & env, Expr * e, const Pos & pos)
inline void EvalState::evalAttrs(Env & env, Expr * e, Value & v)
{
e->eval(*this, env, v);
- if (v.type != tAttrs)
+ if (v.type() != nAttrs)
throwTypeError("value is %1% while a set was expected", v);
}
@@ -1036,7 +1047,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v)
Value nameVal;
i.nameExpr->eval(state, *dynamicEnv, nameVal);
state.forceValue(nameVal, i.pos);
- if (nameVal.type == tNull)
+ if (nameVal.type() == nNull)
continue;
state.forceStringNoCtx(nameVal);
Symbol nameSym = state.symbols.create(nameVal.string.s);
@@ -1121,7 +1132,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v)
Symbol name = getName(i, state, env);
if (def) {
state.forceValue(*vAttrs, pos);
- if (vAttrs->type != tAttrs ||
+ if (vAttrs->type() != nAttrs ||
(j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
{
def->eval(state, env, v);
@@ -1161,7 +1172,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
state.forceValue(*vAttrs);
Bindings::iterator j;
Symbol name = getName(i, state, env);
- if (vAttrs->type != tAttrs ||
+ if (vAttrs->type() != nAttrs ||
(j = vAttrs->attrs->find(name)) == vAttrs->attrs->end())
{
mkBool(v, false);
@@ -1177,9 +1188,7 @@ void ExprOpHasAttr::eval(EvalState & state, Env & env, Value & v)
void ExprLambda::eval(EvalState & state, Env & env, Value & v)
{
- v.type = tLambda;
- v.lambda.env = &env;
- v.lambda.fun = this;
+ v.mkLambda(&env, this);
}
@@ -1197,11 +1206,11 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
/* Figure out the number of arguments still needed. */
size_t argsDone = 0;
Value * primOp = &fun;
- while (primOp->type == tPrimOpApp) {
+ while (primOp->isPrimOpApp()) {
argsDone++;
primOp = primOp->primOpApp.left;
}
- assert(primOp->type == tPrimOp);
+ assert(primOp->isPrimOp());
auto arity = primOp->primOp->arity;
auto argsLeft = arity - argsDone;
@@ -1212,7 +1221,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
Value * vArgs[arity];
auto n = arity - 1;
vArgs[n--] = &arg;
- for (Value * arg = &fun; arg->type == tPrimOpApp; arg = arg->primOpApp.left)
+ for (Value * arg = &fun; arg->isPrimOpApp(); arg = arg->primOpApp.left)
vArgs[n--] = arg->primOpApp.right;
/* And call the primop. */
@@ -1222,9 +1231,7 @@ void EvalState::callPrimOp(Value & fun, Value & arg, Value & v, const Pos & pos)
} else {
Value * fun2 = allocValue();
*fun2 = fun;
- v.type = tPrimOpApp;
- v.primOpApp.left = fun2;
- v.primOpApp.right = &arg;
+ v.mkPrimOpApp(fun2, &arg);
}
}
@@ -1234,12 +1241,12 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
forceValue(fun, pos);
- if (fun.type == tPrimOp || fun.type == tPrimOpApp) {
+ if (fun.isPrimOp() || fun.isPrimOpApp()) {
callPrimOp(fun, arg, v, pos);
return;
}
- if (fun.type == tAttrs) {
+ if (fun.type() == nAttrs) {
auto found = fun.attrs->find(sFunctor);
if (found != fun.attrs->end()) {
/* fun may be allocated on the stack of the calling function,
@@ -1255,7 +1262,7 @@ void EvalState::callFunction(Value & fun, Value & arg, Value & v, const Pos & po
}
}
- if (fun.type != tLambda)
+ if (!fun.isLambda())
throwTypeError(pos, "attempt to call something which is not a function but %1%", fun);
ExprLambda & lambda(*fun.lambda.fun);
@@ -1338,7 +1345,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
{
forceValue(fun);
- if (fun.type == tAttrs) {
+ if (fun.type() == nAttrs) {
auto found = fun.attrs->find(sFunctor);
if (found != fun.attrs->end()) {
Value * v = allocValue();
@@ -1348,7 +1355,7 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
}
}
- if (fun.type != tLambda || !fun.lambda.fun->matchAttrs) {
+ if (!fun.isLambda() || !fun.lambda.fun->matchAttrs) {
res = fun;
return;
}
@@ -1370,7 +1377,13 @@ void EvalState::autoCallFunction(Bindings & args, Value & fun, Value & res)
if (j != args.end()) {
actualArgs->attrs->push_back(*j);
} else if (!i.def) {
- throwTypeError("cannot auto-call a function that has an argument without a default value ('%1%')", i.name);
+ throwMissingArgumentError(i.pos, R"(cannot evaluate a function that has an argument without a value ('%1%')
+
+Nix attempted to evaluate a function as a top level expression; in
+this case it must have its arguments supplied either by default
+values, or passed explicitly with '--arg' or '--argstr'. See
+https://nixos.org/manual/nix/stable/#ss-functions.)", i.name);
+
}
}
}
@@ -1404,7 +1417,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v)
if (!state.evalBool(env, cond, pos)) {
std::ostringstream out;
cond->show(out);
- throwAssertionError(pos, "assertion '%1%' failed at %2%", out.str());
+ throwAssertionError(pos, "assertion '%1%' failed", out.str());
}
body->eval(state, env, v);
}
@@ -1532,7 +1545,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
NixFloat nf = 0;
bool first = !forceString;
- ValueType firstType = tString;
+ ValueType firstType = nString;
for (auto & i : *es) {
Value vTmp;
@@ -1543,36 +1556,36 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v)
since paths are copied when they are used in a derivation),
and none of the strings are allowed to have contexts. */
if (first) {
- firstType = vTmp.type;
+ firstType = vTmp.type();
first = false;
}
- if (firstType == tInt) {
- if (vTmp.type == tInt) {
+ if (firstType == nInt) {
+ if (vTmp.type() == nInt) {
n += vTmp.integer;
- } else if (vTmp.type == tFloat) {
+ } else if (vTmp.type() == nFloat) {
// Upgrade the type from int to float;
- firstType = tFloat;
+ firstType = nFloat;
nf = n;
nf += vTmp.fpoint;
} else
throwEvalError(pos, "cannot add %1% to an integer", showType(vTmp));
- } else if (firstType == tFloat) {
- if (vTmp.type == tInt) {
+ } else if (firstType == nFloat) {
+ if (vTmp.type() == nInt) {
nf += vTmp.integer;
- } else if (vTmp.type == tFloat) {
+ } else if (vTmp.type() == nFloat) {
nf += vTmp.fpoint;
} else
throwEvalError(pos, "cannot add %1% to a float", showType(vTmp));
} else
- s << state.coerceToString(pos, vTmp, context, false, firstType == tString);
+ s << state.coerceToString(pos, vTmp, context, false, firstType == nString);
}
- if (firstType == tInt)
+ if (firstType == nInt)
mkInt(v, n);
- else if (firstType == tFloat)
+ else if (firstType == nFloat)
mkFloat(v, nf);
- else if (firstType == tPath) {
+ else if (firstType == nPath) {
if (!context.empty())
throwEvalError(pos, "a string that refers to a store path cannot be appended to a path");
auto path = canonPath(s.str());
@@ -1599,7 +1612,7 @@ void EvalState::forceValueDeep(Value & v)
forceValue(v);
- if (v.type == tAttrs) {
+ if (v.type() == nAttrs) {
for (auto & i : *v.attrs)
try {
recurse(*i.value);
@@ -1622,7 +1635,7 @@ void EvalState::forceValueDeep(Value & v)
NixInt EvalState::forceInt(Value & v, const Pos & pos)
{
forceValue(v, pos);
- if (v.type != tInt)
+ if (v.type() != nInt)
throwTypeError(pos, "value is %1% while an integer was expected", v);
return v.integer;
}
@@ -1631,9 +1644,9 @@ NixInt EvalState::forceInt(Value & v, const Pos & pos)
NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
{
forceValue(v, pos);
- if (v.type == tInt)
+ if (v.type() == nInt)
return v.integer;
- else if (v.type != tFloat)
+ else if (v.type() != nFloat)
throwTypeError(pos, "value is %1% while a float was expected", v);
return v.fpoint;
}
@@ -1642,7 +1655,7 @@ NixFloat EvalState::forceFloat(Value & v, const Pos & pos)
bool EvalState::forceBool(Value & v, const Pos & pos)
{
forceValue(v, pos);
- if (v.type != tBool)
+ if (v.type() != nBool)
throwTypeError(pos, "value is %1% while a Boolean was expected", v);
return v.boolean;
}
@@ -1650,14 +1663,14 @@ bool EvalState::forceBool(Value & v, const Pos & pos)
bool EvalState::isFunctor(Value & fun)
{
- return fun.type == tAttrs && fun.attrs->find(sFunctor) != fun.attrs->end();
+ return fun.type() == nAttrs && fun.attrs->find(sFunctor) != fun.attrs->end();
}
void EvalState::forceFunction(Value & v, const Pos & pos)
{
forceValue(v, pos);
- if (v.type != tLambda && v.type != tPrimOp && v.type != tPrimOpApp && !isFunctor(v))
+ if (v.type() != nFunction && !isFunctor(v))
throwTypeError(pos, "value is %1% while a function was expected", v);
}
@@ -1665,7 +1678,7 @@ void EvalState::forceFunction(Value & v, const Pos & pos)
string EvalState::forceString(Value & v, const Pos & pos)
{
forceValue(v, pos);
- if (v.type != tString) {
+ if (v.type() != nString) {
if (pos)
throwTypeError(pos, "value is %1% while a string was expected", v);
else
@@ -1698,7 +1711,7 @@ void copyContext(const Value & v, PathSet & context)
std::vector<std::pair<Path, std::string>> Value::getContext()
{
std::vector<std::pair<Path, std::string>> res;
- assert(type == tString);
+ assert(internalType == tString);
if (string.context)
for (const char * * p = string.context; *p; ++p)
res.push_back(decodeContext(*p));
@@ -1731,11 +1744,11 @@ string EvalState::forceStringNoCtx(Value & v, const Pos & pos)
bool EvalState::isDerivation(Value & v)
{
- if (v.type != tAttrs) return false;
+ if (v.type() != nAttrs) return false;
Bindings::iterator i = v.attrs->find(sType);
if (i == v.attrs->end()) return false;
forceValue(*i->value);
- if (i->value->type != tString) return false;
+ if (i->value->type() != nString) return false;
return strcmp(i->value->string.s, "derivation") == 0;
}
@@ -1760,17 +1773,17 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
string s;
- if (v.type == tString) {
+ if (v.type() == nString) {
copyContext(v, context);
return v.string.s;
}
- if (v.type == tPath) {
+ if (v.type() == nPath) {
Path path(canonPath(v.path));
return copyToStore ? copyPathToStore(context, path) : path;
}
- if (v.type == tAttrs) {
+ if (v.type() == nAttrs) {
auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore);
if (maybeString) {
return *maybeString;
@@ -1780,18 +1793,18 @@ string EvalState::coerceToString(const Pos & pos, Value & v, PathSet & context,
return coerceToString(pos, *i->value, context, coerceMore, copyToStore);
}
- if (v.type == tExternal)
+ if (v.type() == nExternal)
return v.external->coerceToString(pos, context, coerceMore, copyToStore);
if (coerceMore) {
/* Note that `false' is represented as an empty string for
shell scripting convenience, just like `null'. */
- if (v.type == tBool && v.boolean) return "1";
- if (v.type == tBool && !v.boolean) return "";
- if (v.type == tInt) return std::to_string(v.integer);
- if (v.type == tFloat) return std::to_string(v.fpoint);
- if (v.type == tNull) return "";
+ if (v.type() == nBool && v.boolean) return "1";
+ if (v.type() == nBool && !v.boolean) return "";
+ if (v.type() == nInt) return std::to_string(v.integer);
+ if (v.type() == nFloat) return std::to_string(v.fpoint);
+ if (v.type() == nNull) return "";
if (v.isList()) {
string result;
@@ -1854,40 +1867,38 @@ bool EvalState::eqValues(Value & v1, Value & v2)
if (&v1 == &v2) return true;
// Special case type-compatibility between float and int
- if (v1.type == tInt && v2.type == tFloat)
+ if (v1.type() == nInt && v2.type() == nFloat)
return v1.integer == v2.fpoint;
- if (v1.type == tFloat && v2.type == tInt)
+ if (v1.type() == nFloat && v2.type() == nInt)
return v1.fpoint == v2.integer;
// All other types are not compatible with each other.
- if (v1.type != v2.type) return false;
+ if (v1.type() != v2.type()) return false;
- switch (v1.type) {
+ switch (v1.type()) {
- case tInt:
+ case nInt:
return v1.integer == v2.integer;
- case tBool:
+ case nBool:
return v1.boolean == v2.boolean;
- case tString:
+ case nString:
return strcmp(v1.string.s, v2.string.s) == 0;
- case tPath:
+ case nPath:
return strcmp(v1.path, v2.path) == 0;
- case tNull:
+ case nNull:
return true;
- case tList1:
- case tList2:
- case tListN:
+ case nList:
if (v1.listSize() != v2.listSize()) return false;
for (size_t n = 0; n < v1.listSize(); ++n)
if (!eqValues(*v1.listElems()[n], *v2.listElems()[n])) return false;
return true;
- case tAttrs: {
+ case nAttrs: {
/* If both sets denote a derivation (type = "derivation"),
then compare their outPaths. */
if (isDerivation(v1) && isDerivation(v2)) {
@@ -1909,15 +1920,13 @@ bool EvalState::eqValues(Value & v1, Value & v2)
}
/* Functions are incomparable. */
- case tLambda:
- case tPrimOp:
- case tPrimOpApp:
+ case nFunction:
return false;
- case tExternal:
+ case nExternal:
return *v1.external == *v2.external;
- case tFloat:
+ case nFloat:
return v1.fpoint == v2.fpoint;
default:
@@ -2046,7 +2055,7 @@ void EvalState::printStats()
string ExternalValueBase::coerceToString(const Pos & pos, PathSet & context, bool copyMore, bool copyToStore) const
{
throw TypeError({
- .hint = hintfmt("cannot coerce %1% to a string", showType()),
+ .msg = hintfmt("cannot coerce %1% to a string", showType()),
.errPos = pos
});
}
@@ -2072,10 +2081,19 @@ EvalSettings::EvalSettings()
Strings EvalSettings::getDefaultNixPath()
{
Strings res;
- auto add = [&](const Path & p) { if (pathExists(p)) { res.push_back(p); } };
+ auto add = [&](const Path & p, const std::string & s = std::string()) {
+ if (pathExists(p)) {
+ if (s.empty()) {
+ res.push_back(p);
+ } else {
+ res.push_back(s + "=" + p);
+ }
+ }
+ };
+
add(getHome() + "/.nix-defexpr/channels");
- add("nixpkgs=" + settings.nixStateDir + "/nix/profiles/per-user/root/channels/nixpkgs");
- add(settings.nixStateDir + "/nix/profiles/per-user/root/channels");
+ add(settings.nixStateDir + "/profiles/per-user/root/channels/nixpkgs", "nixpkgs");
+ add(settings.nixStateDir + "/profiles/per-user/root/channels");
return res;
}
diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh
index 0e1f61baa..e3eaed6d3 100644
--- a/src/libexpr/eval.hh
+++ b/src/libexpr/eval.hh
@@ -432,4 +432,6 @@ struct EvalSettings : Config
extern EvalSettings evalSettings;
+static const std::string corepkgsPrefix{"/__corepkgs__/"};
+
}
diff --git a/src/libexpr/fetchurl.nix b/src/libexpr/fetchurl.nix
new file mode 100644
index 000000000..02531103b
--- /dev/null
+++ b/src/libexpr/fetchurl.nix
@@ -0,0 +1,41 @@
+{ system ? "" # obsolete
+, url
+, hash ? "" # an SRI hash
+
+# Legacy hash specification
+, md5 ? "", sha1 ? "", sha256 ? "", sha512 ? ""
+, outputHash ?
+ if hash != "" then hash else if sha512 != "" then sha512 else if sha1 != "" then sha1 else if md5 != "" then md5 else sha256
+, outputHashAlgo ?
+ if hash != "" then "" else if sha512 != "" then "sha512" else if sha1 != "" then "sha1" else if md5 != "" then "md5" else "sha256"
+
+, executable ? false
+, unpack ? false
+, name ? baseNameOf (toString url)
+}:
+
+derivation {
+ builder = "builtin:fetchurl";
+
+ # New-style output content requirements.
+ inherit outputHashAlgo outputHash;
+ outputHashMode = if unpack || executable then "recursive" else "flat";
+
+ inherit name url executable unpack;
+
+ system = "builtin";
+
+ # No need to double the amount of network traffic
+ preferLocalBuild = true;
+
+ impureEnvVars = [
+ # We borrow these environment variables from the caller to allow
+ # easy proxy configuration. This is impure, but a fixed-output
+ # derivation like fetchurl is allowed to do so since its result is
+ # by definition pure.
+ "http_proxy" "https_proxy" "ftp_proxy" "all_proxy" "no_proxy"
+ ];
+
+ # To make "nix-prefetch-url" work.
+ urls = [ url ];
+}
diff --git a/src/libexpr/flake/config.cc b/src/libexpr/flake/config.cc
new file mode 100644
index 000000000..63566131e
--- /dev/null
+++ b/src/libexpr/flake/config.cc
@@ -0,0 +1,81 @@
+#include "flake.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix::flake {
+
+// setting name -> setting value -> allow or ignore.
+typedef std::map<std::string, std::map<std::string, bool>> TrustedList;
+
+Path trustedListPath()
+{
+ return getDataDir() + "/nix/trusted-settings.json";
+}
+
+static TrustedList readTrustedList()
+{
+ auto path = trustedListPath();
+ if (!pathExists(path)) return {};
+ auto json = nlohmann::json::parse(readFile(path));
+ return json;
+}
+
+static void writeTrustedList(const TrustedList & trustedList)
+{
+ writeFile(trustedListPath(), nlohmann::json(trustedList).dump());
+}
+
+void ConfigFile::apply()
+{
+ std::set<std::string> whitelist{"bash-prompt", "bash-prompt-suffix"};
+
+ for (auto & [name, value] : settings) {
+
+ auto baseName = hasPrefix(name, "extra-") ? std::string(name, 6) : name;
+
+ // FIXME: Move into libutil/config.cc.
+ std::string valueS;
+ if (auto s = std::get_if<std::string>(&value))
+ valueS = *s;
+ else if (auto n = std::get_if<int64_t>(&value))
+ valueS = fmt("%d", n);
+ else if (auto b = std::get_if<Explicit<bool>>(&value))
+ valueS = b->t ? "true" : "false";
+ else if (auto ss = std::get_if<std::vector<std::string>>(&value))
+ valueS = concatStringsSep(" ", *ss); // FIXME: evil
+ else
+ assert(false);
+
+ if (!whitelist.count(baseName)) {
+ auto trustedList = readTrustedList();
+
+ bool trusted = false;
+
+ if (auto saved = get(get(trustedList, name).value_or(std::map<std::string, bool>()), valueS)) {
+ trusted = *saved;
+ } else {
+ // FIXME: filter ANSI escapes, newlines, \r, etc.
+ if (std::tolower(logger->ask(fmt("do you want to allow configuration setting '%s' to be set to '" ANSI_RED "%s" ANSI_NORMAL "' (y/N)?", name, valueS)).value_or('n')) != 'y') {
+ if (std::tolower(logger->ask("do you want to permanently mark this value as untrusted (y/N)?").value_or('n')) == 'y') {
+ trustedList[name][valueS] = false;
+ writeTrustedList(trustedList);
+ }
+ } else {
+ if (std::tolower(logger->ask("do you want to permanently mark this value as trusted (y/N)?").value_or('n')) == 'y') {
+ trustedList[name][valueS] = trusted = true;
+ writeTrustedList(trustedList);
+ }
+ }
+ }
+
+ if (!trusted) {
+ warn("ignoring untrusted flake configuration setting '%s'", name);
+ continue;
+ }
+ }
+
+ globalConfig.set(name, valueS);
+ }
+}
+
+}
diff --git a/src/libexpr/flake/flake.cc b/src/libexpr/flake/flake.cc
index bae4d65e5..2e94490d4 100644
--- a/src/libexpr/flake/flake.cc
+++ b/src/libexpr/flake/flake.cc
@@ -71,14 +71,20 @@ static std::tuple<fetchers::Tree, FlakeRef, FlakeRef> fetchOrSubstituteTree(
return {std::move(tree), resolvedRef, lockedRef};
}
+static void forceTrivialValue(EvalState & state, Value & value, const Pos & pos)
+{
+ if (value.isThunk() && value.isTrivial())
+ state.forceValue(value, pos);
+}
+
+
static void expectType(EvalState & state, ValueType type,
Value & value, const Pos & pos)
{
- if (value.type == tThunk && value.isTrivial())
- state.forceValue(value, pos);
- if (value.type != type)
+ forceTrivialValue(state, value, pos);
+ if (value.type() != type)
throw Error("expected %s but got %s at %s",
- showType(type), showType(value.type), pos);
+ showType(type), showType(value.type()), pos);
}
static std::map<FlakeId, FlakeInput> parseFlakeInputs(
@@ -87,7 +93,7 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
static FlakeInput parseFlakeInput(EvalState & state,
const std::string & inputName, Value * value, const Pos & pos)
{
- expectType(state, tAttrs, *value, pos);
+ expectType(state, nAttrs, *value, pos);
FlakeInput input;
@@ -102,24 +108,32 @@ static FlakeInput parseFlakeInput(EvalState & state,
for (nix::Attr attr : *(value->attrs)) {
try {
if (attr.name == sUrl) {
- expectType(state, tString, *attr.value, *attr.pos);
+ expectType(state, nString, *attr.value, *attr.pos);
url = attr.value->string.s;
attrs.emplace("url", *url);
} else if (attr.name == sFlake) {
- expectType(state, tBool, *attr.value, *attr.pos);
+ expectType(state, nBool, *attr.value, *attr.pos);
input.isFlake = attr.value->boolean;
} else if (attr.name == sInputs) {
input.overrides = parseFlakeInputs(state, attr.value, *attr.pos);
} else if (attr.name == sFollows) {
- expectType(state, tString, *attr.value, *attr.pos);
+ expectType(state, nString, *attr.value, *attr.pos);
input.follows = parseInputPath(attr.value->string.s);
} else {
- state.forceValue(*attr.value);
- if (attr.value->type == tString)
- attrs.emplace(attr.name, attr.value->string.s);
- else
- throw TypeError("flake input attribute '%s' is %s while a string is expected",
- attr.name, showType(*attr.value));
+ switch (attr.value->type()) {
+ case nString:
+ attrs.emplace(attr.name, attr.value->string.s);
+ break;
+ case nBool:
+ attrs.emplace(attr.name, Explicit<bool> { attr.value->boolean });
+ break;
+ case nInt:
+ attrs.emplace(attr.name, (long unsigned int)attr.value->integer);
+ break;
+ default:
+ throw TypeError("flake input attribute '%s' is %s while a string, Boolean, or integer is expected",
+ attr.name, showType(*attr.value));
+ }
}
} catch (Error & e) {
e.addTrace(*attr.pos, hintfmt("in flake attribute '%s'", attr.name));
@@ -153,7 +167,7 @@ static std::map<FlakeId, FlakeInput> parseFlakeInputs(
{
std::map<FlakeId, FlakeInput> inputs;
- expectType(state, tAttrs, *value, pos);
+ expectType(state, nAttrs, *value, pos);
for (nix::Attr & inputAttr : *(*value).attrs) {
inputs.emplace(inputAttr.name,
@@ -194,15 +208,10 @@ static Flake getFlake(
Value vInfo;
state.evalFile(flakeFile, vInfo, true); // FIXME: symlink attack
- expectType(state, tAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0));
-
- auto sEdition = state.symbols.create("edition"); // FIXME: remove soon
-
- if (vInfo.attrs->get(sEdition))
- warn("flake '%s' has deprecated attribute 'edition'", lockedRef);
+ expectType(state, nAttrs, vInfo, Pos(foFile, state.symbols.create(flakeFile), 0, 0));
if (auto description = vInfo.attrs->get(state.sDescription)) {
- expectType(state, tString, *description->value, *description->pos);
+ expectType(state, nString, *description->value, *description->pos);
flake.description = description->value->string.s;
}
@@ -214,9 +223,9 @@ static Flake getFlake(
auto sOutputs = state.symbols.create("outputs");
if (auto outputs = vInfo.attrs->get(sOutputs)) {
- expectType(state, tLambda, *outputs->value, *outputs->pos);
+ expectType(state, nFunction, *outputs->value, *outputs->pos);
- if (outputs->value->lambda.fun->matchAttrs) {
+ if (outputs->value->isLambda() && outputs->value->lambda.fun->matchAttrs) {
for (auto & formal : outputs->value->lambda.fun->formals->formals) {
if (formal.name != state.sSelf)
flake.inputs.emplace(formal.name, FlakeInput {
@@ -228,11 +237,41 @@ static Flake getFlake(
} else
throw Error("flake '%s' lacks attribute 'outputs'", lockedRef);
+ auto sNixConfig = state.symbols.create("nixConfig");
+
+ if (auto nixConfig = vInfo.attrs->get(sNixConfig)) {
+ expectType(state, nAttrs, *nixConfig->value, *nixConfig->pos);
+
+ for (auto & setting : *nixConfig->value->attrs) {
+ forceTrivialValue(state, *setting.value, *setting.pos);
+ if (setting.value->type() == nString)
+ flake.config.settings.insert({setting.name, state.forceStringNoCtx(*setting.value, *setting.pos)});
+ else if (setting.value->type() == nInt)
+ flake.config.settings.insert({setting.name, state.forceInt(*setting.value, *setting.pos)});
+ else if (setting.value->type() == nBool)
+ flake.config.settings.insert({setting.name, state.forceBool(*setting.value, *setting.pos)});
+ else if (setting.value->type() == nList) {
+ std::vector<std::string> ss;
+ for (unsigned int n = 0; n < setting.value->listSize(); ++n) {
+ auto elem = setting.value->listElems()[n];
+ if (elem->type() != nString)
+ throw TypeError("list element in flake configuration setting '%s' is %s while a string is expected",
+ setting.name, showType(*setting.value));
+ ss.push_back(state.forceStringNoCtx(*elem, *setting.pos));
+ }
+ flake.config.settings.insert({setting.name, ss});
+ }
+ else
+ throw TypeError("flake configuration setting '%s' is %s",
+ setting.name, showType(*setting.value));
+ }
+ }
+
for (auto & attr : *vInfo.attrs) {
- if (attr.name != sEdition &&
- attr.name != state.sDescription &&
+ if (attr.name != state.sDescription &&
attr.name != sInputs &&
- attr.name != sOutputs)
+ attr.name != sOutputs &&
+ attr.name != sNixConfig)
throw Error("flake '%s' has an unsupported attribute '%s', at %s",
lockedRef, attr.name, *attr.pos);
}
@@ -259,284 +298,298 @@ LockedFlake lockFlake(
auto flake = getFlake(state, topRef, lockFlags.useRegistries, flakeCache);
- // FIXME: symlink attack
- auto oldLockFile = LockFile::read(
- flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock");
-
- debug("old lock file: %s", oldLockFile);
-
- // FIXME: check whether all overrides are used.
- std::map<InputPath, FlakeInput> overrides;
- std::set<InputPath> overridesUsed, updatesUsed;
-
- for (auto & i : lockFlags.inputOverrides)
- overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
-
- LockFile newLockFile;
-
- std::vector<FlakeRef> parents;
-
- std::function<void(
- const FlakeInputs & flakeInputs,
- std::shared_ptr<Node> node,
- const InputPath & inputPathPrefix,
- std::shared_ptr<const Node> oldNode)>
- computeLocks;
-
- computeLocks = [&](
- const FlakeInputs & flakeInputs,
- std::shared_ptr<Node> node,
- const InputPath & inputPathPrefix,
- std::shared_ptr<const Node> oldNode)
- {
- debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
-
- /* Get the overrides (i.e. attributes of the form
- 'inputs.nixops.inputs.nixpkgs.url = ...'). */
- // FIXME: check this
- for (auto & [id, input] : flake.inputs) {
- for (auto & [idOverride, inputOverride] : input.overrides) {
- auto inputPath(inputPathPrefix);
- inputPath.push_back(id);
- inputPath.push_back(idOverride);
- overrides.insert_or_assign(inputPath, inputOverride);
- }
- }
-
- /* Go over the flake inputs, resolve/fetch them if
- necessary (i.e. if they're new or the flakeref changed
- from what's in the lock file). */
- for (auto & [id, input2] : flakeInputs) {
- auto inputPath(inputPathPrefix);
- inputPath.push_back(id);
- auto inputPathS = printInputPath(inputPath);
- debug("computing input '%s'", inputPathS);
-
- /* Do we have an override for this input from one of the
- ancestors? */
- auto i = overrides.find(inputPath);
- bool hasOverride = i != overrides.end();
- if (hasOverride) overridesUsed.insert(inputPath);
- auto & input = hasOverride ? i->second : input2;
-
- /* Resolve 'follows' later (since it may refer to an input
- path we haven't processed yet. */
- if (input.follows) {
- InputPath target;
- if (hasOverride || input.absolute)
- /* 'follows' from an override is relative to the
- root of the graph. */
- target = *input.follows;
- else {
- /* Otherwise, it's relative to the current flake. */
- target = inputPathPrefix;
- for (auto & i : *input.follows) target.push_back(i);
+ try {
+
+ // FIXME: symlink attack
+ auto oldLockFile = LockFile::read(
+ flake.sourceInfo->actualPath + "/" + flake.lockedRef.subdir + "/flake.lock");
+
+ debug("old lock file: %s", oldLockFile);
+
+ // FIXME: check whether all overrides are used.
+ std::map<InputPath, FlakeInput> overrides;
+ std::set<InputPath> overridesUsed, updatesUsed;
+
+ for (auto & i : lockFlags.inputOverrides)
+ overrides.insert_or_assign(i.first, FlakeInput { .ref = i.second });
+
+ LockFile newLockFile;
+
+ std::vector<FlakeRef> parents;
+
+ std::function<void(
+ const FlakeInputs & flakeInputs,
+ std::shared_ptr<Node> node,
+ const InputPath & inputPathPrefix,
+ std::shared_ptr<const Node> oldNode)>
+ computeLocks;
+
+ computeLocks = [&](
+ const FlakeInputs & flakeInputs,
+ std::shared_ptr<Node> node,
+ const InputPath & inputPathPrefix,
+ std::shared_ptr<const Node> oldNode)
+ {
+ debug("computing lock file node '%s'", printInputPath(inputPathPrefix));
+
+ /* Get the overrides (i.e. attributes of the form
+ 'inputs.nixops.inputs.nixpkgs.url = ...'). */
+ // FIXME: check this
+ for (auto & [id, input] : flake.inputs) {
+ for (auto & [idOverride, inputOverride] : input.overrides) {
+ auto inputPath(inputPathPrefix);
+ inputPath.push_back(id);
+ inputPath.push_back(idOverride);
+ overrides.insert_or_assign(inputPath, inputOverride);
}
- debug("input '%s' follows '%s'", inputPathS, printInputPath(target));
- node->inputs.insert_or_assign(id, target);
- continue;
}
- assert(input.ref);
-
- /* Do we have an entry in the existing lock file? And we
- don't have a --update-input flag for this input? */
- std::shared_ptr<LockedNode> oldLock;
-
- updatesUsed.insert(inputPath);
-
- if (oldNode && !lockFlags.inputUpdates.count(inputPath))
- if (auto oldLock2 = get(oldNode->inputs, id))
- if (auto oldLock3 = std::get_if<0>(&*oldLock2))
- oldLock = *oldLock3;
-
- if (oldLock
- && oldLock->originalRef == *input.ref
- && !hasOverride)
- {
- debug("keeping existing input '%s'", inputPathS);
-
- /* Copy the input from the old lock since its flakeref
- didn't change and there is no override from a
- higher level flake. */
- auto childNode = std::make_shared<LockedNode>(
- oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake);
-
- node->inputs.insert_or_assign(id, childNode);
-
- /* If we have an --update-input flag for an input
- of this input, then we must fetch the flake to
- update it. */
- auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
-
- auto hasChildUpdate =
- lb != lockFlags.inputUpdates.end()
- && lb->size() > inputPath.size()
- && std::equal(inputPath.begin(), inputPath.end(), lb->begin());
-
- if (hasChildUpdate) {
- auto inputFlake = getFlake(
- state, oldLock->lockedRef, false, flakeCache);
- computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
- } else {
- /* No need to fetch this flake, we can be
- lazy. However there may be new overrides on the
- inputs of this flake, so we need to check
- those. */
- FlakeInputs fakeInputs;
-
- for (auto & i : oldLock->inputs) {
- if (auto lockedNode = std::get_if<0>(&i.second)) {
- fakeInputs.emplace(i.first, FlakeInput {
- .ref = (*lockedNode)->originalRef,
- .isFlake = (*lockedNode)->isFlake,
- });
- } else if (auto follows = std::get_if<1>(&i.second)) {
- fakeInputs.emplace(i.first, FlakeInput {
- .follows = *follows,
- .absolute = true
- });
+ /* Go over the flake inputs, resolve/fetch them if
+ necessary (i.e. if they're new or the flakeref changed
+ from what's in the lock file). */
+ for (auto & [id, input2] : flakeInputs) {
+ auto inputPath(inputPathPrefix);
+ inputPath.push_back(id);
+ auto inputPathS = printInputPath(inputPath);
+ debug("computing input '%s'", inputPathS);
+
+ try {
+
+ /* Do we have an override for this input from one of the
+ ancestors? */
+ auto i = overrides.find(inputPath);
+ bool hasOverride = i != overrides.end();
+ if (hasOverride) overridesUsed.insert(inputPath);
+ auto & input = hasOverride ? i->second : input2;
+
+ /* Resolve 'follows' later (since it may refer to an input
+ path we haven't processed yet. */
+ if (input.follows) {
+ InputPath target;
+ if (hasOverride || input.absolute)
+ /* 'follows' from an override is relative to the
+ root of the graph. */
+ target = *input.follows;
+ else {
+ /* Otherwise, it's relative to the current flake. */
+ target = inputPathPrefix;
+ for (auto & i : *input.follows) target.push_back(i);
}
+ debug("input '%s' follows '%s'", inputPathS, printInputPath(target));
+ node->inputs.insert_or_assign(id, target);
+ continue;
}
- computeLocks(fakeInputs, childNode, inputPath, oldLock);
- }
+ assert(input.ref);
+
+ /* Do we have an entry in the existing lock file? And we
+ don't have a --update-input flag for this input? */
+ std::shared_ptr<LockedNode> oldLock;
+
+ updatesUsed.insert(inputPath);
+
+ if (oldNode && !lockFlags.inputUpdates.count(inputPath))
+ if (auto oldLock2 = get(oldNode->inputs, id))
+ if (auto oldLock3 = std::get_if<0>(&*oldLock2))
+ oldLock = *oldLock3;
+
+ if (oldLock
+ && oldLock->originalRef == *input.ref
+ && !hasOverride)
+ {
+ debug("keeping existing input '%s'", inputPathS);
+
+ /* Copy the input from the old lock since its flakeref
+ didn't change and there is no override from a
+ higher level flake. */
+ auto childNode = std::make_shared<LockedNode>(
+ oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake);
+
+ node->inputs.insert_or_assign(id, childNode);
+
+ /* If we have an --update-input flag for an input
+ of this input, then we must fetch the flake to
+ update it. */
+ auto lb = lockFlags.inputUpdates.lower_bound(inputPath);
+
+ auto hasChildUpdate =
+ lb != lockFlags.inputUpdates.end()
+ && lb->size() > inputPath.size()
+ && std::equal(inputPath.begin(), inputPath.end(), lb->begin());
+
+ if (hasChildUpdate) {
+ auto inputFlake = getFlake(
+ state, oldLock->lockedRef, false, flakeCache);
+ computeLocks(inputFlake.inputs, childNode, inputPath, oldLock);
+ } else {
+ /* No need to fetch this flake, we can be
+ lazy. However there may be new overrides on the
+ inputs of this flake, so we need to check
+ those. */
+ FlakeInputs fakeInputs;
+
+ for (auto & i : oldLock->inputs) {
+ if (auto lockedNode = std::get_if<0>(&i.second)) {
+ fakeInputs.emplace(i.first, FlakeInput {
+ .ref = (*lockedNode)->originalRef,
+ .isFlake = (*lockedNode)->isFlake,
+ });
+ } else if (auto follows = std::get_if<1>(&i.second)) {
+ fakeInputs.emplace(i.first, FlakeInput {
+ .follows = *follows,
+ .absolute = true
+ });
+ }
+ }
+
+ computeLocks(fakeInputs, childNode, inputPath, oldLock);
+ }
- } else {
- /* We need to create a new lock file entry. So fetch
- this input. */
- debug("creating new input '%s'", inputPathS);
-
- if (!lockFlags.allowMutable && !input.ref->input.isImmutable())
- throw Error("cannot update flake input '%s' in pure mode", inputPathS);
-
- if (input.isFlake) {
- auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache);
-
- /* Note: in case of an --override-input, we use
- the *original* ref (input2.ref) for the
- "original" field, rather than the
- override. This ensures that the override isn't
- nuked the next time we update the lock
- file. That is, overrides are sticky unless you
- use --no-write-lock-file. */
- auto childNode = std::make_shared<LockedNode>(
- inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref);
-
- node->inputs.insert_or_assign(id, childNode);
-
- /* Guard against circular flake imports. */
- for (auto & parent : parents)
- if (parent == *input.ref)
- throw Error("found circular import of flake '%s'", parent);
- parents.push_back(*input.ref);
- Finally cleanup([&]() { parents.pop_back(); });
-
- /* Recursively process the inputs of this
- flake. Also, unless we already have this flake
- in the top-level lock file, use this flake's
- own lock file. */
- computeLocks(
- inputFlake.inputs, childNode, inputPath,
- oldLock
- ? std::dynamic_pointer_cast<const Node>(oldLock)
- : LockFile::read(
- inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root);
- }
+ } else {
+ /* We need to create a new lock file entry. So fetch
+ this input. */
+ debug("creating new input '%s'", inputPathS);
+
+ if (!lockFlags.allowMutable && !input.ref->input.isImmutable())
+ throw Error("cannot update flake input '%s' in pure mode", inputPathS);
+
+ if (input.isFlake) {
+ auto inputFlake = getFlake(state, *input.ref, lockFlags.useRegistries, flakeCache);
+
+ /* Note: in case of an --override-input, we use
+ the *original* ref (input2.ref) for the
+ "original" field, rather than the
+ override. This ensures that the override isn't
+ nuked the next time we update the lock
+ file. That is, overrides are sticky unless you
+ use --no-write-lock-file. */
+ auto childNode = std::make_shared<LockedNode>(
+ inputFlake.lockedRef, input2.ref ? *input2.ref : *input.ref);
+
+ node->inputs.insert_or_assign(id, childNode);
+
+ /* Guard against circular flake imports. */
+ for (auto & parent : parents)
+ if (parent == *input.ref)
+ throw Error("found circular import of flake '%s'", parent);
+ parents.push_back(*input.ref);
+ Finally cleanup([&]() { parents.pop_back(); });
+
+ /* Recursively process the inputs of this
+ flake. Also, unless we already have this flake
+ in the top-level lock file, use this flake's
+ own lock file. */
+ computeLocks(
+ inputFlake.inputs, childNode, inputPath,
+ oldLock
+ ? std::dynamic_pointer_cast<const Node>(oldLock)
+ : LockFile::read(
+ inputFlake.sourceInfo->actualPath + "/" + inputFlake.lockedRef.subdir + "/flake.lock").root);
+ }
+
+ else {
+ auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
+ state, *input.ref, lockFlags.useRegistries, flakeCache);
+ node->inputs.insert_or_assign(id,
+ std::make_shared<LockedNode>(lockedRef, *input.ref, false));
+ }
+ }
- else {
- auto [sourceInfo, resolvedRef, lockedRef] = fetchOrSubstituteTree(
- state, *input.ref, lockFlags.useRegistries, flakeCache);
- node->inputs.insert_or_assign(id,
- std::make_shared<LockedNode>(lockedRef, *input.ref, false));
+ } catch (Error & e) {
+ e.addTrace({}, "while updating the flake input '%s'", inputPathS);
+ throw;
}
}
+ };
+
+ computeLocks(
+ flake.inputs, newLockFile.root, {},
+ lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
+
+ for (auto & i : lockFlags.inputOverrides)
+ if (!overridesUsed.count(i.first))
+ warn("the flag '--override-input %s %s' does not match any input",
+ printInputPath(i.first), i.second);
+
+ for (auto & i : lockFlags.inputUpdates)
+ if (!updatesUsed.count(i))
+ warn("the flag '--update-input %s' does not match any input", printInputPath(i));
+
+ /* Check 'follows' inputs. */
+ newLockFile.check();
+
+ debug("new lock file: %s", newLockFile);
+
+ /* Check whether we need to / can write the new lock file. */
+ if (!(newLockFile == oldLockFile)) {
+
+ auto diff = LockFile::diff(oldLockFile, newLockFile);
+
+ if (lockFlags.writeLockFile) {
+ if (auto sourcePath = topRef.input.getSourcePath()) {
+ if (!newLockFile.isImmutable()) {
+ if (settings.warnDirty)
+ warn("will not write lock file of flake '%s' because it has a mutable input", topRef);
+ } else {
+ if (!lockFlags.updateLockFile)
+ throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
+
+ auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
+
+ auto path = *sourcePath + "/" + relPath;
+
+ bool lockFileExists = pathExists(path);
+
+ if (lockFileExists) {
+ auto s = chomp(diff);
+ if (s.empty())
+ warn("updating lock file '%s'", path);
+ else
+ warn("updating lock file '%s':\n%s", path, s);
+ } else
+ warn("creating lock file '%s'", path);
+
+ newLockFile.write(path);
+
+ topRef.input.markChangedFile(
+ (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
+ lockFlags.commitLockFile
+ ? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s",
+ relPath, lockFileExists ? "Update" : "Add", diff))
+ : std::nullopt);
+
+ /* Rewriting the lockfile changed the top-level
+ repo, so we should re-read it. FIXME: we could
+ also just clear the 'rev' field... */
+ auto prevLockedRef = flake.lockedRef;
+ FlakeCache dummyCache;
+ flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache);
+
+ if (lockFlags.commitLockFile &&
+ flake.lockedRef.input.getRev() &&
+ prevLockedRef.input.getRev() != flake.lockedRef.input.getRev())
+ warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev());
+
+ /* Make sure that we picked up the change,
+ i.e. the tree should usually be dirty
+ now. Corner case: we could have reverted from a
+ dirty to a clean tree! */
+ if (flake.lockedRef.input == prevLockedRef.input
+ && !flake.lockedRef.input.isImmutable())
+ throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
+ }
+ } else
+ throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
+ } else
+ warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff));
}
- };
- computeLocks(
- flake.inputs, newLockFile.root, {},
- lockFlags.recreateLockFile ? nullptr : oldLockFile.root);
-
- for (auto & i : lockFlags.inputOverrides)
- if (!overridesUsed.count(i.first))
- warn("the flag '--override-input %s %s' does not match any input",
- printInputPath(i.first), i.second);
-
- for (auto & i : lockFlags.inputUpdates)
- if (!updatesUsed.count(i))
- warn("the flag '--update-input %s' does not match any input", printInputPath(i));
-
- /* Check 'follows' inputs. */
- newLockFile.check();
-
- debug("new lock file: %s", newLockFile);
-
- /* Check whether we need to / can write the new lock file. */
- if (!(newLockFile == oldLockFile)) {
-
- auto diff = LockFile::diff(oldLockFile, newLockFile);
-
- if (lockFlags.writeLockFile) {
- if (auto sourcePath = topRef.input.getSourcePath()) {
- if (!newLockFile.isImmutable()) {
- if (settings.warnDirty)
- warn("will not write lock file of flake '%s' because it has a mutable input", topRef);
- } else {
- if (!lockFlags.updateLockFile)
- throw Error("flake '%s' requires lock file changes but they're not allowed due to '--no-update-lock-file'", topRef);
-
- auto relPath = (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock";
-
- auto path = *sourcePath + "/" + relPath;
-
- bool lockFileExists = pathExists(path);
-
- if (lockFileExists) {
- auto s = chomp(diff);
- if (s.empty())
- warn("updating lock file '%s'", path);
- else
- warn("updating lock file '%s':\n%s", path, s);
- } else
- warn("creating lock file '%s'", path);
-
- newLockFile.write(path);
-
- topRef.input.markChangedFile(
- (topRef.subdir == "" ? "" : topRef.subdir + "/") + "flake.lock",
- lockFlags.commitLockFile
- ? std::optional<std::string>(fmt("%s: %s\n\nFlake input changes:\n\n%s",
- relPath, lockFileExists ? "Update" : "Add", diff))
- : std::nullopt);
-
- /* Rewriting the lockfile changed the top-level
- repo, so we should re-read it. FIXME: we could
- also just clear the 'rev' field... */
- auto prevLockedRef = flake.lockedRef;
- FlakeCache dummyCache;
- flake = getFlake(state, topRef, lockFlags.useRegistries, dummyCache);
-
- if (lockFlags.commitLockFile &&
- flake.lockedRef.input.getRev() &&
- prevLockedRef.input.getRev() != flake.lockedRef.input.getRev())
- warn("committed new revision '%s'", flake.lockedRef.input.getRev()->gitRev());
-
- /* Make sure that we picked up the change,
- i.e. the tree should usually be dirty
- now. Corner case: we could have reverted from a
- dirty to a clean tree! */
- if (flake.lockedRef.input == prevLockedRef.input
- && !flake.lockedRef.input.isImmutable())
- throw Error("'%s' did not change after I updated its 'flake.lock' file; is 'flake.lock' under version control?", flake.originalRef);
- }
- } else
- throw Error("cannot write modified lock file of flake '%s' (use '--no-write-lock-file' to ignore)", topRef);
- } else
- warn("not writing modified lock file of flake '%s':\n%s", topRef, chomp(diff));
- }
+ return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
- return LockedFlake { .flake = std::move(flake), .lockFile = std::move(newLockFile) };
+ } catch (Error & e) {
+ e.addTrace({}, "while updating the lock file of flake '%s'", flake.lockedRef.to_string());
+ throw;
+ }
}
void callFlake(EvalState & state,
diff --git a/src/libexpr/flake/flake.hh b/src/libexpr/flake/flake.hh
index 69c779af8..d17d5e183 100644
--- a/src/libexpr/flake/flake.hh
+++ b/src/libexpr/flake/flake.hh
@@ -17,23 +17,55 @@ struct FlakeInput;
typedef std::map<FlakeId, FlakeInput> FlakeInputs;
+/* FlakeInput is the 'Flake'-level parsed form of the "input" entries
+ * in the flake file.
+ *
+ * A FlakeInput is normally constructed by the 'parseFlakeInput'
+ * function which parses the input specification in the '.flake' file
+ * to create a 'FlakeRef' (a fetcher, the fetcher-specific
+ * representation of the input specification, and possibly the fetched
+ * local store path result) and then creating this FlakeInput to hold
+ * that FlakeRef, along with anything that might override that
+ * FlakeRef (like command-line overrides or "follows" specifications).
+ *
+ * A FlakeInput is also sometimes constructed directly from a FlakeRef
+ * instead of starting at the flake-file input specification
+ * (e.g. overrides, follows, and implicit inputs).
+ *
+ * A FlakeInput will usually have one of either "ref" or "follows"
+ * set. If not otherwise specified, a "ref" will be generated to a
+ * 'type="indirect"' flake, which is treated as simply the name of a
+ * flake to be resolved in the registry.
+ */
+
struct FlakeInput
{
std::optional<FlakeRef> ref;
- bool isFlake = true;
+ bool isFlake = true; // true = process flake to get outputs, false = (fetched) static source path
std::optional<InputPath> follows;
bool absolute = false; // whether 'follows' is relative to the flake root
FlakeInputs overrides;
};
+struct ConfigFile
+{
+ using ConfigValue = std::variant<std::string, int64_t, Explicit<bool>, std::vector<std::string>>;
+
+ std::map<std::string, ConfigValue> settings;
+
+ void apply();
+};
+
+/* The contents of a flake.nix file. */
struct Flake
{
- FlakeRef originalRef;
- FlakeRef resolvedRef;
- FlakeRef lockedRef;
+ FlakeRef originalRef; // the original flake specification (by the user)
+ FlakeRef resolvedRef; // registry references and caching resolved to the specific underlying flake
+ FlakeRef lockedRef; // the specific local store result of invoking the fetcher
std::optional<std::string> description;
std::shared_ptr<const fetchers::Tree> sourceInfo;
FlakeInputs inputs;
+ ConfigFile config; // 'nixConfig' attribute
~Flake();
};
@@ -81,7 +113,7 @@ struct LockFlags
/* Whether to commit changes to flake.lock. */
bool commitLockFile = false;
- /* Flake inputs to be overriden. */
+ /* Flake inputs to be overridden. */
std::map<InputPath, FlakeRef> inputOverrides;
/* Flake inputs to be updated. This means that any existing lock
diff --git a/src/libexpr/flake/flakeref.hh b/src/libexpr/flake/flakeref.hh
index f4eb825a6..0292eb210 100644
--- a/src/libexpr/flake/flakeref.hh
+++ b/src/libexpr/flake/flakeref.hh
@@ -12,10 +12,33 @@ class Store;
typedef std::string FlakeId;
+/* A flake reference specifies how to fetch a flake or raw source
+ * (e.g. from a Git repository). It is created from a URL-like syntax
+ * (e.g. 'github:NixOS/patchelf'), an attrset representation (e.g. '{
+ * type="github"; owner = "NixOS"; repo = "patchelf"; }'), or a local
+ * path.
+ *
+ * Each flake will have a number of FlakeRef objects: one for each
+ * input to the flake.
+ *
+ * The normal method of constructing a FlakeRef is by starting with an
+ * input description (usually the attrs or a url from the flake file),
+ * locating a fetcher for that input, and then capturing the Input
+ * object that fetcher generates (usually via
+ * FlakeRef::fromAttrs(attrs) or parseFlakeRef(url) calls).
+ *
+ * The actual fetch not have been performed yet (i.e. a FlakeRef may
+ * be lazy), but the fetcher can be invoked at any time via the
+ * FlakeRef to ensure the store is populated with this input.
+ */
+
struct FlakeRef
{
+ /* fetcher-specific representation of the input, sufficient to
+ perform the fetch operation. */
fetchers::Input input;
+ /* sub-path within the fetched input that represents this input */
Path subdir;
bool operator==(const FlakeRef & other) const;
diff --git a/src/libexpr/flake/lockfile.cc b/src/libexpr/flake/lockfile.cc
index bb46e1bb4..6089d1363 100644
--- a/src/libexpr/flake/lockfile.cc
+++ b/src/libexpr/flake/lockfile.cc
@@ -34,7 +34,8 @@ LockedNode::LockedNode(const nlohmann::json & json)
, isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true)
{
if (!lockedRef.input.isImmutable())
- throw Error("lockfile contains mutable lock '%s'", attrsToJson(lockedRef.input.toAttrs()));
+ throw Error("lockfile contains mutable lock '%s'",
+ fetchers::attrsToJSON(lockedRef.input.toAttrs()));
}
StorePath LockedNode::computeStorePath(Store & store) const
@@ -77,7 +78,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
{
if (jsonNode.find("inputs") == jsonNode.end()) return;
for (auto & i : jsonNode["inputs"].items()) {
- if (i.value().is_array()) {
+ if (i.value().is_array()) { // FIXME: remove, obsolete
InputPath path;
for (auto & j : i.value())
path.push_back(j);
@@ -86,10 +87,13 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
std::string inputKey = i.value();
auto k = nodeMap.find(inputKey);
if (k == nodeMap.end()) {
- auto jsonNode2 = json["nodes"][inputKey];
- auto input = std::make_shared<LockedNode>(jsonNode2);
+ auto nodes = json["nodes"];
+ auto jsonNode2 = nodes.find(inputKey);
+ if (jsonNode2 == nodes.end())
+ throw Error("lock file references missing node '%s'", inputKey);
+ auto input = std::make_shared<LockedNode>(*jsonNode2);
k = nodeMap.insert_or_assign(inputKey, input).first;
- getInputs(*input, jsonNode2);
+ getInputs(*input, *jsonNode2);
}
if (auto child = std::dynamic_pointer_cast<LockedNode>(k->second))
node.inputs.insert_or_assign(i.key(), child);
@@ -110,7 +114,7 @@ LockFile::LockFile(const nlohmann::json & json, const Path & path)
// a bit since we don't need to worry about cycles.
}
-nlohmann::json LockFile::toJson() const
+nlohmann::json LockFile::toJSON() const
{
nlohmann::json nodes;
std::unordered_map<std::shared_ptr<const Node>, std::string> nodeKeys;
@@ -154,8 +158,8 @@ nlohmann::json LockFile::toJson() const
}
if (auto lockedNode = std::dynamic_pointer_cast<const LockedNode>(node)) {
- n["original"] = fetchers::attrsToJson(lockedNode->originalRef.toAttrs());
- n["locked"] = fetchers::attrsToJson(lockedNode->lockedRef.toAttrs());
+ n["original"] = fetchers::attrsToJSON(lockedNode->originalRef.toAttrs());
+ n["locked"] = fetchers::attrsToJSON(lockedNode->lockedRef.toAttrs());
if (!lockedNode->isFlake) n["flake"] = false;
}
@@ -174,7 +178,7 @@ nlohmann::json LockFile::toJson() const
std::string LockFile::to_string() const
{
- return toJson().dump(2);
+ return toJSON().dump(2);
}
LockFile LockFile::read(const Path & path)
@@ -185,7 +189,7 @@ LockFile LockFile::read(const Path & path)
std::ostream & operator <<(std::ostream & stream, const LockFile & lockFile)
{
- stream << lockFile.toJson().dump(2);
+ stream << lockFile.toJSON().dump(2);
return stream;
}
@@ -223,7 +227,7 @@ bool LockFile::isImmutable() const
bool LockFile::operator ==(const LockFile & other) const
{
// FIXME: slow
- return toJson() == other.toJson();
+ return toJSON() == other.toJSON();
}
InputPath parseInputPath(std::string_view s)
diff --git a/src/libexpr/flake/lockfile.hh b/src/libexpr/flake/lockfile.hh
index 627794d8c..96f1edc76 100644
--- a/src/libexpr/flake/lockfile.hh
+++ b/src/libexpr/flake/lockfile.hh
@@ -52,7 +52,7 @@ struct LockFile
LockFile() {};
LockFile(const nlohmann::json & json, const Path & path);
- nlohmann::json toJson() const;
+ nlohmann::json toJSON() const;
std::string to_string() const;
diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc
index 91916e8bf..f774e6493 100644
--- a/src/libexpr/get-drvs.cc
+++ b/src/libexpr/get-drvs.cc
@@ -2,6 +2,7 @@
#include "util.hh"
#include "eval-inline.hh"
#include "store-api.hh"
+#include "path-with-outputs.hh"
#include <cstring>
#include <regex>
@@ -19,7 +20,7 @@ DrvInfo::DrvInfo(EvalState & state, const string & attrPath, Bindings * attrs)
DrvInfo::DrvInfo(EvalState & state, ref<Store> store, const std::string & drvPathWithOutputs)
: state(&state), attrs(nullptr), attrPath("")
{
- auto [drvPath, selectedOutputs] = store->parsePathWithOutputs(drvPathWithOutputs);
+ auto [drvPath, selectedOutputs] = parsePathWithOutputs(*store, drvPathWithOutputs);
this->drvPath = store->printStorePath(drvPath);
@@ -128,7 +129,7 @@ DrvInfo::Outputs DrvInfo::queryOutputs(bool onlyOutputsToInstall)
if (!outTI->isList()) throw errMsg;
Outputs result;
for (auto i = outTI->listElems(); i != outTI->listElems() + outTI->listSize(); ++i) {
- if ((*i)->type != tString) throw errMsg;
+ if ((*i)->type() != nString) throw errMsg;
auto out = outputs.find((*i)->string.s);
if (out == outputs.end()) throw errMsg;
result.insert(*out);
@@ -172,20 +173,20 @@ StringSet DrvInfo::queryMetaNames()
bool DrvInfo::checkMeta(Value & v)
{
state->forceValue(v);
- if (v.isList()) {
+ if (v.type() == nList) {
for (unsigned int n = 0; n < v.listSize(); ++n)
if (!checkMeta(*v.listElems()[n])) return false;
return true;
}
- else if (v.type == tAttrs) {
+ else if (v.type() == nAttrs) {
Bindings::iterator i = v.attrs->find(state->sOutPath);
if (i != v.attrs->end()) return false;
for (auto & i : *v.attrs)
if (!checkMeta(*i.value)) return false;
return true;
}
- else return v.type == tInt || v.type == tBool || v.type == tString ||
- v.type == tFloat;
+ else return v.type() == nInt || v.type() == nBool || v.type() == nString ||
+ v.type() == nFloat;
}
@@ -201,7 +202,7 @@ Value * DrvInfo::queryMeta(const string & name)
string DrvInfo::queryMetaString(const string & name)
{
Value * v = queryMeta(name);
- if (!v || v->type != tString) return "";
+ if (!v || v->type() != nString) return "";
return v->string.s;
}
@@ -210,12 +211,12 @@ NixInt DrvInfo::queryMetaInt(const string & name, NixInt def)
{
Value * v = queryMeta(name);
if (!v) return def;
- if (v->type == tInt) return v->integer;
- if (v->type == tString) {
+ if (v->type() == nInt) return v->integer;
+ if (v->type() == nString) {
/* Backwards compatibility with before we had support for
integer meta fields. */
- NixInt n;
- if (string2Int(v->string.s, n)) return n;
+ if (auto n = string2Int<NixInt>(v->string.s))
+ return *n;
}
return def;
}
@@ -224,12 +225,12 @@ NixFloat DrvInfo::queryMetaFloat(const string & name, NixFloat def)
{
Value * v = queryMeta(name);
if (!v) return def;
- if (v->type == tFloat) return v->fpoint;
- if (v->type == tString) {
+ if (v->type() == nFloat) return v->fpoint;
+ if (v->type() == nString) {
/* Backwards compatibility with before we had support for
float meta fields. */
- NixFloat n;
- if (string2Float(v->string.s, n)) return n;
+ if (auto n = string2Float<NixFloat>(v->string.s))
+ return *n;
}
return def;
}
@@ -239,8 +240,8 @@ bool DrvInfo::queryMetaBool(const string & name, bool def)
{
Value * v = queryMeta(name);
if (!v) return def;
- if (v->type == tBool) return v->boolean;
- if (v->type == tString) {
+ if (v->type() == nBool) return v->boolean;
+ if (v->type() == nString) {
/* Backwards compatibility with before we had support for
Boolean meta fields. */
if (strcmp(v->string.s, "true") == 0) return true;
@@ -331,7 +332,7 @@ static void getDerivations(EvalState & state, Value & vIn,
/* Process the expression. */
if (!getDerivation(state, v, pathPrefix, drvs, done, ignoreAssertionFailures)) ;
- else if (v.type == tAttrs) {
+ else if (v.type() == nAttrs) {
/* !!! undocumented hackery to support combining channels in
nix-env.cc. */
@@ -353,7 +354,7 @@ static void getDerivations(EvalState & state, Value & vIn,
/* If the value of this attribute is itself a set,
should we recurse into it? => Only if it has a
`recurseForDerivations = true' attribute. */
- if (i->value->type == tAttrs) {
+ if (i->value->type() == nAttrs) {
Bindings::iterator j = i->value->attrs->find(state.sRecurseForDerivations);
if (j != i->value->attrs->end() && state.forceBool(*j->value, *j->pos))
getDerivations(state, *i->value, pathPrefix2, autoArgs, drvs, done, ignoreAssertionFailures);
@@ -362,7 +363,7 @@ static void getDerivations(EvalState & state, Value & vIn,
}
}
- else if (v.isList()) {
+ else if (v.type() == nList) {
for (unsigned int n = 0; n < v.listSize(); ++n) {
string pathPrefix2 = addToPath(pathPrefix, (format("%1%") % n).str());
if (getDerivation(state, *v.listElems()[n], pathPrefix2, drvs, done, ignoreAssertionFailures))
diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l
index f6e83926b..7298419d9 100644
--- a/src/libexpr/lexer.l
+++ b/src/libexpr/lexer.l
@@ -12,6 +12,10 @@
%{
+#ifdef __clang__
+#pragma clang diagnostic ignored "-Wunneeded-internal-declaration"
+#endif
+
#include <boost/lexical_cast.hpp>
#include "nixexpr.hh"
diff --git a/src/libexpr/local.mk b/src/libexpr/local.mk
index 687a8ccda..26c53d301 100644
--- a/src/libexpr/local.mk
+++ b/src/libexpr/local.mk
@@ -15,7 +15,7 @@ libexpr_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/lib
libexpr_LIBS = libutil libstore libfetchers
-libexpr_LDFLAGS =
+libexpr_LDFLAGS = -lboost_context
ifneq ($(OS), FreeBSD)
libexpr_LDFLAGS += -ldl
endif
@@ -35,13 +35,11 @@ $(d)/lexer-tab.cc $(d)/lexer-tab.hh: $(d)/lexer.l
clean-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
-dist-files += $(d)/parser-tab.cc $(d)/parser-tab.hh $(d)/lexer-tab.cc $(d)/lexer-tab.hh
-
$(eval $(call install-file-in, $(d)/nix-expr.pc, $(prefix)/lib/pkgconfig, 0644))
$(foreach i, $(wildcard src/libexpr/flake/*.hh), \
$(eval $(call install-file-in, $(i), $(includedir)/nix/flake, 0644)))
-$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh
+$(d)/primops.cc: $(d)/imported-drv-to-derivation.nix.gen.hh $(d)/primops/derivation.nix.gen.hh $(d)/fetchurl.nix.gen.hh
$(d)/flake/flake.cc: $(d)/flake/call-flake.nix.gen.hh
diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc
index d5698011f..492b819e7 100644
--- a/src/libexpr/nixexpr.cc
+++ b/src/libexpr/nixexpr.cc
@@ -284,7 +284,7 @@ void ExprVar::bindVars(const StaticEnv & env)
"undefined variable" error now. */
if (withLevel == -1)
throw UndefinedVarError({
- .hint = hintfmt("undefined variable '%1%'", name),
+ .msg = hintfmt("undefined variable '%1%'", name),
.errPos = pos
});
fromWith = true;
diff --git a/src/libexpr/nixexpr.hh b/src/libexpr/nixexpr.hh
index e4cbc660f..8df8055b3 100644
--- a/src/libexpr/nixexpr.hh
+++ b/src/libexpr/nixexpr.hh
@@ -17,6 +17,7 @@ MakeError(ThrownError, AssertionError);
MakeError(Abort, EvalError);
MakeError(TypeError, EvalError);
MakeError(UndefinedVarError, Error);
+MakeError(MissingArgumentError, EvalError);
MakeError(RestrictedPathError, Error);
@@ -129,7 +130,7 @@ struct ExprPath : Expr
{
string s;
Value v;
- ExprPath(const string & s) : s(s) { mkPathNoCopy(v, this->s.c_str()); };
+ ExprPath(const string & s) : s(s) { v.mkPath(this->s.c_str()); };
COMMON_METHODS
Value * maybeThunk(EvalState & state, Env & env);
};
@@ -238,7 +239,7 @@ struct ExprLambda : Expr
{
if (!arg.empty() && formals && formals->argNames.find(arg) != formals->argNames.end())
throw ParseError({
- .hint = hintfmt("duplicate formal function argument '%1%'", arg),
+ .msg = hintfmt("duplicate formal function argument '%1%'", arg),
.errPos = pos
});
};
diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y
index a4c84c526..49d995bb9 100644
--- a/src/libexpr/parser.y
+++ b/src/libexpr/parser.y
@@ -32,7 +32,7 @@ namespace nix {
Path basePath;
Symbol file;
FileOrigin origin;
- ErrorInfo error;
+ std::optional<ErrorInfo> error;
Symbol sLetBody;
ParseData(EvalState & state)
: state(state)
@@ -66,8 +66,8 @@ namespace nix {
static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prevPos)
{
throw ParseError({
- .hint = hintfmt("attribute '%1%' already defined at %2%",
- showAttrPath(attrPath), prevPos),
+ .msg = hintfmt("attribute '%1%' already defined at %2%",
+ showAttrPath(attrPath), prevPos),
.errPos = pos
});
}
@@ -75,7 +75,7 @@ static void dupAttr(const AttrPath & attrPath, const Pos & pos, const Pos & prev
static void dupAttr(Symbol attr, const Pos & pos, const Pos & prevPos)
{
throw ParseError({
- .hint = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
+ .msg = hintfmt("attribute '%1%' already defined at %2%", attr, prevPos),
.errPos = pos
});
}
@@ -146,7 +146,7 @@ static void addFormal(const Pos & pos, Formals * formals, const Formal & formal)
{
if (!formals->argNames.insert(formal.name).second)
throw ParseError({
- .hint = hintfmt("duplicate formal function argument '%1%'",
+ .msg = hintfmt("duplicate formal function argument '%1%'",
formal.name),
.errPos = pos
});
@@ -258,7 +258,7 @@ static inline Pos makeCurPos(const YYLTYPE & loc, ParseData * data)
void yyerror(YYLTYPE * loc, yyscan_t scanner, ParseData * data, const char * error)
{
data->error = {
- .hint = hintfmt(error),
+ .msg = hintfmt(error),
.errPos = makeCurPos(*loc, data)
};
}
@@ -338,7 +338,7 @@ expr_function
| LET binds IN expr_function
{ if (!$2->dynamicAttrs.empty())
throw ParseError({
- .hint = hintfmt("dynamic attributes not allowed in let"),
+ .msg = hintfmt("dynamic attributes not allowed in let"),
.errPos = CUR_POS
});
$$ = new ExprLet($2, $4);
@@ -418,7 +418,7 @@ expr_simple
static bool noURLLiterals = settings.isExperimentalFeatureEnabled("no-url-literals");
if (noURLLiterals)
throw ParseError({
- .hint = hintfmt("URL literals are disabled"),
+ .msg = hintfmt("URL literals are disabled"),
.errPos = CUR_POS
});
$$ = new ExprString(data->symbols.create($1));
@@ -491,7 +491,7 @@ attrs
delete str;
} else
throw ParseError({
- .hint = hintfmt("dynamic attributes not allowed in inherit"),
+ .msg = hintfmt("dynamic attributes not allowed in inherit"),
.errPos = makeCurPos(@2, data)
});
}
@@ -576,7 +576,7 @@ Expr * EvalState::parse(const char * text, FileOrigin origin,
ParseData data(*this);
data.origin = origin;
switch (origin) {
- case foFile:
+ case foFile:
data.file = data.symbols.create(path);
break;
case foStdin:
@@ -593,7 +593,7 @@ Expr * EvalState::parse(const char * text, FileOrigin origin,
int res = yyparse(scanner, &data);
yylex_destroy(scanner);
- if (res) throw ParseError(data.error);
+ if (res) throw ParseError(data.error.value());
data.result->bindVars(staticEnv);
@@ -698,8 +698,12 @@ Path EvalState::findFile(SearchPath & searchPath, const string & path, const Pos
Path res = r.second + suffix;
if (pathExists(res)) return canonPath(res);
}
+
+ if (hasPrefix(path, "nix/"))
+ return corepkgsPrefix + path.substr(4);
+
throw ThrownError({
- .hint = hintfmt(evalSettings.pureEval
+ .msg = hintfmt(evalSettings.pureEval
? "cannot look up '<%s>' in pure evaluation mode (use '--impure' to override)"
: "file '%s' was not found in the Nix search path (add it using $NIX_PATH or -I)",
path),
@@ -721,8 +725,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
store, resolveUri(elem.second), "source", false).first.storePath) };
} catch (FileTransferError & e) {
logWarning({
- .name = "Entry download",
- .hint = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
+ .msg = hintfmt("Nix search path entry '%1%' cannot be downloaded, ignoring", elem.second)
});
res = { false, "" };
}
@@ -732,8 +735,7 @@ std::pair<bool, std::string> EvalState::resolveSearchPathElem(const SearchPathEl
res = { true, path };
else {
logWarning({
- .name = "Entry not found",
- .hint = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
+ .msg = hintfmt("warning: Nix search path entry '%1%' does not exist, ignoring", elem.second)
});
res = { false, "" };
}
diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc
index 2b304aab0..428adf4c2 100644
--- a/src/libexpr/primops.cc
+++ b/src/libexpr/primops.cc
@@ -35,7 +35,7 @@ InvalidPathError::InvalidPathError(const Path & path) :
void EvalState::realiseContext(const PathSet & context)
{
- std::vector<StorePathWithOutputs> drvs;
+ std::vector<DerivedPath::Built> drvs;
for (auto & i : context) {
auto [ctxS, outputName] = decodeContext(i);
@@ -43,7 +43,7 @@ void EvalState::realiseContext(const PathSet & context)
if (!store->isValidPath(ctx))
throw InvalidPathError(store->printStorePath(ctx));
if (!outputName.empty() && ctx.isDerivation()) {
- drvs.push_back(StorePathWithOutputs{ctx, {outputName}});
+ drvs.push_back({ctx, {outputName}});
}
}
@@ -51,14 +51,16 @@ void EvalState::realiseContext(const PathSet & context)
if (!evalSettings.enableImportFromDerivation)
throw EvalError("attempted to realize '%1%' during evaluation but 'allow-import-from-derivation' is false",
- store->printStorePath(drvs.begin()->path));
+ store->printStorePath(drvs.begin()->drvPath));
/* For performance, prefetch all substitute info. */
StorePathSet willBuild, willSubstitute, unknown;
uint64_t downloadSize, narSize;
- store->queryMissing(drvs, willBuild, willSubstitute, unknown, downloadSize, narSize);
+ std::vector<DerivedPath> buildReqs;
+ for (auto & d : drvs) buildReqs.emplace_back(DerivedPath { d });
+ store->queryMissing(buildReqs, willBuild, willSubstitute, unknown, downloadSize, narSize);
- store->buildPaths(drvs);
+ store->buildPaths(buildReqs);
/* Add the output of this derivations to the allowed
paths. */
@@ -115,9 +117,12 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
state.realiseContext(context);
} catch (InvalidPathError & e) {
throw EvalError({
- .hint = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path),
+ .msg = hintfmt("cannot import '%1%', since path '%2%' is not valid", path, e.path),
.errPos = pos
});
+ } catch (Error & e) {
+ e.addTrace(pos, "while importing '%s'", path);
+ throw e;
}
Path realPath = state.checkSourcePath(state.toRealPath(path, context));
@@ -164,7 +169,15 @@ static void import(EvalState & state, const Pos & pos, Value & vPath, Value * vS
state.forceFunction(**fun, pos);
mkApp(v, **fun, w);
state.forceAttrs(v, pos);
- } else {
+ }
+
+ else if (path == corepkgsPrefix + "fetchurl.nix") {
+ state.eval(state.parseExprFromString(
+ #include "fetchurl.nix.gen.hh"
+ , "/"), v);
+ }
+
+ else {
if (!vScope)
state.evalFile(realPath, v);
else {
@@ -274,7 +287,7 @@ void prim_importNative(EvalState & state, const Pos & pos, Value * * args, Value
state.realiseContext(context);
} catch (InvalidPathError & e) {
throw EvalError({
- .hint = hintfmt(
+ .msg = hintfmt(
"cannot import '%1%', since path '%2%' is not valid",
path, e.path),
.errPos = pos
@@ -314,7 +327,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
auto count = args[0]->listSize();
if (count == 0) {
throw EvalError({
- .hint = hintfmt("at least one argument to 'exec' required"),
+ .msg = hintfmt("at least one argument to 'exec' required"),
.errPos = pos
});
}
@@ -328,7 +341,7 @@ void prim_exec(EvalState & state, const Pos & pos, Value * * args, Value & v)
state.realiseContext(context);
} catch (InvalidPathError & e) {
throw EvalError({
- .hint = hintfmt("cannot execute '%1%', since path '%2%' is not valid",
+ .msg = hintfmt("cannot execute '%1%', since path '%2%' is not valid",
program, e.path),
.errPos = pos
});
@@ -356,24 +369,20 @@ static void prim_typeOf(EvalState & state, const Pos & pos, Value * * args, Valu
{
state.forceValue(*args[0], pos);
string t;
- switch (args[0]->type) {
- case tInt: t = "int"; break;
- case tBool: t = "bool"; break;
- case tString: t = "string"; break;
- case tPath: t = "path"; break;
- case tNull: t = "null"; break;
- case tAttrs: t = "set"; break;
- case tList1: case tList2: case tListN: t = "list"; break;
- case tLambda:
- case tPrimOp:
- case tPrimOpApp:
- t = "lambda";
- break;
- case tExternal:
+ switch (args[0]->type()) {
+ case nInt: t = "int"; break;
+ case nBool: t = "bool"; break;
+ case nString: t = "string"; break;
+ case nPath: t = "path"; break;
+ case nNull: t = "null"; break;
+ case nAttrs: t = "set"; break;
+ case nList: t = "list"; break;
+ case nFunction: t = "lambda"; break;
+ case nExternal:
t = args[0]->external->typeOf();
break;
- case tFloat: t = "float"; break;
- default: abort();
+ case nFloat: t = "float"; break;
+ case nThunk: abort();
}
mkString(v, state.symbols.create(t));
}
@@ -393,7 +402,7 @@ static RegisterPrimOp primop_typeOf({
static void prim_isNull(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
state.forceValue(*args[0], pos);
- mkBool(v, args[0]->type == tNull);
+ mkBool(v, args[0]->type() == nNull);
}
static RegisterPrimOp primop_isNull({
@@ -413,18 +422,7 @@ static RegisterPrimOp primop_isNull({
static void prim_isFunction(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
state.forceValue(*args[0], pos);
- bool res;
- switch (args[0]->type) {
- case tLambda:
- case tPrimOp:
- case tPrimOpApp:
- res = true;
- break;
- default:
- res = false;
- break;
- }
- mkBool(v, res);
+ mkBool(v, args[0]->type() == nFunction);
}
static RegisterPrimOp primop_isFunction({
@@ -440,7 +438,7 @@ static RegisterPrimOp primop_isFunction({
static void prim_isInt(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
state.forceValue(*args[0], pos);
- mkBool(v, args[0]->type == tInt);
+ mkBool(v, args[0]->type() == nInt);
}
static RegisterPrimOp primop_isInt({
@@ -456,7 +454,7 @@ static RegisterPrimOp primop_isInt({
static void prim_isFloat(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
state.forceValue(*args[0], pos);
- mkBool(v, args[0]->type == tFloat);
+ mkBool(v, args[0]->type() == nFloat);
}
static RegisterPrimOp primop_isFloat({
@@ -472,7 +470,7 @@ static RegisterPrimOp primop_isFloat({
static void prim_isString(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
state.forceValue(*args[0], pos);
- mkBool(v, args[0]->type == tString);
+ mkBool(v, args[0]->type() == nString);
}
static RegisterPrimOp primop_isString({
@@ -488,7 +486,7 @@ static RegisterPrimOp primop_isString({
static void prim_isBool(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
state.forceValue(*args[0], pos);
- mkBool(v, args[0]->type == tBool);
+ mkBool(v, args[0]->type() == nBool);
}
static RegisterPrimOp primop_isBool({
@@ -504,7 +502,7 @@ static RegisterPrimOp primop_isBool({
static void prim_isPath(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
state.forceValue(*args[0], pos);
- mkBool(v, args[0]->type == tPath);
+ mkBool(v, args[0]->type() == nPath);
}
static RegisterPrimOp primop_isPath({
@@ -520,20 +518,20 @@ struct CompareValues
{
bool operator () (const Value * v1, const Value * v2) const
{
- if (v1->type == tFloat && v2->type == tInt)
+ if (v1->type() == nFloat && v2->type() == nInt)
return v1->fpoint < v2->integer;
- if (v1->type == tInt && v2->type == tFloat)
+ if (v1->type() == nInt && v2->type() == nFloat)
return v1->integer < v2->fpoint;
- if (v1->type != v2->type)
+ if (v1->type() != v2->type())
throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2));
- switch (v1->type) {
- case tInt:
+ switch (v1->type()) {
+ case nInt:
return v1->integer < v2->integer;
- case tFloat:
+ case nFloat:
return v1->fpoint < v2->fpoint;
- case tString:
+ case nString:
return strcmp(v1->string.s, v2->string.s) < 0;
- case tPath:
+ case nPath:
return strcmp(v1->path, v2->path) < 0;
default:
throw EvalError("cannot compare %1% with %2%", showType(*v1), showType(*v2));
@@ -558,7 +556,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
args[0]->attrs->find(state.symbols.create("startSet"));
if (startSet == args[0]->attrs->end())
throw EvalError({
- .hint = hintfmt("attribute 'startSet' required"),
+ .msg = hintfmt("attribute 'startSet' required"),
.errPos = pos
});
state.forceList(*startSet->value, pos);
@@ -572,7 +570,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
args[0]->attrs->find(state.symbols.create("operator"));
if (op == args[0]->attrs->end())
throw EvalError({
- .hint = hintfmt("attribute 'operator' required"),
+ .msg = hintfmt("attribute 'operator' required"),
.errPos = pos
});
state.forceValue(*op->value, pos);
@@ -594,7 +592,7 @@ static void prim_genericClosure(EvalState & state, const Pos & pos, Value * * ar
e->attrs->find(state.symbols.create("key"));
if (key == e->attrs->end())
throw EvalError({
- .hint = hintfmt("attribute 'key' required"),
+ .msg = hintfmt("attribute 'key' required"),
.errPos = pos
});
state.forceValue(*key->value, pos);
@@ -700,10 +698,14 @@ static RegisterPrimOp primop_tryEval({
Try to shallowly evaluate *e*. Return a set containing the
attributes `success` (`true` if *e* evaluated successfully,
`false` if an error was thrown) and `value`, equalling *e* if
- successful and `false` otherwise. Note that this doesn't evaluate
- *e* deeply, so ` let e = { x = throw ""; }; in (builtins.tryEval
- e).success ` will be `true`. Using ` builtins.deepSeq ` one can
- get the expected result: `let e = { x = throw ""; }; in
+ successful and `false` otherwise. `tryEval` will only prevent
+ errors created by `throw` or `assert` from being thrown.
+ Errors `tryEval` will not catch are for example those created
+ by `abort` and type errors generated by builtins. Also note that
+ this doesn't evaluate *e* deeply, so `let e = { x = throw ""; };
+ in (builtins.tryEval e).success` will be `true`. Using
+ `builtins.deepSeq` one can get the expected result:
+ `let e = { x = throw ""; }; in
(builtins.tryEval (builtins.deepSeq e e)).success` will be
`false`.
)",
@@ -777,7 +779,7 @@ static RegisterPrimOp primop_deepSeq({
static void prim_trace(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
state.forceValue(*args[0], pos);
- if (args[0]->type == tString)
+ if (args[0]->type() == nString)
printError("trace: %1%", args[0]->string.s);
else
printError("trace: %1%", *args[0]);
@@ -817,7 +819,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
Bindings::iterator attr = args[0]->attrs->find(state.sName);
if (attr == args[0]->attrs->end())
throw EvalError({
- .hint = hintfmt("required attribute 'name' missing"),
+ .msg = hintfmt("required attribute 'name' missing"),
.errPos = pos
});
string drvName;
@@ -866,7 +868,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
else if (s == "flat") ingestionMethod = FileIngestionMethod::Flat;
else
throw EvalError({
- .hint = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
+ .msg = hintfmt("invalid value '%s' for 'outputHashMode' attribute", s),
.errPos = posDrvName
});
};
@@ -876,7 +878,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
for (auto & j : ss) {
if (outputs.find(j) != outputs.end())
throw EvalError({
- .hint = hintfmt("duplicate derivation output '%1%'", j),
+ .msg = hintfmt("duplicate derivation output '%1%'", j),
.errPos = posDrvName
});
/* !!! Check whether j is a valid attribute
@@ -886,14 +888,14 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
the resulting set. */
if (j == "drv")
throw EvalError({
- .hint = hintfmt("invalid derivation output name 'drv'" ),
+ .msg = hintfmt("invalid derivation output name 'drv'" ),
.errPos = posDrvName
});
outputs.insert(j);
}
if (outputs.empty())
throw EvalError({
- .hint = hintfmt("derivation cannot have an empty set of outputs"),
+ .msg = hintfmt("derivation cannot have an empty set of outputs"),
.errPos = posDrvName
});
};
@@ -902,7 +904,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
if (ignoreNulls) {
state.forceValue(*i->value, pos);
- if (i->value->type == tNull) continue;
+ if (i->value->type() == nNull) continue;
}
if (i->name == state.sContentAddressed) {
@@ -1014,20 +1016,20 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
/* Do we have all required attributes? */
if (drv.builder == "")
throw EvalError({
- .hint = hintfmt("required attribute 'builder' missing"),
+ .msg = hintfmt("required attribute 'builder' missing"),
.errPos = posDrvName
});
if (drv.platform == "")
throw EvalError({
- .hint = hintfmt("required attribute 'system' missing"),
+ .msg = hintfmt("required attribute 'system' missing"),
.errPos = posDrvName
});
/* Check whether the derivation name is valid. */
if (isDerivation(drvName))
throw EvalError({
- .hint = hintfmt("derivation names are not allowed to end in '%s'", drvExtension),
+ .msg = hintfmt("derivation names are not allowed to end in '%s'", drvExtension),
.errPos = posDrvName
});
@@ -1038,7 +1040,7 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
already content addressed. */
if (outputs.size() != 1 || *(outputs.begin()) != "out")
throw Error({
- .hint = hintfmt("multiple outputs are not supported in fixed-output derivations"),
+ .msg = hintfmt("multiple outputs are not supported in fixed-output derivations"),
.errPos = posDrvName
});
@@ -1089,18 +1091,35 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
// Regular, non-CA derivation should always return a single hash and not
// hash per output.
- Hash h = std::get<0>(hashDerivationModulo(*state.store, Derivation(drv), true));
+ auto hashModulo = hashDerivationModulo(*state.store, Derivation(drv), true);
+ std::visit(overloaded {
+ [&](Hash h) {
+ for (auto & i : outputs) {
+ auto outPath = state.store->makeOutputPath(i, h, drvName);
+ drv.env[i] = state.store->printStorePath(outPath);
+ drv.outputs.insert_or_assign(i,
+ DerivationOutput {
+ .output = DerivationOutputInputAddressed {
+ .path = std::move(outPath),
+ },
+ });
+ }
+ },
+ [&](CaOutputHashes) {
+ // Shouldn't happen as the toplevel derivation is not CA.
+ assert(false);
+ },
+ [&](DeferredHash _) {
+ for (auto & i : outputs) {
+ drv.outputs.insert_or_assign(i,
+ DerivationOutput {
+ .output = DerivationOutputDeferred{},
+ });
+ }
+ },
+ },
+ hashModulo);
- for (auto & i : outputs) {
- auto outPath = state.store->makeOutputPath(i, h, drvName);
- drv.env[i] = state.store->printStorePath(outPath);
- drv.outputs.insert_or_assign(i,
- DerivationOutput {
- .output = DerivationOutputInputAddressed {
- .path = std::move(outPath),
- },
- });
- }
}
/* Write the resulting term into the Nix store directory. */
@@ -1115,9 +1134,10 @@ static void prim_derivationStrict(EvalState & state, const Pos & pos, Value * *
However, we don't bother doing this for floating CA derivations because
their "hash modulo" is indeterminate until built. */
- if (drv.type() != DerivationType::CAFloating)
- drvHashes.insert_or_assign(drvPath,
- hashDerivationModulo(*state.store, Derivation(drv), false));
+ if (drv.type() != DerivationType::CAFloating) {
+ auto h = hashDerivationModulo(*state.store, Derivation(drv), false);
+ drvHashes.lock()->insert_or_assign(drvPath, h);
+ }
state.mkAttrs(v, 1 + drv.outputs.size());
mkString(*state.allocAttr(v, state.sDrvPath), drvPathS, {"=" + drvPathS});
@@ -1200,7 +1220,7 @@ static void prim_storePath(EvalState & state, const Pos & pos, Value * * args, V
if (!state.store->isStorePath(path)) path = canonPath(path, true);
if (!state.store->isInStore(path))
throw EvalError({
- .hint = hintfmt("path '%1%' is not in the Nix store", path),
+ .msg = hintfmt("path '%1%' is not in the Nix store", path),
.errPos = pos
});
auto path2 = state.store->toStorePath(path).first;
@@ -1236,7 +1256,7 @@ static void prim_pathExists(EvalState & state, const Pos & pos, Value * * args,
state.realiseContext(context);
} catch (InvalidPathError & e) {
throw EvalError({
- .hint = hintfmt(
+ .msg = hintfmt(
"cannot check the existence of '%1%', since path '%2%' is not valid",
path, e.path),
.errPos = pos
@@ -1290,7 +1310,7 @@ static void prim_dirOf(EvalState & state, const Pos & pos, Value * * args, Value
{
PathSet context;
Path dir = dirOf(state.coerceToString(pos, *args[0], context, false, false));
- if (args[0]->type == tPath) mkPath(v, dir.c_str()); else mkString(v, dir, context);
+ if (args[0]->type() == nPath) mkPath(v, dir.c_str()); else mkString(v, dir, context);
}
static RegisterPrimOp primop_dirOf({
@@ -1313,7 +1333,7 @@ static void prim_readFile(EvalState & state, const Pos & pos, Value * * args, Va
state.realiseContext(context);
} catch (InvalidPathError & e) {
throw EvalError({
- .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
+ .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
.errPos = pos
});
}
@@ -1352,7 +1372,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
i = v2.attrs->find(state.symbols.create("path"));
if (i == v2.attrs->end())
throw EvalError({
- .hint = hintfmt("attribute 'path' missing"),
+ .msg = hintfmt("attribute 'path' missing"),
.errPos = pos
});
@@ -1363,7 +1383,7 @@ static void prim_findFile(EvalState & state, const Pos & pos, Value * * args, Va
state.realiseContext(context);
} catch (InvalidPathError & e) {
throw EvalError({
- .hint = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path),
+ .msg = hintfmt("cannot find '%1%', since path '%2%' is not valid", path, e.path),
.errPos = pos
});
}
@@ -1389,7 +1409,7 @@ static void prim_hashFile(EvalState & state, const Pos & pos, Value * * args, Va
std::optional<HashType> ht = parseHashType(type);
if (!ht)
throw Error({
- .hint = hintfmt("unknown hash type '%1%'", type),
+ .msg = hintfmt("unknown hash type '%1%'", type),
.errPos = pos
});
@@ -1419,7 +1439,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
state.realiseContext(ctx);
} catch (InvalidPathError & e) {
throw EvalError({
- .hint = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
+ .msg = hintfmt("cannot read '%1%', since path '%2%' is not valid", path, e.path),
.errPos = pos
});
}
@@ -1431,7 +1451,7 @@ static void prim_readDir(EvalState & state, const Pos & pos, Value * * args, Val
Value * ent_val = state.allocAttr(v, state.symbols.create(ent.name));
if (ent.type == DT_UNKNOWN)
ent.type = getFileType(path + "/" + ent.name);
- mkStringNoCopy(*ent_val,
+ ent_val->mkString(
ent.type == DT_REG ? "regular" :
ent.type == DT_DIR ? "directory" :
ent.type == DT_LNK ? "symlink" :
@@ -1603,7 +1623,12 @@ static RegisterPrimOp primop_toJSON({
static void prim_fromJSON(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
string s = state.forceStringNoCtx(*args[0], pos);
- parseJSON(state, s, v);
+ try {
+ parseJSON(state, s, v);
+ } catch (JSONParseError &e) {
+ e.addTrace(pos, "while decoding a JSON string");
+ throw e;
+ }
}
static RegisterPrimOp primop_fromJSON({
@@ -1634,7 +1659,7 @@ static void prim_toFile(EvalState & state, const Pos & pos, Value * * args, Valu
for (auto path : context) {
if (path.at(0) != '/')
throw EvalError( {
- .hint = hintfmt(
+ .msg = hintfmt(
"in 'toFile': the file named '%1%' must not contain a reference "
"to a derivation but contains (%2%)",
name, path),
@@ -1785,14 +1810,14 @@ static void prim_filterSource(EvalState & state, const Pos & pos, Value * * args
Path path = state.coerceToPath(pos, *args[1], context);
if (!context.empty())
throw EvalError({
- .hint = hintfmt("string '%1%' cannot refer to other paths", path),
+ .msg = hintfmt("string '%1%' cannot refer to other paths", path),
.errPos = pos
});
state.forceValue(*args[0], pos);
- if (args[0]->type != tLambda)
+ if (args[0]->type() != nFunction)
throw TypeError({
- .hint = hintfmt(
+ .msg = hintfmt(
"first argument in call to 'filterSource' is not a function but %1%",
showType(*args[0])),
.errPos = pos
@@ -1859,7 +1884,7 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
path = state.coerceToPath(*attr.pos, *attr.value, context);
if (!context.empty())
throw EvalError({
- .hint = hintfmt("string '%1%' cannot refer to other paths", path),
+ .msg = hintfmt("string '%1%' cannot refer to other paths", path),
.errPos = *attr.pos
});
} else if (attr.name == state.sName)
@@ -1873,13 +1898,13 @@ static void prim_path(EvalState & state, const Pos & pos, Value * * args, Value
expectedHash = newHashAllowEmpty(state.forceStringNoCtx(*attr.value, *attr.pos), htSHA256);
else
throw EvalError({
- .hint = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name),
+ .msg = hintfmt("unsupported argument '%1%' to 'addPath'", attr.name),
.errPos = *attr.pos
});
}
if (path.empty())
throw EvalError({
- .hint = hintfmt("'path' required"),
+ .msg = hintfmt("'path' required"),
.errPos = pos
});
if (name.empty())
@@ -1994,7 +2019,7 @@ void prim_getAttr(EvalState & state, const Pos & pos, Value * * args, Value & v)
Bindings::iterator i = args[1]->attrs->find(state.symbols.create(attr));
if (i == args[1]->attrs->end())
throw EvalError({
- .hint = hintfmt("attribute '%1%' missing", attr),
+ .msg = hintfmt("attribute '%1%' missing", attr),
.errPos = pos
});
// !!! add to stack trace?
@@ -2056,7 +2081,7 @@ static RegisterPrimOp primop_hasAttr({
static void prim_isAttrs(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
state.forceValue(*args[0], pos);
- mkBool(v, args[0]->type == tAttrs);
+ mkBool(v, args[0]->type() == nAttrs);
}
static RegisterPrimOp primop_isAttrs({
@@ -2126,7 +2151,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
Bindings::iterator j = v2.attrs->find(state.sName);
if (j == v2.attrs->end())
throw TypeError({
- .hint = hintfmt("'name' attribute missing in a call to 'listToAttrs'"),
+ .msg = hintfmt("'name' attribute missing in a call to 'listToAttrs'"),
.errPos = pos
});
string name = state.forceStringNoCtx(*j->value, pos);
@@ -2136,7 +2161,7 @@ static void prim_listToAttrs(EvalState & state, const Pos & pos, Value * * args,
Bindings::iterator j2 = v2.attrs->find(state.symbols.create(state.sValue));
if (j2 == v2.attrs->end())
throw TypeError({
- .hint = hintfmt("'value' attribute missing in a call to 'listToAttrs'"),
+ .msg = hintfmt("'value' attribute missing in a call to 'listToAttrs'"),
.errPos = pos
});
v.attrs->push_back(Attr(sym, j2->value, j2->pos));
@@ -2236,13 +2261,13 @@ static RegisterPrimOp primop_catAttrs({
static void prim_functionArgs(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
state.forceValue(*args[0], pos);
- if (args[0]->type == tPrimOpApp || args[0]->type == tPrimOp) {
+ if (args[0]->isPrimOpApp() || args[0]->isPrimOp()) {
state.mkAttrs(v, 0);
return;
}
- if (args[0]->type != tLambda)
+ if (!args[0]->isLambda())
throw TypeError({
- .hint = hintfmt("'functionArgs' requires a function"),
+ .msg = hintfmt("'functionArgs' requires a function"),
.errPos = pos
});
@@ -2319,7 +2344,7 @@ static RegisterPrimOp primop_mapAttrs({
static void prim_isList(EvalState & state, const Pos & pos, Value * * args, Value & v)
{
state.forceValue(*args[0], pos);
- mkBool(v, args[0]->isList());
+ mkBool(v, args[0]->type() == nList);
}
static RegisterPrimOp primop_isList({
@@ -2336,7 +2361,7 @@ static void elemAt(EvalState & state, const Pos & pos, Value & list, int n, Valu
state.forceList(list, pos);
if (n < 0 || (unsigned int) n >= list.listSize())
throw Error({
- .hint = hintfmt("list index %1% is out of bounds", n),
+ .msg = hintfmt("list index %1% is out of bounds", n),
.errPos = pos
});
state.forceValue(*list.listElems()[n], pos);
@@ -2384,7 +2409,7 @@ static void prim_tail(EvalState & state, const Pos & pos, Value * * args, Value
state.forceList(*args[0], pos);
if (args[0]->listSize() == 0)
throw Error({
- .hint = hintfmt("'tail' called on an empty list"),
+ .msg = hintfmt("'tail' called on an empty list"),
.errPos = pos
});
@@ -2623,7 +2648,7 @@ static void prim_genList(EvalState & state, const Pos & pos, Value * * args, Val
if (len < 0)
throw EvalError({
- .hint = hintfmt("cannot create list of size %1%", len),
+ .msg = hintfmt("cannot create list of size %1%", len),
.errPos = pos
});
@@ -2671,7 +2696,7 @@ static void prim_sort(EvalState & state, const Pos & pos, Value * * args, Value
auto comparator = [&](Value * a, Value * b) {
/* Optimization: if the comparator is lessThan, bypass
callFunction. */
- if (args[0]->type == tPrimOp && args[0]->primOp->fun == prim_lessThan)
+ if (args[0]->isPrimOp() && args[0]->primOp->fun == prim_lessThan)
return CompareValues()(a, b);
Value vTmp1, vTmp2;
@@ -2813,7 +2838,7 @@ static void prim_add(EvalState & state, const Pos & pos, Value * * args, Value &
{
state.forceValue(*args[0], pos);
state.forceValue(*args[1], pos);
- if (args[0]->type == tFloat || args[1]->type == tFloat)
+ if (args[0]->type() == nFloat || args[1]->type() == nFloat)
mkFloat(v, state.forceFloat(*args[0], pos) + state.forceFloat(*args[1], pos));
else
mkInt(v, state.forceInt(*args[0], pos) + state.forceInt(*args[1], pos));
@@ -2832,7 +2857,7 @@ static void prim_sub(EvalState & state, const Pos & pos, Value * * args, Value &
{
state.forceValue(*args[0], pos);
state.forceValue(*args[1], pos);
- if (args[0]->type == tFloat || args[1]->type == tFloat)
+ if (args[0]->type() == nFloat || args[1]->type() == nFloat)
mkFloat(v, state.forceFloat(*args[0], pos) - state.forceFloat(*args[1], pos));
else
mkInt(v, state.forceInt(*args[0], pos) - state.forceInt(*args[1], pos));
@@ -2851,7 +2876,7 @@ static void prim_mul(EvalState & state, const Pos & pos, Value * * args, Value &
{
state.forceValue(*args[0], pos);
state.forceValue(*args[1], pos);
- if (args[0]->type == tFloat || args[1]->type == tFloat)
+ if (args[0]->type() == nFloat || args[1]->type() == nFloat)
mkFloat(v, state.forceFloat(*args[0], pos) * state.forceFloat(*args[1], pos));
else
mkInt(v, state.forceInt(*args[0], pos) * state.forceInt(*args[1], pos));
@@ -2874,11 +2899,11 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
NixFloat f2 = state.forceFloat(*args[1], pos);
if (f2 == 0)
throw EvalError({
- .hint = hintfmt("division by zero"),
+ .msg = hintfmt("division by zero"),
.errPos = pos
});
- if (args[0]->type == tFloat || args[1]->type == tFloat) {
+ if (args[0]->type() == nFloat || args[1]->type() == nFloat) {
mkFloat(v, state.forceFloat(*args[0], pos) / state.forceFloat(*args[1], pos));
} else {
NixInt i1 = state.forceInt(*args[0], pos);
@@ -2886,7 +2911,7 @@ static void prim_div(EvalState & state, const Pos & pos, Value * * args, Value &
/* Avoid division overflow as it might raise SIGFPE. */
if (i1 == std::numeric_limits<NixInt>::min() && i2 == -1)
throw EvalError({
- .hint = hintfmt("overflow in integer division"),
+ .msg = hintfmt("overflow in integer division"),
.errPos = pos
});
@@ -3017,7 +3042,7 @@ static void prim_substring(EvalState & state, const Pos & pos, Value * * args, V
if (start < 0)
throw EvalError({
- .hint = hintfmt("negative start position in 'substring'"),
+ .msg = hintfmt("negative start position in 'substring'"),
.errPos = pos
});
@@ -3068,7 +3093,7 @@ static void prim_hashString(EvalState & state, const Pos & pos, Value * * args,
std::optional<HashType> ht = parseHashType(type);
if (!ht)
throw Error({
- .hint = hintfmt("unknown hash type '%1%'", type),
+ .msg = hintfmt("unknown hash type '%1%'", type),
.errPos = pos
});
@@ -3132,12 +3157,12 @@ void prim_match(EvalState & state, const Pos & pos, Value * * args, Value & v)
if (e.code() == std::regex_constants::error_space) {
// limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
throw EvalError({
- .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
+ .msg = hintfmt("memory limit exceeded by regular expression '%s'", re),
.errPos = pos
});
} else {
throw EvalError({
- .hint = hintfmt("invalid regular expression '%s'", re),
+ .msg = hintfmt("invalid regular expression '%s'", re),
.errPos = pos
});
}
@@ -3240,12 +3265,12 @@ static void prim_split(EvalState & state, const Pos & pos, Value * * args, Value
if (e.code() == std::regex_constants::error_space) {
// limit is _GLIBCXX_REGEX_STATE_LIMIT for libstdc++
throw EvalError({
- .hint = hintfmt("memory limit exceeded by regular expression '%s'", re),
+ .msg = hintfmt("memory limit exceeded by regular expression '%s'", re),
.errPos = pos
});
} else {
throw EvalError({
- .hint = hintfmt("invalid regular expression '%s'", re),
+ .msg = hintfmt("invalid regular expression '%s'", re),
.errPos = pos
});
}
@@ -3325,7 +3350,7 @@ static void prim_replaceStrings(EvalState & state, const Pos & pos, Value * * ar
state.forceList(*args[1], pos);
if (args[0]->listSize() != args[1]->listSize())
throw EvalError({
- .hint = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"),
+ .msg = hintfmt("'from' and 'to' arguments to 'replaceStrings' have different lengths"),
.errPos = pos
});
diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc
index b570fca31..31cf812b4 100644
--- a/src/libexpr/primops/context.cc
+++ b/src/libexpr/primops/context.cc
@@ -147,7 +147,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
for (auto & i : *args[1]->attrs) {
if (!state.store->isStorePath(i.name))
throw EvalError({
- .hint = hintfmt("Context key '%s' is not a store path", i.name),
+ .msg = hintfmt("Context key '%s' is not a store path", i.name),
.errPos = *i.pos
});
if (!settings.readOnlyMode)
@@ -164,7 +164,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
if (state.forceBool(*iter->value, *iter->pos)) {
if (!isDerivation(i.name)) {
throw EvalError({
- .hint = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
+ .msg = hintfmt("Tried to add all-outputs context of %s, which is not a derivation, to a string", i.name),
.errPos = *i.pos
});
}
@@ -177,7 +177,7 @@ static void prim_appendContext(EvalState & state, const Pos & pos, Value * * arg
state.forceList(*iter->value, *iter->pos);
if (iter->value->listSize() && !isDerivation(i.name)) {
throw EvalError({
- .hint = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
+ .msg = hintfmt("Tried to add derivation output context of %s, which is not a derivation, to a string", i.name),
.errPos = *i.pos
});
}
diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc
index a77035c16..4830ebec3 100644
--- a/src/libexpr/primops/fetchMercurial.cc
+++ b/src/libexpr/primops/fetchMercurial.cc
@@ -17,7 +17,7 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
state.forceValue(*args[0]);
- if (args[0]->type == tAttrs) {
+ if (args[0]->type() == nAttrs) {
state.forceAttrs(*args[0], pos);
@@ -38,14 +38,14 @@ static void prim_fetchMercurial(EvalState & state, const Pos & pos, Value * * ar
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else
throw EvalError({
- .hint = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
+ .msg = hintfmt("unsupported argument '%s' to 'fetchMercurial'", attr.name),
.errPos = *attr.pos
});
}
if (url.empty())
throw EvalError({
- .hint = hintfmt("'url' argument required"),
+ .msg = hintfmt("'url' argument required"),
.errPos = pos
});
diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc
index 7cd4d0fbf..27d8ddf35 100644
--- a/src/libexpr/primops/fetchTree.cc
+++ b/src/libexpr/primops/fetchTree.cc
@@ -39,11 +39,12 @@ void emitTreeAttrs(
// Backwards compat for `builtins.fetchGit`: dirty repos return an empty sha1 as rev
auto emptyHash = Hash(htSHA1);
mkString(*state.allocAttr(v, state.symbols.create("rev")), emptyHash.gitRev());
- mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitRev());
+ mkString(*state.allocAttr(v, state.symbols.create("shortRev")), emptyHash.gitShortRev());
}
if (input.getType() == "git")
- mkBool(*state.allocAttr(v, state.symbols.create("submodules")), maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
+ mkBool(*state.allocAttr(v, state.symbols.create("submodules")),
+ fetchers::maybeGetBoolAttr(input.attrs, "submodules").value_or(false));
if (auto revCount = input.getRevCount())
mkInt(*state.allocAttr(v, state.symbols.create("revCount")), *revCount);
@@ -84,26 +85,26 @@ static void fetchTree(
state.forceValue(*args[0]);
- if (args[0]->type == tAttrs) {
+ if (args[0]->type() == nAttrs) {
state.forceAttrs(*args[0], pos);
fetchers::Attrs attrs;
for (auto & attr : *args[0]->attrs) {
state.forceValue(*attr.value);
- if (attr.value->type == tPath || attr.value->type == tString)
+ if (attr.value->type() == nPath || attr.value->type() == nString)
addURI(
state,
attrs,
attr.name,
state.coerceToString(*attr.pos, *attr.value, context, false, false)
);
- else if (attr.value->type == tString)
+ else if (attr.value->type() == nString)
addURI(state, attrs, attr.name, attr.value->string.s);
- else if (attr.value->type == tBool)
- attrs.emplace(attr.name, fetchers::Explicit<bool>{attr.value->boolean});
- else if (attr.value->type == tInt)
- attrs.emplace(attr.name, attr.value->integer);
+ else if (attr.value->type() == nBool)
+ attrs.emplace(attr.name, Explicit<bool>{attr.value->boolean});
+ else if (attr.value->type() == nInt)
+ attrs.emplace(attr.name, uint64_t(attr.value->integer));
else
throw TypeError("fetchTree argument '%s' is %s while a string, Boolean or integer is expected",
attr.name, showType(*attr.value));
@@ -114,7 +115,7 @@ static void fetchTree(
if (!attrs.count("type"))
throw Error({
- .hint = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
+ .msg = hintfmt("attribute 'type' is missing in call to 'fetchTree'"),
.errPos = pos
});
@@ -152,6 +153,7 @@ static void prim_fetchTree(EvalState & state, const Pos & pos, Value * * args, V
fetchTree(state, pos, args, v, std::nullopt);
}
+// FIXME: document
static RegisterPrimOp primop_fetchTree("fetchTree", 1, prim_fetchTree);
static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
@@ -162,7 +164,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
state.forceValue(*args[0]);
- if (args[0]->type == tAttrs) {
+ if (args[0]->type() == nAttrs) {
state.forceAttrs(*args[0], pos);
@@ -176,14 +178,14 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
name = state.forceStringNoCtx(*attr.value, *attr.pos);
else
throw EvalError({
- .hint = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
+ .msg = hintfmt("unsupported argument '%s' to '%s'", attr.name, who),
.errPos = *attr.pos
});
}
if (!url)
throw EvalError({
- .hint = hintfmt("'url' argument required"),
+ .msg = hintfmt("'url' argument required"),
.errPos = pos
});
} else
@@ -211,7 +213,7 @@ static void fetch(EvalState & state, const Pos & pos, Value * * args, Value & v,
? state.store->queryPathInfo(storePath)->narHash
: hashFile(htSHA256, path);
if (hash != *expectedHash)
- throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
+ throw Error((unsigned int) 102, "hash mismatch in file downloaded from '%s':\n specified: %s\n got: %s",
*url, expectedHash->to_string(Base32, true), hash.to_string(Base32, true));
}
@@ -323,6 +325,11 @@ static RegisterPrimOp primop_fetchGit({
A Boolean parameter that specifies whether submodules should be
checked out. Defaults to `false`.
+ - allRefs
+ Whether to fetch all refs of the repository. With this argument being
+ true, it's possible to load a `rev` from *any* `ref` (by default only
+ `rev`s from the specified `ref` are supported).
+
Here are some examples of how to use `fetchGit`.
- To fetch a private repository over SSH:
diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc
index 77bff44ae..4c6682dfd 100644
--- a/src/libexpr/primops/fromTOML.cc
+++ b/src/libexpr/primops/fromTOML.cc
@@ -82,7 +82,7 @@ static void prim_fromTOML(EvalState & state, const Pos & pos, Value * * args, Va
visit(v, parser(tomlStream).parse());
} catch (std::runtime_error & e) {
throw EvalError({
- .hint = hintfmt("while parsing a TOML string: %s", e.what()),
+ .msg = hintfmt("while parsing a TOML string: %s", e.what()),
.errPos = pos
});
}
diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc
index 6ec8315ba..bfea24d40 100644
--- a/src/libexpr/value-to-json.cc
+++ b/src/libexpr/value-to-json.cc
@@ -16,30 +16,30 @@ void printValueAsJSON(EvalState & state, bool strict,
if (strict) state.forceValue(v);
- switch (v.type) {
+ switch (v.type()) {
- case tInt:
+ case nInt:
out.write(v.integer);
break;
- case tBool:
+ case nBool:
out.write(v.boolean);
break;
- case tString:
+ case nString:
copyContext(v, context);
out.write(v.string.s);
break;
- case tPath:
+ case nPath:
out.write(state.copyPathToStore(context, v.path));
break;
- case tNull:
+ case nNull:
out.write(nullptr);
break;
- case tAttrs: {
+ case nAttrs: {
auto maybeString = state.tryAttrsToString(noPos, v, context, false, false);
if (maybeString) {
out.write(*maybeString);
@@ -61,7 +61,7 @@ void printValueAsJSON(EvalState & state, bool strict,
break;
}
- case tList1: case tList2: case tListN: {
+ case nList: {
auto list(out.list());
for (unsigned int n = 0; n < v.listSize(); ++n) {
auto placeholder(list.placeholder());
@@ -70,15 +70,18 @@ void printValueAsJSON(EvalState & state, bool strict,
break;
}
- case tExternal:
+ case nExternal:
v.external->printValueAsJSON(state, strict, out, context);
break;
- case tFloat:
+ case nFloat:
out.write(v.fpoint);
break;
- default:
+ case nThunk:
+ throw TypeError("cannot convert %1% to JSON", showType(v));
+
+ case nFunction:
throw TypeError("cannot convert %1% to JSON", showType(v));
}
}
diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc
index 1f0b1541d..7464455d8 100644
--- a/src/libexpr/value-to-xml.cc
+++ b/src/libexpr/value-to-xml.cc
@@ -58,31 +58,31 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
if (strict) state.forceValue(v);
- switch (v.type) {
+ switch (v.type()) {
- case tInt:
+ case nInt:
doc.writeEmptyElement("int", singletonAttrs("value", (format("%1%") % v.integer).str()));
break;
- case tBool:
+ case nBool:
doc.writeEmptyElement("bool", singletonAttrs("value", v.boolean ? "true" : "false"));
break;
- case tString:
+ case nString:
/* !!! show the context? */
copyContext(v, context);
doc.writeEmptyElement("string", singletonAttrs("value", v.string.s));
break;
- case tPath:
+ case nPath:
doc.writeEmptyElement("path", singletonAttrs("value", v.path));
break;
- case tNull:
+ case nNull:
doc.writeEmptyElement("null");
break;
- case tAttrs:
+ case nAttrs:
if (state.isDerivation(v)) {
XMLAttrs xmlAttrs;
@@ -92,14 +92,14 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
a = v.attrs->find(state.sDrvPath);
if (a != v.attrs->end()) {
if (strict) state.forceValue(*a->value);
- if (a->value->type == tString)
+ if (a->value->type() == nString)
xmlAttrs["drvPath"] = drvPath = a->value->string.s;
}
a = v.attrs->find(state.sOutPath);
if (a != v.attrs->end()) {
if (strict) state.forceValue(*a->value);
- if (a->value->type == tString)
+ if (a->value->type() == nString)
xmlAttrs["outPath"] = a->value->string.s;
}
@@ -118,14 +118,19 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
break;
- case tList1: case tList2: case tListN: {
+ case nList: {
XMLOpenElement _(doc, "list");
for (unsigned int n = 0; n < v.listSize(); ++n)
printValueAsXML(state, strict, location, *v.listElems()[n], doc, context, drvsSeen);
break;
}
- case tLambda: {
+ case nFunction: {
+ if (!v.isLambda()) {
+ // FIXME: Serialize primops and primopapps
+ doc.writeEmptyElement("unevaluated");
+ break;
+ }
XMLAttrs xmlAttrs;
if (location) posToXML(xmlAttrs, v.lambda.fun->pos);
XMLOpenElement _(doc, "function", xmlAttrs);
@@ -143,15 +148,15 @@ static void printValueAsXML(EvalState & state, bool strict, bool location,
break;
}
- case tExternal:
+ case nExternal:
v.external->printValueAsXML(state, strict, location, doc, context, drvsSeen);
break;
- case tFloat:
+ case nFloat:
doc.writeEmptyElement("float", singletonAttrs("value", (format("%1%") % v.fpoint).str()));
break;
- default:
+ case nThunk:
doc.writeEmptyElement("unevaluated");
}
}
diff --git a/src/libexpr/value.hh b/src/libexpr/value.hh
index fe11bb2ed..b317c1898 100644
--- a/src/libexpr/value.hh
+++ b/src/libexpr/value.hh
@@ -27,8 +27,24 @@ typedef enum {
tPrimOpApp,
tExternal,
tFloat
-} ValueType;
+} InternalType;
+// This type abstracts over all actual value types in the language,
+// grouping together implementation details like tList*, different function
+// types, and types in non-normal form (so thunks and co.)
+typedef enum {
+ nThunk,
+ nInt,
+ nFloat,
+ nBool,
+ nString,
+ nPath,
+ nNull,
+ nAttrs,
+ nList,
+ nFunction,
+ nExternal
+} ValueType;
class Bindings;
struct Env;
@@ -90,7 +106,28 @@ std::ostream & operator << (std::ostream & str, const ExternalValueBase & v);
struct Value
{
- ValueType type;
+private:
+ InternalType internalType;
+
+friend std::string showType(const Value & v);
+friend void printValue(std::ostream & str, std::set<const Value *> & active, const Value & v);
+
+public:
+
+ // Functions needed to distinguish the type
+ // These should be removed eventually, by putting the functionality that's
+ // needed by callers into methods of this type
+
+ // type() == nThunk
+ inline bool isThunk() const { return internalType == tThunk; };
+ inline bool isApp() const { return internalType == tApp; };
+ inline bool isBlackhole() const { return internalType == tBlackhole; };
+
+ // type() == nFunction
+ inline bool isLambda() const { return internalType == tLambda; };
+ inline bool isPrimOp() const { return internalType == tPrimOp; };
+ inline bool isPrimOpApp() const { return internalType == tPrimOpApp; };
+
union
{
NixInt integer;
@@ -147,24 +184,161 @@ struct Value
NixFloat fpoint;
};
+ // Returns the normal type of a Value. This only returns nThunk if the
+ // Value hasn't been forceValue'd
+ inline ValueType type() const
+ {
+ switch (internalType) {
+ case tInt: return nInt;
+ case tBool: return nBool;
+ case tString: return nString;
+ case tPath: return nPath;
+ case tNull: return nNull;
+ case tAttrs: return nAttrs;
+ case tList1: case tList2: case tListN: return nList;
+ case tLambda: case tPrimOp: case tPrimOpApp: return nFunction;
+ case tExternal: return nExternal;
+ case tFloat: return nFloat;
+ case tThunk: case tApp: case tBlackhole: return nThunk;
+ }
+ abort();
+ }
+
+ /* After overwriting an app node, be sure to clear pointers in the
+ Value to ensure that the target isn't kept alive unnecessarily. */
+ inline void clearValue()
+ {
+ app.left = app.right = 0;
+ }
+
+ inline void mkInt(NixInt n)
+ {
+ clearValue();
+ internalType = tInt;
+ integer = n;
+ }
+
+ inline void mkBool(bool b)
+ {
+ clearValue();
+ internalType = tBool;
+ boolean = b;
+ }
+
+ inline void mkString(const char * s, const char * * context = 0)
+ {
+ internalType = tString;
+ string.s = s;
+ string.context = context;
+ }
+
+ inline void mkPath(const char * s)
+ {
+ clearValue();
+ internalType = tPath;
+ path = s;
+ }
+
+ inline void mkNull()
+ {
+ clearValue();
+ internalType = tNull;
+ }
+
+ inline void mkAttrs(Bindings * a)
+ {
+ clearValue();
+ internalType = tAttrs;
+ attrs = a;
+ }
+
+ inline void mkList(size_t size)
+ {
+ clearValue();
+ if (size == 1)
+ internalType = tList1;
+ else if (size == 2)
+ internalType = tList2;
+ else {
+ internalType = tListN;
+ bigList.size = size;
+ }
+ }
+
+ inline void mkThunk(Env * e, Expr * ex)
+ {
+ internalType = tThunk;
+ thunk.env = e;
+ thunk.expr = ex;
+ }
+
+ inline void mkApp(Value * l, Value * r)
+ {
+ internalType = tApp;
+ app.left = l;
+ app.right = r;
+ }
+
+ inline void mkLambda(Env * e, ExprLambda * f)
+ {
+ internalType = tLambda;
+ lambda.env = e;
+ lambda.fun = f;
+ }
+
+ inline void mkBlackhole()
+ {
+ internalType = tBlackhole;
+ // Value will be overridden anyways
+ }
+
+ inline void mkPrimOp(PrimOp * p)
+ {
+ clearValue();
+ internalType = tPrimOp;
+ primOp = p;
+ }
+
+
+ inline void mkPrimOpApp(Value * l, Value * r)
+ {
+ internalType = tPrimOpApp;
+ app.left = l;
+ app.right = r;
+ }
+
+ inline void mkExternal(ExternalValueBase * e)
+ {
+ clearValue();
+ internalType = tExternal;
+ external = e;
+ }
+
+ inline void mkFloat(NixFloat n)
+ {
+ clearValue();
+ internalType = tFloat;
+ fpoint = n;
+ }
+
bool isList() const
{
- return type == tList1 || type == tList2 || type == tListN;
+ return internalType == tList1 || internalType == tList2 || internalType == tListN;
}
Value * * listElems()
{
- return type == tList1 || type == tList2 ? smallList : bigList.elems;
+ return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems;
}
const Value * const * listElems() const
{
- return type == tList1 || type == tList2 ? smallList : bigList.elems;
+ return internalType == tList1 || internalType == tList2 ? smallList : bigList.elems;
}
size_t listSize() const
{
- return type == tList1 ? 1 : type == tList2 ? 2 : bigList.size;
+ return internalType == tList1 ? 1 : internalType == tList2 ? 2 : bigList.size;
}
/* Check whether forcing this value requires a trivial amount of
@@ -176,86 +350,42 @@ struct Value
};
-/* After overwriting an app node, be sure to clear pointers in the
- Value to ensure that the target isn't kept alive unnecessarily. */
-static inline void clearValue(Value & v)
-{
- v.app.left = v.app.right = 0;
-}
-
+// TODO: Remove these static functions, replace call sites with v.mk* instead
static inline void mkInt(Value & v, NixInt n)
{
- clearValue(v);
- v.type = tInt;
- v.integer = n;
+ v.mkInt(n);
}
-
static inline void mkFloat(Value & v, NixFloat n)
{
- clearValue(v);
- v.type = tFloat;
- v.fpoint = n;
+ v.mkFloat(n);
}
-
static inline void mkBool(Value & v, bool b)
{
- clearValue(v);
- v.type = tBool;
- v.boolean = b;
+ v.mkBool(b);
}
-
static inline void mkNull(Value & v)
{
- clearValue(v);
- v.type = tNull;
+ v.mkNull();
}
-
static inline void mkApp(Value & v, Value & left, Value & right)
{
- v.type = tApp;
- v.app.left = &left;
- v.app.right = &right;
-}
-
-
-static inline void mkPrimOpApp(Value & v, Value & left, Value & right)
-{
- v.type = tPrimOpApp;
- v.app.left = &left;
- v.app.right = &right;
-}
-
-
-static inline void mkStringNoCopy(Value & v, const char * s)
-{
- v.type = tString;
- v.string.s = s;
- v.string.context = 0;
+ v.mkApp(&left, &right);
}
-
static inline void mkString(Value & v, const Symbol & s)
{
- mkStringNoCopy(v, ((const string &) s).c_str());
+ v.mkString(((const string &) s).c_str());
}
void mkString(Value & v, const char * s);
-static inline void mkPathNoCopy(Value & v, const char * s)
-{
- clearValue(v);
- v.type = tPath;
- v.path = s;
-}
-
-
void mkPath(Value & v, const char * s);
diff --git a/src/libfetchers/attrs.cc b/src/libfetchers/attrs.cc
index 1e59faa73..a565d19d4 100644
--- a/src/libfetchers/attrs.cc
+++ b/src/libfetchers/attrs.cc
@@ -11,11 +11,11 @@ Attrs jsonToAttrs(const nlohmann::json & json)
for (auto & i : json.items()) {
if (i.value().is_number())
- attrs.emplace(i.key(), i.value().get<int64_t>());
+ attrs.emplace(i.key(), i.value().get<uint64_t>());
else if (i.value().is_string())
attrs.emplace(i.key(), i.value().get<std::string>());
else if (i.value().is_boolean())
- attrs.emplace(i.key(), i.value().get<bool>());
+ attrs.emplace(i.key(), Explicit<bool> { i.value().get<bool>() });
else
throw Error("unsupported input attribute type in lock file");
}
@@ -23,7 +23,7 @@ Attrs jsonToAttrs(const nlohmann::json & json)
return attrs;
}
-nlohmann::json attrsToJson(const Attrs & attrs)
+nlohmann::json attrsToJSON(const Attrs & attrs)
{
nlohmann::json json;
for (auto & attr : attrs) {
@@ -44,7 +44,7 @@ std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::strin
if (i == attrs.end()) return {};
if (auto v = std::get_if<std::string>(&i->second))
return *v;
- throw Error("input attribute '%s' is not a string %s", name, attrsToJson(attrs).dump());
+ throw Error("input attribute '%s' is not a string %s", name, attrsToJSON(attrs).dump());
}
std::string getStrAttr(const Attrs & attrs, const std::string & name)
diff --git a/src/libfetchers/attrs.hh b/src/libfetchers/attrs.hh
index 4b4630c80..a2d53a7bf 100644
--- a/src/libfetchers/attrs.hh
+++ b/src/libfetchers/attrs.hh
@@ -8,24 +8,12 @@
namespace nix::fetchers {
-/* Wrap bools to prevent string literals (i.e. 'char *') from being
- cast to a bool in Attr. */
-template<typename T>
-struct Explicit {
- T t;
-
- bool operator ==(const Explicit<T> & other) const
- {
- return t == other.t;
- }
-};
-
typedef std::variant<std::string, uint64_t, Explicit<bool>> Attr;
typedef std::map<std::string, Attr> Attrs;
Attrs jsonToAttrs(const nlohmann::json & json);
-nlohmann::json attrsToJson(const Attrs & attrs);
+nlohmann::json attrsToJSON(const Attrs & attrs);
std::optional<std::string> maybeGetStrAttr(const Attrs & attrs, const std::string & name);
diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc
index e1c7f3dee..34ff6f85b 100644
--- a/src/libfetchers/cache.cc
+++ b/src/libfetchers/cache.cc
@@ -55,8 +55,8 @@ struct CacheImpl : Cache
bool immutable) override
{
_state.lock()->add.use()
- (attrsToJson(inAttrs).dump())
- (attrsToJson(infoAttrs).dump())
+ (attrsToJSON(inAttrs).dump())
+ (attrsToJSON(infoAttrs).dump())
(store->printStorePath(storePath))
(immutable)
(time(0)).exec();
@@ -70,7 +70,7 @@ struct CacheImpl : Cache
if (!res->expired)
return std::make_pair(std::move(res->infoAttrs), std::move(res->storePath));
debug("ignoring expired cache entry '%s'",
- attrsToJson(inAttrs).dump());
+ attrsToJSON(inAttrs).dump());
}
return {};
}
@@ -81,15 +81,15 @@ struct CacheImpl : Cache
{
auto state(_state.lock());
- auto inAttrsJson = attrsToJson(inAttrs).dump();
+ auto inAttrsJSON = attrsToJSON(inAttrs).dump();
- auto stmt(state->lookup.use()(inAttrsJson));
+ auto stmt(state->lookup.use()(inAttrsJSON));
if (!stmt.next()) {
- debug("did not find cache entry for '%s'", inAttrsJson);
+ debug("did not find cache entry for '%s'", inAttrsJSON);
return {};
}
- auto infoJson = stmt.getStr(0);
+ auto infoJSON = stmt.getStr(0);
auto storePath = store->parseStorePath(stmt.getStr(1));
auto immutable = stmt.getInt(2) != 0;
auto timestamp = stmt.getInt(3);
@@ -97,16 +97,16 @@ struct CacheImpl : Cache
store->addTempRoot(storePath);
if (!store->isValidPath(storePath)) {
// FIXME: we could try to substitute 'storePath'.
- debug("ignoring disappeared cache entry '%s'", inAttrsJson);
+ debug("ignoring disappeared cache entry '%s'", inAttrsJSON);
return {};
}
debug("using cache entry '%s' -> '%s', '%s'",
- inAttrsJson, infoJson, store->printStorePath(storePath));
+ inAttrsJSON, infoJSON, store->printStorePath(storePath));
return Result {
.expired = !immutable && (settings.tarballTtl.get() == 0 || timestamp + settings.tarballTtl < time(0)),
- .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJson)),
+ .infoAttrs = jsonToAttrs(nlohmann::json::parse(infoJSON)),
.storePath = std::move(storePath)
};
}
diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc
index 49851f7bc..916e0a8e8 100644
--- a/src/libfetchers/fetchers.cc
+++ b/src/libfetchers/fetchers.cc
@@ -65,7 +65,7 @@ Input Input::fromAttrs(Attrs && attrs)
ParsedURL Input::toURL() const
{
if (!scheme)
- throw Error("cannot show unsupported input '%s'", attrsToJson(attrs));
+ throw Error("cannot show unsupported input '%s'", attrsToJSON(attrs));
return scheme->toURL(*this);
}
@@ -110,7 +110,7 @@ bool Input::contains(const Input & other) const
std::pair<Tree, Input> Input::fetch(ref<Store> store) const
{
if (!scheme)
- throw Error("cannot fetch unsupported input '%s'", attrsToJson(toAttrs()));
+ throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs()));
/* The tree may already be in the Nix store, or it could be
substituted (which is often faster than fetching from the
@@ -132,7 +132,14 @@ std::pair<Tree, Input> Input::fetch(ref<Store> store) const
}
}
- auto [tree, input] = scheme->fetch(store, *this);
+ auto [tree, input] = [&]() -> std::pair<Tree, Input> {
+ try {
+ return scheme->fetch(store, *this);
+ } catch (Error & e) {
+ e.addTrace({}, "while fetching the input '%s'", to_string());
+ throw;
+ }
+ }();
if (tree.actualPath == "")
tree.actualPath = store->toRealPath(tree.storePath);
@@ -247,7 +254,7 @@ std::optional<time_t> Input::getLastModified() const
ParsedURL InputScheme::toURL(const Input & input)
{
- throw Error("don't know how to convert input '%s' to a URL", attrsToJson(input.attrs));
+ throw Error("don't know how to convert input '%s' to a URL", attrsToJSON(input.attrs));
}
Input InputScheme::applyOverrides(
diff --git a/src/libfetchers/fetchers.hh b/src/libfetchers/fetchers.hh
index e8ae59143..c6b219c02 100644
--- a/src/libfetchers/fetchers.hh
+++ b/src/libfetchers/fetchers.hh
@@ -21,6 +21,14 @@ struct Tree
struct InputScheme;
+/* The Input object is generated by a specific fetcher, based on the
+ * user-supplied input attribute in the flake.nix file, and contains
+ * the information that the specific fetcher needs to perform the
+ * actual fetch. The Input object is most commonly created via the
+ * "fromURL()" or "fromAttrs()" static functions which are provided
+ * the url or attrset specified in the flake file.
+ */
+
struct Input
{
friend struct InputScheme;
@@ -84,6 +92,16 @@ public:
std::optional<time_t> getLastModified() const;
};
+
+/* The InputScheme represents a type of fetcher. Each fetcher
+ * registers with nix at startup time. When processing an input for a
+ * flake, each scheme is given an opportunity to "recognize" that
+ * input from the url or attributes in the flake file's specification
+ * and return an Input object to represent the input if it is
+ * recognized. The Input object contains the information the fetcher
+ * needs to actually perform the "fetch()" when called.
+ */
+
struct InputScheme
{
virtual ~InputScheme()
@@ -127,7 +145,13 @@ DownloadFileResult downloadFile(
bool immutable,
const Headers & headers = {});
-std::pair<Tree, time_t> downloadTarball(
+struct DownloadTarballMeta
+{
+ time_t lastModified;
+ std::string effectiveUrl;
+};
+
+std::pair<Tree, DownloadTarballMeta> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc
index a6411b02b..b9a240b13 100644
--- a/src/libfetchers/git.cc
+++ b/src/libfetchers/git.cc
@@ -59,12 +59,13 @@ struct GitInputScheme : InputScheme
if (maybeGetStrAttr(attrs, "type") != "git") return {};
for (auto & [name, value] : attrs)
- if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash")
+ if (name != "type" && name != "url" && name != "ref" && name != "rev" && name != "shallow" && name != "submodules" && name != "lastModified" && name != "revCount" && name != "narHash" && name != "allRefs")
throw Error("unsupported Git input attribute '%s'", name);
parseURL(getStrAttr(attrs, "url"));
maybeGetBoolAttr(attrs, "shallow");
maybeGetBoolAttr(attrs, "submodules");
+ maybeGetBoolAttr(attrs, "allRefs");
if (auto ref = maybeGetStrAttr(attrs, "ref")) {
if (std::regex_search(*ref, badGitRefRegex))
@@ -152,12 +153,14 @@ struct GitInputScheme : InputScheme
std::pair<bool, std::string> getActualUrl(const Input & input) const
{
- // Don't clone file:// URIs (but otherwise treat them the
- // same as remote URIs, i.e. don't use the working tree or
- // HEAD).
+ // file:// URIs are normally not cloned (but otherwise treated the
+ // same as remote URIs, i.e. we don't use the working tree or
+ // HEAD). Exception: If _NIX_FORCE_HTTP is set, or the repo is a bare git
+ // repo, treat as a remote URI to force a clone.
static bool forceHttp = getEnv("_NIX_FORCE_HTTP") == "1"; // for testing
auto url = parseURL(getStrAttr(input.attrs, "url"));
- bool isLocal = url.scheme == "file" && !forceHttp;
+ bool isBareRepository = url.scheme == "file" && !pathExists(url.path + "/.git");
+ bool isLocal = url.scheme == "file" && !forceHttp && !isBareRepository;
return {isLocal, isLocal ? url.path : url.base};
}
@@ -169,10 +172,12 @@ struct GitInputScheme : InputScheme
bool shallow = maybeGetBoolAttr(input.attrs, "shallow").value_or(false);
bool submodules = maybeGetBoolAttr(input.attrs, "submodules").value_or(false);
+ bool allRefs = maybeGetBoolAttr(input.attrs, "allRefs").value_or(false);
std::string cacheType = "git";
if (shallow) cacheType += "-shallow";
if (submodules) cacheType += "-submodules";
+ if (allRefs) cacheType += "-all-refs";
auto getImmutableAttrs = [&]()
{
@@ -273,7 +278,7 @@ struct GitInputScheme : InputScheme
haveCommits ? std::stoull(runProgram("git", true, { "-C", actualUrl, "log", "-1", "--format=%ct", "--no-show-signature", "HEAD" })) : 0);
return {
- Tree(store->printStorePath(storePath), std::move(storePath)),
+ Tree(store->toRealPath(storePath), std::move(storePath)),
input
};
}
@@ -338,11 +343,15 @@ struct GitInputScheme : InputScheme
}
}
} else {
- /* If the local ref is older than ‘tarball-ttl’ seconds, do a
- git fetch to update the local ref to the remote ref. */
- struct stat st;
- doFetch = stat(localRefFile.c_str(), &st) != 0 ||
- (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
+ if (allRefs) {
+ doFetch = true;
+ } else {
+ /* If the local ref is older than ‘tarball-ttl’ seconds, do a
+ git fetch to update the local ref to the remote ref. */
+ struct stat st;
+ doFetch = stat(localRefFile.c_str(), &st) != 0 ||
+ (uint64_t) st.st_mtime + settings.tarballTtl <= (uint64_t) now;
+ }
}
if (doFetch) {
@@ -352,9 +361,13 @@ struct GitInputScheme : InputScheme
// we're using --quiet for now. Should process its stderr.
try {
auto ref = input.getRef();
- auto fetchRef = ref->compare(0, 5, "refs/") == 0
- ? *ref
- : "refs/heads/" + *ref;
+ auto fetchRef = allRefs
+ ? "refs/*"
+ : ref->compare(0, 5, "refs/") == 0
+ ? *ref
+ : ref == "HEAD"
+ ? *ref
+ : "refs/heads/" + *ref;
runProgram("git", true, { "-C", repoDir, "fetch", "--quiet", "--force", "--", actualUrl, fmt("%s:%s", fetchRef, fetchRef) });
} catch (Error & e) {
if (!pathExists(localRefFile)) throw;
@@ -392,6 +405,28 @@ struct GitInputScheme : InputScheme
AutoDelete delTmpDir(tmpDir, true);
PathFilter filter = defaultPathFilter;
+ RunOptions checkCommitOpts(
+ "git",
+ { "-C", repoDir, "cat-file", "commit", input.getRev()->gitRev() }
+ );
+ checkCommitOpts.searchPath = true;
+ checkCommitOpts.mergeStderrToStdout = true;
+
+ auto result = runProgram(checkCommitOpts);
+ if (WEXITSTATUS(result.first) == 128
+ && result.second.find("bad file") != std::string::npos
+ ) {
+ throw Error(
+ "Cannot find Git revision '%s' in ref '%s' of repository '%s'! "
+ "Please make sure that the " ANSI_BOLD "rev" ANSI_NORMAL " exists on the "
+ ANSI_BOLD "ref" ANSI_NORMAL " you've specified or add " ANSI_BOLD
+ "allRefs = true;" ANSI_NORMAL " to " ANSI_BOLD "fetchGit" ANSI_NORMAL ".",
+ input.getRev()->gitRev(),
+ *input.getRef(),
+ actualUrl
+ );
+ }
+
if (submodules) {
Path tmpGitDir = createTempDir();
AutoDelete delTmpGitDir(tmpGitDir, true);
diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc
index 92ff224f7..3e5ad75a8 100644
--- a/src/libfetchers/github.cc
+++ b/src/libfetchers/github.cc
@@ -37,15 +37,29 @@ struct GitArchiveInputScheme : InputScheme
std::optional<std::string> ref;
std::optional<std::string> host_url;
- if (path.size() == 2) {
- } else if (path.size() == 3) {
+ auto size = path.size();
+ if (size == 3) {
if (std::regex_match(path[2], revRegex))
rev = Hash::parseAny(path[2], htSHA1);
else if (std::regex_match(path[2], refRegex))
ref = path[2];
else
throw BadURL("in URL '%s', '%s' is not a commit hash or branch/tag name", url.url, path[2]);
- } else
+ } else if (size > 3) {
+ std::string rs;
+ for (auto i = std::next(path.begin(), 2); i != path.end(); i++) {
+ rs += *i;
+ if (std::next(i) != path.end()) {
+ rs += "/";
+ }
+ }
+
+ if (std::regex_match(rs, refRegex)) {
+ ref = rs;
+ } else {
+ throw BadURL("in URL '%s', '%s' is not a branch/tag name", url.url, rs);
+ }
+ } else if (size < 2)
throw BadURL("URL '%s' is invalid", url.url);
for (auto &[name, value] : url.query) {
@@ -193,16 +207,16 @@ struct GitArchiveInputScheme : InputScheme
auto url = getDownloadUrl(input);
- auto [tree, lastModified] = downloadTarball(store, url.url, "source", true, url.headers);
+ auto [tree, meta] = downloadTarball(store, url.url, "source", true, url.headers);
- input.attrs.insert_or_assign("lastModified", lastModified);
+ input.attrs.insert_or_assign("lastModified", uint64_t(meta.lastModified));
getCache()->add(
store,
immutableAttrs,
{
{"rev", rev->gitRev()},
- {"lastModified", lastModified}
+ {"lastModified", uint64_t(meta.lastModified)}
},
tree.storePath,
true);
diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc
index 7d3d52751..0eb401e10 100644
--- a/src/libfetchers/mercurial.cc
+++ b/src/libfetchers/mercurial.cc
@@ -11,6 +11,36 @@ using namespace std::string_literals;
namespace nix::fetchers {
+namespace {
+
+RunOptions hgOptions(const Strings & args) {
+ RunOptions opts("hg", args);
+ opts.searchPath = true;
+
+ auto env = getEnv();
+ // Set HGPLAIN: this means we get consistent output from hg and avoids leakage from a user or system .hgrc.
+ env["HGPLAIN"] = "";
+ opts.environment = env;
+
+ return opts;
+}
+
+// runProgram wrapper that uses hgOptions instead of stock RunOptions.
+string runHg(const Strings & args, const std::optional<std::string> & input = {})
+{
+ RunOptions opts = hgOptions(args);
+ opts.input = input;
+
+ auto res = runProgram(opts);
+
+ if (!statusOk(res.first))
+ throw ExecError(res.first, fmt("hg %1%", statusToString(res.first)));
+
+ return res.second;
+}
+
+}
+
struct MercurialInputScheme : InputScheme
{
std::optional<Input> inputFromURL(const ParsedURL & url) override
@@ -100,11 +130,11 @@ struct MercurialInputScheme : InputScheme
assert(sourcePath);
// FIXME: shut up if file is already tracked.
- runProgram("hg", true,
+ runHg(
{ "add", *sourcePath + "/" + std::string(file) });
if (commitMsg)
- runProgram("hg", true,
+ runHg(
{ "commit", *sourcePath + "/" + std::string(file), "-m", *commitMsg });
}
@@ -130,7 +160,7 @@ struct MercurialInputScheme : InputScheme
if (!input.getRef() && !input.getRev() && isLocal && pathExists(actualUrl + "/.hg")) {
- bool clean = runProgram("hg", true, { "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
+ bool clean = runHg({ "status", "-R", actualUrl, "--modified", "--added", "--removed" }) == "";
if (!clean) {
@@ -143,10 +173,10 @@ struct MercurialInputScheme : InputScheme
if (settings.warnDirty)
warn("Mercurial tree '%s' is unclean", actualUrl);
- input.attrs.insert_or_assign("ref", chomp(runProgram("hg", true, { "branch", "-R", actualUrl })));
+ input.attrs.insert_or_assign("ref", chomp(runHg({ "branch", "-R", actualUrl })));
auto files = tokenizeString<std::set<std::string>>(
- runProgram("hg", true, { "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
+ runHg({ "status", "-R", actualUrl, "--clean", "--modified", "--added", "--no-status", "--print0" }), "\0"s);
PathFilter filter = [&](const Path & p) -> bool {
assert(hasPrefix(p, actualUrl));
@@ -166,7 +196,7 @@ struct MercurialInputScheme : InputScheme
auto storePath = store->addToStore("source", actualUrl, FileIngestionMethod::Recursive, htSHA256, filter);
return {
- Tree(store->printStorePath(storePath), std::move(storePath)),
+ Tree(store->toRealPath(storePath), std::move(storePath)),
input
};
}
@@ -224,33 +254,33 @@ struct MercurialInputScheme : InputScheme
if (!(input.getRev()
&& pathExists(cacheDir)
&& runProgram(
- RunOptions("hg", { "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
+ hgOptions({ "log", "-R", cacheDir, "-r", input.getRev()->gitRev(), "--template", "1" })
.killStderr(true)).second == "1"))
{
Activity act(*logger, lvlTalkative, actUnknown, fmt("fetching Mercurial repository '%s'", actualUrl));
if (pathExists(cacheDir)) {
try {
- runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
+ runHg({ "pull", "-R", cacheDir, "--", actualUrl });
}
catch (ExecError & e) {
string transJournal = cacheDir + "/.hg/store/journal";
/* hg throws "abandoned transaction" error only if this file exists */
if (pathExists(transJournal)) {
- runProgram("hg", true, { "recover", "-R", cacheDir });
- runProgram("hg", true, { "pull", "-R", cacheDir, "--", actualUrl });
+ runHg({ "recover", "-R", cacheDir });
+ runHg({ "pull", "-R", cacheDir, "--", actualUrl });
} else {
throw ExecError(e.status, fmt("'hg pull' %s", statusToString(e.status)));
}
}
} else {
createDirs(dirOf(cacheDir));
- runProgram("hg", true, { "clone", "--noupdate", "--", actualUrl, cacheDir });
+ runHg({ "clone", "--noupdate", "--", actualUrl, cacheDir });
}
}
auto tokens = tokenizeString<std::vector<std::string>>(
- runProgram("hg", true, { "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
+ runHg({ "log", "-R", cacheDir, "-r", revOrRef, "--template", "{node} {rev} {branch}" }));
assert(tokens.size() == 3);
input.attrs.insert_or_assign("rev", Hash::parseAny(tokens[0], htSHA1).gitRev());
@@ -263,7 +293,7 @@ struct MercurialInputScheme : InputScheme
Path tmpDir = createTempDir();
AutoDelete delTmpDir(tmpDir, true);
- runProgram("hg", true, { "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir });
+ runHg({ "archive", "-R", cacheDir, "-r", input.getRev()->gitRev(), tmpDir });
deletePath(tmpDir + "/.hg_archival.txt");
@@ -271,7 +301,7 @@ struct MercurialInputScheme : InputScheme
Attrs infoAttrs({
{"rev", input.getRev()->gitRev()},
- {"revCount", (int64_t) revCount},
+ {"revCount", (uint64_t) revCount},
});
if (!_input.getRev())
diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc
index bcb904c0d..d1003de57 100644
--- a/src/libfetchers/path.cc
+++ b/src/libfetchers/path.cc
@@ -20,10 +20,10 @@ struct PathInputScheme : InputScheme
if (name == "rev" || name == "narHash")
input.attrs.insert_or_assign(name, value);
else if (name == "revCount" || name == "lastModified") {
- uint64_t n;
- if (!string2Int(value, n))
+ if (auto n = string2Int<uint64_t>(value))
+ input.attrs.insert_or_assign(name, *n);
+ else
throw Error("path URL '%s' has invalid parameter '%s'", url.to_string(), name);
- input.attrs.insert_or_assign(name, n);
}
else
throw Error("path URL '%s' has unsupported parameter '%s'", url.to_string(), name);
diff --git a/src/libfetchers/registry.cc b/src/libfetchers/registry.cc
index 2426882ca..74376adc0 100644
--- a/src/libfetchers/registry.cc
+++ b/src/libfetchers/registry.cc
@@ -60,10 +60,10 @@ void Registry::write(const Path & path)
nlohmann::json arr;
for (auto & entry : entries) {
nlohmann::json obj;
- obj["from"] = attrsToJson(entry.from.toAttrs());
- obj["to"] = attrsToJson(entry.to.toAttrs());
+ obj["from"] = attrsToJSON(entry.from.toAttrs());
+ obj["to"] = attrsToJSON(entry.to.toAttrs());
if (!entry.extraAttrs.empty())
- obj["to"].update(attrsToJson(entry.extraAttrs));
+ obj["to"].update(attrsToJSON(entry.extraAttrs));
if (entry.exact)
obj["exact"] = true;
arr.emplace_back(std::move(obj));
@@ -114,7 +114,7 @@ static std::shared_ptr<Registry> getSystemRegistry()
Path getUserRegistryPath()
{
- return getHome() + "/.config/nix/registry.json";
+ return getConfigDir() + "/nix/registry.json";
}
std::shared_ptr<Registry> getUserRegistry()
diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc
index 8c0f20475..bd05bb2f1 100644
--- a/src/libfetchers/tarball.cc
+++ b/src/libfetchers/tarball.cc
@@ -64,7 +64,6 @@ DownloadFileResult downloadFile(
if (res.cached) {
assert(cached);
- assert(request.expectedETag == res.etag);
storePath = std::move(cached->storePath);
} else {
StringSink sink;
@@ -110,7 +109,7 @@ DownloadFileResult downloadFile(
};
}
-std::pair<Tree, time_t> downloadTarball(
+std::pair<Tree, DownloadTarballMeta> downloadTarball(
ref<Store> store,
const std::string & url,
const std::string & name,
@@ -128,7 +127,10 @@ std::pair<Tree, time_t> downloadTarball(
if (cached && !cached->expired)
return {
Tree(store->toRealPath(cached->storePath), std::move(cached->storePath)),
- getIntAttr(cached->infoAttrs, "lastModified")
+ {
+ .lastModified = time_t(getIntAttr(cached->infoAttrs, "lastModified")),
+ .effectiveUrl = maybeGetStrAttr(cached->infoAttrs, "effectiveUrl").value_or(url),
+ },
};
auto res = downloadFile(store, url, name, immutable, headers);
@@ -152,7 +154,8 @@ std::pair<Tree, time_t> downloadTarball(
}
Attrs infoAttrs({
- {"lastModified", lastModified},
+ {"lastModified", uint64_t(lastModified)},
+ {"effectiveUrl", res.effectiveUrl},
{"etag", res.etag},
});
@@ -165,7 +168,10 @@ std::pair<Tree, time_t> downloadTarball(
return {
Tree(store->toRealPath(*unpackedStorePath), std::move(*unpackedStorePath)),
- lastModified,
+ {
+ .lastModified = lastModified,
+ .effectiveUrl = res.effectiveUrl,
+ },
};
}
@@ -224,9 +230,11 @@ struct TarballInputScheme : InputScheme
return true;
}
- std::pair<Tree, Input> fetch(ref<Store> store, const Input & input) override
+ std::pair<Tree, Input> fetch(ref<Store> store, const Input & _input) override
{
- auto tree = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false).first;
+ Input input(_input);
+ auto [tree, meta] = downloadTarball(store, getStrAttr(input.attrs, "url"), "source", false);
+ input.attrs.insert_or_assign("url", meta.effectiveUrl);
return {std::move(tree), input};
}
};
diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index 9151a0344..c43e9ebd2 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -10,25 +10,28 @@ MixCommonArgs::MixCommonArgs(const string & programName)
addFlag({
.longName = "verbose",
.shortName = 'v',
- .description = "increase verbosity level",
+ .description = "Increase the logging verbosity level.",
+ .category = loggingCategory,
.handler = {[]() { verbosity = (Verbosity) (verbosity + 1); }},
});
addFlag({
.longName = "quiet",
- .description = "decrease verbosity level",
+ .description = "Decrease the logging verbosity level.",
+ .category = loggingCategory,
.handler = {[]() { verbosity = verbosity > lvlError ? (Verbosity) (verbosity - 1) : lvlError; }},
});
addFlag({
.longName = "debug",
- .description = "enable debug output",
+ .description = "Set the logging verbosity level to 'debug'.",
+ .category = loggingCategory,
.handler = {[]() { verbosity = lvlDebug; }},
});
addFlag({
.longName = "option",
- .description = "set a Nix configuration option (overriding `nix.conf`)",
+ .description = "Set the Nix configuration setting *name* to *value* (overriding `nix.conf`).",
.labels = {"name", "value"},
.handler = {[](std::string name, std::string value) {
try {
@@ -44,15 +47,15 @@ MixCommonArgs::MixCommonArgs(const string & programName)
globalConfig.getSettings(settings);
for (auto & s : settings)
if (hasPrefix(s.first, prefix))
- completions->add(s.first, s.second.description);
+ completions->add(s.first, fmt("Set the `%s` setting.", s.first));
}
}
});
addFlag({
.longName = "log-format",
- .description = "format of log output; `raw`, `internal-json`, `bar` "
- "or `bar-with-logs`",
+ .description = "Set the format of log output; one of `raw`, `internal-json`, `bar` or `bar-with-logs`.",
+ .category = loggingCategory,
.labels = {"format"},
.handler = {[](std::string format) { setLogFormat(format); }},
});
@@ -60,14 +63,14 @@ MixCommonArgs::MixCommonArgs(const string & programName)
addFlag({
.longName = "max-jobs",
.shortName = 'j',
- .description = "maximum number of parallel builds",
+ .description = "The maximum number of parallel builds.",
.labels = Strings{"jobs"},
.handler = {[=](std::string s) {
settings.set("max-jobs", s);
}}
});
- std::string cat = "config";
+ std::string cat = "Options to override configuration settings";
globalConfig.convertToArgs(*this, cat);
// Backward compatibility hack: nix-env already had a --system flag.
@@ -76,4 +79,11 @@ MixCommonArgs::MixCommonArgs(const string & programName)
hiddenCategories.insert(cat);
}
+void MixCommonArgs::initialFlagsProcessed()
+{
+ initPlugins();
+ pluginsInited();
+}
+
+
}
diff --git a/src/libmain/common-args.hh b/src/libmain/common-args.hh
index a4de3dccf..31bdf527a 100644
--- a/src/libmain/common-args.hh
+++ b/src/libmain/common-args.hh
@@ -4,10 +4,17 @@
namespace nix {
-struct MixCommonArgs : virtual Args
+//static constexpr auto commonArgsCategory = "Miscellaneous common options";
+static constexpr auto loggingCategory = "Logging-related options";
+
+class MixCommonArgs : public virtual Args
{
+ void initialFlagsProcessed() override;
+public:
string programName;
MixCommonArgs(const string & programName);
+protected:
+ virtual void pluginsInited() {}
};
struct MixDryRun : virtual Args
@@ -16,7 +23,12 @@ struct MixDryRun : virtual Args
MixDryRun()
{
- mkFlag(0, "dry-run", "show what this command would do without doing it", &dryRun);
+ addFlag({
+ .longName = "dry-run",
+ .description = "Show what this command would do without doing it.",
+ //.category = commonArgsCategory,
+ .handler = {&dryRun, true},
+ });
}
};
@@ -26,7 +38,12 @@ struct MixJSON : virtual Args
MixJSON()
{
- mkFlag(0, "json", "produce JSON output", &json);
+ addFlag({
+ .longName = "json",
+ .description = "Produce output in JSON format, suitable for consumption by another program.",
+ //.category = commonArgsCategory,
+ .handler = {&json, true},
+ });
}
};
diff --git a/src/libmain/loggers.cc b/src/libmain/loggers.cc
index 0a7291780..cdf23859b 100644
--- a/src/libmain/loggers.cc
+++ b/src/libmain/loggers.cc
@@ -12,7 +12,7 @@ LogFormat parseLogFormat(const std::string & logFormatStr) {
else if (logFormatStr == "raw-with-logs")
return LogFormat::rawWithLogs;
else if (logFormatStr == "internal-json")
- return LogFormat::internalJson;
+ return LogFormat::internalJSON;
else if (logFormatStr == "bar")
return LogFormat::bar;
else if (logFormatStr == "bar-with-logs")
@@ -26,7 +26,7 @@ Logger * makeDefaultLogger() {
return makeSimpleLogger(false);
case LogFormat::rawWithLogs:
return makeSimpleLogger(true);
- case LogFormat::internalJson:
+ case LogFormat::internalJSON:
return makeJSONLogger(*makeSimpleLogger(true));
case LogFormat::bar:
return makeProgressBar();
diff --git a/src/libmain/loggers.hh b/src/libmain/loggers.hh
index cada03110..f3c759193 100644
--- a/src/libmain/loggers.hh
+++ b/src/libmain/loggers.hh
@@ -7,7 +7,7 @@ namespace nix {
enum class LogFormat {
raw,
rawWithLogs,
- internalJson,
+ internalJSON,
bar,
barWithLogs,
};
diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc
index 07b45b3b5..15354549a 100644
--- a/src/libmain/progress-bar.cc
+++ b/src/libmain/progress-bar.cc
@@ -122,6 +122,7 @@ public:
void log(Verbosity lvl, const FormatOrString & fs) override
{
+ if (lvl > verbosity) return;
auto state(state_.lock());
log(*state, lvl, fs.s);
}
@@ -466,6 +467,17 @@ public:
Logger::writeToStdout(s);
}
}
+
+ std::optional<char> ask(std::string_view msg) override
+ {
+ auto state(state_.lock());
+ if (!state->active || !isatty(STDIN_FILENO)) return {};
+ std::cerr << fmt("\r\e[K%s ", msg);
+ auto s = trim(readLine(STDIN_FILENO));
+ if (s.size() != 1) return {};
+ draw(*state);
+ return s[0];
+ }
};
Logger * makeProgressBar(bool printBuildLogs)
diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc
index 2247aeca4..86930c2e3 100644
--- a/src/libmain/shared.cc
+++ b/src/libmain/shared.cc
@@ -18,6 +18,8 @@
#include <openssl/crypto.h>
+#include <sodium.h>
+
namespace nix {
@@ -34,7 +36,7 @@ void printGCWarning()
}
-void printMissing(ref<Store> store, const std::vector<StorePathWithOutputs> & paths, Verbosity lvl)
+void printMissing(ref<Store> store, const std::vector<DerivedPath> & paths, Verbosity lvl)
{
uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
@@ -126,6 +128,9 @@ void initNix()
CRYPTO_set_locking_callback(opensslLockCallback);
#endif
+ if (sodium_init() == -1)
+ throw Error("could not initialise libsodium");
+
loadConfFile();
startSignalHandlerThread();
@@ -181,50 +186,64 @@ LegacyArgs::LegacyArgs(const std::string & programName,
addFlag({
.longName = "no-build-output",
.shortName = 'Q',
- .description = "do not show build output",
+ .description = "Do not show build output.",
.handler = {[&]() {setLogFormat(LogFormat::raw); }},
});
addFlag({
.longName = "keep-failed",
.shortName ='K',
- .description = "keep temporary directories of failed builds",
+ .description = "Keep temporary directories of failed builds.",
.handler = {&(bool&) settings.keepFailed, true},
});
addFlag({
.longName = "keep-going",
.shortName ='k',
- .description = "keep going after a build fails",
+ .description = "Keep going after a build fails.",
.handler = {&(bool&) settings.keepGoing, true},
});
addFlag({
.longName = "fallback",
- .description = "build from source if substitution fails",
+ .description = "Build from source if substitution fails.",
.handler = {&(bool&) settings.tryFallback, true},
});
auto intSettingAlias = [&](char shortName, const std::string & longName,
- const std::string & description, const std::string & dest) {
- mkFlag<unsigned int>(shortName, longName, description, [=](unsigned int n) {
- settings.set(dest, std::to_string(n));
+ const std::string & description, const std::string & dest)
+ {
+ addFlag({
+ .longName = longName,
+ .shortName = shortName,
+ .description = description,
+ .labels = {"n"},
+ .handler = {[=](std::string s) {
+ auto n = string2IntWithUnitPrefix<uint64_t>(s);
+ settings.set(dest, std::to_string(n));
+ }}
});
};
- intSettingAlias(0, "cores", "maximum number of CPU cores to use inside a build", "cores");
- intSettingAlias(0, "max-silent-time", "number of seconds of silence before a build is killed", "max-silent-time");
- intSettingAlias(0, "timeout", "number of seconds before a build is killed", "timeout");
+ intSettingAlias(0, "cores", "Maximum number of CPU cores to use inside a build.", "cores");
+ intSettingAlias(0, "max-silent-time", "Number of seconds of silence before a build is killed.", "max-silent-time");
+ intSettingAlias(0, "timeout", "Number of seconds before a build is killed.", "timeout");
- mkFlag(0, "readonly-mode", "do not write to the Nix store",
- &settings.readOnlyMode);
+ addFlag({
+ .longName = "readonly-mode",
+ .description = "Do not write to the Nix store.",
+ .handler = {&settings.readOnlyMode, true},
+ });
- mkFlag(0, "no-gc-warning", "disable warning about not using '--add-root'",
- &gcWarning, false);
+ addFlag({
+ .longName = "no-gc-warning",
+ .description = "Disable warnings about not using `--add-root`.",
+ .handler = {&gcWarning, true},
+ });
addFlag({
.longName = "store",
- .description = "URI of the Nix store to use",
+ .description = "The URL of the Nix store to use.",
.labels = {"store-uri"},
.handler = {&(std::string&) settings.storeUri},
});
@@ -274,9 +293,7 @@ void printVersion(const string & programName)
#if HAVE_BOEHMGC
cfg.push_back("gc");
#endif
-#if HAVE_SODIUM
cfg.push_back("signed-caches");
-#endif
std::cout << "System type: " << settings.thisSystem << "\n";
std::cout << "Additional system types: " << concatStringsSep(", ", settings.extraPlatforms.get()) << "\n";
std::cout << "Features: " << concatStringsSep(", ", cfg) << "\n";
@@ -293,7 +310,7 @@ void printVersion(const string & programName)
void showManPage(const string & name)
{
- restoreSignals();
+ restoreProcessContext();
setenv("MANPATH", settings.nixManDir.c_str(), 1);
execlp("man", "man", name.c_str(), nullptr);
throw SysError("command 'man %1%' failed", name.c_str());
@@ -356,7 +373,7 @@ RunPager::RunPager()
throw SysError("dupping stdin");
if (!getenv("LESS"))
setenv("LESS", "FRSXMK", 1);
- restoreSignals();
+ restoreProcessContext();
if (pager)
execl("/bin/sh", "sh", "-c", pager, nullptr);
execlp("pager", "pager", nullptr);
diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh
index ffae5d796..05277d90a 100644
--- a/src/libmain/shared.hh
+++ b/src/libmain/shared.hh
@@ -4,6 +4,7 @@
#include "args.hh"
#include "common-args.hh"
#include "path.hh"
+#include "derived-path.hh"
#include <signal.h>
@@ -42,7 +43,7 @@ struct StorePathWithOutputs;
void printMissing(
ref<Store> store,
- const std::vector<StorePathWithOutputs> & paths,
+ const std::vector<DerivedPath> & paths,
Verbosity lvl = lvlInfo);
void printMissing(ref<Store> store, const StorePathSet & willBuild,
@@ -57,23 +58,7 @@ template<class N> N getIntArg(const string & opt,
{
++i;
if (i == end) throw UsageError("'%1%' requires an argument", opt);
- string s = *i;
- N multiplier = 1;
- if (allowUnit && !s.empty()) {
- char u = std::toupper(*s.rbegin());
- if (std::isalpha(u)) {
- if (u == 'K') multiplier = 1ULL << 10;
- else if (u == 'M') multiplier = 1ULL << 20;
- else if (u == 'G') multiplier = 1ULL << 30;
- else if (u == 'T') multiplier = 1ULL << 40;
- else throw UsageError("invalid unit specifier '%1%'", u);
- s.resize(s.size() - 1);
- }
- }
- N n;
- if (!string2Int(s, n))
- throw UsageError("'%1%' requires an integer argument", opt);
- return n * multiplier;
+ return string2IntWithUnitPrefix<N>(*i);
}
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index f6224d6a0..4f5f8607d 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -86,8 +86,7 @@ void BinaryCacheStore::getFile(const std::string & path, Sink & sink)
promise.set_exception(std::current_exception());
}
}});
- auto data = promise.get_future().get();
- sink((unsigned char *) data->data(), data->size());
+ sink(*promise.get_future().get());
}
std::shared_ptr<std::string> BinaryCacheStore::getFile(const std::string & path)
@@ -434,7 +433,9 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
if (!repair && isValidPath(path))
return path;
- auto source = StringSource { s };
+ StringSink sink;
+ dumpString(s, sink);
+ auto source = StringSource { *sink.s };
return addToStoreCommon(source, repair, CheckSigs, [&](HashResult nar) {
ValidPathInfo info { path, nar.first };
info.narSize = nar.second;
@@ -444,6 +445,24 @@ StorePath BinaryCacheStore::addTextToStore(const string & name, const string & s
})->path;
}
+std::optional<const Realisation> BinaryCacheStore::queryRealisation(const DrvOutput & id)
+{
+ auto outputInfoFilePath = realisationsPrefix + "/" + id.to_string() + ".doi";
+ auto rawOutputInfo = getFile(outputInfoFilePath);
+
+ if (rawOutputInfo) {
+ return {Realisation::fromJSON(
+ nlohmann::json::parse(*rawOutputInfo), outputInfoFilePath)};
+ } else {
+ return std::nullopt;
+ }
+}
+
+void BinaryCacheStore::registerDrvOutput(const Realisation& info) {
+ auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi";
+ upsertFile(filePath, info.toJSON().dump(), "application/json");
+}
+
ref<FSAccessor> BinaryCacheStore::getFSAccessor()
{
return make_ref<RemoteFSAccessor>(ref<Store>(shared_from_this()), localNarCache);
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 5224d7ec8..c2163166c 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -24,7 +24,7 @@ struct BinaryCacheStoreConfig : virtual StoreConfig
"enable multi-threading compression, available for xz only currently"};
};
-class BinaryCacheStore : public Store, public virtual BinaryCacheStoreConfig
+class BinaryCacheStore : public virtual BinaryCacheStoreConfig, public virtual Store
{
private:
@@ -33,6 +33,9 @@ private:
protected:
+ // The prefix under which realisation infos will be stored
+ const std::string realisationsPrefix = "/realisations";
+
BinaryCacheStore(const Params & params);
public:
@@ -99,14 +102,11 @@ public:
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
- void narFromPath(const StorePath & path, Sink & sink) override;
+ void registerDrvOutput(const Realisation & info) override;
- BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
- BuildMode buildMode) override
- { unsupported("buildDerivation"); }
+ std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
- void ensurePath(const StorePath & path) override
- { unsupported("ensurePath"); }
+ void narFromPath(const StorePath & path, Sink & sink) override;
ref<FSAccessor> getFSAccessor() override;
diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc
index 5600d7e03..3ce538f77 100644
--- a/src/libstore/build/derivation-goal.cc
+++ b/src/libstore/build/derivation-goal.cc
@@ -9,10 +9,10 @@
#include "archive.hh"
#include "json.hh"
#include "compression.hh"
-#include "daemon.hh"
#include "worker-protocol.hh"
#include "topo-sort.hh"
#include "callback.hh"
+#include "local-store.hh" // TODO remove, along with remaining downcasts
#include <regex>
#include <queue>
@@ -50,6 +50,11 @@
#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
#endif
+#if __APPLE__
+#include <spawn.h>
+#include <sys/sysctl.h>
+#endif
+
#include <pwd.h>
#include <grp.h>
@@ -57,40 +62,6 @@
namespace nix {
-void handleDiffHook(
- uid_t uid, uid_t gid,
- const Path & tryA, const Path & tryB,
- const Path & drvPath, const Path & tmpDir)
-{
- auto diffHook = settings.diffHook;
- if (diffHook != "" && settings.runDiffHook) {
- try {
- RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
- diffHookOptions.searchPath = true;
- diffHookOptions.uid = uid;
- diffHookOptions.gid = gid;
- diffHookOptions.chdir = "/";
-
- auto diffRes = runProgram(diffHookOptions);
- if (!statusOk(diffRes.first))
- throw ExecError(diffRes.first,
- "diff-hook program '%1%' %2%",
- diffHook,
- statusToString(diffRes.first));
-
- if (diffRes.second != "")
- printError(chomp(diffRes.second));
- } catch (Error & error) {
- ErrorInfo ei = error.info();
- ei.hint = hintfmt("diff hook execution failed: %s",
- (error.info().hint.has_value() ? error.info().hint->str() : ""));
- logError(ei);
- }
- }
-}
-
-const Path DerivationGoal::homeDir = "/homeless-shelter";
-
DerivationGoal::DerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, Worker & worker, BuildMode buildMode)
: Goal(worker)
@@ -102,7 +73,7 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath,
state = &DerivationGoal::getDerivation;
name = fmt(
"building of '%s' from .drv file",
- StorePathWithOutputs { drvPath, wantedOutputs }.to_string(worker.store));
+ DerivedPath::Built { drvPath, wantedOutputs }.to_string(worker.store));
trace("created");
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@@ -118,11 +89,12 @@ DerivationGoal::DerivationGoal(const StorePath & drvPath, const BasicDerivation
, wantedOutputs(wantedOutputs)
, buildMode(buildMode)
{
- this->drv = std::make_unique<BasicDerivation>(BasicDerivation(drv));
+ this->drv = std::make_unique<Derivation>(drv);
+
state = &DerivationGoal::haveDerivation;
name = fmt(
"building of '%s' from in-memory derivation",
- StorePathWithOutputs { drvPath, drv.outputNames() }.to_string(worker.store));
+ DerivedPath::Built { drvPath, drv.outputNames() }.to_string(worker.store));
trace("created");
mcExpectedBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.expectedBuilds);
@@ -138,9 +110,6 @@ DerivationGoal::~DerivationGoal()
{
/* Careful: we should never ever throw an exception from a
destructor. */
- try { killChild(); } catch (...) { ignoreException(); }
- try { stopDaemon(); } catch (...) { ignoreException(); }
- try { deleteTmpDir(false); } catch (...) { ignoreException(); }
try { closeLogFile(); } catch (...) { ignoreException(); }
}
@@ -155,38 +124,8 @@ string DerivationGoal::key()
}
-inline bool DerivationGoal::needsHashRewrite()
-{
-#if __linux__
- return !useChroot;
-#else
- /* Darwin requires hash rewriting even when sandboxing is enabled. */
- return true;
-#endif
-}
-
-
void DerivationGoal::killChild()
{
- if (pid != -1) {
- worker.childTerminated(this);
-
- if (buildUser) {
- /* If we're using a build user, then there is a tricky
- race condition: if we kill the build user before the
- child has done its setuid() to the build user uid, then
- it won't be killed, and we'll potentially lock up in
- pid.wait(). So also send a conventional kill to the
- child. */
- ::kill(-pid, SIGKILL); /* ignore the result */
- buildUser->kill();
- pid.wait();
- } else
- pid.kill();
-
- assert(pid == -1);
- }
-
hook.reset();
}
@@ -231,7 +170,7 @@ void DerivationGoal::getDerivation()
return;
}
- addWaitee(worker.makeSubstitutionGoal(drvPath));
+ addWaitee(upcast_goal(worker.makePathSubstitutionGoal(drvPath)));
state = &DerivationGoal::loadDerivation;
}
@@ -254,7 +193,7 @@ void DerivationGoal::loadDerivation()
assert(worker.store.isValidPath(drvPath));
/* Get the derivation. */
- drv = std::unique_ptr<BasicDerivation>(new Derivation(worker.store.derivationFromPath(drvPath)));
+ drv = std::make_unique<Derivation>(worker.store.derivationFromPath(drvPath));
haveDerivation();
}
@@ -273,6 +212,16 @@ void DerivationGoal::haveDerivation()
if (i.second.second)
worker.store.addTempRoot(*i.second.second);
+ auto outputHashes = staticOutputHashes(worker.store, *drv);
+ for (auto &[outputName, outputHash] : outputHashes)
+ initialOutputs.insert({
+ outputName,
+ InitialOutput{
+ .wanted = true, // Will be refined later
+ .outputHash = outputHash
+ }
+ });
+
/* Check what outputs paths are not already valid. */
checkPathValidity();
bool allValid = true;
@@ -297,17 +246,22 @@ void DerivationGoal::haveDerivation()
through substitutes. If that doesn't work, we'll build
them. */
if (settings.useSubstitutes && parsedDrv->substitutesAllowed())
- for (auto & [_, status] : initialOutputs) {
+ for (auto & [outputName, status] : initialOutputs) {
if (!status.wanted) continue;
- if (!status.known) {
- warn("do not know how to query for unknown floating content-addressed derivation output yet");
- /* Nothing to wait for; tail call */
- return DerivationGoal::gaveUpOnSubstitution();
- }
- addWaitee(worker.makeSubstitutionGoal(
- status.known->path,
- buildMode == bmRepair ? Repair : NoRepair,
- getDerivationCA(*drv)));
+ if (!status.known)
+ addWaitee(
+ upcast_goal(
+ worker.makeDrvOutputSubstitutionGoal(
+ DrvOutput{status.outputHash, outputName},
+ buildMode == bmRepair ? Repair : NoRepair
+ )
+ )
+ );
+ else
+ addWaitee(upcast_goal(worker.makePathSubstitutionGoal(
+ status.known->path,
+ buildMode == bmRepair ? Repair : NoRepair,
+ getDerivationCA(*drv))));
}
if (waitees.empty()) /* to prevent hang (no wake-up event) */
@@ -330,8 +284,13 @@ void DerivationGoal::outputsSubstitutionTried()
/* If the substitutes form an incomplete closure, then we should
build the dependencies of this derivation, but after that, we
- can still use the substitutes for this derivation itself. */
- if (nrIncompleteClosure > 0) retrySubstitution = true;
+ can still use the substitutes for this derivation itself.
+
+ If the nrIncompleteClosure != nrFailed, we have another issue as well.
+ In particular, it may be the case that the hole in the closure is
+ an output of the current derivation, which causes a loop if retried.
+ */
+ if (nrIncompleteClosure > 0 && nrIncompleteClosure == nrFailed) retrySubstitution = true;
nrFailed = nrNoSubstituters = nrIncompleteClosure = 0;
@@ -383,7 +342,7 @@ void DerivationGoal::gaveUpOnSubstitution()
if (!settings.useSubstitutes)
throw Error("dependency '%s' of '%s' does not exist, and substitution is disabled",
worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
- addWaitee(worker.makeSubstitutionGoal(i));
+ addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i)));
}
if (waitees.empty()) /* to prevent hang (no wake-up event) */
@@ -429,15 +388,12 @@ void DerivationGoal::repairClosure()
/* Check each path (slow!). */
for (auto & i : outputClosure) {
if (worker.pathContentsGood(i)) continue;
- logError({
- .name = "Corrupt path in closure",
- .hint = hintfmt(
- "found corrupted or missing path '%s' in the output closure of '%s'",
- worker.store.printStorePath(i), worker.store.printStorePath(drvPath))
- });
+ printError(
+ "found corrupted or missing path '%s' in the output closure of '%s'",
+ worker.store.printStorePath(i), worker.store.printStorePath(drvPath));
auto drvPath2 = outputsToDrv.find(i);
if (drvPath2 == outputsToDrv.end())
- addWaitee(worker.makeSubstitutionGoal(i, Repair));
+ addWaitee(upcast_goal(worker.makePathSubstitutionGoal(i, Repair)));
else
addWaitee(worker.makeDerivationGoal(drvPath2->second, StringSet(), bmRepair));
}
@@ -488,7 +444,9 @@ void DerivationGoal::inputsRealised()
if (useDerivation) {
auto & fullDrv = *dynamic_cast<Derivation *>(drv.get());
- if (!fullDrv.inputDrvs.empty() && fullDrv.type() == DerivationType::CAFloating) {
+ if (settings.isExperimentalFeatureEnabled("ca-derivations") &&
+ ((!fullDrv.inputDrvs.empty() && derivationIsCA(fullDrv.type()))
+ || fullDrv.type() == DerivationType::DeferredInputAddressed)) {
/* We are be able to resolve this derivation based on the
now-known results of dependencies. If so, we become a stub goal
aliasing that resolved derivation goal */
@@ -497,9 +455,7 @@ void DerivationGoal::inputsRealised()
Derivation drvResolved { *std::move(attempt) };
auto pathResolved = writeDerivation(worker.store, drvResolved);
- /* Add to memotable to speed up downstream goal's queries with the
- original derivation. */
- drvPathResolutions.lock()->insert_or_assign(drvPath, pathResolved);
+ resolvedDrv = drvResolved;
auto msg = fmt("Resolved derivation: '%s' -> '%s'",
worker.store.printStorePath(drvPath),
@@ -530,12 +486,12 @@ void DerivationGoal::inputsRealised()
if (!optRealizedInput)
throw Error(
"derivation '%s' requires output '%s' from input derivation '%s', which is supposedly realized already, yet we still don't know what path corresponds to that output",
- worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
+ worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
worker.store.computeFSClosure(*optRealizedInput, inputPaths);
} else
throw Error(
"derivation '%s' requires non-existent output '%s' from input derivation '%s'",
- worker.store.printStorePath(drvPath), j, worker.store.printStorePath(drvPath));
+ worker.store.printStorePath(drvPath), j, worker.store.printStorePath(depDrvPath));
}
}
}
@@ -561,7 +517,6 @@ void DerivationGoal::inputsRealised()
result = BuildResult();
}
-
void DerivationGoal::started() {
auto msg = fmt(
buildMode == bmRepair ? "repairing outputs of '%s'" :
@@ -576,7 +531,6 @@ void DerivationGoal::started() {
worker.updateProgress();
}
-
void DerivationGoal::tryToBuild()
{
trace("trying to build");
@@ -590,9 +544,17 @@ void DerivationGoal::tryToBuild()
PathSet lockFiles;
/* FIXME: Should lock something like the drv itself so we don't build same
CA drv concurrently */
- for (auto & i : drv->outputsAndOptPaths(worker.store))
- if (i.second.second)
- lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
+ if (dynamic_cast<LocalStore *>(&worker.store))
+ /* If we aren't a local store, we might need to use the local store as
+ a build remote, but that would cause a deadlock. */
+ /* FIXME: Make it so we can use ourselves as a build remote even if we
+ are the local store (separate locking for building vs scheduling? */
+ /* FIXME: find some way to lock for scheduling for the other stores so
+ a forking daemon with --store still won't farm out redundant builds.
+ */
+ for (auto & i : drv->outputsAndOptPaths(worker.store))
+ if (i.second.second)
+ lockFiles.insert(worker.store.Store::toRealPath(*i.second.second));
if (!outputLocks.lockPaths(lockFiles, "", false)) {
if (!actLock)
@@ -673,55 +635,10 @@ void DerivationGoal::tryToBuild()
}
void DerivationGoal::tryLocalBuild() {
- bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally(worker.store);
-
- /* Make sure that we are allowed to start a build. If this
- derivation prefers to be done locally, do it even if
- maxBuildJobs is 0. */
- unsigned int curBuilds = worker.getNrLocalBuilds();
- if (curBuilds >= settings.maxBuildJobs && !(buildLocally && curBuilds == 0)) {
- worker.waitForBuildSlot(shared_from_this());
- outputLocks.unlock();
- return;
- }
-
- if (useBuildUsers()) {
- if (!buildUser)
- buildUser = acquireUserLock();
-
- if (!buildUser) {
- if (!actLock)
- actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
- fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
- worker.waitForAWhile(shared_from_this());
- return;
- }
-
- /* Make sure that no other processes are executing under this
- uid. */
- buildUser->kill();
- }
-
- actLock.reset();
-
- try {
-
- /* Okay, we have to build. */
- startBuilder();
-
- } catch (BuildError & e) {
- outputLocks.unlock();
- buildUser.reset();
- worker.permanentFailure = true;
- done(BuildResult::InputRejected, e);
- return;
- }
-
- /* This state will be reached when we get EOF on the child's
- log pipe. */
- state = &DerivationGoal::buildDone;
-
- started();
+ throw Error(
+ "unable to build with a primary store that isn't a local store; "
+ "either pass a different '--store' or enable remote builds."
+ "\nhttps://nixos.org/nix/manual/#chap-distributed-builds");
}
@@ -778,25 +695,63 @@ void replaceValidPath(const Path & storePath, const Path & tmpPath)
}
-MakeError(NotDeterministic, BuildError);
+int DerivationGoal::getChildStatus()
+{
+ return hook->pid.kill();
+}
+
+
+void DerivationGoal::closeReadPipes()
+{
+ hook->builderOut.readSide = -1;
+ hook->fromHook.readSide = -1;
+}
+
+
+void DerivationGoal::cleanupHookFinally()
+{
+}
+
+
+void DerivationGoal::cleanupPreChildKill()
+{
+}
+
+
+void DerivationGoal::cleanupPostChildKill()
+{
+}
+
+
+bool DerivationGoal::cleanupDecideWhetherDiskFull()
+{
+ return false;
+}
+
+
+void DerivationGoal::cleanupPostOutputsRegisteredModeCheck()
+{
+}
+
+
+void DerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
+{
+}
void DerivationGoal::buildDone()
{
trace("build done");
- /* Release the build user at the end of this function. We don't do
- it right away because we don't want another build grabbing this
- uid and then messing around with our output. */
- Finally releaseBuildUser([&]() { buildUser.reset(); });
+ Finally releaseBuildUser([&](){ this->cleanupHookFinally(); });
- sandboxMountNamespace = -1;
+ cleanupPreChildKill();
/* Since we got an EOF on the logger pipe, the builder is presumed
to have terminated. In fact, the builder could also have
simply have closed its end of the pipe, so just to be sure,
kill it. */
- int status = hook ? hook->pid.kill() : pid.kill();
+ int status = getChildStatus();
debug("builder process for '%s' finished", worker.store.printStorePath(drvPath));
@@ -807,24 +762,12 @@ void DerivationGoal::buildDone()
worker.childTerminated(this);
/* Close the read side of the logger pipe. */
- if (hook) {
- hook->builderOut.readSide = -1;
- hook->fromHook.readSide = -1;
- } else
- builderOut.readSide = -1;
+ closeReadPipes();
/* Close the log file. */
closeLogFile();
- /* When running under a build user, make sure that all processes
- running under that uid are gone. This is to prevent a
- malicious user from leaving behind a process that keeps files
- open and modifies them after they have been chown'ed to
- root. */
- if (buildUser) buildUser->kill();
-
- /* Terminate the recursive Nix daemon. */
- stopDaemon();
+ cleanupPostChildKill();
bool diskFull = false;
@@ -833,43 +776,21 @@ void DerivationGoal::buildDone()
/* Check the exit status. */
if (!statusOk(status)) {
- /* Heuristically check whether the build failure may have
- been caused by a disk full condition. We have no way
- of knowing whether the build actually got an ENOSPC.
- So instead, check if the disk is (nearly) full now. If
- so, we don't mark this build as a permanent failure. */
-#if HAVE_STATVFS
- uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
- struct statvfs st;
- if (statvfs(worker.store.realStoreDir.c_str(), &st) == 0 &&
- (uint64_t) st.f_bavail * st.f_bsize < required)
- diskFull = true;
- if (statvfs(tmpDir.c_str(), &st) == 0 &&
- (uint64_t) st.f_bavail * st.f_bsize < required)
- diskFull = true;
-#endif
-
- deleteTmpDir(false);
-
- /* Move paths out of the chroot for easier debugging of
- build failures. */
- if (useChroot && buildMode == bmNormal)
- for (auto & [_, status] : initialOutputs) {
- if (!status.known) continue;
- if (buildMode != bmCheck && status.known->isValid()) continue;
- auto p = worker.store.printStorePath(status.known->path);
- if (pathExists(chrootRootDir + p))
- rename((chrootRootDir + p).c_str(), p.c_str());
- }
+ diskFull |= cleanupDecideWhetherDiskFull();
auto msg = fmt("builder for '%s' %s",
yellowtxt(worker.store.printStorePath(drvPath)),
statusToString(status));
if (!logger->isVerbose() && !logTail.empty()) {
- msg += (format("; last %d log lines:") % logTail.size()).str();
- for (auto & line : logTail)
- msg += "\n " + line;
+ msg += fmt(";\nlast %d log lines:\n", logTail.size());
+ for (auto & line : logTail) {
+ msg += "> ";
+ msg += line;
+ msg += "\n";
+ }
+ msg += fmt("For full logs, run '" ANSI_BOLD "nix log %s" ANSI_NORMAL "'.",
+ worker.store.printStorePath(drvPath));
}
if (diskFull)
@@ -905,10 +826,8 @@ void DerivationGoal::buildDone()
LogSink(Activity & act) : act(act) { }
- void operator() (const unsigned char * data, size_t len) override {
- for (size_t i = 0; i < len; i++) {
- auto c = data[i];
-
+ void operator() (std::string_view data) override {
+ for (auto c : data) {
if (c == '\n') {
flushLine();
} else {
@@ -937,19 +856,12 @@ void DerivationGoal::buildDone()
}
if (buildMode == bmCheck) {
- deleteTmpDir(true);
+ cleanupPostOutputsRegisteredModeCheck();
done(BuildResult::Built);
return;
}
- /* Delete unused redirected outputs (when doing hash rewriting). */
- for (auto & i : redirectedOutputs)
- deletePath(worker.store.Store::toRealPath(i.second));
-
- /* Delete the chroot (if we were using one). */
- autoDelChroot.reset(); /* this runs the destructor */
-
- deleteTmpDir(true);
+ cleanupPostOutputsRegisteredModeNonCheck();
/* Repeat the build if necessary. */
if (curRound++ < nrRounds) {
@@ -993,7 +905,39 @@ void DerivationGoal::buildDone()
}
void DerivationGoal::resolvedFinished() {
- done(BuildResult::Built);
+ assert(resolvedDrv);
+
+ auto resolvedHashes = staticOutputHashes(worker.store, *resolvedDrv);
+
+ // `wantedOutputs` might be empty, which means “all the outputs”
+ auto realWantedOutputs = wantedOutputs;
+ if (realWantedOutputs.empty())
+ realWantedOutputs = resolvedDrv->outputNames();
+
+ for (auto & wantedOutput : realWantedOutputs) {
+ assert(initialOutputs.count(wantedOutput) != 0);
+ assert(resolvedHashes.count(wantedOutput) != 0);
+ auto realisation = worker.store.queryRealisation(
+ DrvOutput{resolvedHashes.at(wantedOutput), wantedOutput}
+ );
+ // We've just built it, but maybe the build failed, in which case the
+ // realisation won't be there
+ if (realisation) {
+ auto newRealisation = *realisation;
+ newRealisation.id = DrvOutput{initialOutputs.at(wantedOutput).outputHash, wantedOutput};
+ newRealisation.signatures.clear();
+ signRealisation(newRealisation);
+ worker.store.registerDrvOutput(newRealisation);
+ } else {
+ // If we don't have a realisation, then it must mean that something
+ // failed when building the resolved drv
+ assert(!result.success());
+ }
+ }
+
+ // This is potentially a bit fishy in terms of error reporting. Not sure
+ // how to do it in a cleaner way
+ amDone(nrFailed == 0 ? ecSuccess : ecFailed, ex);
}
HookReply DerivationGoal::tryBuildHook()
@@ -1018,7 +962,14 @@ HookReply DerivationGoal::tryBuildHook()
whether the hook wishes to perform the build. */
string reply;
while (true) {
- string s = readLine(worker.hook->fromHook.readSide.get());
+ auto s = [&]() {
+ try {
+ return readLine(worker.hook->fromHook.readSide.get());
+ } catch (Error & e) {
+ e.addTrace({}, "while reading the response from the build hook");
+ throw e;
+ }
+ }();
if (handleJSONLogMessage(s, worker.act, worker.hook->activities, true))
;
else if (string(s, 0, 2) == "# ") {
@@ -1047,12 +998,9 @@ HookReply DerivationGoal::tryBuildHook()
} catch (SysError & e) {
if (e.errNo == EPIPE) {
- logError({
- .name = "Build hook died",
- .hint = hintfmt(
- "build hook died unexpectedly: %s",
- chomp(drainFD(worker.hook->fromHook.readSide.get())))
- });
+ printError(
+ "build hook died unexpectedly: %s",
+ chomp(drainFD(worker.hook->fromHook.readSide.get())));
worker.hook = 0;
return rpDecline;
} else
@@ -1061,7 +1009,12 @@ HookReply DerivationGoal::tryBuildHook()
hook = std::move(worker.hook);
- machineName = readLine(hook->fromHook.readSide.get());
+ try {
+ machineName = readLine(hook->fromHook.readSide.get());
+ } catch (Error & e) {
+ e.addTrace({}, "while reading the machine name from the build hook");
+ throw e;
+ }
/* Tell the hook all the inputs that have to be copied to the
remote system. */
@@ -1070,13 +1023,13 @@ HookReply DerivationGoal::tryBuildHook()
/* Tell the hooks the missing outputs that have to be copied back
from the remote system. */
{
- StorePathSet missingPaths;
- for (auto & [_, status] : initialOutputs) {
- if (!status.known) continue;
- if (buildMode != bmCheck && status.known->isValid()) continue;
- missingPaths.insert(status.known->path);
+ StringSet missingOutputs;
+ for (auto & [outputName, status] : initialOutputs) {
+ // XXX: Does this include known CA outputs?
+ if (buildMode != bmCheck && status.known && status.known->isValid()) continue;
+ missingOutputs.insert(outputName);
}
- worker_proto::write(worker.store, hook->sink, missingPaths);
+ worker_proto::write(worker.store, hook->sink, missingOutputs);
}
hook->sink = FdSink();
@@ -1094,13 +1047,6 @@ HookReply DerivationGoal::tryBuildHook()
}
-int childEntry(void * arg)
-{
- ((DerivationGoal *) arg)->runChild();
- return 1;
-}
-
-
StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
{
StorePathSet paths;
@@ -1136,1763 +1082,6 @@ StorePathSet DerivationGoal::exportReferences(const StorePathSet & storePaths)
return paths;
}
-static std::once_flag dns_resolve_flag;
-
-static void preloadNSS() {
- /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
- one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
- been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
- load its lookup libraries in the parent before any child gets a chance to. */
- std::call_once(dns_resolve_flag, []() {
- struct addrinfo *res = NULL;
-
- if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
- if (res) freeaddrinfo(res);
- }
- });
-}
-
-
-void linkOrCopy(const Path & from, const Path & to)
-{
- if (link(from.c_str(), to.c_str()) == -1) {
- /* Hard-linking fails if we exceed the maximum link count on a
- file (e.g. 32000 of ext3), which is quite possible after a
- 'nix-store --optimise'. FIXME: actually, why don't we just
- bind-mount in this case?
-
- It can also fail with EPERM in BeegFS v7 and earlier versions
- which don't allow hard-links to other directories */
- if (errno != EMLINK && errno != EPERM)
- throw SysError("linking '%s' to '%s'", to, from);
- copyPath(from, to);
- }
-}
-
-
-void DerivationGoal::startBuilder()
-{
- /* Right platform? */
- if (!parsedDrv->canBuildLocally(worker.store))
- throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
- drv->platform,
- concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
- worker.store.printStorePath(drvPath),
- settings.thisSystem,
- concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
-
- if (drv->isBuiltin())
- preloadNSS();
-
-#if __APPLE__
- additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
-#endif
-
- /* Are we doing a chroot build? */
- {
- auto noChroot = parsedDrv->getBoolAttr("__noChroot");
- if (settings.sandboxMode == smEnabled) {
- if (noChroot)
- throw Error("derivation '%s' has '__noChroot' set, "
- "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
-#if __APPLE__
- if (additionalSandboxProfile != "")
- throw Error("derivation '%s' specifies a sandbox profile, "
- "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
-#endif
- useChroot = true;
- }
- else if (settings.sandboxMode == smDisabled)
- useChroot = false;
- else if (settings.sandboxMode == smRelaxed)
- useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
- }
-
- if (worker.store.storeDir != worker.store.realStoreDir) {
- #if __linux__
- useChroot = true;
- #else
- throw Error("building using a diverted store is not supported on this platform");
- #endif
- }
-
- /* Create a temporary directory where the build will take
- place. */
- tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
-
- chownToBuilder(tmpDir);
-
- for (auto & [outputName, status] : initialOutputs) {
- /* Set scratch path we'll actually use during the build.
-
- If we're not doing a chroot build, but we have some valid
- output paths. Since we can't just overwrite or delete
- them, we have to do hash rewriting: i.e. in the
- environment/arguments passed to the build, we replace the
- hashes of the valid outputs with unique dummy strings;
- after the build, we discard the redirected outputs
- corresponding to the valid outputs, and rewrite the
- contents of the new outputs to replace the dummy strings
- with the actual hashes. */
- auto scratchPath =
- !status.known
- ? makeFallbackPath(outputName)
- : !needsHashRewrite()
- /* Can always use original path in sandbox */
- ? status.known->path
- : !status.known->isPresent()
- /* If path doesn't yet exist can just use it */
- ? status.known->path
- : buildMode != bmRepair && !status.known->isValid()
- /* If we aren't repairing we'll delete a corrupted path, so we
- can use original path */
- ? status.known->path
- : /* If we are repairing or the path is totally valid, we'll need
- to use a temporary path */
- makeFallbackPath(status.known->path);
- scratchOutputs.insert_or_assign(outputName, scratchPath);
-
- /* A non-removed corrupted path needs to be stored here, too */
- if (buildMode == bmRepair && !status.known->isValid())
- redirectedBadOutputs.insert(status.known->path);
-
- /* Substitute output placeholders with the scratch output paths.
- We'll use during the build. */
- inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath);
-
- /* Additional tasks if we know the final path a priori. */
- if (!status.known) continue;
- auto fixedFinalPath = status.known->path;
-
- /* Additional tasks if the final and scratch are both known and
- differ. */
- if (fixedFinalPath == scratchPath) continue;
-
- /* Ensure scratch path is ours to use. */
- deletePath(worker.store.printStorePath(scratchPath));
-
- /* Rewrite and unrewrite paths */
- {
- std::string h1 { fixedFinalPath.hashPart() };
- std::string h2 { scratchPath.hashPart() };
- inputRewrites[h1] = h2;
- }
-
- redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath));
- }
-
- /* Construct the environment passed to the builder. */
- initEnv();
-
- writeStructuredAttrs();
-
- /* Handle exportReferencesGraph(), if set. */
- if (!parsedDrv->getStructuredAttrs()) {
- /* The `exportReferencesGraph' feature allows the references graph
- to be passed to a builder. This attribute should be a list of
- pairs [name1 path1 name2 path2 ...]. The references graph of
- each `pathN' will be stored in a text file `nameN' in the
- temporary build directory. The text files have the format used
- by `nix-store --register-validity'. However, the deriver
- fields are left empty. */
- string s = get(drv->env, "exportReferencesGraph").value_or("");
- Strings ss = tokenizeString<Strings>(s);
- if (ss.size() % 2 != 0)
- throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
- for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
- string fileName = *i++;
- static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
- if (!std::regex_match(fileName, regex))
- throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
-
- auto storePathS = *i++;
- if (!worker.store.isInStore(storePathS))
- throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
- auto storePath = worker.store.toStorePath(storePathS).first;
-
- /* Write closure info to <fileName>. */
- writeFile(tmpDir + "/" + fileName,
- worker.store.makeValidityRegistration(
- exportReferences({storePath}), false, false));
- }
- }
-
- useUidRange = parsedDrv->getRequiredSystemFeatures().count("uid-range");
- useSystemdCgroup = parsedDrv->getRequiredSystemFeatures().count("systemd-cgroup");
-
- if (useChroot) {
-
- /* Allow a user-configurable set of directories from the
- host file system. */
- PathSet dirs = settings.sandboxPaths;
- PathSet dirs2 = settings.extraSandboxPaths;
- dirs.insert(dirs2.begin(), dirs2.end());
-
- dirsInChroot.clear();
-
- for (auto i : dirs) {
- if (i.empty()) continue;
- bool optional = false;
- if (i[i.size() - 1] == '?') {
- optional = true;
- i.pop_back();
- }
- size_t p = i.find('=');
- if (p == string::npos)
- dirsInChroot[i] = {i, optional};
- else
- dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional};
- }
- dirsInChroot[tmpDirInSandbox] = tmpDir;
-
- /* Add the closure of store paths to the chroot. */
- StorePathSet closure;
- for (auto & i : dirsInChroot)
- try {
- if (worker.store.isInStore(i.second.source))
- worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
- } catch (InvalidPath & e) {
- } catch (Error & e) {
- e.addTrace({}, "while processing 'sandbox-paths'");
- throw;
- }
- for (auto & i : closure) {
- auto p = worker.store.printStorePath(i);
- dirsInChroot.insert_or_assign(p, p);
- }
-
- PathSet allowedPaths = settings.allowedImpureHostPrefixes;
-
- /* This works like the above, except on a per-derivation level */
- auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings());
-
- for (auto & i : impurePaths) {
- bool found = false;
- /* Note: we're not resolving symlinks here to prevent
- giving a non-root user info about inaccessible
- files. */
- Path canonI = canonPath(i);
- /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */
- for (auto & a : allowedPaths) {
- Path canonA = canonPath(a);
- if (canonI == canonA || isInDir(canonI, canonA)) {
- found = true;
- break;
- }
- }
- if (!found)
- throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps",
- worker.store.printStorePath(drvPath), i);
-
- dirsInChroot[i] = i;
- }
-
-#if __linux__
- /* Create a temporary directory in which we set up the chroot
- environment using bind-mounts. We put it in the Nix store
- to ensure that we can create hard-links to non-directory
- inputs in the fake Nix store in the chroot (see below). */
- chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
- deletePath(chrootRootDir);
-
- /* Clean up the chroot directory automatically. */
- autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
-
- printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
-
- if (mkdir(chrootRootDir.c_str(), useUidRange ? 0755 : 0750) == -1)
- throw SysError("cannot create '%1%'", chrootRootDir);
-
- // FIXME: only make root writable for user namespace builds.
- if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
- throw SysError("cannot change ownership of '%1%'", chrootRootDir);
-
- /* Create a writable /tmp in the chroot. Many builders need
- this. (Of course they should really respect $TMPDIR
- instead.) */
- Path chrootTmpDir = chrootRootDir + "/tmp";
- createDirs(chrootTmpDir);
- chmod_(chrootTmpDir, 01777);
-
- /* Create a /etc/passwd with entries for the build user and the
- nobody account. The latter is kind of a hack to support
- Samba-in-QEMU. */
- createDirs(chrootRootDir + "/etc");
- chownToBuilder(chrootRootDir + "/etc");
-
- if (useUidRange && (!buildUser || buildUser->getUIDCount() < 65536))
- throw Error("feature 'uid-range' requires '%s' to be enabled", settings.autoAllocateUids.name);
-
- /* Declare the build user's group so that programs get a consistent
- view of the system (e.g., "id -gn"). */
- writeFile(chrootRootDir + "/etc/group",
- fmt("root:x:0:\n"
- "nixbld:!:%1%:\n"
- "nogroup:x:65534:\n", sandboxGid()));
-
- /* Create /etc/hosts with localhost entry. */
- if (!(derivationIsImpure(derivationType)))
- writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
-
- /* Make the closure of the inputs available in the chroot,
- rather than the whole Nix store. This prevents any access
- to undeclared dependencies. Directories are bind-mounted,
- while other inputs are hard-linked (since only directories
- can be bind-mounted). !!! As an extra security
- precaution, make the fake Nix store only writable by the
- build user. */
- Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
- createDirs(chrootStoreDir);
- chmod_(chrootStoreDir, 01775);
-
- if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1)
- throw SysError("cannot change ownership of '%1%'", chrootStoreDir);
-
- for (auto & i : inputPaths) {
- auto p = worker.store.printStorePath(i);
- Path r = worker.store.toRealPath(p);
- if (S_ISDIR(lstat(r).st_mode))
- dirsInChroot.insert_or_assign(p, r);
- else
- linkOrCopy(r, chrootRootDir + p);
- }
-
- /* If we're repairing, checking or rebuilding part of a
- multiple-outputs derivation, it's possible that we're
- rebuilding a path that is in settings.dirsInChroot
- (typically the dependencies of /bin/sh). Throw them
- out. */
- for (auto & i : drv->outputsAndOptPaths(worker.store)) {
- /* If the name isn't known a priori (i.e. floating
- content-addressed derivation), the temporary location we use
- should be fresh. Freshness means it is impossible that the path
- is already in the sandbox, so we don't need to worry about
- removing it. */
- if (i.second.second)
- dirsInChroot.erase(worker.store.printStorePath(*i.second.second));
- }
-
- if (useSystemdCgroup) {
- settings.requireExperimentalFeature("systemd-cgroup");
- std::optional<Path> cgroup;
- if (!buildUser || !(cgroup = buildUser->getCgroup()))
- throw Error("feature 'systemd-cgroup' requires 'auto-allocate-uids = true' in nix.conf");
- chownToBuilder(*cgroup);
- chownToBuilder(*cgroup + "/cgroup.procs");
- }
-
-#else
- if (useUidRange)
- throw Error("feature 'uid-range' is not supported on this platform");
- if (useSystemdCgroup)
- throw Error("feature 'systemd-cgroup' is not supported on this platform");
- #if __APPLE__
- /* We don't really have any parent prep work to do (yet?)
- All work happens in the child, instead. */
- #else
- throw Error("sandboxing builds is not supported on this platform");
- #endif
-#endif
- } else {
- if (useUidRange)
- throw Error("feature 'uid-range' is only supported in sandboxed builds");
- if (useSystemdCgroup)
- throw Error("feature 'systemd-cgroup' is only supported in sandboxed builds");
- }
-
- if (needsHashRewrite() && pathExists(homeDir))
- throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir);
-
- if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) {
- printMsg(lvlChatty, format("executing pre-build hook '%1%'")
- % settings.preBuildHook);
- auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) :
- Strings({ worker.store.printStorePath(drvPath) });
- enum BuildHookState {
- stBegin,
- stExtraChrootDirs
- };
- auto state = stBegin;
- auto lines = runProgram(settings.preBuildHook, false, args);
- auto lastPos = std::string::size_type{0};
- for (auto nlPos = lines.find('\n'); nlPos != string::npos;
- nlPos = lines.find('\n', lastPos)) {
- auto line = std::string{lines, lastPos, nlPos - lastPos};
- lastPos = nlPos + 1;
- if (state == stBegin) {
- if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
- state = stExtraChrootDirs;
- } else {
- throw Error("unknown pre-build hook command '%1%'", line);
- }
- } else if (state == stExtraChrootDirs) {
- if (line == "") {
- state = stBegin;
- } else {
- auto p = line.find('=');
- if (p == string::npos)
- dirsInChroot[line] = line;
- else
- dirsInChroot[string(line, 0, p)] = string(line, p + 1);
- }
- }
- }
- }
-
- /* Fire up a Nix daemon to process recursive Nix calls from the
- builder. */
- if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix"))
- startDaemon();
-
- /* Run the builder. */
- printMsg(lvlChatty, "executing builder '%1%'", drv->builder);
-
- /* Create the log file. */
- Path logFile = openLogFile();
-
- /* Create a pipe to get the output of the builder. */
- //builderOut.create();
-
- builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
- if (!builderOut.readSide)
- throw SysError("opening pseudoterminal master");
-
- std::string slaveName(ptsname(builderOut.readSide.get()));
-
- if (buildUser) {
- if (chmod(slaveName.c_str(), 0600))
- throw SysError("changing mode of pseudoterminal slave");
-
- if (chown(slaveName.c_str(), buildUser->getUID(), 0))
- throw SysError("changing owner of pseudoterminal slave");
- }
-#if __APPLE__
- else {
- if (grantpt(builderOut.readSide.get()))
- throw SysError("granting access to pseudoterminal slave");
- }
-#endif
-
- #if 0
- // Mount the pt in the sandbox so that the "tty" command works.
- // FIXME: this doesn't work with the new devpts in the sandbox.
- if (useChroot)
- dirsInChroot[slaveName] = {slaveName, false};
- #endif
-
- if (unlockpt(builderOut.readSide.get()))
- throw SysError("unlocking pseudoterminal");
-
- builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY);
- if (!builderOut.writeSide)
- throw SysError("opening pseudoterminal slave");
-
- // Put the pt into raw mode to prevent \n -> \r\n translation.
- struct termios term;
- if (tcgetattr(builderOut.writeSide.get(), &term))
- throw SysError("getting pseudoterminal attributes");
-
- cfmakeraw(&term);
-
- if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
- throw SysError("putting pseudoterminal into raw mode");
-
- result.startTime = time(0);
-
- /* Fork a child to build the package. */
- ProcessOptions options;
-
-#if __linux__
- if (useChroot) {
- /* Set up private namespaces for the build:
-
- - The PID namespace causes the build to start as PID 1.
- Processes outside of the chroot are not visible to those
- on the inside, but processes inside the chroot are
- visible from the outside (though with different PIDs).
-
- - The private mount namespace ensures that all the bind
- mounts we do will only show up in this process and its
- children, and will disappear automatically when we're
- done.
-
- - The private network namespace ensures that the builder
- cannot talk to the outside world (or vice versa). It
- only has a private loopback interface. (Fixed-output
- derivations are not run in a private network namespace
- to allow functions like fetchurl to work.)
-
- - The IPC namespace prevents the builder from communicating
- with outside processes using SysV IPC mechanisms (shared
- memory, message queues, semaphores). It also ensures
- that all IPC objects are destroyed when the builder
- exits.
-
- - The UTS namespace ensures that builders see a hostname of
- localhost rather than the actual hostname.
-
- We use a helper process to do the clone() to work around
- clone() being broken in multi-threaded programs due to
- at-fork handlers not being run. Note that we use
- CLONE_PARENT to ensure that the real builder is parented to
- us.
- */
-
- if (!(derivationIsImpure(derivationType)))
- privateNetwork = true;
-
- userNamespaceSync.create();
-
- options.allowVfork = false;
-
- Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
- static bool userNamespacesEnabled =
- pathExists(maxUserNamespaces)
- && trim(readFile(maxUserNamespaces)) != "0";
-
- usingUserNamespace = userNamespacesEnabled;
-
- Pid helper = startProcess([&]() {
-
- /* Drop additional groups here because we can't do it
- after we've created the new user namespace. FIXME:
- this means that if we're not root in the parent
- namespace, we can't drop additional groups; they will
- be mapped to nogroup in the child namespace. There does
- not seem to be a workaround for this. (But who can tell
- from reading user_namespaces(7)?)
- See also https://lwn.net/Articles/621612/. */
- if (getuid() == 0 && setgroups(0, 0) == -1)
- throw SysError("setgroups failed");
-
- size_t stackSize = 1 * 1024 * 1024;
- char * stack = (char *) mmap(0, stackSize,
- PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
- if (stack == MAP_FAILED) throw SysError("allocating stack");
-
- int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
- if (privateNetwork)
- flags |= CLONE_NEWNET;
- if (usingUserNamespace)
- flags |= CLONE_NEWUSER;
-
- pid_t child = clone(childEntry, stack + stackSize, flags, this);
- if (child == -1 && errno == EINVAL) {
- /* Fallback for Linux < 2.13 where CLONE_NEWPID and
- CLONE_PARENT are not allowed together. */
- flags &= ~CLONE_NEWPID;
- child = clone(childEntry, stack + stackSize, flags, this);
- }
- if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) {
- /* Some distros patch Linux to not allow unprivileged
- * user namespaces. If we get EPERM or EINVAL, try
- * without CLONE_NEWUSER and see if that works.
- */
- usingUserNamespace = false;
- flags &= ~CLONE_NEWUSER;
- child = clone(childEntry, stack + stackSize, flags, this);
- }
- /* Otherwise exit with EPERM so we can handle this in the
- parent. This is only done when sandbox-fallback is set
- to true (the default). */
- if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback)
- _exit(1);
- if (child == -1) throw SysError("cloning builder process");
-
- writeFull(builderOut.writeSide.get(),
- fmt("%d %d\n", usingUserNamespace, child));
- _exit(0);
- }, options);
-
- int res = helper.wait();
- if (res != 0 && settings.sandboxFallback) {
- useChroot = false;
- initTmpDir();
- goto fallback;
- } else if (res != 0)
- throw Error("unable to start build process");
-
- userNamespaceSync.readSide = -1;
-
- /* Close the write side to prevent runChild() from hanging
- reading from this. */
- Finally cleanup([&]() {
- userNamespaceSync.writeSide = -1;
- });
-
- pid_t tmp;
- auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get()));
- assert(ss.size() == 2);
- usingUserNamespace = ss[0] == "1";
- if (!string2Int<pid_t>(ss[1], tmp)) abort();
- pid = tmp;
-
- if (usingUserNamespace) {
- /* Set the UID/GID mapping of the builder's user namespace
- such that the sandbox user maps to the build user, or to
- the calling user (if build users are disabled). */
- uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
- uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
- uint32_t nrIds = buildUser && useUidRange ? buildUser->getUIDCount() : 1;
-
- writeFile("/proc/" + std::to_string(pid) + "/uid_map",
- fmt("%d %d %d", sandboxUid(), hostUid, nrIds));
-
- if (!useUidRange)
- writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
-
- writeFile("/proc/" + std::to_string(pid) + "/gid_map",
- fmt("%d %d %d", sandboxGid(), hostGid, nrIds));
- } else {
- debug("note: not using a user namespace");
- if (!buildUser)
- throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces");
- }
-
- /* Now that we now the sandbox uid, we can write
- /etc/passwd. */
- writeFile(chrootRootDir + "/etc/passwd", fmt(
- "root:x:0:0:Nix build user:%3%:/noshell\n"
- "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
- "nobody:x:65534:65534:Nobody:/:/noshell\n",
- sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
-
- /* Save the mount namespace of the child. We have to do this
- *before* the child does a chroot. */
- sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
- if (sandboxMountNamespace.get() == -1)
- throw SysError("getting sandbox mount namespace");
-
- /* Move the child into its own cgroup. */
- if (buildUser) {
- if (auto cgroup = buildUser->getCgroup())
- writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid));
- }
-
- /* Signal the builder that we've updated its user namespace. */
- writeFull(userNamespaceSync.writeSide.get(), "1");
-
- } else
-#endif
- {
- fallback:
- options.allowVfork = !buildUser && !drv->isBuiltin();
- pid = startProcess([&]() {
- runChild();
- }, options);
- }
-
- /* parent */
- pid.setSeparatePG(true);
- builderOut.writeSide = -1;
- worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true);
-
- /* Check if setting up the build environment failed. */
- while (true) {
- string msg = readLine(builderOut.readSide.get());
- if (string(msg, 0, 1) == "\2") break;
- if (string(msg, 0, 1) == "\1") {
- FdSource source(builderOut.readSide.get());
- auto ex = readError(source);
- ex.addTrace({}, "while setting up the build environment");
- throw ex;
- }
- debug("sandbox setup: " + msg);
- }
-}
-
-
-void DerivationGoal::initTmpDir() {
- /* In a sandbox, for determinism, always use the same temporary
- directory. */
-#if __linux__
- tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
-#else
- tmpDirInSandbox = tmpDir;
-#endif
-
- /* In non-structured mode, add all bindings specified in the
- derivation via the environment, except those listed in the
- passAsFile attribute. Those are passed as file names pointing
- to temporary files containing the contents. Note that
- passAsFile is ignored in structure mode because it's not
- needed (attributes are not passed through the environment, so
- there is no size constraint). */
- if (!parsedDrv->getStructuredAttrs()) {
-
- StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or(""));
- for (auto & i : drv->env) {
- if (passAsFile.find(i.first) == passAsFile.end()) {
- env[i.first] = i.second;
- } else {
- auto hash = hashString(htSHA256, i.first);
- string fn = ".attr-" + hash.to_string(Base32, false);
- Path p = tmpDir + "/" + fn;
- writeFile(p, rewriteStrings(i.second, inputRewrites));
- chownToBuilder(p);
- env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
- }
- }
-
- }
-
- /* For convenience, set an environment pointing to the top build
- directory. */
- env["NIX_BUILD_TOP"] = tmpDirInSandbox;
-
- /* Also set TMPDIR and variants to point to this directory. */
- env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox;
-
- /* Explicitly set PWD to prevent problems with chroot builds. In
- particular, dietlibc cannot figure out the cwd because the
- inode of the current directory doesn't appear in .. (because
- getdents returns the inode of the mount point). */
- env["PWD"] = tmpDirInSandbox;
-}
-
-
-void DerivationGoal::initEnv()
-{
- env.clear();
-
- /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
- PATH is not set. We don't want this, so we fill it in with some dummy
- value. */
- env["PATH"] = "/path-not-set";
-
- /* Set HOME to a non-existing path to prevent certain programs from using
- /etc/passwd (or NIS, or whatever) to locate the home directory (for
- example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd
- if HOME is not set, but they will just assume that the settings file
- they are looking for does not exist if HOME is set but points to some
- non-existing path. */
- env["HOME"] = homeDir;
-
- /* Tell the builder where the Nix store is. Usually they
- shouldn't care, but this is useful for purity checking (e.g.,
- the compiler or linker might only want to accept paths to files
- in the store or in the build directory). */
- env["NIX_STORE"] = worker.store.storeDir;
-
- /* The maximum number of cores to utilize for parallel building. */
- env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
-
- initTmpDir();
-
- /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
- derivation, tell the builder, so that for instance `fetchurl'
- can skip checking the output. On older Nixes, this environment
- variable won't be set, so `fetchurl' will do the check. */
- if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
-
- /* *Only* if this is a fixed-output derivation, propagate the
- values of the environment variables specified in the
- `impureEnvVars' attribute to the builder. This allows for
- instance environment variables for proxy configuration such as
- `http_proxy' to be easily passed to downloaders like
- `fetchurl'. Passing such environment variables from the caller
- to the builder is generally impure, but the output of
- fixed-output derivations is by definition pure (since we
- already know the cryptographic hash of the output). */
- if (derivationIsImpure(derivationType)) {
- for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
- env[i] = getEnv(i).value_or("");
- }
-
- /* Currently structured log messages piggyback on stderr, but we
- may change that in the future. So tell the builder which file
- descriptor to use for that. */
- env["NIX_LOG_FD"] = "2";
-
- /* Trigger colored output in various tools. */
- env["TERM"] = "xterm-256color";
-}
-
-
-static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
-
-
-void DerivationGoal::writeStructuredAttrs()
-{
- auto structuredAttrs = parsedDrv->getStructuredAttrs();
- if (!structuredAttrs) return;
-
- auto json = *structuredAttrs;
-
- /* Add an "outputs" object containing the output paths. */
- nlohmann::json outputs;
- for (auto & i : drv->outputs) {
- /* The placeholder must have a rewrite, so we use it to cover both the
- cases where we know or don't know the output path ahead of time. */
- outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
- }
- json["outputs"] = outputs;
-
- /* Handle exportReferencesGraph. */
- auto e = json.find("exportReferencesGraph");
- if (e != json.end() && e->is_object()) {
- for (auto i = e->begin(); i != e->end(); ++i) {
- std::ostringstream str;
- {
- JSONPlaceholder jsonRoot(str, true);
- StorePathSet storePaths;
- for (auto & p : *i)
- storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
- worker.store.pathInfoToJSON(jsonRoot,
- exportReferences(storePaths), false, true);
- }
- json[i.key()] = nlohmann::json::parse(str.str()); // urgh
- }
- }
-
- writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
- chownToBuilder(tmpDir + "/.attrs.json");
-
- /* As a convenience to bash scripts, write a shell file that
- maps all attributes that are representable in bash -
- namely, strings, integers, nulls, Booleans, and arrays and
- objects consisting entirely of those values. (So nested
- arrays or objects are not supported.) */
-
- auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
- if (value.is_string())
- return shellEscape(value);
-
- if (value.is_number()) {
- auto f = value.get<float>();
- if (std::ceil(f) == f)
- return std::to_string(value.get<int>());
- }
-
- if (value.is_null())
- return std::string("''");
-
- if (value.is_boolean())
- return value.get<bool>() ? std::string("1") : std::string("");
-
- return {};
- };
-
- std::string jsonSh;
-
- for (auto i = json.begin(); i != json.end(); ++i) {
-
- if (!std::regex_match(i.key(), shVarName)) continue;
-
- auto & value = i.value();
-
- auto s = handleSimpleType(value);
- if (s)
- jsonSh += fmt("declare %s=%s\n", i.key(), *s);
-
- else if (value.is_array()) {
- std::string s2;
- bool good = true;
-
- for (auto i = value.begin(); i != value.end(); ++i) {
- auto s3 = handleSimpleType(i.value());
- if (!s3) { good = false; break; }
- s2 += *s3; s2 += ' ';
- }
-
- if (good)
- jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
- }
-
- else if (value.is_object()) {
- std::string s2;
- bool good = true;
-
- for (auto i = value.begin(); i != value.end(); ++i) {
- auto s3 = handleSimpleType(i.value());
- if (!s3) { good = false; break; }
- s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
- }
-
- if (good)
- jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
- }
- }
-
- writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
- chownToBuilder(tmpDir + "/.attrs.sh");
-}
-
-struct RestrictedStoreConfig : LocalFSStoreConfig
-{
- using LocalFSStoreConfig::LocalFSStoreConfig;
- const std::string name() { return "Restricted Store"; }
-};
-
-/* A wrapper around LocalStore that only allows building/querying of
- paths that are in the input closures of the build or were added via
- recursive Nix calls. */
-struct RestrictedStore : public LocalFSStore, public virtual RestrictedStoreConfig
-{
- ref<LocalStore> next;
-
- DerivationGoal & goal;
-
- RestrictedStore(const Params & params, ref<LocalStore> next, DerivationGoal & goal)
- : StoreConfig(params), Store(params), LocalFSStore(params), next(next), goal(goal)
- { }
-
- Path getRealStoreDir() override
- { return next->realStoreDir; }
-
- std::string getUri() override
- { return next->getUri(); }
-
- StorePathSet queryAllValidPaths() override
- {
- StorePathSet paths;
- for (auto & p : goal.inputPaths) paths.insert(p);
- for (auto & p : goal.addedPaths) paths.insert(p);
- return paths;
- }
-
- void queryPathInfoUncached(const StorePath & path,
- Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
- {
- if (goal.isAllowed(path)) {
- try {
- /* Censor impure information. */
- auto info = std::make_shared<ValidPathInfo>(*next->queryPathInfo(path));
- info->deriver.reset();
- info->registrationTime = 0;
- info->ultimate = false;
- info->sigs.clear();
- callback(info);
- } catch (InvalidPath &) {
- callback(nullptr);
- }
- } else
- callback(nullptr);
- };
-
- void queryReferrers(const StorePath & path, StorePathSet & referrers) override
- { }
-
- std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override
- {
- if (!goal.isAllowed(path))
- throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
- return next->queryPartialDerivationOutputMap(path);
- }
-
- std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
- { throw Error("queryPathFromHashPart"); }
-
- StorePath addToStore(const string & name, const Path & srcPath,
- FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
- PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
- { throw Error("addToStore"); }
-
- void addToStore(const ValidPathInfo & info, Source & narSource,
- RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
- {
- next->addToStore(info, narSource, repair, checkSigs);
- goal.addDependency(info.path);
- }
-
- StorePath addTextToStore(const string & name, const string & s,
- const StorePathSet & references, RepairFlag repair = NoRepair) override
- {
- auto path = next->addTextToStore(name, s, references, repair);
- goal.addDependency(path);
- return path;
- }
-
- void narFromPath(const StorePath & path, Sink & sink) override
- {
- if (!goal.isAllowed(path))
- throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path));
- LocalFSStore::narFromPath(path, sink);
- }
-
- void ensurePath(const StorePath & path) override
- {
- if (!goal.isAllowed(path))
- throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path));
- /* Nothing to be done; 'path' must already be valid. */
- }
-
- void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override
- {
- if (buildMode != bmNormal) throw Error("unsupported build mode");
-
- StorePathSet newPaths;
-
- for (auto & path : paths) {
- if (!goal.isAllowed(path.path))
- throw InvalidPath("cannot build unknown path '%s' in recursive Nix", printStorePath(path.path));
- }
-
- next->buildPaths(paths, buildMode);
-
- for (auto & path : paths) {
- if (!path.path.isDerivation()) continue;
- auto outputs = next->queryDerivationOutputMap(path.path);
- for (auto & output : outputs)
- if (wantOutput(output.first, path.outputs))
- newPaths.insert(output.second);
- }
-
- StorePathSet closure;
- next->computeFSClosure(newPaths, closure);
- for (auto & path : closure)
- goal.addDependency(path);
- }
-
- BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
- BuildMode buildMode = bmNormal) override
- { unsupported("buildDerivation"); }
-
- void addTempRoot(const StorePath & path) override
- { }
-
- void addIndirectRoot(const Path & path) override
- { }
-
- Roots findRoots(bool censor) override
- { return Roots(); }
-
- void collectGarbage(const GCOptions & options, GCResults & results) override
- { }
-
- void addSignatures(const StorePath & storePath, const StringSet & sigs) override
- { unsupported("addSignatures"); }
-
- void queryMissing(const std::vector<StorePathWithOutputs> & targets,
- StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
- uint64_t & downloadSize, uint64_t & narSize) override
- {
- /* This is slightly impure since it leaks information to the
- client about what paths will be built/substituted or are
- already present. Probably not a big deal. */
-
- std::vector<StorePathWithOutputs> allowed;
- for (auto & path : targets) {
- if (goal.isAllowed(path.path))
- allowed.emplace_back(path);
- else
- unknown.insert(path.path);
- }
-
- next->queryMissing(allowed, willBuild, willSubstitute,
- unknown, downloadSize, narSize);
- }
-};
-
-
-void DerivationGoal::startDaemon()
-{
- settings.requireExperimentalFeature("recursive-nix");
-
- Store::Params params;
- params["path-info-cache-size"] = "0";
- params["store"] = worker.store.storeDir;
- params["root"] = worker.store.rootDir;
- params["state"] = "/no-such-path";
- params["log"] = "/no-such-path";
- auto store = make_ref<RestrictedStore>(params,
- ref<LocalStore>(std::dynamic_pointer_cast<LocalStore>(worker.store.shared_from_this())),
- *this);
-
- addedPaths.clear();
-
- auto socketName = ".nix-socket";
- Path socketPath = tmpDir + "/" + socketName;
- env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName;
-
- daemonSocket = createUnixDomainSocket(socketPath, 0600);
-
- chownToBuilder(socketPath);
-
- daemonThread = std::thread([this, store]() {
-
- while (true) {
-
- /* Accept a connection. */
- struct sockaddr_un remoteAddr;
- socklen_t remoteAddrLen = sizeof(remoteAddr);
-
- AutoCloseFD remote = accept(daemonSocket.get(),
- (struct sockaddr *) &remoteAddr, &remoteAddrLen);
- if (!remote) {
- if (errno == EINTR) continue;
- if (errno == EINVAL) break;
- throw SysError("accepting connection");
- }
-
- closeOnExec(remote.get());
-
- debug("received daemon connection");
-
- auto workerThread = std::thread([store, remote{std::move(remote)}]() {
- FdSource from(remote.get());
- FdSink to(remote.get());
- try {
- daemon::processConnection(store, from, to,
- daemon::NotTrusted, daemon::Recursive,
- [&](Store & store) { store.createUser("nobody", 65535); });
- debug("terminated daemon connection");
- } catch (SysError &) {
- ignoreException();
- }
- });
-
- daemonWorkerThreads.push_back(std::move(workerThread));
- }
-
- debug("daemon shutting down");
- });
-}
-
-
-void DerivationGoal::stopDaemon()
-{
- if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
- throw SysError("shutting down daemon socket");
-
- if (daemonThread.joinable())
- daemonThread.join();
-
- // FIXME: should prune worker threads more quickly.
- // FIXME: shutdown the client socket to speed up worker termination.
- for (auto & thread : daemonWorkerThreads)
- thread.join();
- daemonWorkerThreads.clear();
-
- daemonSocket = -1;
-}
-
-
-void DerivationGoal::addDependency(const StorePath & path)
-{
- if (isAllowed(path)) return;
-
- addedPaths.insert(path);
-
- /* If we're doing a sandbox build, then we have to make the path
- appear in the sandbox. */
- if (useChroot) {
-
- debug("materialising '%s' in the sandbox", worker.store.printStorePath(path));
-
- #if __linux__
-
- Path source = worker.store.Store::toRealPath(path);
- Path target = chrootRootDir + worker.store.printStorePath(path);
- debug("bind-mounting %s -> %s", target, source);
-
- if (pathExists(target))
- throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
-
- auto st = lstat(source);
-
- if (S_ISDIR(st.st_mode)) {
-
- /* Bind-mount the path into the sandbox. This requires
- entering its mount namespace, which is not possible
- in multithreaded programs. So we do this in a
- child process.*/
- Pid child(startProcess([&]() {
-
- if (setns(sandboxMountNamespace.get(), 0) == -1)
- throw SysError("entering sandbox mount namespace");
-
- createDirs(target);
-
- if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
- throw SysError("bind mount from '%s' to '%s' failed", source, target);
-
- _exit(0);
- }));
-
- int status = child.wait();
- if (status != 0)
- throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path));
-
- } else
- linkOrCopy(source, target);
-
- #else
- throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox",
- worker.store.printStorePath(path));
- #endif
-
- }
-}
-
-
-void DerivationGoal::chownToBuilder(const Path & path)
-{
- if (!buildUser) return;
- if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
- throw SysError("cannot change ownership of '%1%'", path);
-}
-
-
-void setupSeccomp()
-{
-#if __linux__
- if (!settings.filterSyscalls) return;
-#if HAVE_SECCOMP
- scmp_filter_ctx ctx;
-
- if (!(ctx = seccomp_init(SCMP_ACT_ALLOW)))
- throw SysError("unable to initialize seccomp mode 2");
-
- Finally cleanup([&]() {
- seccomp_release(ctx);
- });
-
- if (nativeSystem == "x86_64-linux" &&
- seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
- throw SysError("unable to add 32-bit seccomp architecture");
-
- if (nativeSystem == "x86_64-linux" &&
- seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
- throw SysError("unable to add X32 seccomp architecture");
-
- if (nativeSystem == "aarch64-linux" &&
- seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
- printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
-
- /* Prevent builders from creating setuid/setgid binaries. */
- for (int perm : { S_ISUID, S_ISGID }) {
- if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
- SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
- throw SysError("unable to add seccomp rule");
-
- if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1,
- SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
- throw SysError("unable to add seccomp rule");
-
- if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1,
- SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
- throw SysError("unable to add seccomp rule");
- }
-
- /* Prevent builders from creating EAs or ACLs. Not all filesystems
- support these, and they're not allowed in the Nix store because
- they're not representable in the NAR serialisation. */
- if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 ||
- seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 ||
- seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0)
- throw SysError("unable to add seccomp rule");
-
- if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0)
- throw SysError("unable to set 'no new privileges' seccomp attribute");
-
- if (seccomp_load(ctx) != 0)
- throw SysError("unable to load seccomp BPF program");
-#else
- throw Error(
- "seccomp is not supported on this platform; "
- "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!");
-#endif
-#endif
-}
-
-
-void DerivationGoal::runChild()
-{
- /* Warning: in the child we should absolutely not make any SQLite
- calls! */
-
- try { /* child */
-
- commonChildInit(builderOut);
-
- try {
- setupSeccomp();
- } catch (...) {
- if (buildUser) throw;
- }
-
- bool setUser = true;
-
- /* Make the contents of netrc available to builtin:fetchurl
- (which may run under a different uid and/or in a sandbox). */
- std::string netrcData;
- try {
- if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
- netrcData = readFile(settings.netrcFile);
- } catch (SysError &) { }
-
-#if __linux__
- if (useChroot) {
-
- userNamespaceSync.writeSide = -1;
-
- if (drainFD(userNamespaceSync.readSide.get()) != "1")
- throw Error("user namespace initialisation failed");
-
- userNamespaceSync.readSide = -1;
-
- if (privateNetwork) {
-
- /* Initialise the loopback interface. */
- AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
- if (!fd) throw SysError("cannot open IP socket");
-
- struct ifreq ifr;
- strcpy(ifr.ifr_name, "lo");
- ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
- if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1)
- throw SysError("cannot set loopback interface flags");
- }
-
- /* Set the hostname etc. to fixed values. */
- char hostname[] = "localhost";
- if (sethostname(hostname, sizeof(hostname)) == -1)
- throw SysError("cannot set host name");
- char domainname[] = "(none)"; // kernel default
- if (setdomainname(domainname, sizeof(domainname)) == -1)
- throw SysError("cannot set domain name");
-
- /* Make all filesystems private. This is necessary
- because subtrees may have been mounted as "shared"
- (MS_SHARED). (Systemd does this, for instance.) Even
- though we have a private mount namespace, mounting
- filesystems on top of a shared subtree still propagates
- outside of the namespace. Making a subtree private is
- local to the namespace, though, so setting MS_PRIVATE
- does not affect the outside world. */
- if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1)
- throw SysError("unable to make '/' private");
-
- /* Bind-mount chroot directory to itself, to treat it as a
- different filesystem from /, as needed for pivot_root. */
- if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1)
- throw SysError("unable to bind mount '%1%'", chrootRootDir);
-
- /* Bind-mount the sandbox's Nix store onto itself so that
- we can mark it as a "shared" subtree, allowing bind
- mounts made in *this* mount namespace to be propagated
- into the child namespace created by the
- unshare(CLONE_NEWNS) call below.
-
- Marking chrootRootDir as MS_SHARED causes pivot_root()
- to fail with EINVAL. Don't know why. */
- Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
-
- if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1)
- throw SysError("unable to bind mount the Nix store", chrootStoreDir);
-
- if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1)
- throw SysError("unable to make '%s' shared", chrootStoreDir);
-
- /* Set up a nearly empty /dev, unless the user asked to
- bind-mount the host /dev. */
- Strings ss;
- if (dirsInChroot.find("/dev") == dirsInChroot.end()) {
- createDirs(chrootRootDir + "/dev/shm");
- createDirs(chrootRootDir + "/dev/pts");
- ss.push_back("/dev/full");
- if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
- ss.push_back("/dev/kvm");
- ss.push_back("/dev/null");
- ss.push_back("/dev/random");
- ss.push_back("/dev/tty");
- ss.push_back("/dev/urandom");
- ss.push_back("/dev/zero");
- createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd");
- createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin");
- createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout");
- createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr");
- }
-
- /* Fixed-output derivations typically need to access the
- network, so give them access to /etc/resolv.conf and so
- on. */
- if (derivationIsImpure(derivationType)) {
- ss.push_back("/etc/resolv.conf");
-
- // Only use nss functions to resolve hosts and
- // services. Don’t use it for anything else that may
- // be configured for this system. This limits the
- // potential impurities introduced in fixed-outputs.
- writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
-
- ss.push_back("/etc/services");
- ss.push_back("/etc/hosts");
- if (pathExists("/var/run/nscd/socket"))
- ss.push_back("/var/run/nscd/socket");
- }
-
- for (auto & i : ss) dirsInChroot.emplace(i, i);
-
- /* Bind-mount all the directories from the "host"
- filesystem that we want in the chroot
- environment. */
- auto doBind = [&](const Path & source, const Path & target, bool optional = false) {
- debug("bind mounting '%1%' to '%2%'", source, target);
- struct stat st;
- if (stat(source.c_str(), &st) == -1) {
- if (optional && errno == ENOENT)
- return;
- else
- throw SysError("getting attributes of path '%1%'", source);
- }
- if (S_ISDIR(st.st_mode))
- createDirs(target);
- else {
- createDirs(dirOf(target));
- writeFile(target, "");
- }
- if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
- throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
- };
-
- for (auto & i : dirsInChroot) {
- if (i.second.source == "/proc") continue; // backwards compatibility
- doBind(i.second.source, chrootRootDir + i.first, i.second.optional);
- }
-
- /* Bind a new instance of procfs on /proc. */
- createDirs(chrootRootDir + "/proc");
- if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
- throw SysError("mounting /proc");
-
- /* Mount sysfs on /sys. */
- if (useUidRange) {
- createDirs(chrootRootDir + "/sys");
- if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1)
- throw SysError("mounting /sys");
- }
-
- /* Mount a new tmpfs on /dev/shm to ensure that whatever
- the builder puts in /dev/shm is cleaned up automatically. */
- if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0,
- fmt("size=%s", settings.sandboxShmSize).c_str()) == -1)
- throw SysError("mounting /dev/shm");
-
- /* Mount a new devpts on /dev/pts. Note that this
- requires the kernel to be compiled with
- CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case
- if /dev/ptx/ptmx exists). */
- if (pathExists("/dev/pts/ptmx") &&
- !pathExists(chrootRootDir + "/dev/ptmx")
- && !dirsInChroot.count("/dev/pts"))
- {
- if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0)
- {
- createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx");
-
- /* Make sure /dev/pts/ptmx is world-writable. With some
- Linux versions, it is created with permissions 0. */
- chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
- } else {
- if (errno != EINVAL)
- throw SysError("mounting /dev/pts");
- doBind("/dev/pts", chrootRootDir + "/dev/pts");
- doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx");
- }
- }
-
- /* Unshare this mount namespace. This is necessary because
- pivot_root() below changes the root of the mount
- namespace. This means that the call to setns() in
- addDependency() would hide the host's filesystem,
- making it impossible to bind-mount paths from the host
- Nix store into the sandbox. Therefore, we save the
- pre-pivot_root namespace in
- sandboxMountNamespace. Since we made /nix/store a
- shared subtree above, this allows addDependency() to
- make paths appear in the sandbox. */
- if (unshare(CLONE_NEWNS) == -1)
- throw SysError("unsharing mount namespace");
-
- /* Unshare the cgroup namespace. This means
- /proc/self/cgroup will show the child's cgroup as '/'
- rather than whatever it is in the parent. */
- if (useSystemdCgroup && unshare(CLONE_NEWCGROUP) == -1)
- throw SysError("unsharing cgroup namespace");
-
- /* Do the chroot(). */
- if (chdir(chrootRootDir.c_str()) == -1)
- throw SysError("cannot change directory to '%1%'", chrootRootDir);
-
- if (mkdir("real-root", 0) == -1)
- throw SysError("cannot create real-root directory");
-
- if (pivot_root(".", "real-root") == -1)
- throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root"));
-
- if (chroot(".") == -1)
- throw SysError("cannot change root directory to '%1%'", chrootRootDir);
-
- if (umount2("real-root", MNT_DETACH) == -1)
- throw SysError("cannot unmount real root filesystem");
-
- if (rmdir("real-root") == -1)
- throw SysError("cannot remove real-root directory");
-
- /* Switch to the sandbox uid/gid in the user namespace,
- which corresponds to the build user or calling user in
- the parent namespace. */
- if (setgid(sandboxGid()) == -1)
- throw SysError("setgid failed");
- if (setuid(sandboxUid()) == -1)
- throw SysError("setuid failed");
-
- setUser = false;
- }
-#endif
-
- if (chdir(tmpDirInSandbox.c_str()) == -1)
- throw SysError("changing into '%1%'", tmpDir);
-
- /* Close all other file descriptors. */
- closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO});
-
-#if __linux__
- /* Change the personality to 32-bit if we're doing an
- i686-linux build on an x86_64-linux machine. */
- struct utsname utsbuf;
- uname(&utsbuf);
- if (drv->platform == "i686-linux" &&
- (settings.thisSystem == "x86_64-linux" ||
- (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
- if (personality(PER_LINUX32) == -1)
- throw SysError("cannot set i686-linux personality");
- }
-
- /* Impersonate a Linux 2.6 machine to get some determinism in
- builds that depend on the kernel version. */
- if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) {
- int cur = personality(0xffffffff);
- if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
- }
-
- /* Disable address space randomization for improved
- determinism. */
- int cur = personality(0xffffffff);
- if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
-#endif
-
- /* Disable core dumps by default. */
- struct rlimit limit = { 0, RLIM_INFINITY };
- setrlimit(RLIMIT_CORE, &limit);
-
- // FIXME: set other limits to deterministic values?
-
- /* Fill in the environment. */
- Strings envStrs;
- for (auto & i : env)
- envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites));
-
- /* If we are running in `build-users' mode, then switch to the
- user we allocated above. Make sure that we drop all root
- privileges. Note that above we have closed all file
- descriptors except std*, so that's safe. Also note that
- setuid() when run as root sets the real, effective and
- saved UIDs. */
- if (setUser && buildUser) {
- /* Preserve supplementary groups of the build user, to allow
- admins to specify groups such as "kvm". */
- if (!buildUser->getSupplementaryGIDs().empty() &&
- setgroups(buildUser->getSupplementaryGIDs().size(),
- buildUser->getSupplementaryGIDs().data()) == -1)
- throw SysError("cannot set supplementary groups of build user");
-
- if (setgid(buildUser->getGID()) == -1 ||
- getgid() != buildUser->getGID() ||
- getegid() != buildUser->getGID())
- throw SysError("setgid failed");
-
- if (setuid(buildUser->getUID()) == -1 ||
- getuid() != buildUser->getUID() ||
- geteuid() != buildUser->getUID())
- throw SysError("setuid failed");
- }
-
- /* Fill in the arguments. */
- Strings args;
-
- const char *builder = "invalid";
-
- if (drv->isBuiltin()) {
- ;
- }
-#if __APPLE__
- else {
- /* This has to appear before import statements. */
- std::string sandboxProfile = "(version 1)\n";
-
- if (useChroot) {
-
- /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
- PathSet ancestry;
-
- /* We build the ancestry before adding all inputPaths to the store because we know they'll
- all have the same parents (the store), and there might be lots of inputs. This isn't
- particularly efficient... I doubt it'll be a bottleneck in practice */
- for (auto & i : dirsInChroot) {
- Path cur = i.first;
- while (cur.compare("/") != 0) {
- cur = dirOf(cur);
- ancestry.insert(cur);
- }
- }
-
- /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost
- path component this time, since it's typically /nix/store and we care about that. */
- Path cur = worker.store.storeDir;
- while (cur.compare("/") != 0) {
- ancestry.insert(cur);
- cur = dirOf(cur);
- }
-
- /* Add all our input paths to the chroot */
- for (auto & i : inputPaths) {
- auto p = worker.store.printStorePath(i);
- dirsInChroot[p] = p;
- }
-
- /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
- if (settings.darwinLogSandboxViolations) {
- sandboxProfile += "(deny default)\n";
- } else {
- sandboxProfile += "(deny default (with no-log))\n";
- }
-
- sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
-
- if (derivationIsImpure(derivationType))
- sandboxProfile += "(import \"sandbox-network.sb\")\n";
-
- /* Add the output paths we'll use at build-time to the chroot */
- sandboxProfile += "(allow file-read* file-write* process-exec\n";
- for (auto & [_, path] : scratchOutputs)
- sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path));
-
- sandboxProfile += ")\n";
-
- /* Our inputs (transitive dependencies and any impurities computed above)
-
- without file-write* allowed, access() incorrectly returns EPERM
- */
- sandboxProfile += "(allow file-read* file-write* process-exec\n";
- for (auto & i : dirsInChroot) {
- if (i.first != i.second.source)
- throw Error(
- "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
- i.first, i.second.source);
-
- string path = i.first;
- struct stat st;
- if (lstat(path.c_str(), &st)) {
- if (i.second.optional && errno == ENOENT)
- continue;
- throw SysError("getting attributes of path '%s", path);
- }
- if (S_ISDIR(st.st_mode))
- sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
- else
- sandboxProfile += fmt("\t(literal \"%s\")\n", path);
- }
- sandboxProfile += ")\n";
-
- /* Allow file-read* on full directory hierarchy to self. Allows realpath() */
- sandboxProfile += "(allow file-read*\n";
- for (auto & i : ancestry) {
- sandboxProfile += fmt("\t(literal \"%s\")\n", i);
- }
- sandboxProfile += ")\n";
-
- sandboxProfile += additionalSandboxProfile;
- } else
- sandboxProfile += "(import \"sandbox-minimal.sb\")\n";
-
- debug("Generated sandbox profile:");
- debug(sandboxProfile);
-
- Path sandboxFile = tmpDir + "/.sandbox.sb";
-
- writeFile(sandboxFile, sandboxProfile);
-
- bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
-
- /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
- to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
- Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true);
-
- /* They don't like trailing slashes on subpath directives */
- if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
-
- if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
- builder = "/usr/bin/sandbox-exec";
- args.push_back("sandbox-exec");
- args.push_back("-f");
- args.push_back(sandboxFile);
- args.push_back("-D");
- args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
- args.push_back("-D");
- args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
- if (allowLocalNetworking) {
- args.push_back("-D");
- args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
- }
- args.push_back(drv->builder);
- } else {
- builder = drv->builder.c_str();
- args.push_back(std::string(baseNameOf(drv->builder)));
- }
- }
-#else
- else {
- builder = drv->builder.c_str();
- args.push_back(std::string(baseNameOf(drv->builder)));
- }
-#endif
-
- for (auto & i : drv->args)
- args.push_back(rewriteStrings(i, inputRewrites));
-
- /* Indicate that we managed to set up the build environment. */
- writeFull(STDERR_FILENO, string("\2\n"));
-
- /* Execute the program. This should not return. */
- if (drv->isBuiltin()) {
- try {
- logger = makeJSONLogger(*logger);
-
- BasicDerivation & drv2(*drv);
- for (auto & e : drv2.env)
- e.second = rewriteStrings(e.second, inputRewrites);
-
- if (drv->builder == "builtin:fetchurl")
- builtinFetchurl(drv2, netrcData);
- else if (drv->builder == "builtin:buildenv")
- builtinBuildenv(drv2);
- else if (drv->builder == "builtin:unpack-channel")
- builtinUnpackChannel(drv2);
- else
- throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
- _exit(0);
- } catch (std::exception & e) {
- writeFull(STDERR_FILENO, e.what() + std::string("\n"));
- _exit(1);
- }
- }
-
- execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
-
- throw SysError("executing '%1%'", drv->builder);
-
- } catch (Error & e) {
- writeFull(STDERR_FILENO, "\1\n");
- FdSink sink(STDERR_FILENO);
- sink << e;
- sink.flush();
- _exit(1);
- }
-}
-
void DerivationGoal::registerOutputs()
{
@@ -2903,679 +1092,23 @@ void DerivationGoal::registerOutputs()
We can only early return when the outputs are known a priori. For
floating content-addressed derivations this isn't the case.
*/
- if (hook) {
- bool allValid = true;
- for (auto & i : drv->outputsAndOptPaths(worker.store)) {
- if (!i.second.second || !worker.store.isValidPath(*i.second.second))
- allValid = false;
- }
- if (allValid) return;
- }
-
- std::map<std::string, ValidPathInfo> infos;
-
- /* Set of inodes seen during calls to canonicalisePathMetaData()
- for this build's outputs. This needs to be shared between
- outputs to allow hard links between outputs. */
- InodesSeen inodesSeen;
-
- Path checkSuffix = ".check";
- bool keepPreviousRound = settings.keepFailed || settings.runDiffHook;
-
- std::exception_ptr delayedException;
-
- /* The paths that can be referenced are the input closures, the
- output paths, and any paths that have been built via recursive
- Nix calls. */
- StorePathSet referenceablePaths;
- for (auto & p : inputPaths) referenceablePaths.insert(p);
- for (auto & i : scratchOutputs) referenceablePaths.insert(i.second);
- for (auto & p : addedPaths) referenceablePaths.insert(p);
-
- /* FIXME `needsHashRewrite` should probably be removed and we get to the
- real reason why we aren't using the chroot dir */
- auto toRealPathChroot = [&](const Path & p) -> Path {
- return useChroot && !needsHashRewrite()
- ? chrootRootDir + p
- : worker.store.toRealPath(p);
- };
-
- /* Check whether the output paths were created, and make all
- output paths read-only. Then get the references of each output (that we
- might need to register), so we can topologically sort them. For the ones
- that are most definitely already installed, we just store their final
- name so we can also use it in rewrites. */
- StringSet outputsToSort;
- struct AlreadyRegistered { StorePath path; };
- struct PerhapsNeedToRegister { StorePathSet refs; };
- std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
- std::map<std::string, struct stat> outputStats;
- for (auto & [outputName, _] : drv->outputs) {
- auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
-
- outputsToSort.insert(outputName);
-
- /* Updated wanted info to remove the outputs we definitely don't need to register */
- auto & initialInfo = initialOutputs.at(outputName);
-
- /* Don't register if already valid, and not checking */
- initialInfo.wanted = buildMode == bmCheck
- || !(initialInfo.known && initialInfo.known->isValid());
- if (!initialInfo.wanted) {
- outputReferencesIfUnregistered.insert_or_assign(
- outputName,
- AlreadyRegistered { .path = initialInfo.known->path });
+ for (auto & [outputName, optOutputPath] : worker.store.queryPartialDerivationOutputMap(drvPath)) {
+ if (!wantOutput(outputName, wantedOutputs))
continue;
- }
-
- struct stat st;
- if (lstat(actualPath.c_str(), &st) == -1) {
- if (errno == ENOENT)
- throw BuildError(
- "builder for '%s' failed to produce output path for output '%s' at '%s'",
- worker.store.printStorePath(drvPath), outputName, actualPath);
- throw SysError("getting attributes of path '%s'", actualPath);
- }
-
-#ifndef __CYGWIN__
- /* Check that the output is not group or world writable, as
- that means that someone else can have interfered with the
- build. Also, the output should be owned by the build
- user. */
- if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
- (buildUser && st.st_uid != buildUser->getUID()))
+ if (!optOutputPath)
throw BuildError(
- "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output",
- actualPath, outputName);
-#endif
-
- /* Canonicalise first. This ensures that the path we're
- rewriting doesn't contain a hard link to /etc/shadow or
- something like that. */
- canonicalisePathMetaData(
- actualPath,
- buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt,
- inodesSeen);
-
- debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
-
- /* Pass blank Sink as we are not ready to hash data at this stage. */
- NullSink blank;
- auto references = worker.store.parseStorePathSet(
- scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
-
- outputReferencesIfUnregistered.insert_or_assign(
- outputName,
- PerhapsNeedToRegister { .refs = references });
- outputStats.insert_or_assign(outputName, std::move(st));
- }
-
- auto sortedOutputNames = topoSort(outputsToSort,
- {[&](const std::string & name) {
- return std::visit(overloaded {
- /* Since we'll use the already installed versions of these, we
- can treat them as leaves and ignore any references they
- have. */
- [&](AlreadyRegistered _) { return StringSet {}; },
- [&](PerhapsNeedToRegister refs) {
- StringSet referencedOutputs;
- /* FIXME build inverted map up front so no quadratic waste here */
- for (auto & r : refs.refs)
- for (auto & [o, p] : scratchOutputs)
- if (r == p)
- referencedOutputs.insert(o);
- return referencedOutputs;
- },
- }, outputReferencesIfUnregistered.at(name));
- }},
- {[&](const std::string & path, const std::string & parent) {
- // TODO with more -vvvv also show the temporary paths for manual inspection.
- return BuildError(
- "cycle detected in build of '%s' in the references of output '%s' from output '%s'",
- worker.store.printStorePath(drvPath), path, parent);
- }});
-
- std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
-
- for (auto & outputName : sortedOutputNames) {
- auto output = drv->outputs.at(outputName);
- auto & scratchPath = scratchOutputs.at(outputName);
- auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath));
-
- auto finish = [&](StorePath finalStorePath) {
- /* Store the final path */
- finalOutputs.insert_or_assign(outputName, finalStorePath);
- /* The rewrite rule will be used in downstream outputs that refer to
- use. This is why the topological sort is essential to do first
- before this for loop. */
- if (scratchPath != finalStorePath)
- outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
- };
-
- std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
- [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
- finish(skippedFinalPath.path);
- return std::nullopt;
- },
- [&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> {
- return r.refs;
- },
- }, outputReferencesIfUnregistered.at(outputName));
-
- if (!referencesOpt)
- continue;
- auto references = *referencesOpt;
-
- auto rewriteOutput = [&]() {
- /* Apply hash rewriting if necessary. */
- if (!outputRewrites.empty()) {
- logWarning({
- .name = "Rewriting hashes",
- .hint = hintfmt("rewriting hashes in '%1%'; cross fingers", actualPath),
- });
-
- /* FIXME: this is in-memory. */
- StringSink sink;
- dumpPath(actualPath, sink);
- deletePath(actualPath);
- sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
- StringSource source(*sink.s);
- restorePath(actualPath, source);
-
- /* FIXME: set proper permissions in restorePath() so
- we don't have to do another traversal. */
- canonicalisePathMetaData(actualPath, {}, inodesSeen);
- }
- };
-
- auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> {
- /* In the CA case, we need the rewritten refs to calculate the
- final path, therefore we look for a *non-rewritten
- self-reference, and use a bool rather try to solve the
- computationally intractable fixed point. */
- std::pair<bool, StorePathSet> res {
- false,
- {},
- };
- for (auto & r : references) {
- auto name = r.name();
- auto origHash = std::string { r.hashPart() };
- if (r == scratchPath)
- res.first = true;
- else if (outputRewrites.count(origHash) == 0)
- res.second.insert(r);
- else {
- std::string newRef = outputRewrites.at(origHash);
- newRef += '-';
- newRef += name;
- res.second.insert(StorePath { newRef });
- }
- }
- return res;
- };
-
- auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
- auto & st = outputStats.at(outputName);
- if (outputHash.method == FileIngestionMethod::Flat) {
- /* The output path should be a regular file without execute permission. */
- if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
- throw BuildError(
- "output path '%1%' should be a non-executable regular file "
- "since recursive hashing is not enabled (outputHashMode=flat)",
- actualPath);
- }
- rewriteOutput();
- /* FIXME optimize and deduplicate with addToStore */
- std::string oldHashPart { scratchPath.hashPart() };
- HashModuloSink caSink { outputHash.hashType, oldHashPart };
- switch (outputHash.method) {
- case FileIngestionMethod::Recursive:
- dumpPath(actualPath, caSink);
- break;
- case FileIngestionMethod::Flat:
- readFile(actualPath, caSink);
- break;
- }
- auto got = caSink.finish().first;
- auto refs = rewriteRefs();
- HashModuloSink narSink { htSHA256, oldHashPart };
- dumpPath(actualPath, narSink);
- auto narHashAndSize = narSink.finish();
- ValidPathInfo newInfo0 {
- worker.store.makeFixedOutputPath(
- outputHash.method,
- got,
- outputPathName(drv->name, outputName),
- refs.second,
- refs.first),
- narHashAndSize.first,
- };
- newInfo0.narSize = narHashAndSize.second;
- newInfo0.ca = FixedOutputHash {
- .method = outputHash.method,
- .hash = got,
- };
- newInfo0.references = refs.second;
- if (refs.first)
- newInfo0.references.insert(newInfo0.path);
-
- assert(newInfo0.ca);
- return newInfo0;
- };
-
- ValidPathInfo newInfo = std::visit(overloaded {
- [&](DerivationOutputInputAddressed output) {
- /* input-addressed case */
- auto requiredFinalPath = output.path;
- /* Preemptively add rewrite rule for final hash, as that is
- what the NAR hash will use rather than normalized-self references */
- if (scratchPath != requiredFinalPath)
- outputRewrites.insert_or_assign(
- std::string { scratchPath.hashPart() },
- std::string { requiredFinalPath.hashPart() });
- rewriteOutput();
- auto narHashAndSize = hashPath(htSHA256, actualPath);
- ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
- newInfo0.narSize = narHashAndSize.second;
- auto refs = rewriteRefs();
- newInfo0.references = refs.second;
- if (refs.first)
- newInfo0.references.insert(newInfo0.path);
- return newInfo0;
- },
- [&](DerivationOutputCAFixed dof) {
- auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
- .method = dof.hash.method,
- .hashType = dof.hash.hash.type,
- });
-
- /* Check wanted hash */
- Hash & wanted = dof.hash.hash;
- assert(newInfo0.ca);
- auto got = getContentAddressHash(*newInfo0.ca);
- if (wanted != got) {
- /* Throw an error after registering the path as
- valid. */
- worker.hashMismatch = true;
- delayedException = std::make_exception_ptr(
- BuildError("hash mismatch in fixed-output derivation '%s':\n wanted: %s\n got: %s",
- worker.store.printStorePath(drvPath),
- wanted.to_string(SRI, true),
- got.to_string(SRI, true)));
- }
- return newInfo0;
- },
- [&](DerivationOutputCAFloating dof) {
- return newInfoFromCA(dof);
- },
- }, output.output);
-
- /* Calculate where we'll move the output files. In the checking case we
- will leave leave them where they are, for now, rather than move to
- their usual "final destination" */
- auto finalDestPath = worker.store.printStorePath(newInfo.path);
-
- /* Lock final output path, if not already locked. This happens with
- floating CA derivations and hash-mismatching fixed-output
- derivations. */
- PathLocks dynamicOutputLock;
- auto optFixedPath = output.path(worker.store, drv->name, outputName);
- if (!optFixedPath ||
- worker.store.printStorePath(*optFixedPath) != finalDestPath)
- {
- assert(newInfo.ca);
- dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)});
- }
-
- /* Move files, if needed */
- if (worker.store.toRealPath(finalDestPath) != actualPath) {
- if (buildMode == bmRepair) {
- /* Path already exists, need to replace it */
- replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath);
- actualPath = worker.store.toRealPath(finalDestPath);
- } else if (buildMode == bmCheck) {
- /* Path already exists, and we want to compare, so we leave out
- new path in place. */
- } else if (worker.store.isValidPath(newInfo.path)) {
- /* Path already exists because CA path produced by something
- else. No moving needed. */
- assert(newInfo.ca);
- } else {
- auto destPath = worker.store.toRealPath(finalDestPath);
- movePath(actualPath, destPath);
- actualPath = destPath;
- }
- }
-
- if (buildMode == bmCheck) {
- if (!worker.store.isValidPath(newInfo.path)) continue;
- ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
- if (newInfo.narHash != oldInfo.narHash) {
- worker.checkMismatch = true;
- if (settings.runDiffHook || settings.keepFailed) {
- auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
- deletePath(dst);
- movePath(actualPath, dst);
-
- handleDiffHook(
- buildUser ? buildUser->getUID() : getuid(),
- buildUser ? buildUser->getGID() : getgid(),
- finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir);
-
- throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
- worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst);
- } else
- throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
- worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath));
- }
-
- /* Since we verified the build, it's now ultimately trusted. */
- if (!oldInfo.ultimate) {
- oldInfo.ultimate = true;
- worker.store.signPathInfo(oldInfo);
- worker.store.registerValidPaths({ std::move(oldInfo) });
- }
-
- continue;
- }
-
- /* For debugging, print out the referenced and unreferenced paths. */
- for (auto & i : inputPaths) {
- auto j = references.find(i);
- if (j == references.end())
- debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
- else
- debug("referenced input: '%1%'", worker.store.printStorePath(i));
- }
-
- if (curRound == nrRounds) {
- worker.store.optimisePath(actualPath); // FIXME: combine with scanForReferences()
- worker.markContentsGood(newInfo.path);
- }
-
- newInfo.deriver = drvPath;
- newInfo.ultimate = true;
- worker.store.signPathInfo(newInfo);
-
- finish(newInfo.path);
-
- /* If it's a CA path, register it right away. This is necessary if it
- isn't statically known so that we can safely unlock the path before
- the next iteration */
- if (newInfo.ca)
- worker.store.registerValidPaths({newInfo});
-
- infos.emplace(outputName, std::move(newInfo));
- }
-
- if (buildMode == bmCheck) return;
-
- /* Apply output checks. */
- checkOutputs(infos);
-
- /* Compare the result with the previous round, and report which
- path is different, if any.*/
- if (curRound > 1 && prevInfos != infos) {
- assert(prevInfos.size() == infos.size());
- for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
- if (!(*i == *j)) {
- result.isNonDeterministic = true;
- Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
- bool prevExists = keepPreviousRound && pathExists(prev);
- hintformat hint = prevExists
- ? hintfmt("output '%s' of '%s' differs from '%s' from previous round",
- worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev)
- : hintfmt("output '%s' of '%s' differs from previous round",
- worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath));
-
- handleDiffHook(
- buildUser ? buildUser->getUID() : getuid(),
- buildUser ? buildUser->getGID() : getgid(),
- prev, worker.store.printStorePath(i->second.path),
- worker.store.printStorePath(drvPath), tmpDir);
-
- if (settings.enforceDeterminism)
- throw NotDeterministic(hint);
-
- logError({
- .name = "Output determinism error",
- .hint = hint
- });
-
- curRound = nrRounds; // we know enough, bail out early
- }
- }
-
- /* If this is the first round of several, then move the output out of the way. */
- if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
- for (auto & [_, outputStorePath] : finalOutputs) {
- auto path = worker.store.printStorePath(outputStorePath);
- Path prev = path + checkSuffix;
- deletePath(prev);
- Path dst = path + checkSuffix;
- if (rename(path.c_str(), dst.c_str()))
- throw SysError("renaming '%s' to '%s'", path, dst);
- }
- }
-
- if (curRound < nrRounds) {
- prevInfos = std::move(infos);
- return;
- }
-
- /* Remove the .check directories if we're done. FIXME: keep them
- if the result was not determistic? */
- if (curRound == nrRounds) {
- for (auto & [_, outputStorePath] : finalOutputs) {
- Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix;
- deletePath(prev);
- }
- }
-
- /* Register each output path as valid, and register the sets of
- paths referenced by each of them. If there are cycles in the
- outputs, this will fail. */
- {
- ValidPathInfos infos2;
- for (auto & [outputName, newInfo] : infos) {
- infos2.push_back(newInfo);
- }
- worker.store.registerValidPaths(infos2);
- }
-
- /* In case of a fixed-output derivation hash mismatch, throw an
- exception now that we have registered the output as valid. */
- if (delayedException)
- std::rethrow_exception(delayedException);
-
- /* If we made it this far, we are sure the output matches the derivation
- (since the delayedException would be a fixed output CA mismatch). That
- means it's safe to link the derivation to the output hash. We must do
- that for floating CA derivations, which otherwise couldn't be cached,
- but it's fine to do in all cases. */
- bool isCaFloating = drv->type() == DerivationType::CAFloating;
-
- auto drvPathResolved = drvPath;
- if (!useDerivation && isCaFloating) {
- /* Once a floating CA derivations reaches this point, it
- must already be resolved, so we don't bother trying to
- downcast drv to get would would just be an empty
- inputDrvs field. */
- Derivation drv2 { *drv };
- drvPathResolved = writeDerivation(worker.store, drv2);
- }
-
- if (useDerivation || isCaFloating)
- for (auto & [outputName, newInfo] : infos)
- worker.store.linkDeriverToPath(drvPathResolved, outputName, newInfo.path);
-}
-
-
-void DerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
-{
- std::map<Path, const ValidPathInfo &> outputsByPath;
- for (auto & output : outputs)
- outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second);
-
- for (auto & output : outputs) {
- auto & outputName = output.first;
- auto & info = output.second;
-
- struct Checks
- {
- bool ignoreSelfRefs = false;
- std::optional<uint64_t> maxSize, maxClosureSize;
- std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
- };
-
- /* Compute the closure and closure size of some output. This
- is slightly tricky because some of its references (namely
- other outputs) may not be valid yet. */
- auto getClosure = [&](const StorePath & path)
- {
- uint64_t closureSize = 0;
- StorePathSet pathsDone;
- std::queue<StorePath> pathsLeft;
- pathsLeft.push(path);
-
- while (!pathsLeft.empty()) {
- auto path = pathsLeft.front();
- pathsLeft.pop();
- if (!pathsDone.insert(path).second) continue;
-
- auto i = outputsByPath.find(worker.store.printStorePath(path));
- if (i != outputsByPath.end()) {
- closureSize += i->second.narSize;
- for (auto & ref : i->second.references)
- pathsLeft.push(ref);
- } else {
- auto info = worker.store.queryPathInfo(path);
- closureSize += info->narSize;
- for (auto & ref : info->references)
- pathsLeft.push(ref);
- }
- }
-
- return std::make_pair(std::move(pathsDone), closureSize);
- };
-
- auto applyChecks = [&](const Checks & checks)
- {
- if (checks.maxSize && info.narSize > *checks.maxSize)
- throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
- worker.store.printStorePath(info.path), info.narSize, *checks.maxSize);
-
- if (checks.maxClosureSize) {
- uint64_t closureSize = getClosure(info.path).second;
- if (closureSize > *checks.maxClosureSize)
- throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
- worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize);
- }
-
- auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
- {
- if (!value) return;
-
- /* Parse a list of reference specifiers. Each element must
- either be a store path, or the symbolic name of the output
- of the derivation (such as `out'). */
- StorePathSet spec;
- for (auto & i : *value) {
- if (worker.store.isStorePath(i))
- spec.insert(worker.store.parseStorePath(i));
- else if (finalOutputs.count(i))
- spec.insert(finalOutputs.at(i));
- else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
- }
-
- auto used = recursive
- ? getClosure(info.path).first
- : info.references;
-
- if (recursive && checks.ignoreSelfRefs)
- used.erase(info.path);
-
- StorePathSet badPaths;
-
- for (auto & i : used)
- if (allowed) {
- if (!spec.count(i))
- badPaths.insert(i);
- } else {
- if (spec.count(i))
- badPaths.insert(i);
- }
-
- if (!badPaths.empty()) {
- string badPathsStr;
- for (auto & i : badPaths) {
- badPathsStr += "\n ";
- badPathsStr += worker.store.printStorePath(i);
- }
- throw BuildError("output '%s' is not allowed to refer to the following paths:%s",
- worker.store.printStorePath(info.path), badPathsStr);
- }
- };
-
- checkRefs(checks.allowedReferences, true, false);
- checkRefs(checks.allowedRequisites, true, true);
- checkRefs(checks.disallowedReferences, false, false);
- checkRefs(checks.disallowedRequisites, false, true);
- };
-
- if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
- auto outputChecks = structuredAttrs->find("outputChecks");
- if (outputChecks != structuredAttrs->end()) {
- auto output = outputChecks->find(outputName);
-
- if (output != outputChecks->end()) {
- Checks checks;
-
- auto maxSize = output->find("maxSize");
- if (maxSize != output->end())
- checks.maxSize = maxSize->get<uint64_t>();
-
- auto maxClosureSize = output->find("maxClosureSize");
- if (maxClosureSize != output->end())
- checks.maxClosureSize = maxClosureSize->get<uint64_t>();
-
- auto get = [&](const std::string & name) -> std::optional<Strings> {
- auto i = output->find(name);
- if (i != output->end()) {
- Strings res;
- for (auto j = i->begin(); j != i->end(); ++j) {
- if (!j->is_string())
- throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath));
- res.push_back(j->get<std::string>());
- }
- checks.disallowedRequisites = res;
- return res;
- }
- return {};
- };
-
- checks.allowedReferences = get("allowedReferences");
- checks.allowedRequisites = get("allowedRequisites");
- checks.disallowedReferences = get("disallowedReferences");
- checks.disallowedRequisites = get("disallowedRequisites");
+ "output '%s' from derivation '%s' does not have a known output path",
+ outputName, worker.store.printStorePath(drvPath));
+ auto & outputPath = *optOutputPath;
+ if (!worker.store.isValidPath(outputPath))
+ throw BuildError(
+ "output '%s' from derivation '%s' is supposed to be at '%s' but that path is not valid",
+ outputName, worker.store.printStorePath(drvPath), worker.store.printStorePath(outputPath));
- applyChecks(checks);
- }
- }
- } else {
- // legacy non-structured-attributes case
- Checks checks;
- checks.ignoreSelfRefs = true;
- checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
- checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
- checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
- checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
- applyChecks(checks);
- }
+ finalOutputs.insert_or_assign(outputName, outputPath);
}
}
-
Path DerivationGoal::openLogFile()
{
logSize = 0;
@@ -3585,7 +1118,12 @@ Path DerivationGoal::openLogFile()
auto baseName = std::string(baseNameOf(worker.store.printStorePath(drvPath)));
/* Create a log file. */
- Path dir = fmt("%s/%s/%s/", worker.store.logDir, worker.store.drvsLogDir, string(baseName, 0, 2));
+ Path logDir;
+ if (auto localStore = dynamic_cast<LocalStore *>(&worker.store))
+ logDir = localStore->logDir;
+ else
+ logDir = settings.nixLogDir;
+ Path dir = fmt("%s/%s/%s/", logDir, LocalFSStore::drvsLogDir, string(baseName, 0, 2));
createDirs(dir);
Path logFileName = fmt("%s/%s%s", dir, string(baseName, 2),
@@ -3615,26 +1153,15 @@ void DerivationGoal::closeLogFile()
}
-void DerivationGoal::deleteTmpDir(bool force)
+bool DerivationGoal::isReadDesc(int fd)
{
- if (tmpDir != "") {
- /* Don't keep temporary directories for builtins because they
- might have privileged stuff (like a copy of netrc). */
- if (settings.keepFailed && !force && !drv->isBuiltin()) {
- printError("note: keeping build directory '%s'", tmpDir);
- chmod(tmpDir.c_str(), 0755);
- }
- else
- deletePath(tmpDir);
- tmpDir = "";
- }
+ return fd == hook->builderOut.readSide.get();
}
void DerivationGoal::handleChildOutput(int fd, const string & data)
{
- if ((hook && fd == hook->builderOut.readSide.get()) ||
- (!hook && fd == builderOut.readSide.get()))
+ if (isReadDesc(fd))
{
logSize += data.size();
if (settings.maxLogSize && logSize > settings.maxLogSize) {
@@ -3723,10 +1250,12 @@ OutputPathMap DerivationGoal::queryDerivationOutputMap()
void DerivationGoal::checkPathValidity()
{
bool checkHash = buildMode == bmRepair;
+ auto wantedOutputsLeft = wantedOutputs;
for (auto & i : queryPartialDerivationOutputMap()) {
- InitialOutput info {
- .wanted = wantOutput(i.first, wantedOutputs),
- };
+ InitialOutput & info = initialOutputs.at(i.first);
+ info.wanted = wantOutput(i.first, wantedOutputs);
+ if (info.wanted)
+ wantedOutputsLeft.erase(i.first);
if (i.second) {
auto outputPath = *i.second;
info.known = {
@@ -3738,24 +1267,23 @@ void DerivationGoal::checkPathValidity()
: PathStatus::Corrupt,
};
}
- initialOutputs.insert_or_assign(i.first, info);
+ if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+ if (auto real = worker.store.queryRealisation(
+ DrvOutput{initialOutputs.at(i.first).outputHash, i.first})) {
+ info.known = {
+ .path = real->outPath,
+ .status = PathStatus::Valid,
+ };
+ }
+ }
}
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(std::string_view outputName)
-{
- return worker.store.makeStorePath(
- "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
- Hash(htSHA256), outputPathName(drv->name, outputName));
-}
-
-
-StorePath DerivationGoal::makeFallbackPath(const StorePath & path)
-{
- return worker.store.makeStorePath(
- "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
- Hash(htSHA256), path.name());
+ // If we requested all the outputs via the empty set, we are always fine.
+ // If we requested specific elements, the loop above removes all the valid
+ // ones, so any that are left must be invalid.
+ if (!wantedOutputsLeft.empty())
+ throw Error("derivation '%s' does not have wanted outputs %s",
+ worker.store.printStorePath(drvPath),
+ concatStringsSep(", ", quoteStrings(wantedOutputsLeft)));
}
diff --git a/src/libstore/build/derivation-goal.hh b/src/libstore/build/derivation-goal.hh
index c32681b09..704b77caf 100644
--- a/src/libstore/build/derivation-goal.hh
+++ b/src/libstore/build/derivation-goal.hh
@@ -2,7 +2,8 @@
#include "parsed-derivations.hh"
#include "lock.hh"
-#include "local-store.hh"
+#include "store-api.hh"
+#include "pathlocks.hh"
#include "goal.hh"
namespace nix {
@@ -37,18 +38,21 @@ struct InitialOutputStatus {
struct InitialOutput {
bool wanted;
+ Hash outputHash;
std::optional<InitialOutputStatus> known;
};
-class DerivationGoal : public Goal
+struct DerivationGoal : public Goal
{
-private:
/* Whether to use an on-disk .drv file. */
bool useDerivation;
/* The path of the derivation. */
StorePath drvPath;
+ /* The path of the corresponding resolved derivation */
+ std::optional<BasicDerivation> resolvedDrv;
+
/* The specific outputs that we need to build. Empty means all of
them. */
StringSet wantedOutputs;
@@ -61,7 +65,7 @@ private:
bool retrySubstitution;
/* The derivation stored at drvPath. */
- std::unique_ptr<BasicDerivation> drv;
+ std::unique_ptr<Derivation> drv;
std::unique_ptr<ParsedDerivation> parsedDrv;
@@ -76,18 +80,6 @@ private:
std::map<std::string, InitialOutput> initialOutputs;
- /* User selected for running the builder. */
- std::unique_ptr<UserLock> buildUser;
-
- /* The process ID of the builder. */
- Pid pid;
-
- /* The temporary directory. */
- Path tmpDir;
-
- /* The path of the temporary directory in the sandbox. */
- Path tmpDirInSandbox;
-
/* File descriptor for the log file. */
AutoCloseFD fdLogFile;
std::shared_ptr<BufferedSink> logFileSink, logSink;
@@ -103,86 +95,15 @@ private:
std::string currentHookLine;
- /* Pipe for the builder's standard output/error. */
- Pipe builderOut;
-
- /* Pipe for synchronising updates to the builder namespaces. */
- Pipe userNamespaceSync;
-
- /* The mount namespace of the builder, used to add additional
- paths to the sandbox as a result of recursive Nix calls. */
- AutoCloseFD sandboxMountNamespace;
-
- /* On Linux, whether we're doing the build in its own user
- namespace. */
- bool usingUserNamespace = true;
-
/* The build hook. */
std::unique_ptr<HookInstance> hook;
- /* Whether we're currently doing a chroot build. */
- bool useChroot = false;
-
- Path chrootRootDir;
-
- /* Whether to give the build more than 1 UID. */
- bool useUidRange = false;
-
- /* Whether to make the 'systemd' cgroup controller available to
- the build. */
- bool useSystemdCgroup = false;
-
- /* RAII object to delete the chroot directory. */
- std::shared_ptr<AutoDelete> autoDelChroot;
-
/* The sort of derivation we are building. */
DerivationType derivationType;
- /* Whether to run the build in a private network namespace. */
- bool privateNetwork = false;
-
typedef void (DerivationGoal::*GoalState)();
GoalState state;
- /* Stuff we need to pass to initChild(). */
- struct ChrootPath {
- Path source;
- bool optional;
- ChrootPath(Path source = "", bool optional = false)
- : source(source), optional(optional)
- { }
- };
- typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
- DirsInChroot dirsInChroot;
-
- typedef map<string, string> Environment;
- Environment env;
-
-#if __APPLE__
- typedef string SandboxProfile;
- SandboxProfile additionalSandboxProfile;
-#endif
-
- /* Hash rewriting. */
- StringMap inputRewrites, outputRewrites;
- typedef map<StorePath, StorePath> RedirectedOutputs;
- RedirectedOutputs redirectedOutputs;
-
- /* The outputs paths used during the build.
-
- - Input-addressed derivations or fixed content-addressed outputs are
- sometimes built when some of their outputs already exist, and can not
- be hidden via sandboxing. We use temporary locations instead and
- rewrite after the build. Otherwise the regular predetermined paths are
- put here.
-
- - Floating content-addressed derivations do not know their final build
- output paths until the outputs are hashed, so random locations are
- used, and then renamed. The randomness helps guard against hidden
- self-references.
- */
- OutputPathMap scratchOutputs;
-
/* The final output paths of the build.
- For input-addressed derivations, always the precomputed paths
@@ -195,11 +116,6 @@ private:
BuildMode buildMode;
- /* If we're repairing without a chroot, there may be outputs that
- are valid but corrupt. So we redirect these outputs to
- temporary paths. */
- StorePathSet redirectedBadOutputs;
-
BuildResult result;
/* The current round, if we're building multiple times. */
@@ -207,17 +123,6 @@ private:
size_t nrRounds;
- /* Path registration info from the previous round, if we're
- building multiple times. Since this contains the hash, it
- allows us to compare whether two rounds produced the same
- result. */
- std::map<Path, ValidPathInfo> prevInfos;
-
- uid_t sandboxUid() { return usingUserNamespace ? (useUidRange ? 0 : 1000) : buildUser->getUID(); }
- gid_t sandboxGid() { return usingUserNamespace ? (useUidRange ? 0 : 100) : buildUser->getGID(); }
-
- const static Path homeDir;
-
std::unique_ptr<MaintainCount<uint64_t>> mcExpectedBuilds, mcRunningBuilds;
std::unique_ptr<Activity> act;
@@ -230,40 +135,13 @@ private:
/* The remote machine on which we're building. */
std::string machineName;
- /* The recursive Nix daemon socket. */
- AutoCloseFD daemonSocket;
-
- /* The daemon main thread. */
- std::thread daemonThread;
-
- /* The daemon worker threads. */
- std::vector<std::thread> daemonWorkerThreads;
-
- /* Paths that were added via recursive Nix calls. */
- StorePathSet addedPaths;
-
- /* Recursive Nix calls are only allowed to build or realize paths
- in the original input closure or added via a recursive Nix call
- (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
- /nix/store/<bla> is some arbitrary path in a binary cache). */
- bool isAllowed(const StorePath & path)
- {
- return inputPaths.count(path) || addedPaths.count(path);
- }
-
- friend struct RestrictedStore;
-
-public:
DerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
DerivationGoal(const StorePath & drvPath, const BasicDerivation & drv,
const StringSet & wantedOutputs, Worker & worker,
BuildMode buildMode = bmNormal);
- ~DerivationGoal();
-
- /* Whether we need to perform hash rewriting if there are valid output paths. */
- bool needsHashRewrite();
+ virtual ~DerivationGoal();
void timedOut(Error && ex) override;
@@ -271,17 +149,11 @@ public:
void work() override;
- StorePath getDrvPath()
- {
- return drvPath;
- }
-
/* Add wanted outputs to an already existing derivation goal. */
void addWantedOutputs(const StringSet & outputs);
BuildResult getResult() { return result; }
-private:
/* The states. */
void getDerivation();
void loadDerivation();
@@ -291,7 +163,7 @@ private:
void closureRepaired();
void inputsRealised();
void tryToBuild();
- void tryLocalBuild();
+ virtual void tryLocalBuild();
void buildDone();
void resolvedFinished();
@@ -299,51 +171,33 @@ private:
/* Is the build hook willing to perform the build? */
HookReply tryBuildHook();
- /* Start building a derivation. */
- void startBuilder();
-
- /* Fill in the environment for the builder. */
- void initEnv();
-
- /* Setup tmp dir location. */
- void initTmpDir();
-
- /* Write a JSON file containing the derivation attributes. */
- void writeStructuredAttrs();
-
- void startDaemon();
-
- void stopDaemon();
-
- /* Add 'path' to the set of paths that may be referenced by the
- outputs, and make it appear in the sandbox. */
- void addDependency(const StorePath & path);
-
- /* Make a file owned by the builder. */
- void chownToBuilder(const Path & path);
-
- /* Run the builder's process. */
- void runChild();
-
- friend int childEntry(void *);
+ virtual int getChildStatus();
/* Check that the derivation outputs all exist and register them
as valid. */
- void registerOutputs();
-
- /* Check that an output meets the requirements specified by the
- 'outputChecks' attribute (or the legacy
- '{allowed,disallowed}{References,Requisites}' attributes). */
- void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
+ virtual void registerOutputs();
/* Open a log file and a pipe to it. */
Path openLogFile();
+ /* Sign the newly built realisation if the store allows it */
+ virtual void signRealisation(Realisation&) {}
+
/* Close the log file. */
void closeLogFile();
- /* Delete the temporary directory, if we have one. */
- void deleteTmpDir(bool force);
+ /* Close the read side of the logger pipe. */
+ virtual void closeReadPipes();
+
+ /* Cleanup hooks for buildDone() */
+ virtual void cleanupHookFinally();
+ virtual void cleanupPreChildKill();
+ virtual void cleanupPostChildKill();
+ virtual bool cleanupDecideWhetherDiskFull();
+ virtual void cleanupPostOutputsRegisteredModeCheck();
+ virtual void cleanupPostOutputsRegisteredModeNonCheck();
+
+ virtual bool isReadDesc(int fd);
/* Callback used by the worker to write to the log. */
void handleChildOutput(int fd, const string & data) override;
@@ -360,17 +214,7 @@ private:
void checkPathValidity();
/* Forcibly kill the child process, if any. */
- void killChild();
-
- /* Create alternative path calculated from but distinct from the
- input, so we can avoid overwriting outputs (or other store paths)
- that already exist. */
- StorePath makeFallbackPath(const StorePath & path);
- /* Make a path to another based on the output name along with the
- derivation hash. */
- /* FIXME add option to randomize, so we can audit whether our
- rewrites caught everything */
- StorePath makeFallbackPath(std::string_view outputName);
+ virtual void killChild();
void repairClosure();
@@ -383,4 +227,6 @@ private:
StorePathSet exportReferences(const StorePathSet & storePaths);
};
+MakeError(NotDeterministic, BuildError);
+
}
diff --git a/src/libstore/build/drv-output-substitution-goal.cc b/src/libstore/build/drv-output-substitution-goal.cc
new file mode 100644
index 000000000..a5ac4c49d
--- /dev/null
+++ b/src/libstore/build/drv-output-substitution-goal.cc
@@ -0,0 +1,95 @@
+#include "drv-output-substitution-goal.hh"
+#include "worker.hh"
+#include "substitution-goal.hh"
+
+namespace nix {
+
+DrvOutputSubstitutionGoal::DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
+ : Goal(worker)
+ , id(id)
+{
+ state = &DrvOutputSubstitutionGoal::init;
+ name = fmt("substitution of '%s'", id.to_string());
+ trace("created");
+}
+
+
+void DrvOutputSubstitutionGoal::init()
+{
+ trace("init");
+ subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list<ref<Store>>();
+ tryNext();
+}
+
+void DrvOutputSubstitutionGoal::tryNext()
+{
+ trace("Trying next substituter");
+
+ if (subs.size() == 0) {
+ /* None left. Terminate this goal and let someone else deal
+ with it. */
+ debug("drv output '%s' is required, but there is no substituter that can provide it", id.to_string());
+
+ /* Hack: don't indicate failure if there were no substituters.
+ In that case the calling derivation should just do a
+ build. */
+ amDone(substituterFailed ? ecFailed : ecNoSubstituters);
+
+ if (substituterFailed) {
+ worker.failedSubstitutions++;
+ worker.updateProgress();
+ }
+
+ return;
+ }
+
+ auto sub = subs.front();
+ subs.pop_front();
+
+ // FIXME: Make async
+ outputInfo = sub->queryRealisation(id);
+ if (!outputInfo) {
+ tryNext();
+ return;
+ }
+
+ addWaitee(worker.makePathSubstitutionGoal(outputInfo->outPath));
+
+ if (waitees.empty()) outPathValid();
+ else state = &DrvOutputSubstitutionGoal::outPathValid;
+}
+
+void DrvOutputSubstitutionGoal::outPathValid()
+{
+ assert(outputInfo);
+ trace("Output path substituted");
+
+ if (nrFailed > 0) {
+ debug("The output path of the derivation output '%s' could not be substituted", id.to_string());
+ amDone(nrNoSubstituters > 0 || nrIncompleteClosure > 0 ? ecIncompleteClosure : ecFailed);
+ return;
+ }
+
+ worker.store.registerDrvOutput(*outputInfo);
+ finished();
+}
+
+void DrvOutputSubstitutionGoal::finished()
+{
+ trace("finished");
+ amDone(ecSuccess);
+}
+
+string DrvOutputSubstitutionGoal::key()
+{
+ /* "a$" ensures substitution goals happen before derivation
+ goals. */
+ return "a$" + std::string(id.to_string());
+}
+
+void DrvOutputSubstitutionGoal::work()
+{
+ (this->*state)();
+}
+
+}
diff --git a/src/libstore/build/drv-output-substitution-goal.hh b/src/libstore/build/drv-output-substitution-goal.hh
new file mode 100644
index 000000000..63ab53d89
--- /dev/null
+++ b/src/libstore/build/drv-output-substitution-goal.hh
@@ -0,0 +1,50 @@
+#pragma once
+
+#include "store-api.hh"
+#include "goal.hh"
+#include "realisation.hh"
+
+namespace nix {
+
+class Worker;
+
+// Substitution of a derivation output.
+// This is done in three steps:
+// 1. Fetch the output info from a substituter
+// 2. Substitute the corresponding output path
+// 3. Register the output info
+class DrvOutputSubstitutionGoal : public Goal {
+private:
+ // The drv output we're trying to substitue
+ DrvOutput id;
+
+ // The realisation corresponding to the given output id.
+ // Will be filled once we can get it.
+ std::optional<Realisation> outputInfo;
+
+ /* The remaining substituters. */
+ std::list<ref<Store>> subs;
+
+ /* Whether a substituter failed. */
+ bool substituterFailed = false;
+
+public:
+ DrvOutputSubstitutionGoal(const DrvOutput& id, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+
+ typedef void (DrvOutputSubstitutionGoal::*GoalState)();
+ GoalState state;
+
+ void init();
+ void tryNext();
+ void outPathValid();
+ void finished();
+
+ void timedOut(Error && ex) override { abort(); };
+
+ string key() override;
+
+ void work() override;
+
+};
+
+}
diff --git a/src/libstore/build/local-store-build.cc b/src/libstore/build/entry-points.cc
index a05fb5805..732d4785d 100644
--- a/src/libstore/build/local-store-build.cc
+++ b/src/libstore/build/entry-points.cc
@@ -2,34 +2,24 @@
#include "worker.hh"
#include "substitution-goal.hh"
#include "derivation-goal.hh"
+#include "local-store.hh"
namespace nix {
-static void primeCache(Store & store, const std::vector<StorePathWithOutputs> & paths)
-{
- StorePathSet willBuild, willSubstitute, unknown;
- uint64_t downloadSize, narSize;
- store.queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
-
- if (!willBuild.empty() && 0 == settings.maxBuildJobs && getMachines().empty())
- throw Error(
- "%d derivations need to be built, but neither local builds ('--max-jobs') "
- "nor remote builds ('--builders') are enabled", willBuild.size());
-}
-
-
-void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
+void Store::buildPaths(const std::vector<DerivedPath> & reqs, BuildMode buildMode)
{
Worker worker(*this);
- primeCache(*this, drvPaths);
-
Goals goals;
- for (auto & path : drvPaths) {
- if (path.path.isDerivation())
- goals.insert(worker.makeDerivationGoal(path.path, path.outputs, buildMode));
- else
- goals.insert(worker.makeSubstitutionGoal(path.path, buildMode == bmRepair ? Repair : NoRepair));
+ for (auto & br : reqs) {
+ std::visit(overloaded {
+ [&](DerivedPath::Built bfd) {
+ goals.insert(worker.makeDerivationGoal(bfd.drvPath, bfd.outputs, buildMode));
+ },
+ [&](DerivedPath::Opaque bo) {
+ goals.insert(worker.makePathSubstitutionGoal(bo.path, buildMode == bmRepair ? Repair : NoRepair));
+ },
+ }, br.raw());
}
worker.run(goals);
@@ -44,9 +34,8 @@ void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths,
ex = i->ex;
}
if (i->exitCode != Goal::ecSuccess) {
- DerivationGoal * i2 = dynamic_cast<DerivationGoal *>(i.get());
- if (i2) failed.insert(i2->getDrvPath());
- else failed.insert(dynamic_cast<SubstitutionGoal *>(i.get())->getStorePath());
+ if (auto i2 = dynamic_cast<DerivationGoal *>(i.get())) failed.insert(i2->drvPath);
+ else if (auto i2 = dynamic_cast<PathSubstitutionGoal *>(i.get())) failed.insert(i2->storePath);
}
}
@@ -59,7 +48,7 @@ void LocalStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths,
}
}
-BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
+BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode)
{
Worker worker(*this);
@@ -74,20 +63,38 @@ BuildResult LocalStore::buildDerivation(const StorePath & drvPath, const BasicDe
result.status = BuildResult::MiscFailure;
result.errorMsg = e.msg();
}
+ // XXX: Should use `goal->queryPartialDerivationOutputMap()` once it's
+ // extended to return the full realisation for each output
+ auto staticDrvOutputs = drv.outputsAndOptPaths(*this);
+ auto outputHashes = staticOutputHashes(*this, drv);
+ for (auto & [outputName, staticOutput] : staticDrvOutputs) {
+ auto outputId = DrvOutput{outputHashes.at(outputName), outputName};
+ if (staticOutput.second)
+ result.builtOutputs.insert_or_assign(
+ outputId,
+ Realisation{ outputId, *staticOutput.second}
+ );
+ if (settings.isExperimentalFeatureEnabled("ca-derivations") && !derivationHasKnownOutputPaths(drv.type())) {
+ auto realisation = this->queryRealisation(outputId);
+ if (realisation)
+ result.builtOutputs.insert_or_assign(
+ outputId,
+ *realisation
+ );
+ }
+ }
return result;
}
-void LocalStore::ensurePath(const StorePath & path)
+void Store::ensurePath(const StorePath & path)
{
/* If the path is already valid, we're done. */
if (isValidPath(path)) return;
- primeCache(*this, {{path}});
-
Worker worker(*this);
- GoalPtr goal = worker.makeSubstitutionGoal(path);
+ GoalPtr goal = worker.makePathSubstitutionGoal(path);
Goals goals = {goal};
worker.run(goals);
@@ -105,7 +112,7 @@ void LocalStore::ensurePath(const StorePath & path)
void LocalStore::repairPath(const StorePath & path)
{
Worker worker(*this);
- GoalPtr goal = worker.makeSubstitutionGoal(path, Repair);
+ GoalPtr goal = worker.makePathSubstitutionGoal(path, Repair);
Goals goals = {goal};
worker.run(goals);
diff --git a/src/libstore/build/goal.cc b/src/libstore/build/goal.cc
index 2dd7a4d37..9de40bdf2 100644
--- a/src/libstore/build/goal.cc
+++ b/src/libstore/build/goal.cc
@@ -78,6 +78,8 @@ void Goal::amDone(ExitCode result, std::optional<Error> ex)
}
waiters.clear();
worker.removeGoal(shared_from_this());
+
+ cleanup();
}
diff --git a/src/libstore/build/goal.hh b/src/libstore/build/goal.hh
index 360c160ce..e6bf628cb 100644
--- a/src/libstore/build/goal.hh
+++ b/src/libstore/build/goal.hh
@@ -7,7 +7,7 @@ namespace nix {
/* Forward definition. */
struct Goal;
-struct Worker;
+class Worker;
/* A pointer to a goal. */
typedef std::shared_ptr<Goal> GoalPtr;
@@ -46,7 +46,7 @@ struct Goal : public std::enable_shared_from_this<Goal>
unsigned int nrNoSubstituters;
/* Number of substitution goals we are/were waiting for that
- failed because othey had unsubstitutable references. */
+ failed because they had unsubstitutable references. */
unsigned int nrIncompleteClosure;
/* Name of this goal for debugging purposes. */
@@ -100,6 +100,8 @@ struct Goal : public std::enable_shared_from_this<Goal>
virtual string key() = 0;
void amDone(ExitCode result, std::optional<Error> ex = {});
+
+ virtual void cleanup() { }
};
void addToWeakGoals(WeakGoals & goals, GoalPtr p);
diff --git a/src/libstore/build/local-derivation-goal.cc b/src/libstore/build/local-derivation-goal.cc
new file mode 100644
index 000000000..ab37f9523
--- /dev/null
+++ b/src/libstore/build/local-derivation-goal.cc
@@ -0,0 +1,2906 @@
+#include "local-derivation-goal.hh"
+#include "hook-instance.hh"
+#include "worker.hh"
+#include "builtins.hh"
+#include "builtins/buildenv.hh"
+#include "references.hh"
+#include "finally.hh"
+#include "util.hh"
+#include "archive.hh"
+#include "json.hh"
+#include "compression.hh"
+#include "daemon.hh"
+#include "worker-protocol.hh"
+#include "topo-sort.hh"
+#include "callback.hh"
+
+#include <regex>
+#include <queue>
+
+#include <sys/types.h>
+#include <sys/socket.h>
+#include <sys/un.h>
+#include <netdb.h>
+#include <fcntl.h>
+#include <termios.h>
+#include <unistd.h>
+#include <sys/mman.h>
+#include <sys/utsname.h>
+#include <sys/resource.h>
+
+#if HAVE_STATVFS
+#include <sys/statvfs.h>
+#endif
+
+/* Includes required for chroot support. */
+#if __linux__
+#include <sys/socket.h>
+#include <sys/ioctl.h>
+#include <net/if.h>
+#include <netinet/ip.h>
+#include <sys/personality.h>
+#include <sys/mman.h>
+#include <sched.h>
+#include <sys/param.h>
+#include <sys/mount.h>
+#include <sys/syscall.h>
+#if HAVE_SECCOMP
+#include <seccomp.h>
+#endif
+#define pivot_root(new_root, put_old) (syscall(SYS_pivot_root, new_root, put_old))
+#endif
+
+#if __APPLE__
+#include <spawn.h>
+#include <sys/sysctl.h>
+#endif
+
+#include <pwd.h>
+#include <grp.h>
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+void handleDiffHook(
+ uid_t uid, uid_t gid,
+ const Path & tryA, const Path & tryB,
+ const Path & drvPath, const Path & tmpDir)
+{
+ auto diffHook = settings.diffHook;
+ if (diffHook != "" && settings.runDiffHook) {
+ try {
+ RunOptions diffHookOptions(diffHook,{tryA, tryB, drvPath, tmpDir});
+ diffHookOptions.searchPath = true;
+ diffHookOptions.uid = uid;
+ diffHookOptions.gid = gid;
+ diffHookOptions.chdir = "/";
+ auto diffRes = runProgram(diffHookOptions);
+ if (!statusOk(diffRes.first))
+ throw ExecError(diffRes.first,
+ "diff-hook program '%1%' %2%",
+ diffHook,
+ statusToString(diffRes.first));
+
+ if (diffRes.second != "")
+ printError(chomp(diffRes.second));
+ } catch (Error & error) {
+ ErrorInfo ei = error.info();
+ // FIXME: wrap errors.
+ ei.msg = hintfmt("diff hook execution failed: %s", ei.msg.str());
+ logError(ei);
+ }
+ }
+}
+
+const Path LocalDerivationGoal::homeDir = "/homeless-shelter";
+
+
+LocalDerivationGoal::~LocalDerivationGoal()
+{
+ /* Careful: we should never ever throw an exception from a
+ destructor. */
+ try { deleteTmpDir(false); } catch (...) { ignoreException(); }
+ try { killChild(); } catch (...) { ignoreException(); }
+ try { stopDaemon(); } catch (...) { ignoreException(); }
+}
+
+
+inline bool LocalDerivationGoal::needsHashRewrite()
+{
+#if __linux__
+ return !useChroot;
+#else
+ /* Darwin requires hash rewriting even when sandboxing is enabled. */
+ return true;
+#endif
+}
+
+
+LocalStore & LocalDerivationGoal::getLocalStore()
+{
+ auto p = dynamic_cast<LocalStore *>(&worker.store);
+ assert(p);
+ return *p;
+}
+
+
+void LocalDerivationGoal::killChild()
+{
+ if (pid != -1) {
+ worker.childTerminated(this);
+
+ if (buildUser) {
+ /* If we're using a build user, then there is a tricky
+ race condition: if we kill the build user before the
+ child has done its setuid() to the build user uid, then
+ it won't be killed, and we'll potentially lock up in
+ pid.wait(). So also send a conventional kill to the
+ child. */
+ ::kill(-pid, SIGKILL); /* ignore the result */
+ buildUser->kill();
+ pid.wait();
+ } else
+ pid.kill();
+
+ assert(pid == -1);
+ }
+
+ DerivationGoal::killChild();
+}
+
+
+void LocalDerivationGoal::tryLocalBuild() {
+ unsigned int curBuilds = worker.getNrLocalBuilds();
+ if (curBuilds >= settings.maxBuildJobs) {
+ worker.waitForBuildSlot(shared_from_this());
+ outputLocks.unlock();
+ return;
+ }
+
+ if (useBuildUsers()) {
+ if (!buildUser)
+ buildUser = acquireUserLock();
+
+ if (!buildUser) {
+ if (!actLock)
+ actLock = std::make_unique<Activity>(*logger, lvlWarn, actBuildWaiting,
+ fmt("waiting for UID to build '%s'", yellowtxt(worker.store.printStorePath(drvPath))));
+ worker.waitForAWhile(shared_from_this());
+ return;
+ }
+
+ /* Make sure that no other processes are executing under this
+ uid. */
+ buildUser->kill();
+ }
+
+ actLock.reset();
+
+ try {
+
+ /* Okay, we have to build. */
+ startBuilder();
+
+ } catch (BuildError & e) {
+ outputLocks.unlock();
+ buildUser.reset();
+ worker.permanentFailure = true;
+ done(BuildResult::InputRejected, e);
+ return;
+ }
+
+ /* This state will be reached when we get EOF on the child's
+ log pipe. */
+ state = &DerivationGoal::buildDone;
+
+ started();
+}
+
+static void chmod_(const Path & path, mode_t mode)
+{
+ if (chmod(path.c_str(), mode) == -1)
+ throw SysError("setting permissions on '%s'", path);
+}
+
+
+/* Move/rename path 'src' to 'dst'. Temporarily make 'src' writable if
+ it's a directory and we're not root (to be able to update the
+ directory's parent link ".."). */
+static void movePath(const Path & src, const Path & dst)
+{
+ auto st = lstat(src);
+
+ bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
+
+ if (changePerm)
+ chmod_(src, st.st_mode | S_IWUSR);
+
+ if (rename(src.c_str(), dst.c_str()))
+ throw SysError("renaming '%1%' to '%2%'", src, dst);
+
+ if (changePerm)
+ chmod_(dst, st.st_mode);
+}
+
+
+extern void replaceValidPath(const Path & storePath, const Path & tmpPath);
+
+
+int LocalDerivationGoal::getChildStatus()
+{
+ return hook ? DerivationGoal::getChildStatus() : pid.kill();
+}
+
+void LocalDerivationGoal::closeReadPipes()
+{
+ if (hook) {
+ DerivationGoal::closeReadPipes();
+ } else
+ builderOut.readSide = -1;
+}
+
+
+void LocalDerivationGoal::cleanupHookFinally()
+{
+ /* Release the build user at the end of this function. We don't do
+ it right away because we don't want another build grabbing this
+ uid and then messing around with our output. */
+ buildUser.reset();
+}
+
+
+void LocalDerivationGoal::cleanupPreChildKill()
+{
+ sandboxMountNamespace = -1;
+}
+
+
+void LocalDerivationGoal::cleanupPostChildKill()
+{
+ /* When running under a build user, make sure that all processes
+ running under that uid are gone. This is to prevent a
+ malicious user from leaving behind a process that keeps files
+ open and modifies them after they have been chown'ed to
+ root. */
+ if (buildUser) buildUser->kill();
+
+ /* Terminate the recursive Nix daemon. */
+ stopDaemon();
+}
+
+
+bool LocalDerivationGoal::cleanupDecideWhetherDiskFull()
+{
+ bool diskFull = false;
+
+ /* Heuristically check whether the build failure may have
+ been caused by a disk full condition. We have no way
+ of knowing whether the build actually got an ENOSPC.
+ So instead, check if the disk is (nearly) full now. If
+ so, we don't mark this build as a permanent failure. */
+#if HAVE_STATVFS
+ {
+ auto & localStore = getLocalStore();
+ uint64_t required = 8ULL * 1024 * 1024; // FIXME: make configurable
+ struct statvfs st;
+ if (statvfs(localStore.realStoreDir.c_str(), &st) == 0 &&
+ (uint64_t) st.f_bavail * st.f_bsize < required)
+ diskFull = true;
+ if (statvfs(tmpDir.c_str(), &st) == 0 &&
+ (uint64_t) st.f_bavail * st.f_bsize < required)
+ diskFull = true;
+ }
+#endif
+
+ deleteTmpDir(false);
+
+ /* Move paths out of the chroot for easier debugging of
+ build failures. */
+ if (useChroot && buildMode == bmNormal)
+ for (auto & [_, status] : initialOutputs) {
+ if (!status.known) continue;
+ if (buildMode != bmCheck && status.known->isValid()) continue;
+ auto p = worker.store.printStorePath(status.known->path);
+ if (pathExists(chrootRootDir + p))
+ rename((chrootRootDir + p).c_str(), p.c_str());
+ }
+
+ return diskFull;
+}
+
+
+void LocalDerivationGoal::cleanupPostOutputsRegisteredModeCheck()
+{
+ deleteTmpDir(true);
+}
+
+
+void LocalDerivationGoal::cleanupPostOutputsRegisteredModeNonCheck()
+{
+ /* Delete unused redirected outputs (when doing hash rewriting). */
+ for (auto & i : redirectedOutputs)
+ deletePath(worker.store.Store::toRealPath(i.second));
+
+ /* Delete the chroot (if we were using one). */
+ autoDelChroot.reset(); /* this runs the destructor */
+
+ cleanupPostOutputsRegisteredModeCheck();
+}
+
+
+int childEntry(void * arg)
+{
+ ((LocalDerivationGoal *) arg)->runChild();
+ return 1;
+}
+
+
+static std::once_flag dns_resolve_flag;
+
+static void preloadNSS() {
+ /* builtin:fetchurl can trigger a DNS lookup, which with glibc can trigger a dynamic library load of
+ one of the glibc NSS libraries in a sandboxed child, which will fail unless the library's already
+ been loaded in the parent. So we force a lookup of an invalid domain to force the NSS machinery to
+ load its lookup libraries in the parent before any child gets a chance to. */
+ std::call_once(dns_resolve_flag, []() {
+ struct addrinfo *res = NULL;
+
+ if (getaddrinfo("this.pre-initializes.the.dns.resolvers.invalid.", "http", NULL, &res) != 0) {
+ if (res) freeaddrinfo(res);
+ }
+ });
+}
+
+
+static void linkOrCopy(const Path & from, const Path & to)
+{
+ if (link(from.c_str(), to.c_str()) == -1) {
+ /* Hard-linking fails if we exceed the maximum link count on a
+ file (e.g. 32000 of ext3), which is quite possible after a
+ 'nix-store --optimise'. FIXME: actually, why don't we just
+ bind-mount in this case?
+
+ It can also fail with EPERM in BeegFS v7 and earlier versions
+ which don't allow hard-links to other directories */
+ if (errno != EMLINK && errno != EPERM)
+ throw SysError("linking '%s' to '%s'", to, from);
+ copyPath(from, to);
+ }
+}
+
+
+void LocalDerivationGoal::startBuilder()
+{
+ /* Right platform? */
+ if (!parsedDrv->canBuildLocally(worker.store))
+ throw Error("a '%s' with features {%s} is required to build '%s', but I am a '%s' with features {%s}",
+ drv->platform,
+ concatStringsSep(", ", parsedDrv->getRequiredSystemFeatures()),
+ worker.store.printStorePath(drvPath),
+ settings.thisSystem,
+ concatStringsSep<StringSet>(", ", worker.store.systemFeatures));
+
+ if (drv->isBuiltin())
+ preloadNSS();
+
+#if __APPLE__
+ additionalSandboxProfile = parsedDrv->getStringAttr("__sandboxProfile").value_or("");
+#endif
+
+ /* Are we doing a chroot build? */
+ {
+ auto noChroot = parsedDrv->getBoolAttr("__noChroot");
+ if (settings.sandboxMode == smEnabled) {
+ if (noChroot)
+ throw Error("derivation '%s' has '__noChroot' set, "
+ "but that's not allowed when 'sandbox' is 'true'", worker.store.printStorePath(drvPath));
+#if __APPLE__
+ if (additionalSandboxProfile != "")
+ throw Error("derivation '%s' specifies a sandbox profile, "
+ "but this is only allowed when 'sandbox' is 'relaxed'", worker.store.printStorePath(drvPath));
+#endif
+ useChroot = true;
+ }
+ else if (settings.sandboxMode == smDisabled)
+ useChroot = false;
+ else if (settings.sandboxMode == smRelaxed)
+ useChroot = !(derivationIsImpure(derivationType)) && !noChroot;
+ }
+
+ auto & localStore = getLocalStore();
+ if (localStore.storeDir != localStore.realStoreDir) {
+ #if __linux__
+ useChroot = true;
+ #else
+ throw Error("building using a diverted store is not supported on this platform");
+ #endif
+ }
+
+ /* Create a temporary directory where the build will take
+ place. */
+ tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
+
+ chownToBuilder(tmpDir);
+
+ for (auto & [outputName, status] : initialOutputs) {
+ /* Set scratch path we'll actually use during the build.
+
+ If we're not doing a chroot build, but we have some valid
+ output paths. Since we can't just overwrite or delete
+ them, we have to do hash rewriting: i.e. in the
+ environment/arguments passed to the build, we replace the
+ hashes of the valid outputs with unique dummy strings;
+ after the build, we discard the redirected outputs
+ corresponding to the valid outputs, and rewrite the
+ contents of the new outputs to replace the dummy strings
+ with the actual hashes. */
+ auto scratchPath =
+ !status.known
+ ? makeFallbackPath(outputName)
+ : !needsHashRewrite()
+ /* Can always use original path in sandbox */
+ ? status.known->path
+ : !status.known->isPresent()
+ /* If path doesn't yet exist can just use it */
+ ? status.known->path
+ : buildMode != bmRepair && !status.known->isValid()
+ /* If we aren't repairing we'll delete a corrupted path, so we
+ can use original path */
+ ? status.known->path
+ : /* If we are repairing or the path is totally valid, we'll need
+ to use a temporary path */
+ makeFallbackPath(status.known->path);
+ scratchOutputs.insert_or_assign(outputName, scratchPath);
+
+ /* Substitute output placeholders with the scratch output paths.
+ We'll use during the build. */
+ inputRewrites[hashPlaceholder(outputName)] = worker.store.printStorePath(scratchPath);
+
+ /* Additional tasks if we know the final path a priori. */
+ if (!status.known) continue;
+ auto fixedFinalPath = status.known->path;
+
+ /* Additional tasks if the final and scratch are both known and
+ differ. */
+ if (fixedFinalPath == scratchPath) continue;
+
+ /* Ensure scratch path is ours to use. */
+ deletePath(worker.store.printStorePath(scratchPath));
+
+ /* Rewrite and unrewrite paths */
+ {
+ std::string h1 { fixedFinalPath.hashPart() };
+ std::string h2 { scratchPath.hashPart() };
+ inputRewrites[h1] = h2;
+ }
+
+ redirectedOutputs.insert_or_assign(std::move(fixedFinalPath), std::move(scratchPath));
+ }
+
+ /* Construct the environment passed to the builder. */
+ initEnv();
+
+ writeStructuredAttrs();
+
+ /* Handle exportReferencesGraph(), if set. */
+ if (!parsedDrv->getStructuredAttrs()) {
+ /* The `exportReferencesGraph' feature allows the references graph
+ to be passed to a builder. This attribute should be a list of
+ pairs [name1 path1 name2 path2 ...]. The references graph of
+ each `pathN' will be stored in a text file `nameN' in the
+ temporary build directory. The text files have the format used
+ by `nix-store --register-validity'. However, the deriver
+ fields are left empty. */
+ string s = get(drv->env, "exportReferencesGraph").value_or("");
+ Strings ss = tokenizeString<Strings>(s);
+ if (ss.size() % 2 != 0)
+ throw BuildError("odd number of tokens in 'exportReferencesGraph': '%1%'", s);
+ for (Strings::iterator i = ss.begin(); i != ss.end(); ) {
+ string fileName = *i++;
+ static std::regex regex("[A-Za-z_][A-Za-z0-9_.-]*");
+ if (!std::regex_match(fileName, regex))
+ throw Error("invalid file name '%s' in 'exportReferencesGraph'", fileName);
+
+ auto storePathS = *i++;
+ if (!worker.store.isInStore(storePathS))
+ throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS);
+ auto storePath = worker.store.toStorePath(storePathS).first;
+
+ /* Write closure info to <fileName>. */
+ writeFile(tmpDir + "/" + fileName,
+ worker.store.makeValidityRegistration(
+ exportReferences({storePath}), false, false));
+ }
+ }
+
+ useUidRange = parsedDrv->getRequiredSystemFeatures().count("uid-range");
+ useSystemdCgroup = parsedDrv->getRequiredSystemFeatures().count("systemd-cgroup");
+
+ if (useChroot) {
+
+ /* Allow a user-configurable set of directories from the
+ host file system. */
+ dirsInChroot.clear();
+
+ for (auto i : settings.sandboxPaths.get()) {
+ if (i.empty()) continue;
+ bool optional = false;
+ if (i[i.size() - 1] == '?') {
+ optional = true;
+ i.pop_back();
+ }
+ size_t p = i.find('=');
+ if (p == string::npos)
+ dirsInChroot[i] = {i, optional};
+ else
+ dirsInChroot[string(i, 0, p)] = {string(i, p + 1), optional};
+ }
+ dirsInChroot[tmpDirInSandbox] = tmpDir;
+
+ /* Add the closure of store paths to the chroot. */
+ StorePathSet closure;
+ for (auto & i : dirsInChroot)
+ try {
+ if (worker.store.isInStore(i.second.source))
+ worker.store.computeFSClosure(worker.store.toStorePath(i.second.source).first, closure);
+ } catch (InvalidPath & e) {
+ } catch (Error & e) {
+ e.addTrace({}, "while processing 'sandbox-paths'");
+ throw;
+ }
+ for (auto & i : closure) {
+ auto p = worker.store.printStorePath(i);
+ dirsInChroot.insert_or_assign(p, p);
+ }
+
+ PathSet allowedPaths = settings.allowedImpureHostPrefixes;
+
+ /* This works like the above, except on a per-derivation level */
+ auto impurePaths = parsedDrv->getStringsAttr("__impureHostDeps").value_or(Strings());
+
+ for (auto & i : impurePaths) {
+ bool found = false;
+ /* Note: we're not resolving symlinks here to prevent
+ giving a non-root user info about inaccessible
+ files. */
+ Path canonI = canonPath(i);
+ /* If only we had a trie to do this more efficiently :) luckily, these are generally going to be pretty small */
+ for (auto & a : allowedPaths) {
+ Path canonA = canonPath(a);
+ if (canonI == canonA || isInDir(canonI, canonA)) {
+ found = true;
+ break;
+ }
+ }
+ if (!found)
+ throw Error("derivation '%s' requested impure path '%s', but it was not in allowed-impure-host-deps",
+ worker.store.printStorePath(drvPath), i);
+
+ dirsInChroot[i] = i;
+ }
+
+#if __linux__
+ /* Create a temporary directory in which we set up the chroot
+ environment using bind-mounts. We put it in the Nix store
+ to ensure that we can create hard-links to non-directory
+ inputs in the fake Nix store in the chroot (see below). */
+ chrootRootDir = worker.store.Store::toRealPath(drvPath) + ".chroot";
+ deletePath(chrootRootDir);
+
+ /* Clean up the chroot directory automatically. */
+ autoDelChroot = std::make_shared<AutoDelete>(chrootRootDir);
+
+ printMsg(lvlChatty, format("setting up chroot environment in '%1%'") % chrootRootDir);
+
+ if (mkdir(chrootRootDir.c_str(), useUidRange ? 0755 : 0750) == -1)
+ throw SysError("cannot create '%1%'", chrootRootDir);
+
+ // FIXME: only make root writable for user namespace builds.
+ if (buildUser && chown(chrootRootDir.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
+ throw SysError("cannot change ownership of '%1%'", chrootRootDir);
+
+ /* Create a writable /tmp in the chroot. Many builders need
+ this. (Of course they should really respect $TMPDIR
+ instead.) */
+ Path chrootTmpDir = chrootRootDir + "/tmp";
+ createDirs(chrootTmpDir);
+ chmod_(chrootTmpDir, 01777);
+
+ /* Create a /etc/passwd with entries for the build user and the
+ nobody account. The latter is kind of a hack to support
+ Samba-in-QEMU. */
+ createDirs(chrootRootDir + "/etc");
+ chownToBuilder(chrootRootDir + "/etc");
+
+ if (useUidRange && (!buildUser || buildUser->getUIDCount() < 65536))
+ throw Error("feature 'uid-range' requires '%s' to be enabled", settings.autoAllocateUids.name);
+
+ /* Declare the build user's group so that programs get a consistent
+ view of the system (e.g., "id -gn"). */
+ writeFile(chrootRootDir + "/etc/group",
+ fmt("root:x:0:\n"
+ "nixbld:!:%1%:\n"
+ "nogroup:x:65534:\n", sandboxGid()));
+
+ /* Create /etc/hosts with localhost entry. */
+ if (!(derivationIsImpure(derivationType)))
+ writeFile(chrootRootDir + "/etc/hosts", "127.0.0.1 localhost\n::1 localhost\n");
+
+ /* Make the closure of the inputs available in the chroot,
+ rather than the whole Nix store. This prevents any access
+ to undeclared dependencies. Directories are bind-mounted,
+ while other inputs are hard-linked (since only directories
+ can be bind-mounted). !!! As an extra security
+ precaution, make the fake Nix store only writable by the
+ build user. */
+ Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
+ createDirs(chrootStoreDir);
+ chmod_(chrootStoreDir, 01775);
+
+ if (buildUser && chown(chrootStoreDir.c_str(), 0, buildUser->getGID()) == -1)
+ throw SysError("cannot change ownership of '%1%'", chrootStoreDir);
+
+ for (auto & i : inputPaths) {
+ auto p = worker.store.printStorePath(i);
+ Path r = worker.store.toRealPath(p);
+ if (S_ISDIR(lstat(r).st_mode))
+ dirsInChroot.insert_or_assign(p, r);
+ else
+ linkOrCopy(r, chrootRootDir + p);
+ }
+
+ /* If we're repairing, checking or rebuilding part of a
+ multiple-outputs derivation, it's possible that we're
+ rebuilding a path that is in settings.dirsInChroot
+ (typically the dependencies of /bin/sh). Throw them
+ out. */
+ for (auto & i : drv->outputsAndOptPaths(worker.store)) {
+ /* If the name isn't known a priori (i.e. floating
+ content-addressed derivation), the temporary location we use
+ should be fresh. Freshness means it is impossible that the path
+ is already in the sandbox, so we don't need to worry about
+ removing it. */
+ if (i.second.second)
+ dirsInChroot.erase(worker.store.printStorePath(*i.second.second));
+ }
+
+ if (useSystemdCgroup) {
+ settings.requireExperimentalFeature("systemd-cgroup");
+ std::optional<Path> cgroup;
+ if (!buildUser || !(cgroup = buildUser->getCgroup()))
+ throw Error("feature 'systemd-cgroup' requires 'auto-allocate-uids = true' in nix.conf");
+ chownToBuilder(*cgroup);
+ chownToBuilder(*cgroup + "/cgroup.procs");
+ }
+
+#else
+ if (useUidRange)
+ throw Error("feature 'uid-range' is not supported on this platform");
+ if (useSystemdCgroup)
+ throw Error("feature 'systemd-cgroup' is not supported on this platform");
+ #if __APPLE__
+ /* We don't really have any parent prep work to do (yet?)
+ All work happens in the child, instead. */
+ #else
+ throw Error("sandboxing builds is not supported on this platform");
+ #endif
+#endif
+ } else {
+ if (useUidRange)
+ throw Error("feature 'uid-range' is only supported in sandboxed builds");
+ if (useSystemdCgroup)
+ throw Error("feature 'systemd-cgroup' is only supported in sandboxed builds");
+ }
+
+ if (needsHashRewrite() && pathExists(homeDir))
+ throw Error("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing", homeDir);
+
+ if (useChroot && settings.preBuildHook != "" && dynamic_cast<Derivation *>(drv.get())) {
+ printMsg(lvlChatty, format("executing pre-build hook '%1%'")
+ % settings.preBuildHook);
+ auto args = useChroot ? Strings({worker.store.printStorePath(drvPath), chrootRootDir}) :
+ Strings({ worker.store.printStorePath(drvPath) });
+ enum BuildHookState {
+ stBegin,
+ stExtraChrootDirs
+ };
+ auto state = stBegin;
+ auto lines = runProgram(settings.preBuildHook, false, args);
+ auto lastPos = std::string::size_type{0};
+ for (auto nlPos = lines.find('\n'); nlPos != string::npos;
+ nlPos = lines.find('\n', lastPos)) {
+ auto line = std::string{lines, lastPos, nlPos - lastPos};
+ lastPos = nlPos + 1;
+ if (state == stBegin) {
+ if (line == "extra-sandbox-paths" || line == "extra-chroot-dirs") {
+ state = stExtraChrootDirs;
+ } else {
+ throw Error("unknown pre-build hook command '%1%'", line);
+ }
+ } else if (state == stExtraChrootDirs) {
+ if (line == "") {
+ state = stBegin;
+ } else {
+ auto p = line.find('=');
+ if (p == string::npos)
+ dirsInChroot[line] = line;
+ else
+ dirsInChroot[string(line, 0, p)] = string(line, p + 1);
+ }
+ }
+ }
+ }
+
+ /* Fire up a Nix daemon to process recursive Nix calls from the
+ builder. */
+ if (parsedDrv->getRequiredSystemFeatures().count("recursive-nix"))
+ startDaemon();
+
+ /* Run the builder. */
+ printMsg(lvlChatty, "executing builder '%1%'", drv->builder);
+
+ /* Create the log file. */
+ Path logFile = openLogFile();
+
+ /* Create a pipe to get the output of the builder. */
+ //builderOut.create();
+
+ builderOut.readSide = posix_openpt(O_RDWR | O_NOCTTY);
+ if (!builderOut.readSide)
+ throw SysError("opening pseudoterminal master");
+
+ std::string slaveName(ptsname(builderOut.readSide.get()));
+
+ if (buildUser) {
+ if (chmod(slaveName.c_str(), 0600))
+ throw SysError("changing mode of pseudoterminal slave");
+
+ if (chown(slaveName.c_str(), buildUser->getUID(), 0))
+ throw SysError("changing owner of pseudoterminal slave");
+ }
+#if __APPLE__
+ else {
+ if (grantpt(builderOut.readSide.get()))
+ throw SysError("granting access to pseudoterminal slave");
+ }
+#endif
+
+ #if 0
+ // Mount the pt in the sandbox so that the "tty" command works.
+ // FIXME: this doesn't work with the new devpts in the sandbox.
+ if (useChroot)
+ dirsInChroot[slaveName] = {slaveName, false};
+ #endif
+
+ if (unlockpt(builderOut.readSide.get()))
+ throw SysError("unlocking pseudoterminal");
+
+ builderOut.writeSide = open(slaveName.c_str(), O_RDWR | O_NOCTTY);
+ if (!builderOut.writeSide)
+ throw SysError("opening pseudoterminal slave");
+
+ // Put the pt into raw mode to prevent \n -> \r\n translation.
+ struct termios term;
+ if (tcgetattr(builderOut.writeSide.get(), &term))
+ throw SysError("getting pseudoterminal attributes");
+
+ cfmakeraw(&term);
+
+ if (tcsetattr(builderOut.writeSide.get(), TCSANOW, &term))
+ throw SysError("putting pseudoterminal into raw mode");
+
+ result.startTime = time(0);
+
+ /* Fork a child to build the package. */
+ ProcessOptions options;
+
+#if __linux__
+ if (useChroot) {
+ /* Set up private namespaces for the build:
+
+ - The PID namespace causes the build to start as PID 1.
+ Processes outside of the chroot are not visible to those
+ on the inside, but processes inside the chroot are
+ visible from the outside (though with different PIDs).
+
+ - The private mount namespace ensures that all the bind
+ mounts we do will only show up in this process and its
+ children, and will disappear automatically when we're
+ done.
+
+ - The private network namespace ensures that the builder
+ cannot talk to the outside world (or vice versa). It
+ only has a private loopback interface. (Fixed-output
+ derivations are not run in a private network namespace
+ to allow functions like fetchurl to work.)
+
+ - The IPC namespace prevents the builder from communicating
+ with outside processes using SysV IPC mechanisms (shared
+ memory, message queues, semaphores). It also ensures
+ that all IPC objects are destroyed when the builder
+ exits.
+
+ - The UTS namespace ensures that builders see a hostname of
+ localhost rather than the actual hostname.
+
+ We use a helper process to do the clone() to work around
+ clone() being broken in multi-threaded programs due to
+ at-fork handlers not being run. Note that we use
+ CLONE_PARENT to ensure that the real builder is parented to
+ us.
+ */
+
+ if (!(derivationIsImpure(derivationType)))
+ privateNetwork = true;
+
+ userNamespaceSync.create();
+
+ options.allowVfork = false;
+
+ Path maxUserNamespaces = "/proc/sys/user/max_user_namespaces";
+ static bool userNamespacesEnabled =
+ pathExists(maxUserNamespaces)
+ && trim(readFile(maxUserNamespaces)) != "0";
+
+ usingUserNamespace = userNamespacesEnabled;
+
+ Pid helper = startProcess([&]() {
+
+ /* Drop additional groups here because we can't do it
+ after we've created the new user namespace. FIXME:
+ this means that if we're not root in the parent
+ namespace, we can't drop additional groups; they will
+ be mapped to nogroup in the child namespace. There does
+ not seem to be a workaround for this. (But who can tell
+ from reading user_namespaces(7)?)
+ See also https://lwn.net/Articles/621612/. */
+ if (getuid() == 0 && setgroups(0, 0) == -1)
+ throw SysError("setgroups failed");
+
+ size_t stackSize = 1 * 1024 * 1024;
+ char * stack = (char *) mmap(0, stackSize,
+ PROT_WRITE | PROT_READ, MAP_PRIVATE | MAP_ANONYMOUS | MAP_STACK, -1, 0);
+ if (stack == MAP_FAILED) throw SysError("allocating stack");
+
+ int flags = CLONE_NEWPID | CLONE_NEWNS | CLONE_NEWIPC | CLONE_NEWUTS | CLONE_PARENT | SIGCHLD;
+ if (privateNetwork)
+ flags |= CLONE_NEWNET;
+ if (usingUserNamespace)
+ flags |= CLONE_NEWUSER;
+
+ pid_t child = clone(childEntry, stack + stackSize, flags, this);
+ if (child == -1 && errno == EINVAL) {
+ /* Fallback for Linux < 2.13 where CLONE_NEWPID and
+ CLONE_PARENT are not allowed together. */
+ flags &= ~CLONE_NEWPID;
+ child = clone(childEntry, stack + stackSize, flags, this);
+ }
+ if (usingUserNamespace && child == -1 && (errno == EPERM || errno == EINVAL)) {
+ /* Some distros patch Linux to not allow unprivileged
+ * user namespaces. If we get EPERM or EINVAL, try
+ * without CLONE_NEWUSER and see if that works.
+ */
+ usingUserNamespace = false;
+ flags &= ~CLONE_NEWUSER;
+ child = clone(childEntry, stack + stackSize, flags, this);
+ }
+ /* Otherwise exit with EPERM so we can handle this in the
+ parent. This is only done when sandbox-fallback is set
+ to true (the default). */
+ if (child == -1 && (errno == EPERM || errno == EINVAL) && settings.sandboxFallback)
+ _exit(1);
+ if (child == -1) throw SysError("cloning builder process");
+
+ writeFull(builderOut.writeSide.get(),
+ fmt("%d %d\n", usingUserNamespace, child));
+ _exit(0);
+ }, options);
+
+ int res = helper.wait();
+ if (res != 0 && settings.sandboxFallback) {
+ useChroot = false;
+ initTmpDir();
+ goto fallback;
+ } else if (res != 0)
+ throw Error("unable to start build process");
+
+ userNamespaceSync.readSide = -1;
+
+ /* Close the write side to prevent runChild() from hanging
+ reading from this. */
+ Finally cleanup([&]() {
+ userNamespaceSync.writeSide = -1;
+ });
+
+ auto ss = tokenizeString<std::vector<std::string>>(readLine(builderOut.readSide.get()));
+ assert(ss.size() == 2);
+ usingUserNamespace = ss[0] == "1";
+ pid = string2Int<pid_t>(ss[1]).value();
+
+ if (usingUserNamespace) {
+ /* Set the UID/GID mapping of the builder's user namespace
+ such that the sandbox user maps to the build user, or to
+ the calling user (if build users are disabled). */
+ uid_t hostUid = buildUser ? buildUser->getUID() : getuid();
+ uid_t hostGid = buildUser ? buildUser->getGID() : getgid();
+ uint32_t nrIds = buildUser && useUidRange ? buildUser->getUIDCount() : 1;
+
+ writeFile("/proc/" + std::to_string(pid) + "/uid_map",
+ fmt("%d %d %d", sandboxUid(), hostUid, nrIds));
+
+ if (!useUidRange)
+ writeFile("/proc/" + std::to_string(pid) + "/setgroups", "deny");
+
+ writeFile("/proc/" + std::to_string(pid) + "/gid_map",
+ fmt("%d %d %d", sandboxGid(), hostGid, nrIds));
+ } else {
+ debug("note: not using a user namespace");
+ if (!buildUser)
+ throw Error("cannot perform a sandboxed build because user namespaces are not enabled; check /proc/sys/user/max_user_namespaces");
+ }
+
+ /* Now that we now the sandbox uid, we can write
+ /etc/passwd. */
+ writeFile(chrootRootDir + "/etc/passwd", fmt(
+ "root:x:0:0:Nix build user:%3%:/noshell\n"
+ "nixbld:x:%1%:%2%:Nix build user:%3%:/noshell\n"
+ "nobody:x:65534:65534:Nobody:/:/noshell\n",
+ sandboxUid(), sandboxGid(), settings.sandboxBuildDir));
+
+ /* Save the mount namespace of the child. We have to do this
+ *before* the child does a chroot. */
+ sandboxMountNamespace = open(fmt("/proc/%d/ns/mnt", (pid_t) pid).c_str(), O_RDONLY);
+ if (sandboxMountNamespace.get() == -1)
+ throw SysError("getting sandbox mount namespace");
+
+ /* Move the child into its own cgroup. */
+ if (buildUser) {
+ if (auto cgroup = buildUser->getCgroup())
+ writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid));
+ }
+
+ /* Signal the builder that we've updated its user namespace. */
+ writeFull(userNamespaceSync.writeSide.get(), "1");
+
+ } else
+#endif
+ {
+ fallback:
+ options.allowVfork = !buildUser && !drv->isBuiltin();
+ pid = startProcess([&]() {
+ runChild();
+ }, options);
+ }
+
+ /* parent */
+ pid.setSeparatePG(true);
+ builderOut.writeSide = -1;
+ worker.childStarted(shared_from_this(), {builderOut.readSide.get()}, true, true);
+
+ /* Check if setting up the build environment failed. */
+ std::vector<std::string> msgs;
+ while (true) {
+ string msg = [&]() {
+ try {
+ return readLine(builderOut.readSide.get());
+ } catch (Error & e) {
+ e.addTrace({}, "while waiting for the build environment to initialize (previous messages: %s)",
+ concatStringsSep("|", msgs));
+ throw e;
+ }
+ }();
+ if (string(msg, 0, 1) == "\2") break;
+ if (string(msg, 0, 1) == "\1") {
+ FdSource source(builderOut.readSide.get());
+ auto ex = readError(source);
+ ex.addTrace({}, "while setting up the build environment");
+ throw ex;
+ }
+ debug("sandbox setup: " + msg);
+ msgs.push_back(std::move(msg));
+ }
+}
+
+
+void LocalDerivationGoal::initTmpDir() {
+ /* In a sandbox, for determinism, always use the same temporary
+ directory. */
+#if __linux__
+ tmpDirInSandbox = useChroot ? settings.sandboxBuildDir : tmpDir;
+#else
+ tmpDirInSandbox = tmpDir;
+#endif
+
+ /* In non-structured mode, add all bindings specified in the
+ derivation via the environment, except those listed in the
+ passAsFile attribute. Those are passed as file names pointing
+ to temporary files containing the contents. Note that
+ passAsFile is ignored in structure mode because it's not
+ needed (attributes are not passed through the environment, so
+ there is no size constraint). */
+ if (!parsedDrv->getStructuredAttrs()) {
+
+ StringSet passAsFile = tokenizeString<StringSet>(get(drv->env, "passAsFile").value_or(""));
+ for (auto & i : drv->env) {
+ if (passAsFile.find(i.first) == passAsFile.end()) {
+ env[i.first] = i.second;
+ } else {
+ auto hash = hashString(htSHA256, i.first);
+ string fn = ".attr-" + hash.to_string(Base32, false);
+ Path p = tmpDir + "/" + fn;
+ writeFile(p, rewriteStrings(i.second, inputRewrites));
+ chownToBuilder(p);
+ env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
+ }
+ }
+
+ }
+
+ /* For convenience, set an environment pointing to the top build
+ directory. */
+ env["NIX_BUILD_TOP"] = tmpDirInSandbox;
+
+ /* Also set TMPDIR and variants to point to this directory. */
+ env["TMPDIR"] = env["TEMPDIR"] = env["TMP"] = env["TEMP"] = tmpDirInSandbox;
+
+ /* Explicitly set PWD to prevent problems with chroot builds. In
+ particular, dietlibc cannot figure out the cwd because the
+ inode of the current directory doesn't appear in .. (because
+ getdents returns the inode of the mount point). */
+ env["PWD"] = tmpDirInSandbox;
+}
+
+
+void LocalDerivationGoal::initEnv()
+{
+ env.clear();
+
+ /* Most shells initialise PATH to some default (/bin:/usr/bin:...) when
+ PATH is not set. We don't want this, so we fill it in with some dummy
+ value. */
+ env["PATH"] = "/path-not-set";
+
+ /* Set HOME to a non-existing path to prevent certain programs from using
+ /etc/passwd (or NIS, or whatever) to locate the home directory (for
+ example, wget looks for ~/.wgetrc). I.e., these tools use /etc/passwd
+ if HOME is not set, but they will just assume that the settings file
+ they are looking for does not exist if HOME is set but points to some
+ non-existing path. */
+ env["HOME"] = homeDir;
+
+ /* Tell the builder where the Nix store is. Usually they
+ shouldn't care, but this is useful for purity checking (e.g.,
+ the compiler or linker might only want to accept paths to files
+ in the store or in the build directory). */
+ env["NIX_STORE"] = worker.store.storeDir;
+
+ /* The maximum number of cores to utilize for parallel building. */
+ env["NIX_BUILD_CORES"] = (format("%d") % settings.buildCores).str();
+
+ initTmpDir();
+
+ /* Compatibility hack with Nix <= 0.7: if this is a fixed-output
+ derivation, tell the builder, so that for instance `fetchurl'
+ can skip checking the output. On older Nixes, this environment
+ variable won't be set, so `fetchurl' will do the check. */
+ if (derivationIsFixed(derivationType)) env["NIX_OUTPUT_CHECKED"] = "1";
+
+ /* *Only* if this is a fixed-output derivation, propagate the
+ values of the environment variables specified in the
+ `impureEnvVars' attribute to the builder. This allows for
+ instance environment variables for proxy configuration such as
+ `http_proxy' to be easily passed to downloaders like
+ `fetchurl'. Passing such environment variables from the caller
+ to the builder is generally impure, but the output of
+ fixed-output derivations is by definition pure (since we
+ already know the cryptographic hash of the output). */
+ if (derivationIsImpure(derivationType)) {
+ for (auto & i : parsedDrv->getStringsAttr("impureEnvVars").value_or(Strings()))
+ env[i] = getEnv(i).value_or("");
+ }
+
+ /* Currently structured log messages piggyback on stderr, but we
+ may change that in the future. So tell the builder which file
+ descriptor to use for that. */
+ env["NIX_LOG_FD"] = "2";
+
+ /* Trigger colored output in various tools. */
+ env["TERM"] = "xterm-256color";
+}
+
+
+static std::regex shVarName("[A-Za-z_][A-Za-z0-9_]*");
+
+
+void LocalDerivationGoal::writeStructuredAttrs()
+{
+ auto structuredAttrs = parsedDrv->getStructuredAttrs();
+ if (!structuredAttrs) return;
+
+ auto json = *structuredAttrs;
+
+ /* Add an "outputs" object containing the output paths. */
+ nlohmann::json outputs;
+ for (auto & i : drv->outputs) {
+ /* The placeholder must have a rewrite, so we use it to cover both the
+ cases where we know or don't know the output path ahead of time. */
+ outputs[i.first] = rewriteStrings(hashPlaceholder(i.first), inputRewrites);
+ }
+ json["outputs"] = outputs;
+
+ /* Handle exportReferencesGraph. */
+ auto e = json.find("exportReferencesGraph");
+ if (e != json.end() && e->is_object()) {
+ for (auto i = e->begin(); i != e->end(); ++i) {
+ std::ostringstream str;
+ {
+ JSONPlaceholder jsonRoot(str, true);
+ StorePathSet storePaths;
+ for (auto & p : *i)
+ storePaths.insert(worker.store.parseStorePath(p.get<std::string>()));
+ worker.store.pathInfoToJSON(jsonRoot,
+ exportReferences(storePaths), false, true);
+ }
+ json[i.key()] = nlohmann::json::parse(str.str()); // urgh
+ }
+ }
+
+ writeFile(tmpDir + "/.attrs.json", rewriteStrings(json.dump(), inputRewrites));
+ chownToBuilder(tmpDir + "/.attrs.json");
+
+ /* As a convenience to bash scripts, write a shell file that
+ maps all attributes that are representable in bash -
+ namely, strings, integers, nulls, Booleans, and arrays and
+ objects consisting entirely of those values. (So nested
+ arrays or objects are not supported.) */
+
+ auto handleSimpleType = [](const nlohmann::json & value) -> std::optional<std::string> {
+ if (value.is_string())
+ return shellEscape(value);
+
+ if (value.is_number()) {
+ auto f = value.get<float>();
+ if (std::ceil(f) == f)
+ return std::to_string(value.get<int>());
+ }
+
+ if (value.is_null())
+ return std::string("''");
+
+ if (value.is_boolean())
+ return value.get<bool>() ? std::string("1") : std::string("");
+
+ return {};
+ };
+
+ std::string jsonSh;
+
+ for (auto i = json.begin(); i != json.end(); ++i) {
+
+ if (!std::regex_match(i.key(), shVarName)) continue;
+
+ auto & value = i.value();
+
+ auto s = handleSimpleType(value);
+ if (s)
+ jsonSh += fmt("declare %s=%s\n", i.key(), *s);
+
+ else if (value.is_array()) {
+ std::string s2;
+ bool good = true;
+
+ for (auto i = value.begin(); i != value.end(); ++i) {
+ auto s3 = handleSimpleType(i.value());
+ if (!s3) { good = false; break; }
+ s2 += *s3; s2 += ' ';
+ }
+
+ if (good)
+ jsonSh += fmt("declare -a %s=(%s)\n", i.key(), s2);
+ }
+
+ else if (value.is_object()) {
+ std::string s2;
+ bool good = true;
+
+ for (auto i = value.begin(); i != value.end(); ++i) {
+ auto s3 = handleSimpleType(i.value());
+ if (!s3) { good = false; break; }
+ s2 += fmt("[%s]=%s ", shellEscape(i.key()), *s3);
+ }
+
+ if (good)
+ jsonSh += fmt("declare -A %s=(%s)\n", i.key(), s2);
+ }
+ }
+
+ writeFile(tmpDir + "/.attrs.sh", rewriteStrings(jsonSh, inputRewrites));
+ chownToBuilder(tmpDir + "/.attrs.sh");
+}
+
+
+static StorePath pathPartOfReq(const DerivedPath & req)
+{
+ return std::visit(overloaded {
+ [&](DerivedPath::Opaque bo) {
+ return bo.path;
+ },
+ [&](DerivedPath::Built bfd) {
+ return bfd.drvPath;
+ },
+ }, req.raw());
+}
+
+
+bool LocalDerivationGoal::isAllowed(const DerivedPath & req)
+{
+ return this->isAllowed(pathPartOfReq(req));
+}
+
+
+struct RestrictedStoreConfig : virtual LocalFSStoreConfig
+{
+ using LocalFSStoreConfig::LocalFSStoreConfig;
+ const std::string name() { return "Restricted Store"; }
+};
+
+/* A wrapper around LocalStore that only allows building/querying of
+ paths that are in the input closures of the build or were added via
+ recursive Nix calls. */
+struct RestrictedStore : public virtual RestrictedStoreConfig, public virtual LocalFSStore
+{
+ ref<LocalStore> next;
+
+ LocalDerivationGoal & goal;
+
+ RestrictedStore(const Params & params, ref<LocalStore> next, LocalDerivationGoal & goal)
+ : StoreConfig(params)
+ , LocalFSStoreConfig(params)
+ , RestrictedStoreConfig(params)
+ , Store(params)
+ , LocalFSStore(params)
+ , next(next), goal(goal)
+ { }
+
+ Path getRealStoreDir() override
+ { return next->realStoreDir; }
+
+ std::string getUri() override
+ { return next->getUri(); }
+
+ StorePathSet queryAllValidPaths() override
+ {
+ StorePathSet paths;
+ for (auto & p : goal.inputPaths) paths.insert(p);
+ for (auto & p : goal.addedPaths) paths.insert(p);
+ return paths;
+ }
+
+ void queryPathInfoUncached(const StorePath & path,
+ Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept override
+ {
+ if (goal.isAllowed(path)) {
+ try {
+ /* Censor impure information. */
+ auto info = std::make_shared<ValidPathInfo>(*next->queryPathInfo(path));
+ info->deriver.reset();
+ info->registrationTime = 0;
+ info->ultimate = false;
+ info->sigs.clear();
+ callback(info);
+ } catch (InvalidPath &) {
+ callback(nullptr);
+ }
+ } else
+ callback(nullptr);
+ };
+
+ void queryReferrers(const StorePath & path, StorePathSet & referrers) override
+ { }
+
+ std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path) override
+ {
+ if (!goal.isAllowed(path))
+ throw InvalidPath("cannot query output map for unknown path '%s' in recursive Nix", printStorePath(path));
+ return next->queryPartialDerivationOutputMap(path);
+ }
+
+ std::optional<StorePath> queryPathFromHashPart(const std::string & hashPart) override
+ { throw Error("queryPathFromHashPart"); }
+
+ StorePath addToStore(const string & name, const Path & srcPath,
+ FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256,
+ PathFilter & filter = defaultPathFilter, RepairFlag repair = NoRepair) override
+ { throw Error("addToStore"); }
+
+ void addToStore(const ValidPathInfo & info, Source & narSource,
+ RepairFlag repair = NoRepair, CheckSigsFlag checkSigs = CheckSigs) override
+ {
+ next->addToStore(info, narSource, repair, checkSigs);
+ goal.addDependency(info.path);
+ }
+
+ StorePath addTextToStore(const string & name, const string & s,
+ const StorePathSet & references, RepairFlag repair = NoRepair) override
+ {
+ auto path = next->addTextToStore(name, s, references, repair);
+ goal.addDependency(path);
+ return path;
+ }
+
+ StorePath addToStoreFromDump(Source & dump, const string & name,
+ FileIngestionMethod method = FileIngestionMethod::Recursive, HashType hashAlgo = htSHA256, RepairFlag repair = NoRepair) override
+ {
+ auto path = next->addToStoreFromDump(dump, name, method, hashAlgo, repair);
+ goal.addDependency(path);
+ return path;
+ }
+
+ void narFromPath(const StorePath & path, Sink & sink) override
+ {
+ if (!goal.isAllowed(path))
+ throw InvalidPath("cannot dump unknown path '%s' in recursive Nix", printStorePath(path));
+ LocalFSStore::narFromPath(path, sink);
+ }
+
+ void ensurePath(const StorePath & path) override
+ {
+ if (!goal.isAllowed(path))
+ throw InvalidPath("cannot substitute unknown path '%s' in recursive Nix", printStorePath(path));
+ /* Nothing to be done; 'path' must already be valid. */
+ }
+
+ void registerDrvOutput(const Realisation & info) override
+ // XXX: This should probably be allowed as a no-op if the realisation
+ // corresponds to an allowed derivation
+ { throw Error("registerDrvOutput"); }
+
+ std::optional<const Realisation> queryRealisation(const DrvOutput & id) override
+ // XXX: This should probably be allowed if the realisation corresponds to
+ // an allowed derivation
+ { throw Error("queryRealisation"); }
+
+ void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override
+ {
+ if (buildMode != bmNormal) throw Error("unsupported build mode");
+
+ StorePathSet newPaths;
+
+ for (auto & req : paths) {
+ if (!goal.isAllowed(req))
+ throw InvalidPath("cannot build '%s' in recursive Nix because path is unknown", req.to_string(*next));
+ }
+
+ next->buildPaths(paths, buildMode);
+
+ for (auto & path : paths) {
+ auto p = std::get_if<DerivedPath::Built>(&path);
+ if (!p) continue;
+ auto & bfd = *p;
+ auto outputs = next->queryDerivationOutputMap(bfd.drvPath);
+ for (auto & [outputName, outputPath] : outputs)
+ if (wantOutput(outputName, bfd.outputs))
+ newPaths.insert(outputPath);
+ }
+
+ StorePathSet closure;
+ next->computeFSClosure(newPaths, closure);
+ for (auto & path : closure)
+ goal.addDependency(path);
+ }
+
+ BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
+ BuildMode buildMode = bmNormal) override
+ { unsupported("buildDerivation"); }
+
+ void addTempRoot(const StorePath & path) override
+ { }
+
+ void addIndirectRoot(const Path & path) override
+ { }
+
+ Roots findRoots(bool censor) override
+ { return Roots(); }
+
+ void collectGarbage(const GCOptions & options, GCResults & results) override
+ { }
+
+ void addSignatures(const StorePath & storePath, const StringSet & sigs) override
+ { unsupported("addSignatures"); }
+
+ void queryMissing(const std::vector<DerivedPath> & targets,
+ StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
+ uint64_t & downloadSize, uint64_t & narSize) override
+ {
+ /* This is slightly impure since it leaks information to the
+ client about what paths will be built/substituted or are
+ already present. Probably not a big deal. */
+
+ std::vector<DerivedPath> allowed;
+ for (auto & req : targets) {
+ if (goal.isAllowed(req))
+ allowed.emplace_back(req);
+ else
+ unknown.insert(pathPartOfReq(req));
+ }
+
+ next->queryMissing(allowed, willBuild, willSubstitute,
+ unknown, downloadSize, narSize);
+ }
+};
+
+
+void LocalDerivationGoal::startDaemon()
+{
+ settings.requireExperimentalFeature("recursive-nix");
+
+ Store::Params params;
+ params["path-info-cache-size"] = "0";
+ params["store"] = worker.store.storeDir;
+ params["root"] = getLocalStore().rootDir;
+ params["state"] = "/no-such-path";
+ params["log"] = "/no-such-path";
+ auto store = make_ref<RestrictedStore>(params,
+ ref<LocalStore>(std::dynamic_pointer_cast<LocalStore>(worker.store.shared_from_this())),
+ *this);
+
+ addedPaths.clear();
+
+ auto socketName = ".nix-socket";
+ Path socketPath = tmpDir + "/" + socketName;
+ env["NIX_REMOTE"] = "unix://" + tmpDirInSandbox + "/" + socketName;
+
+ daemonSocket = createUnixDomainSocket(socketPath, 0600);
+
+ chownToBuilder(socketPath);
+
+ daemonThread = std::thread([this, store]() {
+
+ while (true) {
+
+ /* Accept a connection. */
+ struct sockaddr_un remoteAddr;
+ socklen_t remoteAddrLen = sizeof(remoteAddr);
+
+ AutoCloseFD remote = accept(daemonSocket.get(),
+ (struct sockaddr *) &remoteAddr, &remoteAddrLen);
+ if (!remote) {
+ if (errno == EINTR) continue;
+ if (errno == EINVAL) break;
+ throw SysError("accepting connection");
+ }
+
+ closeOnExec(remote.get());
+
+ debug("received daemon connection");
+
+ auto workerThread = std::thread([store, remote{std::move(remote)}]() {
+ FdSource from(remote.get());
+ FdSink to(remote.get());
+ try {
+ daemon::processConnection(store, from, to,
+ daemon::NotTrusted, daemon::Recursive,
+ [&](Store & store) { store.createUser("nobody", 65535); });
+ debug("terminated daemon connection");
+ } catch (SysError &) {
+ ignoreException();
+ }
+ });
+
+ daemonWorkerThreads.push_back(std::move(workerThread));
+ }
+
+ debug("daemon shutting down");
+ });
+}
+
+
+void LocalDerivationGoal::stopDaemon()
+{
+ if (daemonSocket && shutdown(daemonSocket.get(), SHUT_RDWR) == -1)
+ throw SysError("shutting down daemon socket");
+
+ if (daemonThread.joinable())
+ daemonThread.join();
+
+ // FIXME: should prune worker threads more quickly.
+ // FIXME: shutdown the client socket to speed up worker termination.
+ for (auto & thread : daemonWorkerThreads)
+ thread.join();
+ daemonWorkerThreads.clear();
+
+ daemonSocket = -1;
+}
+
+
+void LocalDerivationGoal::addDependency(const StorePath & path)
+{
+ if (isAllowed(path)) return;
+
+ addedPaths.insert(path);
+
+ /* If we're doing a sandbox build, then we have to make the path
+ appear in the sandbox. */
+ if (useChroot) {
+
+ debug("materialising '%s' in the sandbox", worker.store.printStorePath(path));
+
+ #if __linux__
+
+ Path source = worker.store.Store::toRealPath(path);
+ Path target = chrootRootDir + worker.store.printStorePath(path);
+ debug("bind-mounting %s -> %s", target, source);
+
+ if (pathExists(target))
+ throw Error("store path '%s' already exists in the sandbox", worker.store.printStorePath(path));
+
+ auto st = lstat(source);
+
+ if (S_ISDIR(st.st_mode)) {
+
+ /* Bind-mount the path into the sandbox. This requires
+ entering its mount namespace, which is not possible
+ in multithreaded programs. So we do this in a
+ child process.*/
+ Pid child(startProcess([&]() {
+
+ if (setns(sandboxMountNamespace.get(), 0) == -1)
+ throw SysError("entering sandbox mount namespace");
+
+ createDirs(target);
+
+ if (mount(source.c_str(), target.c_str(), "", MS_BIND, 0) == -1)
+ throw SysError("bind mount from '%s' to '%s' failed", source, target);
+
+ _exit(0);
+ }));
+
+ int status = child.wait();
+ if (status != 0)
+ throw Error("could not add path '%s' to sandbox", worker.store.printStorePath(path));
+
+ } else
+ linkOrCopy(source, target);
+
+ #else
+ throw Error("don't know how to make path '%s' (produced by a recursive Nix call) appear in the sandbox",
+ worker.store.printStorePath(path));
+ #endif
+
+ }
+}
+
+void LocalDerivationGoal::chownToBuilder(const Path & path)
+{
+ if (!buildUser) return;
+ if (chown(path.c_str(), buildUser->getUID(), buildUser->getGID()) == -1)
+ throw SysError("cannot change ownership of '%1%'", path);
+}
+
+
+void setupSeccomp()
+{
+#if __linux__
+ if (!settings.filterSyscalls) return;
+#if HAVE_SECCOMP
+ scmp_filter_ctx ctx;
+
+ if (!(ctx = seccomp_init(SCMP_ACT_ALLOW)))
+ throw SysError("unable to initialize seccomp mode 2");
+
+ Finally cleanup([&]() {
+ seccomp_release(ctx);
+ });
+
+ if (nativeSystem == "x86_64-linux" &&
+ seccomp_arch_add(ctx, SCMP_ARCH_X86) != 0)
+ throw SysError("unable to add 32-bit seccomp architecture");
+
+ if (nativeSystem == "x86_64-linux" &&
+ seccomp_arch_add(ctx, SCMP_ARCH_X32) != 0)
+ throw SysError("unable to add X32 seccomp architecture");
+
+ if (nativeSystem == "aarch64-linux" &&
+ seccomp_arch_add(ctx, SCMP_ARCH_ARM) != 0)
+ printError("unable to add ARM seccomp architecture; this may result in spurious build failures if running 32-bit ARM processes");
+
+ /* Prevent builders from creating setuid/setgid binaries. */
+ for (int perm : { S_ISUID, S_ISGID }) {
+ if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(chmod), 1,
+ SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
+ throw SysError("unable to add seccomp rule");
+
+ if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmod), 1,
+ SCMP_A1(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
+ throw SysError("unable to add seccomp rule");
+
+ if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(EPERM), SCMP_SYS(fchmodat), 1,
+ SCMP_A2(SCMP_CMP_MASKED_EQ, (scmp_datum_t) perm, (scmp_datum_t) perm)) != 0)
+ throw SysError("unable to add seccomp rule");
+ }
+
+ /* Prevent builders from creating EAs or ACLs. Not all filesystems
+ support these, and they're not allowed in the Nix store because
+ they're not representable in the NAR serialisation. */
+ if (seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(setxattr), 0) != 0 ||
+ seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(lsetxattr), 0) != 0 ||
+ seccomp_rule_add(ctx, SCMP_ACT_ERRNO(ENOTSUP), SCMP_SYS(fsetxattr), 0) != 0)
+ throw SysError("unable to add seccomp rule");
+
+ if (seccomp_attr_set(ctx, SCMP_FLTATR_CTL_NNP, settings.allowNewPrivileges ? 0 : 1) != 0)
+ throw SysError("unable to set 'no new privileges' seccomp attribute");
+
+ if (seccomp_load(ctx) != 0)
+ throw SysError("unable to load seccomp BPF program");
+#else
+ throw Error(
+ "seccomp is not supported on this platform; "
+ "you can bypass this error by setting the option 'filter-syscalls' to false, but note that untrusted builds can then create setuid binaries!");
+#endif
+#endif
+}
+
+
+void LocalDerivationGoal::runChild()
+{
+ /* Warning: in the child we should absolutely not make any SQLite
+ calls! */
+
+ try { /* child */
+
+ commonChildInit(builderOut);
+
+ try {
+ setupSeccomp();
+ } catch (...) {
+ if (buildUser) throw;
+ }
+
+ bool setUser = true;
+
+ /* Make the contents of netrc available to builtin:fetchurl
+ (which may run under a different uid and/or in a sandbox). */
+ std::string netrcData;
+ try {
+ if (drv->isBuiltin() && drv->builder == "builtin:fetchurl")
+ netrcData = readFile(settings.netrcFile);
+ } catch (SysError &) { }
+
+#if __linux__
+ if (useChroot) {
+
+ userNamespaceSync.writeSide = -1;
+
+ if (drainFD(userNamespaceSync.readSide.get()) != "1")
+ throw Error("user namespace initialisation failed");
+
+ userNamespaceSync.readSide = -1;
+
+ if (privateNetwork) {
+
+ /* Initialise the loopback interface. */
+ AutoCloseFD fd(socket(PF_INET, SOCK_DGRAM, IPPROTO_IP));
+ if (!fd) throw SysError("cannot open IP socket");
+
+ struct ifreq ifr;
+ strcpy(ifr.ifr_name, "lo");
+ ifr.ifr_flags = IFF_UP | IFF_LOOPBACK | IFF_RUNNING;
+ if (ioctl(fd.get(), SIOCSIFFLAGS, &ifr) == -1)
+ throw SysError("cannot set loopback interface flags");
+ }
+
+ /* Set the hostname etc. to fixed values. */
+ char hostname[] = "localhost";
+ if (sethostname(hostname, sizeof(hostname)) == -1)
+ throw SysError("cannot set host name");
+ char domainname[] = "(none)"; // kernel default
+ if (setdomainname(domainname, sizeof(domainname)) == -1)
+ throw SysError("cannot set domain name");
+
+ /* Make all filesystems private. This is necessary
+ because subtrees may have been mounted as "shared"
+ (MS_SHARED). (Systemd does this, for instance.) Even
+ though we have a private mount namespace, mounting
+ filesystems on top of a shared subtree still propagates
+ outside of the namespace. Making a subtree private is
+ local to the namespace, though, so setting MS_PRIVATE
+ does not affect the outside world. */
+ if (mount(0, "/", 0, MS_PRIVATE | MS_REC, 0) == -1)
+ throw SysError("unable to make '/' private");
+
+ /* Bind-mount chroot directory to itself, to treat it as a
+ different filesystem from /, as needed for pivot_root. */
+ if (mount(chrootRootDir.c_str(), chrootRootDir.c_str(), 0, MS_BIND, 0) == -1)
+ throw SysError("unable to bind mount '%1%'", chrootRootDir);
+
+ /* Bind-mount the sandbox's Nix store onto itself so that
+ we can mark it as a "shared" subtree, allowing bind
+ mounts made in *this* mount namespace to be propagated
+ into the child namespace created by the
+ unshare(CLONE_NEWNS) call below.
+
+ Marking chrootRootDir as MS_SHARED causes pivot_root()
+ to fail with EINVAL. Don't know why. */
+ Path chrootStoreDir = chrootRootDir + worker.store.storeDir;
+
+ if (mount(chrootStoreDir.c_str(), chrootStoreDir.c_str(), 0, MS_BIND, 0) == -1)
+ throw SysError("unable to bind mount the Nix store", chrootStoreDir);
+
+ if (mount(0, chrootStoreDir.c_str(), 0, MS_SHARED, 0) == -1)
+ throw SysError("unable to make '%s' shared", chrootStoreDir);
+
+ /* Set up a nearly empty /dev, unless the user asked to
+ bind-mount the host /dev. */
+ Strings ss;
+ if (dirsInChroot.find("/dev") == dirsInChroot.end()) {
+ createDirs(chrootRootDir + "/dev/shm");
+ createDirs(chrootRootDir + "/dev/pts");
+ ss.push_back("/dev/full");
+ if (worker.store.systemFeatures.get().count("kvm") && pathExists("/dev/kvm"))
+ ss.push_back("/dev/kvm");
+ ss.push_back("/dev/null");
+ ss.push_back("/dev/random");
+ ss.push_back("/dev/tty");
+ ss.push_back("/dev/urandom");
+ ss.push_back("/dev/zero");
+ createSymlink("/proc/self/fd", chrootRootDir + "/dev/fd");
+ createSymlink("/proc/self/fd/0", chrootRootDir + "/dev/stdin");
+ createSymlink("/proc/self/fd/1", chrootRootDir + "/dev/stdout");
+ createSymlink("/proc/self/fd/2", chrootRootDir + "/dev/stderr");
+ }
+
+ /* Fixed-output derivations typically need to access the
+ network, so give them access to /etc/resolv.conf and so
+ on. */
+ if (derivationIsImpure(derivationType)) {
+ // Only use nss functions to resolve hosts and
+ // services. Don’t use it for anything else that may
+ // be configured for this system. This limits the
+ // potential impurities introduced in fixed-outputs.
+ writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
+
+ /* N.B. it is realistic that these paths might not exist. It
+ happens when testing Nix building fixed-output derivations
+ within a pure derivation. */
+ for (auto & path : { "/etc/resolv.conf", "/etc/services", "/etc/hosts", "/var/run/nscd/socket" })
+ if (pathExists(path))
+ ss.push_back(path);
+ }
+
+ for (auto & i : ss) dirsInChroot.emplace(i, i);
+
+ /* Bind-mount all the directories from the "host"
+ filesystem that we want in the chroot
+ environment. */
+ auto doBind = [&](const Path & source, const Path & target, bool optional = false) {
+ debug("bind mounting '%1%' to '%2%'", source, target);
+ struct stat st;
+ if (stat(source.c_str(), &st) == -1) {
+ if (optional && errno == ENOENT)
+ return;
+ else
+ throw SysError("getting attributes of path '%1%'", source);
+ }
+ if (S_ISDIR(st.st_mode))
+ createDirs(target);
+ else {
+ createDirs(dirOf(target));
+ writeFile(target, "");
+ }
+ if (mount(source.c_str(), target.c_str(), "", MS_BIND | MS_REC, 0) == -1)
+ throw SysError("bind mount from '%1%' to '%2%' failed", source, target);
+ };
+
+ for (auto & i : dirsInChroot) {
+ if (i.second.source == "/proc") continue; // backwards compatibility
+ doBind(i.second.source, chrootRootDir + i.first, i.second.optional);
+ }
+
+ /* Bind a new instance of procfs on /proc. */
+ createDirs(chrootRootDir + "/proc");
+ if (mount("none", (chrootRootDir + "/proc").c_str(), "proc", 0, 0) == -1)
+ throw SysError("mounting /proc");
+
+ /* Mount sysfs on /sys. */
+ if (useUidRange) {
+ createDirs(chrootRootDir + "/sys");
+ if (mount("none", (chrootRootDir + "/sys").c_str(), "sysfs", 0, 0) == -1)
+ throw SysError("mounting /sys");
+ }
+
+ /* Mount a new tmpfs on /dev/shm to ensure that whatever
+ the builder puts in /dev/shm is cleaned up automatically. */
+ if (pathExists("/dev/shm") && mount("none", (chrootRootDir + "/dev/shm").c_str(), "tmpfs", 0,
+ fmt("size=%s", settings.sandboxShmSize).c_str()) == -1)
+ throw SysError("mounting /dev/shm");
+
+ /* Mount a new devpts on /dev/pts. Note that this
+ requires the kernel to be compiled with
+ CONFIG_DEVPTS_MULTIPLE_INSTANCES=y (which is the case
+ if /dev/ptx/ptmx exists). */
+ if (pathExists("/dev/pts/ptmx") &&
+ !pathExists(chrootRootDir + "/dev/ptmx")
+ && !dirsInChroot.count("/dev/pts"))
+ {
+ if (mount("none", (chrootRootDir + "/dev/pts").c_str(), "devpts", 0, "newinstance,mode=0620") == 0)
+ {
+ createSymlink("/dev/pts/ptmx", chrootRootDir + "/dev/ptmx");
+
+ /* Make sure /dev/pts/ptmx is world-writable. With some
+ Linux versions, it is created with permissions 0. */
+ chmod_(chrootRootDir + "/dev/pts/ptmx", 0666);
+ } else {
+ if (errno != EINVAL)
+ throw SysError("mounting /dev/pts");
+ doBind("/dev/pts", chrootRootDir + "/dev/pts");
+ doBind("/dev/ptmx", chrootRootDir + "/dev/ptmx");
+ }
+ }
+
+ /* Unshare this mount namespace. This is necessary because
+ pivot_root() below changes the root of the mount
+ namespace. This means that the call to setns() in
+ addDependency() would hide the host's filesystem,
+ making it impossible to bind-mount paths from the host
+ Nix store into the sandbox. Therefore, we save the
+ pre-pivot_root namespace in
+ sandboxMountNamespace. Since we made /nix/store a
+ shared subtree above, this allows addDependency() to
+ make paths appear in the sandbox. */
+ if (unshare(CLONE_NEWNS) == -1)
+ throw SysError("unsharing mount namespace");
+
+ /* Unshare the cgroup namespace. This means
+ /proc/self/cgroup will show the child's cgroup as '/'
+ rather than whatever it is in the parent. */
+ if (useSystemdCgroup && unshare(CLONE_NEWCGROUP) == -1)
+ throw SysError("unsharing cgroup namespace");
+
+ /* Do the chroot(). */
+ if (chdir(chrootRootDir.c_str()) == -1)
+ throw SysError("cannot change directory to '%1%'", chrootRootDir);
+
+ if (mkdir("real-root", 0) == -1)
+ throw SysError("cannot create real-root directory");
+
+ if (pivot_root(".", "real-root") == -1)
+ throw SysError("cannot pivot old root directory onto '%1%'", (chrootRootDir + "/real-root"));
+
+ if (chroot(".") == -1)
+ throw SysError("cannot change root directory to '%1%'", chrootRootDir);
+
+ if (umount2("real-root", MNT_DETACH) == -1)
+ throw SysError("cannot unmount real root filesystem");
+
+ if (rmdir("real-root") == -1)
+ throw SysError("cannot remove real-root directory");
+
+ /* Switch to the sandbox uid/gid in the user namespace,
+ which corresponds to the build user or calling user in
+ the parent namespace. */
+ if (setgid(sandboxGid()) == -1)
+ throw SysError("setgid failed");
+ if (setuid(sandboxUid()) == -1)
+ throw SysError("setuid failed");
+
+ setUser = false;
+ }
+#endif
+
+ if (chdir(tmpDirInSandbox.c_str()) == -1)
+ throw SysError("changing into '%1%'", tmpDir);
+
+ /* Close all other file descriptors. */
+ closeMostFDs({STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO});
+
+#if __linux__
+ /* Change the personality to 32-bit if we're doing an
+ i686-linux build on an x86_64-linux machine. */
+ struct utsname utsbuf;
+ uname(&utsbuf);
+ if (drv->platform == "i686-linux" &&
+ (settings.thisSystem == "x86_64-linux" ||
+ (!strcmp(utsbuf.sysname, "Linux") && !strcmp(utsbuf.machine, "x86_64")))) {
+ if (personality(PER_LINUX32) == -1)
+ throw SysError("cannot set i686-linux personality");
+ }
+
+ /* Impersonate a Linux 2.6 machine to get some determinism in
+ builds that depend on the kernel version. */
+ if ((drv->platform == "i686-linux" || drv->platform == "x86_64-linux") && settings.impersonateLinux26) {
+ int cur = personality(0xffffffff);
+ if (cur != -1) personality(cur | 0x0020000 /* == UNAME26 */);
+ }
+
+ /* Disable address space randomization for improved
+ determinism. */
+ int cur = personality(0xffffffff);
+ if (cur != -1) personality(cur | ADDR_NO_RANDOMIZE);
+#endif
+
+ /* Disable core dumps by default. */
+ struct rlimit limit = { 0, RLIM_INFINITY };
+ setrlimit(RLIMIT_CORE, &limit);
+
+ // FIXME: set other limits to deterministic values?
+
+ /* Fill in the environment. */
+ Strings envStrs;
+ for (auto & i : env)
+ envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites));
+
+ /* If we are running in `build-users' mode, then switch to the
+ user we allocated above. Make sure that we drop all root
+ privileges. Note that above we have closed all file
+ descriptors except std*, so that's safe. Also note that
+ setuid() when run as root sets the real, effective and
+ saved UIDs. */
+ if (setUser && buildUser) {
+ /* Preserve supplementary groups of the build user, to allow
+ admins to specify groups such as "kvm". */
+ if (!buildUser->getSupplementaryGIDs().empty() &&
+ setgroups(buildUser->getSupplementaryGIDs().size(),
+ buildUser->getSupplementaryGIDs().data()) == -1)
+ throw SysError("cannot set supplementary groups of build user");
+
+ if (setgid(buildUser->getGID()) == -1 ||
+ getgid() != buildUser->getGID() ||
+ getegid() != buildUser->getGID())
+ throw SysError("setgid failed");
+
+ if (setuid(buildUser->getUID()) == -1 ||
+ getuid() != buildUser->getUID() ||
+ geteuid() != buildUser->getUID())
+ throw SysError("setuid failed");
+ }
+
+ /* Fill in the arguments. */
+ Strings args;
+
+ const char *builder = "invalid";
+
+ if (drv->isBuiltin()) {
+ ;
+ }
+#if __APPLE__
+ else {
+ /* This has to appear before import statements. */
+ std::string sandboxProfile = "(version 1)\n";
+
+ if (useChroot) {
+
+ /* Lots and lots and lots of file functions freak out if they can't stat their full ancestry */
+ PathSet ancestry;
+
+ /* We build the ancestry before adding all inputPaths to the store because we know they'll
+ all have the same parents (the store), and there might be lots of inputs. This isn't
+ particularly efficient... I doubt it'll be a bottleneck in practice */
+ for (auto & i : dirsInChroot) {
+ Path cur = i.first;
+ while (cur.compare("/") != 0) {
+ cur = dirOf(cur);
+ ancestry.insert(cur);
+ }
+ }
+
+ /* And we want the store in there regardless of how empty dirsInChroot. We include the innermost
+ path component this time, since it's typically /nix/store and we care about that. */
+ Path cur = worker.store.storeDir;
+ while (cur.compare("/") != 0) {
+ ancestry.insert(cur);
+ cur = dirOf(cur);
+ }
+
+ /* Add all our input paths to the chroot */
+ for (auto & i : inputPaths) {
+ auto p = worker.store.printStorePath(i);
+ dirsInChroot[p] = p;
+ }
+
+ /* Violations will go to the syslog if you set this. Unfortunately the destination does not appear to be configurable */
+ if (settings.darwinLogSandboxViolations) {
+ sandboxProfile += "(deny default)\n";
+ } else {
+ sandboxProfile += "(deny default (with no-log))\n";
+ }
+
+ sandboxProfile += "(import \"sandbox-defaults.sb\")\n";
+
+ if (derivationIsImpure(derivationType))
+ sandboxProfile += "(import \"sandbox-network.sb\")\n";
+
+ /* Add the output paths we'll use at build-time to the chroot */
+ sandboxProfile += "(allow file-read* file-write* process-exec\n";
+ for (auto & [_, path] : scratchOutputs)
+ sandboxProfile += fmt("\t(subpath \"%s\")\n", worker.store.printStorePath(path));
+
+ sandboxProfile += ")\n";
+
+ /* Our inputs (transitive dependencies and any impurities computed above)
+
+ without file-write* allowed, access() incorrectly returns EPERM
+ */
+ sandboxProfile += "(allow file-read* file-write* process-exec\n";
+ for (auto & i : dirsInChroot) {
+ if (i.first != i.second.source)
+ throw Error(
+ "can't map '%1%' to '%2%': mismatched impure paths not supported on Darwin",
+ i.first, i.second.source);
+
+ string path = i.first;
+ struct stat st;
+ if (lstat(path.c_str(), &st)) {
+ if (i.second.optional && errno == ENOENT)
+ continue;
+ throw SysError("getting attributes of path '%s", path);
+ }
+ if (S_ISDIR(st.st_mode))
+ sandboxProfile += fmt("\t(subpath \"%s\")\n", path);
+ else
+ sandboxProfile += fmt("\t(literal \"%s\")\n", path);
+ }
+ sandboxProfile += ")\n";
+
+ /* Allow file-read* on full directory hierarchy to self. Allows realpath() */
+ sandboxProfile += "(allow file-read*\n";
+ for (auto & i : ancestry) {
+ sandboxProfile += fmt("\t(literal \"%s\")\n", i);
+ }
+ sandboxProfile += ")\n";
+
+ sandboxProfile += additionalSandboxProfile;
+ } else
+ sandboxProfile += "(import \"sandbox-minimal.sb\")\n";
+
+ debug("Generated sandbox profile:");
+ debug(sandboxProfile);
+
+ Path sandboxFile = tmpDir + "/.sandbox.sb";
+
+ writeFile(sandboxFile, sandboxProfile);
+
+ bool allowLocalNetworking = parsedDrv->getBoolAttr("__darwinAllowLocalNetworking");
+
+ /* The tmpDir in scope points at the temporary build directory for our derivation. Some packages try different mechanisms
+ to find temporary directories, so we want to open up a broader place for them to dump their files, if needed. */
+ Path globalTmpDir = canonPath(getEnv("TMPDIR").value_or("/tmp"), true);
+
+ /* They don't like trailing slashes on subpath directives */
+ if (globalTmpDir.back() == '/') globalTmpDir.pop_back();
+
+ if (getEnv("_NIX_TEST_NO_SANDBOX") != "1") {
+ builder = "/usr/bin/sandbox-exec";
+ args.push_back("sandbox-exec");
+ args.push_back("-f");
+ args.push_back(sandboxFile);
+ args.push_back("-D");
+ args.push_back("_GLOBAL_TMP_DIR=" + globalTmpDir);
+ args.push_back("-D");
+ args.push_back("IMPORT_DIR=" + settings.nixDataDir + "/nix/sandbox/");
+ if (allowLocalNetworking) {
+ args.push_back("-D");
+ args.push_back(string("_ALLOW_LOCAL_NETWORKING=1"));
+ }
+ args.push_back(drv->builder);
+ } else {
+ builder = drv->builder.c_str();
+ args.push_back(std::string(baseNameOf(drv->builder)));
+ }
+ }
+#else
+ else {
+ builder = drv->builder.c_str();
+ args.push_back(std::string(baseNameOf(drv->builder)));
+ }
+#endif
+
+ for (auto & i : drv->args)
+ args.push_back(rewriteStrings(i, inputRewrites));
+
+ /* Indicate that we managed to set up the build environment. */
+ writeFull(STDERR_FILENO, string("\2\n"));
+
+ /* Execute the program. This should not return. */
+ if (drv->isBuiltin()) {
+ try {
+ logger = makeJSONLogger(*logger);
+
+ BasicDerivation & drv2(*drv);
+ for (auto & e : drv2.env)
+ e.second = rewriteStrings(e.second, inputRewrites);
+
+ if (drv->builder == "builtin:fetchurl")
+ builtinFetchurl(drv2, netrcData);
+ else if (drv->builder == "builtin:buildenv")
+ builtinBuildenv(drv2);
+ else if (drv->builder == "builtin:unpack-channel")
+ builtinUnpackChannel(drv2);
+ else
+ throw Error("unsupported builtin function '%1%'", string(drv->builder, 8));
+ _exit(0);
+ } catch (std::exception & e) {
+ writeFull(STDERR_FILENO, e.what() + std::string("\n"));
+ _exit(1);
+ }
+ }
+
+#if __APPLE__
+ posix_spawnattr_t attrp;
+
+ if (posix_spawnattr_init(&attrp))
+ throw SysError("failed to initialize builder");
+
+ if (posix_spawnattr_setflags(&attrp, POSIX_SPAWN_SETEXEC))
+ throw SysError("failed to initialize builder");
+
+ if (drv->platform == "aarch64-darwin") {
+ // Unset kern.curproc_arch_affinity so we can escape Rosetta
+ int affinity = 0;
+ sysctlbyname("kern.curproc_arch_affinity", NULL, NULL, &affinity, sizeof(affinity));
+
+ cpu_type_t cpu = CPU_TYPE_ARM64;
+ posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
+ } else if (drv->platform == "x86_64-darwin") {
+ cpu_type_t cpu = CPU_TYPE_X86_64;
+ posix_spawnattr_setbinpref_np(&attrp, 1, &cpu, NULL);
+ }
+
+ posix_spawn(NULL, builder, NULL, &attrp, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
+#else
+ execve(builder, stringsToCharPtrs(args).data(), stringsToCharPtrs(envStrs).data());
+#endif
+
+ throw SysError("executing '%1%'", drv->builder);
+
+ } catch (Error & e) {
+ writeFull(STDERR_FILENO, "\1\n");
+ FdSink sink(STDERR_FILENO);
+ sink << e;
+ sink.flush();
+ _exit(1);
+ }
+}
+
+
+void LocalDerivationGoal::registerOutputs()
+{
+ /* When using a build hook, the build hook can register the output
+ as valid (by doing `nix-store --import'). If so we don't have
+ to do anything here.
+
+ We can only early return when the outputs are known a priori. For
+ floating content-addressed derivations this isn't the case.
+ */
+ if (hook) {
+ DerivationGoal::registerOutputs();
+ return;
+ }
+
+ std::map<std::string, ValidPathInfo> infos;
+
+ /* Set of inodes seen during calls to canonicalisePathMetaData()
+ for this build's outputs. This needs to be shared between
+ outputs to allow hard links between outputs. */
+ InodesSeen inodesSeen;
+
+ Path checkSuffix = ".check";
+ bool keepPreviousRound = settings.keepFailed || settings.runDiffHook;
+
+ std::exception_ptr delayedException;
+
+ /* The paths that can be referenced are the input closures, the
+ output paths, and any paths that have been built via recursive
+ Nix calls. */
+ StorePathSet referenceablePaths;
+ for (auto & p : inputPaths) referenceablePaths.insert(p);
+ for (auto & i : scratchOutputs) referenceablePaths.insert(i.second);
+ for (auto & p : addedPaths) referenceablePaths.insert(p);
+
+ /* FIXME `needsHashRewrite` should probably be removed and we get to the
+ real reason why we aren't using the chroot dir */
+ auto toRealPathChroot = [&](const Path & p) -> Path {
+ return useChroot && !needsHashRewrite()
+ ? chrootRootDir + p
+ : worker.store.toRealPath(p);
+ };
+
+ /* Check whether the output paths were created, and make all
+ output paths read-only. Then get the references of each output (that we
+ might need to register), so we can topologically sort them. For the ones
+ that are most definitely already installed, we just store their final
+ name so we can also use it in rewrites. */
+ StringSet outputsToSort;
+ struct AlreadyRegistered { StorePath path; };
+ struct PerhapsNeedToRegister { StorePathSet refs; };
+ std::map<std::string, std::variant<AlreadyRegistered, PerhapsNeedToRegister>> outputReferencesIfUnregistered;
+ std::map<std::string, struct stat> outputStats;
+ for (auto & [outputName, _] : drv->outputs) {
+ auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchOutputs.at(outputName)));
+
+ outputsToSort.insert(outputName);
+
+ /* Updated wanted info to remove the outputs we definitely don't need to register */
+ auto & initialInfo = initialOutputs.at(outputName);
+
+ /* Don't register if already valid, and not checking */
+ initialInfo.wanted = buildMode == bmCheck
+ || !(initialInfo.known && initialInfo.known->isValid());
+ if (!initialInfo.wanted) {
+ outputReferencesIfUnregistered.insert_or_assign(
+ outputName,
+ AlreadyRegistered { .path = initialInfo.known->path });
+ continue;
+ }
+
+ struct stat st;
+ if (lstat(actualPath.c_str(), &st) == -1) {
+ if (errno == ENOENT)
+ throw BuildError(
+ "builder for '%s' failed to produce output path for output '%s' at '%s'",
+ worker.store.printStorePath(drvPath), outputName, actualPath);
+ throw SysError("getting attributes of path '%s'", actualPath);
+ }
+
+#ifndef __CYGWIN__
+ /* Check that the output is not group or world writable, as
+ that means that someone else can have interfered with the
+ build. Also, the output should be owned by the build
+ user. */
+ if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) ||
+ (buildUser && st.st_uid != buildUser->getUID()))
+ throw BuildError(
+ "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output",
+ actualPath, outputName);
+#endif
+
+ /* Canonicalise first. This ensures that the path we're
+ rewriting doesn't contain a hard link to /etc/shadow or
+ something like that. */
+ canonicalisePathMetaData(
+ actualPath,
+ buildUser ? std::optional(buildUser->getUIDRange()) : std::nullopt,
+ inodesSeen);
+
+ debug("scanning for references for output '%s' in temp location '%s'", outputName, actualPath);
+
+ /* Pass blank Sink as we are not ready to hash data at this stage. */
+ NullSink blank;
+ auto references = worker.store.parseStorePathSet(
+ scanForReferences(blank, actualPath, worker.store.printStorePathSet(referenceablePaths)));
+
+ outputReferencesIfUnregistered.insert_or_assign(
+ outputName,
+ PerhapsNeedToRegister { .refs = references });
+ outputStats.insert_or_assign(outputName, std::move(st));
+ }
+
+ auto sortedOutputNames = topoSort(outputsToSort,
+ {[&](const std::string & name) {
+ return std::visit(overloaded {
+ /* Since we'll use the already installed versions of these, we
+ can treat them as leaves and ignore any references they
+ have. */
+ [&](AlreadyRegistered _) { return StringSet {}; },
+ [&](PerhapsNeedToRegister refs) {
+ StringSet referencedOutputs;
+ /* FIXME build inverted map up front so no quadratic waste here */
+ for (auto & r : refs.refs)
+ for (auto & [o, p] : scratchOutputs)
+ if (r == p)
+ referencedOutputs.insert(o);
+ return referencedOutputs;
+ },
+ }, outputReferencesIfUnregistered.at(name));
+ }},
+ {[&](const std::string & path, const std::string & parent) {
+ // TODO with more -vvvv also show the temporary paths for manual inspection.
+ return BuildError(
+ "cycle detected in build of '%s' in the references of output '%s' from output '%s'",
+ worker.store.printStorePath(drvPath), path, parent);
+ }});
+
+ std::reverse(sortedOutputNames.begin(), sortedOutputNames.end());
+
+ for (auto & outputName : sortedOutputNames) {
+ auto output = drv->outputs.at(outputName);
+ auto & scratchPath = scratchOutputs.at(outputName);
+ auto actualPath = toRealPathChroot(worker.store.printStorePath(scratchPath));
+
+ auto finish = [&](StorePath finalStorePath) {
+ /* Store the final path */
+ finalOutputs.insert_or_assign(outputName, finalStorePath);
+ /* The rewrite rule will be used in downstream outputs that refer to
+ use. This is why the topological sort is essential to do first
+ before this for loop. */
+ if (scratchPath != finalStorePath)
+ outputRewrites[std::string { scratchPath.hashPart() }] = std::string { finalStorePath.hashPart() };
+ };
+
+ std::optional<StorePathSet> referencesOpt = std::visit(overloaded {
+ [&](AlreadyRegistered skippedFinalPath) -> std::optional<StorePathSet> {
+ finish(skippedFinalPath.path);
+ return std::nullopt;
+ },
+ [&](PerhapsNeedToRegister r) -> std::optional<StorePathSet> {
+ return r.refs;
+ },
+ }, outputReferencesIfUnregistered.at(outputName));
+
+ if (!referencesOpt)
+ continue;
+ auto references = *referencesOpt;
+
+ auto rewriteOutput = [&]() {
+ /* Apply hash rewriting if necessary. */
+ if (!outputRewrites.empty()) {
+ warn("rewriting hashes in '%1%'; cross fingers", actualPath);
+
+ /* FIXME: this is in-memory. */
+ StringSink sink;
+ dumpPath(actualPath, sink);
+ deletePath(actualPath);
+ sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
+ StringSource source(*sink.s);
+ restorePath(actualPath, source);
+
+ /* FIXME: set proper permissions in restorePath() so
+ we don't have to do another traversal. */
+ canonicalisePathMetaData(actualPath, {}, inodesSeen);
+ }
+ };
+
+ auto rewriteRefs = [&]() -> std::pair<bool, StorePathSet> {
+ /* In the CA case, we need the rewritten refs to calculate the
+ final path, therefore we look for a *non-rewritten
+ self-reference, and use a bool rather try to solve the
+ computationally intractable fixed point. */
+ std::pair<bool, StorePathSet> res {
+ false,
+ {},
+ };
+ for (auto & r : references) {
+ auto name = r.name();
+ auto origHash = std::string { r.hashPart() };
+ if (r == scratchPath)
+ res.first = true;
+ else if (outputRewrites.count(origHash) == 0)
+ res.second.insert(r);
+ else {
+ std::string newRef = outputRewrites.at(origHash);
+ newRef += '-';
+ newRef += name;
+ res.second.insert(StorePath { newRef });
+ }
+ }
+ return res;
+ };
+
+ auto newInfoFromCA = [&](const DerivationOutputCAFloating outputHash) -> ValidPathInfo {
+ auto & st = outputStats.at(outputName);
+ if (outputHash.method == FileIngestionMethod::Flat) {
+ /* The output path should be a regular file without execute permission. */
+ if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
+ throw BuildError(
+ "output path '%1%' should be a non-executable regular file "
+ "since recursive hashing is not enabled (outputHashMode=flat)",
+ actualPath);
+ }
+ rewriteOutput();
+ /* FIXME optimize and deduplicate with addToStore */
+ std::string oldHashPart { scratchPath.hashPart() };
+ HashModuloSink caSink { outputHash.hashType, oldHashPart };
+ switch (outputHash.method) {
+ case FileIngestionMethod::Recursive:
+ dumpPath(actualPath, caSink);
+ break;
+ case FileIngestionMethod::Flat:
+ readFile(actualPath, caSink);
+ break;
+ }
+ auto got = caSink.finish().first;
+ auto refs = rewriteRefs();
+ HashModuloSink narSink { htSHA256, oldHashPart };
+ dumpPath(actualPath, narSink);
+ auto narHashAndSize = narSink.finish();
+ ValidPathInfo newInfo0 {
+ worker.store.makeFixedOutputPath(
+ outputHash.method,
+ got,
+ outputPathName(drv->name, outputName),
+ refs.second,
+ refs.first),
+ narHashAndSize.first,
+ };
+ newInfo0.narSize = narHashAndSize.second;
+ newInfo0.ca = FixedOutputHash {
+ .method = outputHash.method,
+ .hash = got,
+ };
+ newInfo0.references = refs.second;
+ if (refs.first)
+ newInfo0.references.insert(newInfo0.path);
+ if (scratchPath != newInfo0.path) {
+ // Also rewrite the output path
+ auto source = sinkToSource([&](Sink & nextSink) {
+ StringSink sink;
+ dumpPath(actualPath, sink);
+ RewritingSink rsink2(oldHashPart, std::string(newInfo0.path.hashPart()), nextSink);
+ rsink2(*sink.s);
+ rsink2.flush();
+ });
+ Path tmpPath = actualPath + ".tmp";
+ restorePath(tmpPath, *source);
+ deletePath(actualPath);
+ movePath(tmpPath, actualPath);
+ }
+
+ assert(newInfo0.ca);
+ return newInfo0;
+ };
+
+ ValidPathInfo newInfo = std::visit(overloaded {
+ [&](DerivationOutputInputAddressed output) {
+ /* input-addressed case */
+ auto requiredFinalPath = output.path;
+ /* Preemptively add rewrite rule for final hash, as that is
+ what the NAR hash will use rather than normalized-self references */
+ if (scratchPath != requiredFinalPath)
+ outputRewrites.insert_or_assign(
+ std::string { scratchPath.hashPart() },
+ std::string { requiredFinalPath.hashPart() });
+ rewriteOutput();
+ auto narHashAndSize = hashPath(htSHA256, actualPath);
+ ValidPathInfo newInfo0 { requiredFinalPath, narHashAndSize.first };
+ newInfo0.narSize = narHashAndSize.second;
+ auto refs = rewriteRefs();
+ newInfo0.references = refs.second;
+ if (refs.first)
+ newInfo0.references.insert(newInfo0.path);
+ return newInfo0;
+ },
+ [&](DerivationOutputCAFixed dof) {
+ auto newInfo0 = newInfoFromCA(DerivationOutputCAFloating {
+ .method = dof.hash.method,
+ .hashType = dof.hash.hash.type,
+ });
+
+ /* Check wanted hash */
+ Hash & wanted = dof.hash.hash;
+ assert(newInfo0.ca);
+ auto got = getContentAddressHash(*newInfo0.ca);
+ if (wanted != got) {
+ /* Throw an error after registering the path as
+ valid. */
+ worker.hashMismatch = true;
+ delayedException = std::make_exception_ptr(
+ BuildError("hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s",
+ worker.store.printStorePath(drvPath),
+ wanted.to_string(SRI, true),
+ got.to_string(SRI, true)));
+ }
+ return newInfo0;
+ },
+ [&](DerivationOutputCAFloating dof) {
+ return newInfoFromCA(dof);
+ },
+ [&](DerivationOutputDeferred) {
+ // No derivation should reach that point without having been
+ // rewritten first
+ assert(false);
+ // Ugly, but the compiler insists on having this return a value
+ // of type `ValidPathInfo` despite the `assert(false)`, so
+ // let's provide it
+ return *(ValidPathInfo*)0;
+ },
+ }, output.output);
+
+ /* Calculate where we'll move the output files. In the checking case we
+ will leave leave them where they are, for now, rather than move to
+ their usual "final destination" */
+ auto finalDestPath = worker.store.printStorePath(newInfo.path);
+
+ /* Lock final output path, if not already locked. This happens with
+ floating CA derivations and hash-mismatching fixed-output
+ derivations. */
+ PathLocks dynamicOutputLock;
+ auto optFixedPath = output.path(worker.store, drv->name, outputName);
+ if (!optFixedPath ||
+ worker.store.printStorePath(*optFixedPath) != finalDestPath)
+ {
+ assert(newInfo.ca);
+ dynamicOutputLock.lockPaths({worker.store.toRealPath(finalDestPath)});
+ }
+
+ /* Move files, if needed */
+ if (worker.store.toRealPath(finalDestPath) != actualPath) {
+ if (buildMode == bmRepair) {
+ /* Path already exists, need to replace it */
+ replaceValidPath(worker.store.toRealPath(finalDestPath), actualPath);
+ actualPath = worker.store.toRealPath(finalDestPath);
+ } else if (buildMode == bmCheck) {
+ /* Path already exists, and we want to compare, so we leave out
+ new path in place. */
+ } else if (worker.store.isValidPath(newInfo.path)) {
+ /* Path already exists because CA path produced by something
+ else. No moving needed. */
+ assert(newInfo.ca);
+ } else {
+ auto destPath = worker.store.toRealPath(finalDestPath);
+ movePath(actualPath, destPath);
+ actualPath = destPath;
+ }
+ }
+
+ auto & localStore = getLocalStore();
+
+ if (buildMode == bmCheck) {
+
+ if (!worker.store.isValidPath(newInfo.path)) continue;
+ ValidPathInfo oldInfo(*worker.store.queryPathInfo(newInfo.path));
+ if (newInfo.narHash != oldInfo.narHash) {
+ worker.checkMismatch = true;
+ if (settings.runDiffHook || settings.keepFailed) {
+ auto dst = worker.store.toRealPath(finalDestPath + checkSuffix);
+ deletePath(dst);
+ movePath(actualPath, dst);
+
+ handleDiffHook(
+ buildUser ? buildUser->getUID() : getuid(),
+ buildUser ? buildUser->getGID() : getgid(),
+ finalDestPath, dst, worker.store.printStorePath(drvPath), tmpDir);
+
+ throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
+ worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath), dst);
+ } else
+ throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
+ worker.store.printStorePath(drvPath), worker.store.toRealPath(finalDestPath));
+ }
+
+ /* Since we verified the build, it's now ultimately trusted. */
+ if (!oldInfo.ultimate) {
+ oldInfo.ultimate = true;
+ localStore.signPathInfo(oldInfo);
+ localStore.registerValidPaths({{oldInfo.path, oldInfo}});
+ }
+
+ continue;
+ }
+
+ /* For debugging, print out the referenced and unreferenced paths. */
+ for (auto & i : inputPaths) {
+ auto j = references.find(i);
+ if (j == references.end())
+ debug("unreferenced input: '%1%'", worker.store.printStorePath(i));
+ else
+ debug("referenced input: '%1%'", worker.store.printStorePath(i));
+ }
+
+ if (curRound == nrRounds) {
+ localStore.optimisePath(actualPath); // FIXME: combine with scanForReferences()
+ worker.markContentsGood(newInfo.path);
+ }
+
+ newInfo.deriver = drvPath;
+ newInfo.ultimate = true;
+ localStore.signPathInfo(newInfo);
+
+ finish(newInfo.path);
+
+ /* If it's a CA path, register it right away. This is necessary if it
+ isn't statically known so that we can safely unlock the path before
+ the next iteration */
+ if (newInfo.ca)
+ localStore.registerValidPaths({{newInfo.path, newInfo}});
+
+ infos.emplace(outputName, std::move(newInfo));
+ }
+
+ if (buildMode == bmCheck) return;
+
+ /* Apply output checks. */
+ checkOutputs(infos);
+
+ /* Compare the result with the previous round, and report which
+ path is different, if any.*/
+ if (curRound > 1 && prevInfos != infos) {
+ assert(prevInfos.size() == infos.size());
+ for (auto i = prevInfos.begin(), j = infos.begin(); i != prevInfos.end(); ++i, ++j)
+ if (!(*i == *j)) {
+ result.isNonDeterministic = true;
+ Path prev = worker.store.printStorePath(i->second.path) + checkSuffix;
+ bool prevExists = keepPreviousRound && pathExists(prev);
+ hintformat hint = prevExists
+ ? hintfmt("output '%s' of '%s' differs from '%s' from previous round",
+ worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath), prev)
+ : hintfmt("output '%s' of '%s' differs from previous round",
+ worker.store.printStorePath(i->second.path), worker.store.printStorePath(drvPath));
+
+ handleDiffHook(
+ buildUser ? buildUser->getUID() : getuid(),
+ buildUser ? buildUser->getGID() : getgid(),
+ prev, worker.store.printStorePath(i->second.path),
+ worker.store.printStorePath(drvPath), tmpDir);
+
+ if (settings.enforceDeterminism)
+ throw NotDeterministic(hint);
+
+ printError(hint);
+
+ curRound = nrRounds; // we know enough, bail out early
+ }
+ }
+
+ /* If this is the first round of several, then move the output out of the way. */
+ if (nrRounds > 1 && curRound == 1 && curRound < nrRounds && keepPreviousRound) {
+ for (auto & [_, outputStorePath] : finalOutputs) {
+ auto path = worker.store.printStorePath(outputStorePath);
+ Path prev = path + checkSuffix;
+ deletePath(prev);
+ Path dst = path + checkSuffix;
+ if (rename(path.c_str(), dst.c_str()))
+ throw SysError("renaming '%s' to '%s'", path, dst);
+ }
+ }
+
+ if (curRound < nrRounds) {
+ prevInfos = std::move(infos);
+ return;
+ }
+
+ /* Remove the .check directories if we're done. FIXME: keep them
+ if the result was not determistic? */
+ if (curRound == nrRounds) {
+ for (auto & [_, outputStorePath] : finalOutputs) {
+ Path prev = worker.store.printStorePath(outputStorePath) + checkSuffix;
+ deletePath(prev);
+ }
+ }
+
+ /* Register each output path as valid, and register the sets of
+ paths referenced by each of them. If there are cycles in the
+ outputs, this will fail. */
+ {
+ auto & localStore = getLocalStore();
+
+ ValidPathInfos infos2;
+ for (auto & [outputName, newInfo] : infos) {
+ infos2.insert_or_assign(newInfo.path, newInfo);
+ }
+ localStore.registerValidPaths(infos2);
+ }
+
+ /* In case of a fixed-output derivation hash mismatch, throw an
+ exception now that we have registered the output as valid. */
+ if (delayedException)
+ std::rethrow_exception(delayedException);
+
+ /* If we made it this far, we are sure the output matches the derivation
+ (since the delayedException would be a fixed output CA mismatch). That
+ means it's safe to link the derivation to the output hash. We must do
+ that for floating CA derivations, which otherwise couldn't be cached,
+ but it's fine to do in all cases. */
+
+ if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+ for (auto& [outputName, newInfo] : infos) {
+ auto thisRealisation = Realisation{
+ .id = DrvOutput{initialOutputs.at(outputName).outputHash,
+ outputName},
+ .outPath = newInfo.path};
+ signRealisation(thisRealisation);
+ worker.store.registerDrvOutput(thisRealisation);
+ }
+ }
+}
+
+void LocalDerivationGoal::signRealisation(Realisation & realisation)
+{
+ getLocalStore().signRealisation(realisation);
+}
+
+
+void LocalDerivationGoal::checkOutputs(const std::map<Path, ValidPathInfo> & outputs)
+{
+ std::map<Path, const ValidPathInfo &> outputsByPath;
+ for (auto & output : outputs)
+ outputsByPath.emplace(worker.store.printStorePath(output.second.path), output.second);
+
+ for (auto & output : outputs) {
+ auto & outputName = output.first;
+ auto & info = output.second;
+
+ struct Checks
+ {
+ bool ignoreSelfRefs = false;
+ std::optional<uint64_t> maxSize, maxClosureSize;
+ std::optional<Strings> allowedReferences, allowedRequisites, disallowedReferences, disallowedRequisites;
+ };
+
+ /* Compute the closure and closure size of some output. This
+ is slightly tricky because some of its references (namely
+ other outputs) may not be valid yet. */
+ auto getClosure = [&](const StorePath & path)
+ {
+ uint64_t closureSize = 0;
+ StorePathSet pathsDone;
+ std::queue<StorePath> pathsLeft;
+ pathsLeft.push(path);
+
+ while (!pathsLeft.empty()) {
+ auto path = pathsLeft.front();
+ pathsLeft.pop();
+ if (!pathsDone.insert(path).second) continue;
+
+ auto i = outputsByPath.find(worker.store.printStorePath(path));
+ if (i != outputsByPath.end()) {
+ closureSize += i->second.narSize;
+ for (auto & ref : i->second.references)
+ pathsLeft.push(ref);
+ } else {
+ auto info = worker.store.queryPathInfo(path);
+ closureSize += info->narSize;
+ for (auto & ref : info->references)
+ pathsLeft.push(ref);
+ }
+ }
+
+ return std::make_pair(std::move(pathsDone), closureSize);
+ };
+
+ auto applyChecks = [&](const Checks & checks)
+ {
+ if (checks.maxSize && info.narSize > *checks.maxSize)
+ throw BuildError("path '%s' is too large at %d bytes; limit is %d bytes",
+ worker.store.printStorePath(info.path), info.narSize, *checks.maxSize);
+
+ if (checks.maxClosureSize) {
+ uint64_t closureSize = getClosure(info.path).second;
+ if (closureSize > *checks.maxClosureSize)
+ throw BuildError("closure of path '%s' is too large at %d bytes; limit is %d bytes",
+ worker.store.printStorePath(info.path), closureSize, *checks.maxClosureSize);
+ }
+
+ auto checkRefs = [&](const std::optional<Strings> & value, bool allowed, bool recursive)
+ {
+ if (!value) return;
+
+ /* Parse a list of reference specifiers. Each element must
+ either be a store path, or the symbolic name of the output
+ of the derivation (such as `out'). */
+ StorePathSet spec;
+ for (auto & i : *value) {
+ if (worker.store.isStorePath(i))
+ spec.insert(worker.store.parseStorePath(i));
+ else if (finalOutputs.count(i))
+ spec.insert(finalOutputs.at(i));
+ else throw BuildError("derivation contains an illegal reference specifier '%s'", i);
+ }
+
+ auto used = recursive
+ ? getClosure(info.path).first
+ : info.references;
+
+ if (recursive && checks.ignoreSelfRefs)
+ used.erase(info.path);
+
+ StorePathSet badPaths;
+
+ for (auto & i : used)
+ if (allowed) {
+ if (!spec.count(i))
+ badPaths.insert(i);
+ } else {
+ if (spec.count(i))
+ badPaths.insert(i);
+ }
+
+ if (!badPaths.empty()) {
+ string badPathsStr;
+ for (auto & i : badPaths) {
+ badPathsStr += "\n ";
+ badPathsStr += worker.store.printStorePath(i);
+ }
+ throw BuildError("output '%s' is not allowed to refer to the following paths:%s",
+ worker.store.printStorePath(info.path), badPathsStr);
+ }
+ };
+
+ checkRefs(checks.allowedReferences, true, false);
+ checkRefs(checks.allowedRequisites, true, true);
+ checkRefs(checks.disallowedReferences, false, false);
+ checkRefs(checks.disallowedRequisites, false, true);
+ };
+
+ if (auto structuredAttrs = parsedDrv->getStructuredAttrs()) {
+ auto outputChecks = structuredAttrs->find("outputChecks");
+ if (outputChecks != structuredAttrs->end()) {
+ auto output = outputChecks->find(outputName);
+
+ if (output != outputChecks->end()) {
+ Checks checks;
+
+ auto maxSize = output->find("maxSize");
+ if (maxSize != output->end())
+ checks.maxSize = maxSize->get<uint64_t>();
+
+ auto maxClosureSize = output->find("maxClosureSize");
+ if (maxClosureSize != output->end())
+ checks.maxClosureSize = maxClosureSize->get<uint64_t>();
+
+ auto get = [&](const std::string & name) -> std::optional<Strings> {
+ auto i = output->find(name);
+ if (i != output->end()) {
+ Strings res;
+ for (auto j = i->begin(); j != i->end(); ++j) {
+ if (!j->is_string())
+ throw Error("attribute '%s' of derivation '%s' must be a list of strings", name, worker.store.printStorePath(drvPath));
+ res.push_back(j->get<std::string>());
+ }
+ checks.disallowedRequisites = res;
+ return res;
+ }
+ return {};
+ };
+
+ checks.allowedReferences = get("allowedReferences");
+ checks.allowedRequisites = get("allowedRequisites");
+ checks.disallowedReferences = get("disallowedReferences");
+ checks.disallowedRequisites = get("disallowedRequisites");
+
+ applyChecks(checks);
+ }
+ }
+ } else {
+ // legacy non-structured-attributes case
+ Checks checks;
+ checks.ignoreSelfRefs = true;
+ checks.allowedReferences = parsedDrv->getStringsAttr("allowedReferences");
+ checks.allowedRequisites = parsedDrv->getStringsAttr("allowedRequisites");
+ checks.disallowedReferences = parsedDrv->getStringsAttr("disallowedReferences");
+ checks.disallowedRequisites = parsedDrv->getStringsAttr("disallowedRequisites");
+ applyChecks(checks);
+ }
+ }
+}
+
+
+void LocalDerivationGoal::deleteTmpDir(bool force)
+{
+ if (tmpDir != "") {
+ /* Don't keep temporary directories for builtins because they
+ might have privileged stuff (like a copy of netrc). */
+ if (settings.keepFailed && !force && !drv->isBuiltin()) {
+ printError("note: keeping build directory '%s'", tmpDir);
+ chmod(tmpDir.c_str(), 0755);
+ }
+ else
+ deletePath(tmpDir);
+ tmpDir = "";
+ }
+}
+
+
+bool LocalDerivationGoal::isReadDesc(int fd)
+{
+ return (hook && DerivationGoal::isReadDesc(fd)) ||
+ (!hook && fd == builderOut.readSide.get());
+}
+
+
+StorePath LocalDerivationGoal::makeFallbackPath(std::string_view outputName)
+{
+ return worker.store.makeStorePath(
+ "rewrite:" + std::string(drvPath.to_string()) + ":name:" + std::string(outputName),
+ Hash(htSHA256), outputPathName(drv->name, outputName));
+}
+
+
+StorePath LocalDerivationGoal::makeFallbackPath(const StorePath & path)
+{
+ return worker.store.makeStorePath(
+ "rewrite:" + std::string(drvPath.to_string()) + ":" + std::string(path.to_string()),
+ Hash(htSHA256), path.name());
+}
+
+
+}
diff --git a/src/libstore/build/local-derivation-goal.hh b/src/libstore/build/local-derivation-goal.hh
new file mode 100644
index 000000000..f69d97e99
--- /dev/null
+++ b/src/libstore/build/local-derivation-goal.hh
@@ -0,0 +1,209 @@
+#pragma once
+
+#include "derivation-goal.hh"
+#include "local-store.hh"
+
+namespace nix {
+
+struct LocalDerivationGoal : public DerivationGoal
+{
+ LocalStore & getLocalStore();
+
+ /* User selected for running the builder. */
+ std::unique_ptr<UserLock> buildUser;
+
+ /* The process ID of the builder. */
+ Pid pid;
+
+ /* The temporary directory. */
+ Path tmpDir;
+
+ /* The path of the temporary directory in the sandbox. */
+ Path tmpDirInSandbox;
+
+ /* Pipe for the builder's standard output/error. */
+ Pipe builderOut;
+
+ /* Pipe for synchronising updates to the builder namespaces. */
+ Pipe userNamespaceSync;
+
+ /* The mount namespace of the builder, used to add additional
+ paths to the sandbox as a result of recursive Nix calls. */
+ AutoCloseFD sandboxMountNamespace;
+
+ /* On Linux, whether we're doing the build in its own user
+ namespace. */
+ bool usingUserNamespace = true;
+
+ /* Whether we're currently doing a chroot build. */
+ bool useChroot = false;
+
+ Path chrootRootDir;
+
+ /* Whether to give the build more than 1 UID. */
+ bool useUidRange = false;
+
+ /* Whether to make the 'systemd' cgroup controller available to
+ the build. */
+ bool useSystemdCgroup = false;
+
+ /* RAII object to delete the chroot directory. */
+ std::shared_ptr<AutoDelete> autoDelChroot;
+
+ /* Whether to run the build in a private network namespace. */
+ bool privateNetwork = false;
+
+ /* Stuff we need to pass to initChild(). */
+ struct ChrootPath {
+ Path source;
+ bool optional;
+ ChrootPath(Path source = "", bool optional = false)
+ : source(source), optional(optional)
+ { }
+ };
+ typedef map<Path, ChrootPath> DirsInChroot; // maps target path to source path
+ DirsInChroot dirsInChroot;
+
+ typedef map<string, string> Environment;
+ Environment env;
+
+#if __APPLE__
+ typedef string SandboxProfile;
+ SandboxProfile additionalSandboxProfile;
+#endif
+
+ /* Hash rewriting. */
+ StringMap inputRewrites, outputRewrites;
+ typedef map<StorePath, StorePath> RedirectedOutputs;
+ RedirectedOutputs redirectedOutputs;
+
+ /* The outputs paths used during the build.
+
+ - Input-addressed derivations or fixed content-addressed outputs are
+ sometimes built when some of their outputs already exist, and can not
+ be hidden via sandboxing. We use temporary locations instead and
+ rewrite after the build. Otherwise the regular predetermined paths are
+ put here.
+
+ - Floating content-addressed derivations do not know their final build
+ output paths until the outputs are hashed, so random locations are
+ used, and then renamed. The randomness helps guard against hidden
+ self-references.
+ */
+ OutputPathMap scratchOutputs;
+
+ /* Path registration info from the previous round, if we're
+ building multiple times. Since this contains the hash, it
+ allows us to compare whether two rounds produced the same
+ result. */
+ std::map<Path, ValidPathInfo> prevInfos;
+
+ uid_t sandboxUid() { return usingUserNamespace ? (useUidRange ? 0 : 1000) : buildUser->getUID(); }
+ gid_t sandboxGid() { return usingUserNamespace ? (useUidRange ? 0 : 100) : buildUser->getGID(); }
+
+ const static Path homeDir;
+
+ /* The recursive Nix daemon socket. */
+ AutoCloseFD daemonSocket;
+
+ /* The daemon main thread. */
+ std::thread daemonThread;
+
+ /* The daemon worker threads. */
+ std::vector<std::thread> daemonWorkerThreads;
+
+ /* Paths that were added via recursive Nix calls. */
+ StorePathSet addedPaths;
+
+ /* Recursive Nix calls are only allowed to build or realize paths
+ in the original input closure or added via a recursive Nix call
+ (so e.g. you can't do 'nix-store -r /nix/store/<bla>' where
+ /nix/store/<bla> is some arbitrary path in a binary cache). */
+ bool isAllowed(const StorePath & path)
+ {
+ return inputPaths.count(path) || addedPaths.count(path);
+ }
+ bool isAllowed(const DerivedPath & req);
+
+ friend struct RestrictedStore;
+
+ using DerivationGoal::DerivationGoal;
+
+ virtual ~LocalDerivationGoal() override;
+
+ /* Whether we need to perform hash rewriting if there are valid output paths. */
+ bool needsHashRewrite();
+
+ /* The additional states. */
+ void tryLocalBuild() override;
+
+ /* Start building a derivation. */
+ void startBuilder();
+
+ /* Fill in the environment for the builder. */
+ void initEnv();
+
+ /* Setup tmp dir location. */
+ void initTmpDir();
+
+ /* Write a JSON file containing the derivation attributes. */
+ void writeStructuredAttrs();
+
+ void startDaemon();
+
+ void stopDaemon();
+
+ /* Add 'path' to the set of paths that may be referenced by the
+ outputs, and make it appear in the sandbox. */
+ void addDependency(const StorePath & path);
+
+ /* Make a file owned by the builder. */
+ void chownToBuilder(const Path & path);
+
+ int getChildStatus() override;
+
+ /* Run the builder's process. */
+ void runChild();
+
+ /* Check that the derivation outputs all exist and register them
+ as valid. */
+ void registerOutputs() override;
+
+ void signRealisation(Realisation &) override;
+
+ /* Check that an output meets the requirements specified by the
+ 'outputChecks' attribute (or the legacy
+ '{allowed,disallowed}{References,Requisites}' attributes). */
+ void checkOutputs(const std::map<std::string, ValidPathInfo> & outputs);
+
+ /* Close the read side of the logger pipe. */
+ void closeReadPipes() override;
+
+ /* Cleanup hooks for buildDone() */
+ void cleanupHookFinally() override;
+ void cleanupPreChildKill() override;
+ void cleanupPostChildKill() override;
+ bool cleanupDecideWhetherDiskFull() override;
+ void cleanupPostOutputsRegisteredModeCheck() override;
+ void cleanupPostOutputsRegisteredModeNonCheck() override;
+
+ bool isReadDesc(int fd) override;
+
+ /* Delete the temporary directory, if we have one. */
+ void deleteTmpDir(bool force);
+
+ /* Forcibly kill the child process, if any. */
+ void killChild() override;
+
+ /* Create alternative path calculated from but distinct from the
+ input, so we can avoid overwriting outputs (or other store paths)
+ that already exist. */
+ StorePath makeFallbackPath(const StorePath & path);
+ /* Make a path to another based on the output name along with the
+ derivation hash. */
+ /* FIXME add option to randomize, so we can audit whether our
+ rewrites caught everything */
+ StorePath makeFallbackPath(std::string_view outputName);
+};
+
+}
diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc
index d16584f65..e56cfadbe 100644
--- a/src/libstore/build/substitution-goal.cc
+++ b/src/libstore/build/substitution-goal.cc
@@ -5,40 +5,32 @@
namespace nix {
-SubstitutionGoal::SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
+PathSubstitutionGoal::PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair, std::optional<ContentAddress> ca)
: Goal(worker)
, storePath(storePath)
, repair(repair)
, ca(ca)
{
- state = &SubstitutionGoal::init;
+ state = &PathSubstitutionGoal::init;
name = fmt("substitution of '%s'", worker.store.printStorePath(this->storePath));
trace("created");
maintainExpectedSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.expectedSubstitutions);
}
-SubstitutionGoal::~SubstitutionGoal()
+PathSubstitutionGoal::~PathSubstitutionGoal()
{
- try {
- if (thr.joinable()) {
- // FIXME: signal worker thread to quit.
- thr.join();
- worker.childTerminated(this);
- }
- } catch (...) {
- ignoreException();
- }
+ cleanup();
}
-void SubstitutionGoal::work()
+void PathSubstitutionGoal::work()
{
(this->*state)();
}
-void SubstitutionGoal::init()
+void PathSubstitutionGoal::init()
{
trace("init");
@@ -59,10 +51,12 @@ void SubstitutionGoal::init()
}
-void SubstitutionGoal::tryNext()
+void PathSubstitutionGoal::tryNext()
{
trace("trying next substituter");
+ cleanup();
+
if (subs.size() == 0) {
/* None left. Terminate this goal and let someone else deal
with it. */
@@ -142,15 +136,10 @@ void SubstitutionGoal::tryNext()
/* Bail out early if this substituter lacks a valid
signature. LocalStore::addToStore() also checks for this, but
only after we've downloaded the path. */
- if (worker.store.requireSigs
- && !sub->isTrusted
- && !info->checkSignatures(worker.store, worker.store.getPublicKeys()))
+ if (!sub->isTrusted && worker.store.pathInfoIsUntrusted(*info))
{
- logWarning({
- .name = "Invalid path signature",
- .hint = hintfmt("substituter '%s' does not have a valid signature for path '%s'",
- sub->getUri(), worker.store.printStorePath(storePath))
- });
+ warn("substituter '%s' does not have a valid signature for path '%s'",
+ sub->getUri(), worker.store.printStorePath(storePath));
tryNext();
return;
}
@@ -159,16 +148,16 @@ void SubstitutionGoal::tryNext()
paths referenced by this one. */
for (auto & i : info->references)
if (i != storePath) /* ignore self-references */
- addWaitee(worker.makeSubstitutionGoal(i));
+ addWaitee(worker.makePathSubstitutionGoal(i));
if (waitees.empty()) /* to prevent hang (no wake-up event) */
referencesValid();
else
- state = &SubstitutionGoal::referencesValid;
+ state = &PathSubstitutionGoal::referencesValid;
}
-void SubstitutionGoal::referencesValid()
+void PathSubstitutionGoal::referencesValid()
{
trace("all references realised");
@@ -182,12 +171,12 @@ void SubstitutionGoal::referencesValid()
if (i != storePath) /* ignore self-references */
assert(worker.store.isValidPath(i));
- state = &SubstitutionGoal::tryToRun;
+ state = &PathSubstitutionGoal::tryToRun;
worker.wakeUp(shared_from_this());
}
-void SubstitutionGoal::tryToRun()
+void PathSubstitutionGoal::tryToRun()
{
trace("trying to run");
@@ -210,7 +199,7 @@ void SubstitutionGoal::tryToRun()
thr = std::thread([this]() {
try {
/* Wake up the worker loop when we're done. */
- Finally updateStats([this]() { outPipe.writeSide = -1; });
+ Finally updateStats([this]() { outPipe.writeSide.close(); });
Activity act(*logger, actSubstitute, Logger::Fields{worker.store.printStorePath(storePath), sub->getUri()});
PushActivity pact(act.id);
@@ -226,11 +215,11 @@ void SubstitutionGoal::tryToRun()
worker.childStarted(shared_from_this(), {outPipe.readSide.get()}, true, false);
- state = &SubstitutionGoal::finished;
+ state = &PathSubstitutionGoal::finished;
}
-void SubstitutionGoal::finished()
+void PathSubstitutionGoal::finished()
{
trace("substitute finished");
@@ -254,7 +243,7 @@ void SubstitutionGoal::finished()
}
/* Try the next substitute. */
- state = &SubstitutionGoal::tryNext;
+ state = &PathSubstitutionGoal::tryNext;
worker.wakeUp(shared_from_this());
return;
}
@@ -283,14 +272,31 @@ void SubstitutionGoal::finished()
}
-void SubstitutionGoal::handleChildOutput(int fd, const string & data)
+void PathSubstitutionGoal::handleChildOutput(int fd, const string & data)
{
}
-void SubstitutionGoal::handleEOF(int fd)
+void PathSubstitutionGoal::handleEOF(int fd)
{
if (fd == outPipe.readSide.get()) worker.wakeUp(shared_from_this());
}
+
+void PathSubstitutionGoal::cleanup()
+{
+ try {
+ if (thr.joinable()) {
+ // FIXME: signal worker thread to quit.
+ thr.join();
+ worker.childTerminated(this);
+ }
+
+ outPipe.close();
+ } catch (...) {
+ ignoreException();
+ }
+}
+
+
}
diff --git a/src/libstore/build/substitution-goal.hh b/src/libstore/build/substitution-goal.hh
index 3ae9a9e6b..70c806d23 100644
--- a/src/libstore/build/substitution-goal.hh
+++ b/src/libstore/build/substitution-goal.hh
@@ -8,16 +8,13 @@ namespace nix {
class Worker;
-class SubstitutionGoal : public Goal
+struct PathSubstitutionGoal : public Goal
{
- friend class Worker;
-
-private:
/* The store path that should be realised through a substitute. */
StorePath storePath;
/* The path the substituter refers to the path as. This will be
- * different when the stores have different names. */
+ different when the stores have different names. */
std::optional<StorePath> subPath;
/* The remaining substituters. */
@@ -50,15 +47,15 @@ private:
std::unique_ptr<MaintainCount<uint64_t>> maintainExpectedSubstitutions,
maintainRunningSubstitutions, maintainExpectedNar, maintainExpectedDownload;
- typedef void (SubstitutionGoal::*GoalState)();
+ typedef void (PathSubstitutionGoal::*GoalState)();
GoalState state;
/* Content address for recomputing store path */
std::optional<ContentAddress> ca;
public:
- SubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
- ~SubstitutionGoal();
+ PathSubstitutionGoal(const StorePath & storePath, Worker & worker, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+ ~PathSubstitutionGoal();
void timedOut(Error && ex) override { abort(); };
@@ -83,7 +80,7 @@ public:
void handleChildOutput(int fd, const string & data) override;
void handleEOF(int fd) override;
- StorePath getStorePath() { return storePath; }
+ void cleanup() override;
};
}
diff --git a/src/libstore/build/worker.cc b/src/libstore/build/worker.cc
index 5c3fe2f57..0f2ade348 100644
--- a/src/libstore/build/worker.cc
+++ b/src/libstore/build/worker.cc
@@ -1,14 +1,15 @@
#include "machines.hh"
#include "worker.hh"
#include "substitution-goal.hh"
-#include "derivation-goal.hh"
+#include "drv-output-substitution-goal.hh"
+#include "local-derivation-goal.hh"
#include "hook-instance.hh"
#include <poll.h>
namespace nix {
-Worker::Worker(LocalStore & store)
+Worker::Worker(Store & store)
: act(*logger, actRealise)
, actDerivations(*logger, actBuilds)
, actSubstitutions(*logger, actCopyPaths)
@@ -43,16 +44,13 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
const StringSet & wantedOutputs,
std::function<std::shared_ptr<DerivationGoal>()> mkDrvGoal)
{
- WeakGoalPtr & abstract_goal_weak = derivationGoals[drvPath];
- GoalPtr abstract_goal = abstract_goal_weak.lock(); // FIXME
- std::shared_ptr<DerivationGoal> goal;
- if (!abstract_goal) {
+ std::weak_ptr<DerivationGoal> & goal_weak = derivationGoals[drvPath];
+ std::shared_ptr<DerivationGoal> goal = goal_weak.lock();
+ if (!goal) {
goal = mkDrvGoal();
- abstract_goal_weak = goal;
+ goal_weak = goal;
wakeUp(goal);
} else {
- goal = std::dynamic_pointer_cast<DerivationGoal>(abstract_goal);
- assert(goal);
goal->addWantedOutputs(wantedOutputs);
}
return goal;
@@ -62,8 +60,10 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoalCommon(
std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drvPath,
const StringSet & wantedOutputs, BuildMode buildMode)
{
- return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
- return std::make_shared<DerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
+ return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
+ return !dynamic_cast<LocalStore *>(&store)
+ ? std::make_shared</* */DerivationGoal>(drvPath, wantedOutputs, *this, buildMode)
+ : std::make_shared<LocalDerivationGoal>(drvPath, wantedOutputs, *this, buildMode);
});
}
@@ -71,32 +71,46 @@ std::shared_ptr<DerivationGoal> Worker::makeDerivationGoal(const StorePath & drv
std::shared_ptr<DerivationGoal> Worker::makeBasicDerivationGoal(const StorePath & drvPath,
const BasicDerivation & drv, const StringSet & wantedOutputs, BuildMode buildMode)
{
- return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() {
- return std::make_shared<DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
+ return makeDerivationGoalCommon(drvPath, wantedOutputs, [&]() -> std::shared_ptr<DerivationGoal> {
+ return !dynamic_cast<LocalStore *>(&store)
+ ? std::make_shared</* */DerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode)
+ : std::make_shared<LocalDerivationGoal>(drvPath, drv, wantedOutputs, *this, buildMode);
});
}
-GoalPtr Worker::makeSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
+std::shared_ptr<PathSubstitutionGoal> Worker::makePathSubstitutionGoal(const StorePath & path, RepairFlag repair, std::optional<ContentAddress> ca)
{
- WeakGoalPtr & goal_weak = substitutionGoals[path];
- GoalPtr goal = goal_weak.lock(); // FIXME
+ std::weak_ptr<PathSubstitutionGoal> & goal_weak = substitutionGoals[path];
+ auto goal = goal_weak.lock(); // FIXME
if (!goal) {
- goal = std::make_shared<SubstitutionGoal>(path, *this, repair, ca);
+ goal = std::make_shared<PathSubstitutionGoal>(path, *this, repair, ca);
goal_weak = goal;
wakeUp(goal);
}
return goal;
}
+std::shared_ptr<DrvOutputSubstitutionGoal> Worker::makeDrvOutputSubstitutionGoal(const DrvOutput& id, RepairFlag repair, std::optional<ContentAddress> ca)
+{
+ std::weak_ptr<DrvOutputSubstitutionGoal> & goal_weak = drvOutputSubstitutionGoals[id];
+ auto goal = goal_weak.lock(); // FIXME
+ if (!goal) {
+ goal = std::make_shared<DrvOutputSubstitutionGoal>(id, *this, repair, ca);
+ goal_weak = goal;
+ wakeUp(goal);
+ }
+ return goal;
+}
-static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
+template<typename K, typename G>
+static void removeGoal(std::shared_ptr<G> goal, std::map<K, std::weak_ptr<G>> & goalMap)
{
/* !!! inefficient */
- for (WeakGoalMap::iterator i = goalMap.begin();
+ for (auto i = goalMap.begin();
i != goalMap.end(); )
if (i->second.lock() == goal) {
- WeakGoalMap::iterator j = i; ++j;
+ auto j = i; ++j;
goalMap.erase(i);
i = j;
}
@@ -106,8 +120,15 @@ static void removeGoal(GoalPtr goal, WeakGoalMap & goalMap)
void Worker::removeGoal(GoalPtr goal)
{
- nix::removeGoal(goal, derivationGoals);
- nix::removeGoal(goal, substitutionGoals);
+ if (auto drvGoal = std::dynamic_pointer_cast<DerivationGoal>(goal))
+ nix::removeGoal(drvGoal, derivationGoals);
+ else if (auto subGoal = std::dynamic_pointer_cast<PathSubstitutionGoal>(goal))
+ nix::removeGoal(subGoal, substitutionGoals);
+ else if (auto subGoal = std::dynamic_pointer_cast<DrvOutputSubstitutionGoal>(goal))
+ nix::removeGoal(subGoal, drvOutputSubstitutionGoals);
+ else
+ assert(false);
+
if (topGoals.find(goal) != topGoals.end()) {
topGoals.erase(goal);
/* If a top-level goal failed, then kill all other goals
@@ -206,7 +227,21 @@ void Worker::waitForAWhile(GoalPtr goal)
void Worker::run(const Goals & _topGoals)
{
- for (auto & i : _topGoals) topGoals.insert(i);
+ std::vector<nix::DerivedPath> topPaths;
+
+ for (auto & i : _topGoals) {
+ topGoals.insert(i);
+ if (auto goal = dynamic_cast<DerivationGoal *>(i.get())) {
+ topPaths.push_back(DerivedPath::Built{goal->drvPath, goal->wantedOutputs});
+ } else if (auto goal = dynamic_cast<PathSubstitutionGoal *>(i.get())) {
+ topPaths.push_back(DerivedPath::Opaque{goal->storePath});
+ }
+ }
+
+ /* Call queryMissing() efficiently query substitutes. */
+ StorePathSet willBuild, willSubstitute, unknown;
+ uint64_t downloadSize, narSize;
+ store.queryMissing(topPaths, willBuild, willSubstitute, unknown, downloadSize, narSize);
debug("entered goal loop");
@@ -214,7 +249,9 @@ void Worker::run(const Goals & _topGoals)
checkInterrupt();
- store.autoGC(false);
+ // TODO GC interface?
+ if (auto localStore = dynamic_cast<LocalStore *>(&store))
+ localStore->autoGC(false);
/* Call every wake goal (in the ordering established by
CompareGoalPtrs). */
@@ -439,10 +476,7 @@ bool Worker::pathContentsGood(const StorePath & path)
}
pathContentsGoodCache.insert_or_assign(path, res);
if (!res)
- logError({
- .name = "Corrupted path",
- .hint = hintfmt("path '%s' is corrupted or missing!", store.printStorePath(path))
- });
+ printError("path '%s' is corrupted or missing!", store.printStorePath(path));
return res;
}
@@ -452,4 +486,12 @@ void Worker::markContentsGood(const StorePath & path)
pathContentsGoodCache.insert_or_assign(path, true);
}
+
+GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal) {
+ return subGoal;
+}
+GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal) {
+ return subGoal;
+}
+
}
diff --git a/src/libstore/build/worker.hh b/src/libstore/build/worker.hh
index a54316343..918de35f6 100644
--- a/src/libstore/build/worker.hh
+++ b/src/libstore/build/worker.hh
@@ -2,13 +2,31 @@
#include "types.hh"
#include "lock.hh"
-#include "local-store.hh"
+#include "store-api.hh"
#include "goal.hh"
+#include "realisation.hh"
+
+#include <future>
+#include <thread>
namespace nix {
/* Forward definition. */
-class DerivationGoal;
+struct DerivationGoal;
+struct PathSubstitutionGoal;
+class DrvOutputSubstitutionGoal;
+
+/* Workaround for not being able to declare a something like
+
+ class PathSubstitutionGoal : public Goal;
+
+ even when Goal is a complete type.
+
+ This is still a static cast. The purpose of exporting it is to define it in
+ a place where `PathSubstitutionGoal` is concrete, and use it in a place where it
+ is opaque. */
+GoalPtr upcast_goal(std::shared_ptr<PathSubstitutionGoal> subGoal);
+GoalPtr upcast_goal(std::shared_ptr<DrvOutputSubstitutionGoal> subGoal);
typedef std::chrono::time_point<std::chrono::steady_clock> steady_time_point;
@@ -56,8 +74,9 @@ private:
/* Maps used to prevent multiple instantiations of a goal for the
same derivation / path. */
- WeakGoalMap derivationGoals;
- WeakGoalMap substitutionGoals;
+ std::map<StorePath, std::weak_ptr<DerivationGoal>> derivationGoals;
+ std::map<StorePath, std::weak_ptr<PathSubstitutionGoal>> substitutionGoals;
+ std::map<DrvOutput, std::weak_ptr<DrvOutputSubstitutionGoal>> drvOutputSubstitutionGoals;
/* Goals waiting for busy paths to be unlocked. */
WeakGoals waitingForAnyGoal;
@@ -90,7 +109,7 @@ public:
/* Set if at least one derivation is not deterministic in check mode. */
bool checkMismatch;
- LocalStore & store;
+ Store & store;
std::unique_ptr<HookInstance> hook;
@@ -112,7 +131,7 @@ public:
it answers with "decline-permanently", we don't try again. */
bool tryBuildHook = true;
- Worker(LocalStore & store);
+ Worker(Store & store);
~Worker();
/* Make a goal (with caching). */
@@ -131,7 +150,8 @@ public:
const StringSet & wantedOutputs, BuildMode buildMode = bmNormal);
/* substitution goal */
- GoalPtr makeSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+ std::shared_ptr<PathSubstitutionGoal> makePathSubstitutionGoal(const StorePath & storePath, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
+ std::shared_ptr<DrvOutputSubstitutionGoal> makeDrvOutputSubstitutionGoal(const DrvOutput & id, RepairFlag repair = NoRepair, std::optional<ContentAddress> ca = std::nullopt);
/* Remove a dead goal. */
void removeGoal(GoalPtr goal);
diff --git a/src/libstore/builtins/buildenv.cc b/src/libstore/builtins/buildenv.cc
index 802fb87bc..e88fc687a 100644
--- a/src/libstore/builtins/buildenv.cc
+++ b/src/libstore/builtins/buildenv.cc
@@ -22,10 +22,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
srcFiles = readDirectory(srcDir);
} catch (SysError & e) {
if (e.errNo == ENOTDIR) {
- logWarning({
- .name = "Create links - directory",
- .hint = hintfmt("not including '%s' in the user environment because it's not a directory", srcDir)
- });
+ warn("not including '%s' in the user environment because it's not a directory", srcDir);
return;
}
throw;
@@ -44,10 +41,7 @@ static void createLinks(State & state, const Path & srcDir, const Path & dstDir,
throw SysError("getting status of '%1%'", srcFile);
} catch (SysError & e) {
if (e.errNo == ENOENT || e.errNo == ENOTDIR) {
- logWarning({
- .name = "Create links - skipping symlink",
- .hint = hintfmt("skipping dangling symlink '%s'", dstFile)
- });
+ warn("skipping dangling symlink '%s'", dstFile);
continue;
}
throw;
diff --git a/src/libstore/ca-specific-schema.sql b/src/libstore/ca-specific-schema.sql
new file mode 100644
index 000000000..20ee046a1
--- /dev/null
+++ b/src/libstore/ca-specific-schema.sql
@@ -0,0 +1,12 @@
+-- Extension of the sql schema for content-addressed derivations.
+-- Won't be loaded unless the experimental feature `ca-derivations`
+-- is enabled
+
+create table if not exists Realisations (
+ drvPath text not null,
+ outputName text not null, -- symbolic output id, usually "out"
+ outputPath integer not null,
+ signatures text, -- space-separated list
+ primary key (drvPath, outputName),
+ foreign key (outputPath) references ValidPaths(id) on delete cascade
+);
diff --git a/src/libstore/cgroup.cc b/src/libstore/cgroup.cc
index 0ae45e46d..5d31609da 100644
--- a/src/libstore/cgroup.cc
+++ b/src/libstore/cgroup.cc
@@ -53,7 +53,10 @@ void destroyCgroup(const Path & cgroup)
for (auto & pid_s : pids) {
pid_t pid;
- if (!string2Int(pid_s, pid)) throw Error("invalid pid '%s'", pid);
+ if (auto o = string2Int<pid_t>(pid_s))
+ pid = *o;
+ else
+ throw Error("invalid pid '%s'", pid);
if (pidsShown.insert(pid).second) {
try {
auto cmdline = readFile(fmt("/proc/%d/cmdline", pid));
diff --git a/src/libstore/crypto.cc b/src/libstore/crypto.cc
index 9ec8abd22..1027469c9 100644
--- a/src/libstore/crypto.cc
+++ b/src/libstore/crypto.cc
@@ -2,21 +2,19 @@
#include "util.hh"
#include "globals.hh"
-#if HAVE_SODIUM
#include <sodium.h>
-#endif
namespace nix {
-static std::pair<std::string, std::string> split(const string & s)
+static std::pair<std::string_view, std::string_view> split(std::string_view s)
{
size_t colon = s.find(':');
if (colon == std::string::npos || colon == 0)
return {"", ""};
- return {std::string(s, 0, colon), std::string(s, colon + 1)};
+ return {s.substr(0, colon), s.substr(colon + 1)};
}
-Key::Key(const string & s)
+Key::Key(std::string_view s)
{
auto ss = split(s);
@@ -29,62 +27,57 @@ Key::Key(const string & s)
key = base64Decode(key);
}
-SecretKey::SecretKey(const string & s)
- : Key(s)
+std::string Key::to_string() const
{
-#if HAVE_SODIUM
- if (key.size() != crypto_sign_SECRETKEYBYTES)
- throw Error("secret key is not valid");
-#endif
+ return name + ":" + base64Encode(key);
}
-#if !HAVE_SODIUM
-[[noreturn]] static void noSodium()
+SecretKey::SecretKey(std::string_view s)
+ : Key(s)
{
- throw Error("Nix was not compiled with libsodium, required for signed binary cache support");
+ if (key.size() != crypto_sign_SECRETKEYBYTES)
+ throw Error("secret key is not valid");
}
-#endif
-std::string SecretKey::signDetached(const std::string & data) const
+std::string SecretKey::signDetached(std::string_view data) const
{
-#if HAVE_SODIUM
unsigned char sig[crypto_sign_BYTES];
unsigned long long sigLen;
crypto_sign_detached(sig, &sigLen, (unsigned char *) data.data(), data.size(),
(unsigned char *) key.data());
return name + ":" + base64Encode(std::string((char *) sig, sigLen));
-#else
- noSodium();
-#endif
}
PublicKey SecretKey::toPublicKey() const
{
-#if HAVE_SODIUM
unsigned char pk[crypto_sign_PUBLICKEYBYTES];
crypto_sign_ed25519_sk_to_pk(pk, (unsigned char *) key.data());
return PublicKey(name, std::string((char *) pk, crypto_sign_PUBLICKEYBYTES));
-#else
- noSodium();
-#endif
}
-PublicKey::PublicKey(const string & s)
+SecretKey SecretKey::generate(std::string_view name)
+{
+ unsigned char pk[crypto_sign_PUBLICKEYBYTES];
+ unsigned char sk[crypto_sign_SECRETKEYBYTES];
+ if (crypto_sign_keypair(pk, sk) != 0)
+ throw Error("key generation failed");
+
+ return SecretKey(name, std::string((char *) sk, crypto_sign_SECRETKEYBYTES));
+}
+
+PublicKey::PublicKey(std::string_view s)
: Key(s)
{
-#if HAVE_SODIUM
if (key.size() != crypto_sign_PUBLICKEYBYTES)
throw Error("public key is not valid");
-#endif
}
bool verifyDetached(const std::string & data, const std::string & sig,
const PublicKeys & publicKeys)
{
-#if HAVE_SODIUM
auto ss = split(sig);
- auto key = publicKeys.find(ss.first);
+ auto key = publicKeys.find(std::string(ss.first));
if (key == publicKeys.end()) return false;
auto sig2 = base64Decode(ss.second);
@@ -94,9 +87,6 @@ bool verifyDetached(const std::string & data, const std::string & sig,
return crypto_sign_verify_detached((unsigned char *) sig2.data(),
(unsigned char *) data.data(), data.size(),
(unsigned char *) key->second.key.data()) == 0;
-#else
- noSodium();
-#endif
}
PublicKeys getDefaultPublicKeys()
diff --git a/src/libstore/crypto.hh b/src/libstore/crypto.hh
index 9110af3aa..03f85c103 100644
--- a/src/libstore/crypto.hh
+++ b/src/libstore/crypto.hh
@@ -13,32 +13,40 @@ struct Key
/* Construct Key from a string in the format
‘<name>:<key-in-base64>’. */
- Key(const std::string & s);
+ Key(std::string_view s);
+
+ std::string to_string() const;
protected:
- Key(const std::string & name, const std::string & key)
- : name(name), key(key) { }
+ Key(std::string_view name, std::string && key)
+ : name(name), key(std::move(key)) { }
};
struct PublicKey;
struct SecretKey : Key
{
- SecretKey(const std::string & s);
+ SecretKey(std::string_view s);
/* Return a detached signature of the given string. */
- std::string signDetached(const std::string & s) const;
+ std::string signDetached(std::string_view s) const;
PublicKey toPublicKey() const;
+
+ static SecretKey generate(std::string_view name);
+
+private:
+ SecretKey(std::string_view name, std::string && key)
+ : Key(name, std::move(key)) { }
};
struct PublicKey : Key
{
- PublicKey(const std::string & data);
+ PublicKey(std::string_view data);
private:
- PublicKey(const std::string & name, const std::string & key)
- : Key(name, key) { }
+ PublicKey(std::string_view name, std::string && key)
+ : Key(name, std::move(key)) { }
friend struct SecretKey;
};
diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc
index 99d8add92..affd60472 100644
--- a/src/libstore/daemon.cc
+++ b/src/libstore/daemon.cc
@@ -2,6 +2,7 @@
#include "monitor-fd.hh"
#include "worker-protocol.hh"
#include "store-api.hh"
+#include "path-with-outputs.hh"
#include "finally.hh"
#include "affinity.hh"
#include "archive.hh"
@@ -153,10 +154,10 @@ struct TunnelSink : Sink
{
Sink & to;
TunnelSink(Sink & to) : to(to) { }
- virtual void operator () (const unsigned char * data, size_t len)
+ void operator () (std::string_view data)
{
to << STDERR_WRITE;
- writeString(data, len, to);
+ writeString(data, to);
}
};
@@ -165,7 +166,7 @@ struct TunnelSource : BufferedSource
Source & from;
BufferedSink & to;
TunnelSource(Source & from, BufferedSink & to) : from(from), to(to) { }
- size_t readUnbuffered(unsigned char * data, size_t len) override
+ size_t readUnbuffered(char * data, size_t len) override
{
to << STDERR_READ << len;
to.flush();
@@ -215,6 +216,8 @@ struct ClientSettings
for (auto & s : ss)
if (trusted.count(s))
subs.push_back(s);
+ else if (!hasSuffix(s, "/") && trusted.count(s + "/"))
+ subs.push_back(s + "/");
else
warn("ignoring untrusted substituter '%s'", s);
res = subs;
@@ -231,8 +234,6 @@ struct ClientSettings
settings.set(name, value);
else if (setSubstituters(settings.substituters))
;
- else if (setSubstituters(settings.extraSubstituters))
- ;
else
debug("ignoring the client-specified setting '%s', because it is a restricted setting and you are not a trusted user", name);
} catch (UsageError & e) {
@@ -259,6 +260,18 @@ static void writeValidPathInfo(
}
}
+static std::vector<DerivedPath> readDerivedPaths(Store & store, unsigned int clientVersion, Source & from)
+{
+ std::vector<DerivedPath> reqs;
+ if (GET_PROTOCOL_MINOR(clientVersion) >= 29) {
+ reqs = worker_proto::read(store, from, Phantom<std::vector<DerivedPath>> {});
+ } else {
+ for (auto & s : readStrings<Strings>(from))
+ reqs.push_back(parsePathWithOutputs(store, s).toDerivedPath());
+ }
+ return reqs;
+}
+
static void performOp(TunnelLogger * logger, ref<Store> store,
TrustedFlag trusted, RecursiveFlag recursive, unsigned int clientVersion,
Source & from, BufferedSink & to, unsigned int op)
@@ -276,8 +289,17 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
case wopQueryValidPaths: {
auto paths = worker_proto::read(*store, from, Phantom<StorePathSet> {});
+
+ SubstituteFlag substitute = NoSubstitute;
+ if (GET_PROTOCOL_MINOR(clientVersion) >= 27) {
+ substitute = readInt(from) ? Substitute : NoSubstitute;
+ }
+
logger->startWork();
- auto res = store->queryValidPaths(paths);
+ if (substitute) {
+ store->substitutePaths(paths);
+ }
+ auto res = store->queryValidPaths(paths, substitute);
logger->stopWork();
worker_proto::write(*store, to, res);
break;
@@ -484,9 +506,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopBuildPaths: {
- std::vector<StorePathWithOutputs> drvs;
- for (auto & s : readStrings<Strings>(from))
- drvs.push_back(store->parsePathWithOutputs(s));
+ auto drvs = readDerivedPaths(*store, clientVersion, from);
BuildMode mode = bmNormal;
if (GET_PROTOCOL_MINOR(clientVersion) >= 15) {
mode = (BuildMode) readInt(from);
@@ -566,6 +586,12 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
auto res = store->buildDerivation(drvPath, drv, buildMode);
logger->stopWork();
to << res.status << res.errorMsg;
+ if (GET_PROTOCOL_MINOR(clientVersion) >= 29) {
+ to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime;
+ }
+ if (GET_PROTOCOL_MINOR(clientVersion) >= 28) {
+ worker_proto::write(*store, to, res.builtOutputs);
+ }
break;
}
@@ -844,9 +870,7 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
}
case wopQueryMissing: {
- std::vector<StorePathWithOutputs> targets;
- for (auto & s : readStrings<Strings>(from))
- targets.push_back(store->parsePathWithOutputs(s));
+ auto targets = readDerivedPaths(*store, clientVersion, from);
logger->startWork();
StorePathSet willBuild, willSubstitute, unknown;
uint64_t downloadSize, narSize;
@@ -859,6 +883,28 @@ static void performOp(TunnelLogger * logger, ref<Store> store,
break;
}
+ case wopRegisterDrvOutput: {
+ logger->startWork();
+ auto outputId = DrvOutput::parse(readString(from));
+ auto outputPath = StorePath(readString(from));
+ auto resolvedDrv = StorePath(readString(from));
+ store->registerDrvOutput(Realisation{
+ .id = outputId, .outPath = outputPath});
+ logger->stopWork();
+ break;
+ }
+
+ case wopQueryRealisation: {
+ logger->startWork();
+ auto outputId = DrvOutput::parse(readString(from));
+ auto info = store->queryRealisation(outputId);
+ logger->stopWork();
+ std::set<StorePath> outPaths;
+ if (info) outPaths.insert(info->outPath);
+ worker_proto::write(*store, to, outPaths);
+ break;
+ }
+
default:
throw Error("invalid operation %1%", op);
}
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 07b4e772b..f6defd98f 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -21,6 +21,9 @@ std::optional<StorePath> DerivationOutput::path(const Store & store, std::string
[](DerivationOutputCAFloating dof) -> std::optional<StorePath> {
return std::nullopt;
},
+ [](DerivationOutputDeferred) -> std::optional<StorePath> {
+ return std::nullopt;
+ },
}, output);
}
@@ -37,6 +40,7 @@ bool derivationIsCA(DerivationType dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return true;
+ case DerivationType::DeferredInputAddressed: return false;
};
// Since enums can have non-variant values, but making a `default:` would
// disable exhaustiveness warnings.
@@ -48,15 +52,28 @@ bool derivationIsFixed(DerivationType dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return false;
+ case DerivationType::DeferredInputAddressed: return false;
};
assert(false);
}
+bool derivationHasKnownOutputPaths(DerivationType dt) {
+ switch (dt) {
+ case DerivationType::InputAddressed: return true;
+ case DerivationType::CAFixed: return true;
+ case DerivationType::CAFloating: return false;
+ case DerivationType::DeferredInputAddressed: return false;
+ };
+ assert(false);
+}
+
+
bool derivationIsImpure(DerivationType dt) {
switch (dt) {
case DerivationType::InputAddressed: return false;
case DerivationType::CAFixed: return true;
case DerivationType::CAFloating: return false;
+ case DerivationType::DeferredInputAddressed: return false;
};
assert(false);
}
@@ -180,6 +197,11 @@ static DerivationOutput parseDerivationOutput(const Store & store,
};
}
} else {
+ if (pathS == "") {
+ return DerivationOutput {
+ .output = DerivationOutputDeferred { }
+ };
+ }
validatePath(pathS);
return DerivationOutput {
.output = DerivationOutputInputAddressed {
@@ -325,6 +347,11 @@ string Derivation::unparse(const Store & store, bool maskOutputs,
s += ','; printUnquotedString(s, makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
s += ','; printUnquotedString(s, "");
},
+ [&](DerivationOutputDeferred) {
+ s += ','; printUnquotedString(s, "");
+ s += ','; printUnquotedString(s, "");
+ s += ','; printUnquotedString(s, "");
+ }
}, i.second.output);
s += ')';
}
@@ -389,7 +416,7 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
DerivationType BasicDerivation::type() const
{
- std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs;
+ std::set<std::string_view> inputAddressedOutputs, fixedCAOutputs, floatingCAOutputs, deferredIAOutputs;
std::optional<HashType> floatingHashType;
for (auto & i : outputs) {
std::visit(overloaded {
@@ -408,29 +435,34 @@ DerivationType BasicDerivation::type() const
throw Error("All floating outputs must use the same hash type");
}
},
+ [&](DerivationOutputDeferred _) {
+ deferredIAOutputs.insert(i.first);
+ },
}, i.second.output);
}
- if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+ if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
throw Error("Must have at least one output");
- } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+ } else if (! inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
return DerivationType::InputAddressed;
- } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty()) {
+ } else if (inputAddressedOutputs.empty() && ! fixedCAOutputs.empty() && floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
if (fixedCAOutputs.size() > 1)
// FIXME: Experimental feature?
throw Error("Only one fixed output is allowed for now");
if (*fixedCAOutputs.begin() != "out")
throw Error("Single fixed output must be named \"out\"");
return DerivationType::CAFixed;
- } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty()) {
+ } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && ! floatingCAOutputs.empty() && deferredIAOutputs.empty()) {
return DerivationType::CAFloating;
+ } else if (inputAddressedOutputs.empty() && fixedCAOutputs.empty() && floatingCAOutputs.empty() && !deferredIAOutputs.empty()) {
+ return DerivationType::DeferredInputAddressed;
} else {
throw Error("Can't mix derivation output types");
}
}
-DrvHashes drvHashes;
+Sync<DrvHashes> drvHashes;
/* pathDerivationModulo and hashDerivationModulo are mutually recursive
*/
@@ -438,20 +470,22 @@ DrvHashes drvHashes;
/* Look up the derivation by value and memoize the
`hashDerivationModulo` call.
*/
-static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath & drvPath)
+static const DrvHashModulo pathDerivationModulo(Store & store, const StorePath & drvPath)
{
- auto h = drvHashes.find(drvPath);
- if (h == drvHashes.end()) {
- assert(store.isValidPath(drvPath));
- // Cache it
- h = drvHashes.insert_or_assign(
- drvPath,
- hashDerivationModulo(
- store,
- store.readDerivation(drvPath),
- false)).first;
+ {
+ auto hashes = drvHashes.lock();
+ auto h = hashes->find(drvPath);
+ if (h != hashes->end()) {
+ return h->second;
+ }
}
- return h->second;
+ auto h = hashDerivationModulo(
+ store,
+ store.readInvalidDerivation(drvPath),
+ false);
+ // Cache it
+ drvHashes.lock()->insert_or_assign(drvPath, h);
+ return h;
}
/* See the header for interface details. These are the implementation details.
@@ -473,10 +507,9 @@ static const DrvHashModulo & pathDerivationModulo(Store & store, const StorePath
*/
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs)
{
+ bool isDeferred = false;
/* Return a fixed hash for fixed-output derivations. */
switch (drv.type()) {
- case DerivationType::CAFloating:
- throw Error("Regular input-addressed derivations are not yet allowed to depend on CA derivations");
case DerivationType::CAFixed: {
std::map<std::string, Hash> outputHashes;
for (const auto & i : drv.outputs) {
@@ -489,8 +522,13 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
}
return outputHashes;
}
+ case DerivationType::CAFloating:
+ isDeferred = true;
+ break;
case DerivationType::InputAddressed:
break;
+ case DerivationType::DeferredInputAddressed:
+ break;
}
/* For other derivations, replace the inputs paths with recursive
@@ -503,6 +541,10 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
[&](Hash drvHash) {
inputs2.insert_or_assign(drvHash.to_string(Base16, false), i.second);
},
+ [&](DeferredHash deferredHash) {
+ isDeferred = true;
+ inputs2.insert_or_assign(deferredHash.hash.to_string(Base16, false), i.second);
+ },
// CA derivation's output hashes
[&](CaOutputHashes outputHashes) {
std::set<std::string> justOut = { "out" };
@@ -517,15 +559,34 @@ DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool m
}, res);
}
- return hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
+ auto hash = hashString(htSHA256, drv.unparse(store, maskOutputs, &inputs2));
+
+ if (isDeferred)
+ return DeferredHash { hash };
+ else
+ return hash;
}
-std::string StorePathWithOutputs::to_string(const Store & store) const
+std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv)
{
- return outputs.empty()
- ? store.printStorePath(path)
- : store.printStorePath(path) + "!" + concatStringsSep(",", outputs);
+ std::map<std::string, Hash> res;
+ std::visit(overloaded {
+ [&](Hash drvHash) {
+ for (auto & outputName : drv.outputNames()) {
+ res.insert({outputName, drvHash});
+ }
+ },
+ [&](DeferredHash deferredHash) {
+ for (auto & outputName : drv.outputNames()) {
+ res.insert({outputName, deferredHash.hash});
+ }
+ },
+ [&](CaOutputHashes outputHashes) {
+ res = outputHashes;
+ },
+ }, hashDerivationModulo(store, drv, true));
+ return res;
}
@@ -620,6 +681,11 @@ void writeDerivation(Sink & out, const Store & store, const BasicDerivation & dr
<< (makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType))
<< "";
},
+ [&](DerivationOutputDeferred) {
+ out << ""
+ << ""
+ << "";
+ },
}, i.second.output);
}
worker_proto::write(store, out, drv.inputSrcs);
@@ -645,7 +711,6 @@ std::string downstreamPlaceholder(const Store & store, const StorePath & drvPath
}
-// N.B. Outputs are left unchanged
static void rewriteDerivation(Store & store, BasicDerivation & drv, const StringMap & rewrites) {
debug("Rewriting the derivation");
@@ -666,10 +731,22 @@ static void rewriteDerivation(Store & store, BasicDerivation & drv, const String
newEnv.emplace(envName, envValue);
}
drv.env = newEnv;
-}
+ auto hashModulo = hashDerivationModulo(store, Derivation(drv), true);
+ for (auto & [outputName, output] : drv.outputs) {
+ if (std::holds_alternative<DerivationOutputDeferred>(output.output)) {
+ Hash h = std::get<Hash>(hashModulo);
+ auto outPath = store.makeOutputPath(outputName, h, drv.name);
+ drv.env[outputName] = store.printStorePath(outPath);
+ output = DerivationOutput {
+ .output = DerivationOutputInputAddressed {
+ .path = std::move(outPath),
+ },
+ };
+ }
+ }
-Sync<DrvPathResolutions> drvPathResolutions;
+}
std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
BasicDerivation resolved { *this };
@@ -682,8 +759,13 @@ std::optional<BasicDerivation> Derivation::tryResolve(Store & store) {
StringSet newOutputNames;
for (auto & outputName : input.second) {
auto actualPathOpt = inputDrvOutputs.at(outputName);
- if (!actualPathOpt)
+ if (!actualPathOpt) {
+ warn("output %s of input %s missing, aborting the resolving",
+ outputName,
+ store.printStorePath(input.first)
+ );
return std::nullopt;
+ }
auto actualPath = *actualPathOpt;
inputRewrites.emplace(
downstreamPlaceholder(store, input.first, outputName),
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 6d292b2e5..2df440536 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -18,8 +18,6 @@ namespace nix {
/* The traditional non-fixed-output derivation type. */
struct DerivationOutputInputAddressed
{
- /* Will need to become `std::optional<StorePath>` once input-addressed
- derivations are allowed to depend on cont-addressed derivations */
StorePath path;
};
@@ -41,14 +39,20 @@ struct DerivationOutputCAFloating
HashType hashType;
};
+/* Input-addressed output which depends on a (CA) derivation whose hash isn't
+ * known atm
+ */
+struct DerivationOutputDeferred {};
+
struct DerivationOutput
{
std::variant<
DerivationOutputInputAddressed,
DerivationOutputCAFixed,
- DerivationOutputCAFloating
+ DerivationOutputCAFloating,
+ DerivationOutputDeferred
> output;
- std::optional<HashType> hashAlgoOpt(const Store & store) const;
+
/* Note, when you use this function you should make sure that you're passing
the right derivation name. When in doubt, you should use the safer
interface provided by BasicDerivation::outputsAndOptPaths */
@@ -72,6 +76,7 @@ typedef std::map<string, string> StringPairs;
enum struct DerivationType : uint8_t {
InputAddressed,
+ DeferredInputAddressed,
CAFixed,
CAFloating,
};
@@ -89,6 +94,11 @@ bool derivationIsFixed(DerivationType);
derivation is controlled separately. Never true for non-CA derivations. */
bool derivationIsImpure(DerivationType);
+/* Does the derivation knows its own output paths?
+ * Only true when there's no floating-ca derivation involved in the closure.
+ */
+bool derivationHasKnownOutputPaths(DerivationType);
+
struct BasicDerivation
{
DerivationOutputs outputs; /* keyed on symbolic IDs */
@@ -167,9 +177,12 @@ std::string outputPathName(std::string_view drvName, std::string_view outputName
// whose output hashes are always known since they are fixed up-front.
typedef std::map<std::string, Hash> CaOutputHashes;
+struct DeferredHash { Hash hash; };
+
typedef std::variant<
Hash, // regular DRV normalized hash
- CaOutputHashes
+ CaOutputHashes, // Fixed-output derivation hashes
+ DeferredHash // Deferred hashes for floating outputs drvs and their dependencies
> DrvHashModulo;
/* Returns hashes with the details of fixed-output subderivations
@@ -197,20 +210,17 @@ typedef std::variant<
*/
DrvHashModulo hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutputs);
+/*
+ Return a map associating each output to a hash that uniquely identifies its
+ derivation (modulo the self-references).
+ */
+std::map<std::string, Hash> staticOutputHashes(Store& store, const Derivation& drv);
+
/* Memoisation of hashDerivationModulo(). */
typedef std::map<StorePath, DrvHashModulo> DrvHashes;
-extern DrvHashes drvHashes; // FIXME: global, not thread-safe
-
-/* Memoisation of `readDerivation(..).resove()`. */
-typedef std::map<
- StorePath,
- std::optional<StorePath>
-> DrvPathResolutions;
-
// FIXME: global, though at least thread-safe.
-// FIXME: arguably overlaps with hashDerivationModulo memo table.
-extern Sync<DrvPathResolutions> drvPathResolutions;
+extern Sync<DrvHashes> drvHashes;
bool wantOutput(const string & output, const std::set<string> & wanted);
diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc
new file mode 100644
index 000000000..13833c58e
--- /dev/null
+++ b/src/libstore/derived-path.cc
@@ -0,0 +1,77 @@
+#include "derived-path.hh"
+#include "store-api.hh"
+
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+nlohmann::json DerivedPath::Opaque::toJSON(ref<Store> store) const {
+ nlohmann::json res;
+ res["path"] = store->printStorePath(path);
+ return res;
+}
+
+nlohmann::json DerivedPathWithHints::Built::toJSON(ref<Store> store) const {
+ nlohmann::json res;
+ res["drvPath"] = store->printStorePath(drvPath);
+ for (const auto& [output, path] : outputs) {
+ res["outputs"][output] = path ? store->printStorePath(*path) : "";
+ }
+ return res;
+}
+
+nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref<Store> store) {
+ auto res = nlohmann::json::array();
+ for (const DerivedPathWithHints & buildable : buildables) {
+ std::visit([&res, store](const auto & buildable) {
+ res.push_back(buildable.toJSON(store));
+ }, buildable.raw());
+ }
+ return res;
+}
+
+
+std::string DerivedPath::Opaque::to_string(const Store & store) const {
+ return store.printStorePath(path);
+}
+
+std::string DerivedPath::Built::to_string(const Store & store) const {
+ return store.printStorePath(drvPath)
+ + "!"
+ + (outputs.empty() ? std::string { "*" } : concatStringsSep(",", outputs));
+}
+
+std::string DerivedPath::to_string(const Store & store) const
+{
+ return std::visit(
+ [&](const auto & req) { return req.to_string(store); },
+ this->raw());
+}
+
+
+DerivedPath::Opaque DerivedPath::Opaque::parse(const Store & store, std::string_view s)
+{
+ return {store.parseStorePath(s)};
+}
+
+DerivedPath::Built DerivedPath::Built::parse(const Store & store, std::string_view s)
+{
+ size_t n = s.find("!");
+ assert(n != s.npos);
+ auto drvPath = store.parseStorePath(s.substr(0, n));
+ auto outputsS = s.substr(n + 1);
+ std::set<string> outputs;
+ if (outputsS != "*")
+ outputs = tokenizeString<std::set<string>>(outputsS);
+ return {drvPath, outputs};
+}
+
+DerivedPath DerivedPath::parse(const Store & store, std::string_view s)
+{
+ size_t n = s.find("!");
+ return n == s.npos
+ ? (DerivedPath) DerivedPath::Opaque::parse(store, s)
+ : (DerivedPath) DerivedPath::Built::parse(store, s);
+}
+
+}
diff --git a/src/libstore/derived-path.hh b/src/libstore/derived-path.hh
new file mode 100644
index 000000000..7a2fe59de
--- /dev/null
+++ b/src/libstore/derived-path.hh
@@ -0,0 +1,129 @@
+#pragma once
+
+#include "util.hh"
+#include "path.hh"
+
+#include <optional>
+
+#include <nlohmann/json_fwd.hpp>
+
+namespace nix {
+
+class Store;
+
+/**
+ * An opaque derived path.
+ *
+ * Opaque derived paths are just store paths, and fully evaluated. They
+ * cannot be simplified further. Since they are opaque, they cannot be
+ * built, but they can fetched.
+ */
+struct DerivedPathOpaque {
+ StorePath path;
+
+ nlohmann::json toJSON(ref<Store> store) const;
+ std::string to_string(const Store & store) const;
+ static DerivedPathOpaque parse(const Store & store, std::string_view);
+};
+
+/**
+ * A derived path that is built from a derivation
+ *
+ * Built derived paths are pair of a derivation and some output names.
+ * They are evaluated by building the derivation, and then replacing the
+ * output names with the resulting outputs.
+ *
+ * Note that does mean a derived store paths evaluates to multiple
+ * opaque paths, which is sort of icky as expressions are supposed to
+ * evaluate to single values. Perhaps this should have just a single
+ * output name.
+ */
+struct DerivedPathBuilt {
+ StorePath drvPath;
+ std::set<std::string> outputs;
+
+ std::string to_string(const Store & store) const;
+ static DerivedPathBuilt parse(const Store & store, std::string_view);
+};
+
+using _DerivedPathRaw = std::variant<
+ DerivedPathOpaque,
+ DerivedPathBuilt
+>;
+
+/**
+ * A "derived path" is a very simple sort of expression that evaluates
+ * to (concrete) store path. It is either:
+ *
+ * - opaque, in which case it is just a concrete store path with
+ * possibly no known derivation
+ *
+ * - built, in which case it is a pair of a derivation path and an
+ * output name.
+ */
+struct DerivedPath : _DerivedPathRaw {
+ using Raw = _DerivedPathRaw;
+ using Raw::Raw;
+
+ using Opaque = DerivedPathOpaque;
+ using Built = DerivedPathBuilt;
+
+ inline const Raw & raw() const {
+ return static_cast<const Raw &>(*this);
+ }
+
+ std::string to_string(const Store & store) const;
+ static DerivedPath parse(const Store & store, std::string_view);
+};
+
+/**
+ * A built derived path with hints in the form of optional concrete output paths.
+ *
+ * See 'DerivedPathWithHints' for more an explanation.
+ */
+struct DerivedPathWithHintsBuilt {
+ StorePath drvPath;
+ std::map<std::string, std::optional<StorePath>> outputs;
+
+ nlohmann::json toJSON(ref<Store> store) const;
+ static DerivedPathWithHintsBuilt parse(const Store & store, std::string_view);
+};
+
+using _DerivedPathWithHintsRaw = std::variant<
+ DerivedPath::Opaque,
+ DerivedPathWithHintsBuilt
+>;
+
+/**
+ * A derived path with hints in the form of optional concrete output paths in the built case.
+ *
+ * This type is currently just used by the CLI. The paths are filled in
+ * during evaluation for derivations that know what paths they will
+ * produce in advanced, i.e. input-addressed or fixed-output content
+ * addressed derivations.
+ *
+ * That isn't very good, because it puts floating content-addressed
+ * derivations "at a disadvantage". It would be better to never rely on
+ * the output path of unbuilt derivations, and exclusively use the
+ * realizations types to work with built derivations' concrete output
+ * paths.
+ */
+// FIXME Stop using and delete this, or if that is not possible move out of libstore to libcmd.
+struct DerivedPathWithHints : _DerivedPathWithHintsRaw {
+ using Raw = _DerivedPathWithHintsRaw;
+ using Raw::Raw;
+
+ using Opaque = DerivedPathOpaque;
+ using Built = DerivedPathWithHintsBuilt;
+
+ inline const Raw & raw() const {
+ return static_cast<const Raw &>(*this);
+ }
+
+};
+
+typedef std::vector<DerivedPathWithHints> DerivedPathsWithHints;
+
+nlohmann::json derivedPathsWithHintsToJSON(const DerivedPathsWithHints & buildables, ref<Store> store);
+
+}
diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc
index 98b745c3a..8f26af685 100644
--- a/src/libstore/dummy-store.cc
+++ b/src/libstore/dummy-store.cc
@@ -9,7 +9,7 @@ struct DummyStoreConfig : virtual StoreConfig {
const std::string name() override { return "Dummy Store"; }
};
-struct DummyStore : public Store, public virtual DummyStoreConfig
+struct DummyStore : public virtual DummyStoreConfig, public virtual Store
{
DummyStore(const std::string scheme, const std::string uri, const Params & params)
: DummyStore(params)
@@ -17,6 +17,7 @@ struct DummyStore : public Store, public virtual DummyStoreConfig
DummyStore(const Params & params)
: StoreConfig(params)
+ , DummyStoreConfig(params)
, Store(params)
{ }
@@ -54,12 +55,8 @@ struct DummyStore : public Store, public virtual DummyStoreConfig
void narFromPath(const StorePath & path, Sink & sink) override
{ unsupported("narFromPath"); }
- void ensurePath(const StorePath & path) override
- { unsupported("ensurePath"); }
-
- BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
- BuildMode buildMode) override
- { unsupported("buildDerivation"); }
+ std::optional<const Realisation> queryRealisation(const DrvOutput&) override
+ { unsupported("queryRealisation"); }
};
static RegisterStoreImplementation<DummyStore, DummyStoreConfig> regDummyStore;
diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc
index c2c65af05..8ea5cdc9d 100644
--- a/src/libstore/filetransfer.cc
+++ b/src/libstore/filetransfer.cc
@@ -95,18 +95,18 @@ struct curlFileTransfer : public FileTransfer
fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
{request.uri}, request.parentAct)
, callback(std::move(callback))
- , finalSink([this](const unsigned char * data, size_t len) {
+ , finalSink([this](std::string_view data) {
if (this->request.dataCallback) {
auto httpStatus = getHTTPStatus();
/* Only write data to the sink if this is a
successful response. */
if (successfulStatuses.count(httpStatus)) {
- writtenToSink += len;
- this->request.dataCallback((char *) data, len);
+ writtenToSink += data.size();
+ this->request.dataCallback(data);
}
} else
- this->result.data->append((char *) data, len);
+ this->result.data->append(data);
})
{
if (!request.expectedETag.empty())
@@ -171,8 +171,8 @@ struct curlFileTransfer : public FileTransfer
}
if (errorSink)
- (*errorSink)((unsigned char *) contents, realSize);
- (*decompressionSink)((unsigned char *) contents, realSize);
+ (*errorSink)({(char *) contents, realSize});
+ (*decompressionSink)({(char *) contents, realSize});
return realSize;
} catch (...) {
@@ -375,6 +375,13 @@ struct curlFileTransfer : public FileTransfer
else if (code == CURLE_OK && successfulStatuses.count(httpStatus))
{
result.cached = httpStatus == 304;
+
+ // In 2021, GitHub responds to If-None-Match with 304,
+ // but omits ETag. We just use the If-None-Match etag
+ // since 304 implies they are the same.
+ if (httpStatus == 304 && result.etag == "")
+ result.etag = request.expectedETag;
+
act.progress(result.bodySize, result.bodySize);
done = true;
callback(std::move(result));
@@ -632,11 +639,7 @@ struct curlFileTransfer : public FileTransfer
workerThreadMain();
} catch (nix::Interrupted & e) {
} catch (std::exception & e) {
- logError({
- .name = "File transfer",
- .hint = hintfmt("unexpected error in download thread: %s",
- e.what())
- });
+ printError("unexpected error in download thread: %s", e.what());
}
{
@@ -776,7 +779,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
state->request.notify_one();
});
- request.dataCallback = [_state](char * buf, size_t len) {
+ request.dataCallback = [_state](std::string_view data) {
auto state(_state->lock());
@@ -794,7 +797,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
/* Append data to the buffer and wake up the calling
thread. */
- state->data.append(buf, len);
+ state->data.append(data);
state->avail.notify_one();
};
@@ -840,7 +843,7 @@ void FileTransfer::download(FileTransferRequest && request, Sink & sink)
if it's blocked on a full buffer. We don't hold the state
lock while doing this to prevent blocking the download
thread if sink() takes a long time. */
- sink((unsigned char *) chunk.data(), chunk.size());
+ sink(chunk);
}
}
@@ -852,11 +855,10 @@ FileTransferError::FileTransferError(FileTransfer::Error error, std::shared_ptr<
// FIXME: Due to https://github.com/NixOS/nix/issues/3841 we don't know how
// to print different messages for different verbosity levels. For now
// we add some heuristics for detecting when we want to show the response.
- if (response && (response->size() < 1024 || response->find("<html>") != string::npos)) {
- err.hint = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), *response);
- } else {
- err.hint = hf;
- }
+ if (response && (response->size() < 1024 || response->find("<html>") != string::npos))
+ err.msg = hintfmt("%1%\n\nresponse body:\n\n%2%", normaltxt(hf.str()), chomp(*response));
+ else
+ err.msg = hf;
}
bool isUri(const string & s)
diff --git a/src/libstore/filetransfer.hh b/src/libstore/filetransfer.hh
index c89c51a21..45d9ccf89 100644
--- a/src/libstore/filetransfer.hh
+++ b/src/libstore/filetransfer.hh
@@ -61,9 +61,9 @@ struct FileTransferRequest
bool decompress = true;
std::shared_ptr<std::string> data;
std::string mimeType;
- std::function<void(char *, size_t)> dataCallback;
+ std::function<void(std::string_view data)> dataCallback;
- FileTransferRequest(const std::string & uri)
+ FileTransferRequest(std::string_view uri)
: uri(uri), parentAct(getCurActivity()) { }
std::string verb()
diff --git a/src/libstore/fs-accessor.hh b/src/libstore/fs-accessor.hh
index 64780a6da..c825e84f2 100644
--- a/src/libstore/fs-accessor.hh
+++ b/src/libstore/fs-accessor.hh
@@ -25,7 +25,14 @@ public:
virtual StringSet readDirectory(const Path & path) = 0;
- virtual std::string readFile(const Path & path) = 0;
+ /**
+ * Read a file inside the store.
+ *
+ * If `requireValidPath` is set to `true` (the default), the path must be
+ * inside a valid store path, otherwise it just needs to be physically
+ * present (but not necessarily properly registered)
+ */
+ virtual std::string readFile(const Path & path, bool requireValidPath = true) = 0;
virtual std::string readLink(const Path & path) = 0;
};
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 1238dc530..d3b27d7be 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -3,6 +3,7 @@
#include "archive.hh"
#include "args.hh"
#include "abstract-setting-to-json.hh"
+#include "compute-levels.hh"
#include <algorithm>
#include <map>
@@ -80,12 +81,18 @@ void loadConfFile()
/* We only want to send overrides to the daemon, i.e. stuff from
~/.nix/nix.conf or the command line. */
- globalConfig.resetOverriden();
+ globalConfig.resetOverridden();
auto files = settings.nixUserConfFiles;
for (auto file = files.rbegin(); file != files.rend(); file++) {
globalConfig.applyConfigFile(*file);
}
+
+ auto nixConfEnv = getEnv("NIX_CONFIG");
+ if (nixConfEnv.has_value()) {
+ globalConfig.applyConfig(nixConfEnv.value(), "NIX_CONFIG");
+ }
+
}
std::vector<Path> getUserConfigFiles()
@@ -125,16 +132,48 @@ StringSet Settings::getDefaultSystemFeatures()
return features;
}
+StringSet Settings::getDefaultExtraPlatforms()
+{
+ StringSet extraPlatforms;
+
+ if (std::string{SYSTEM} == "x86_64-linux" && !isWSL1())
+ extraPlatforms.insert("i686-linux");
+
+#if __linux__
+ StringSet levels = computeLevels();
+ for (auto iter = levels.begin(); iter != levels.end(); ++iter)
+ extraPlatforms.insert(*iter + "-linux");
+#elif __APPLE__
+ // Rosetta 2 emulation layer can run x86_64 binaries on aarch64
+ // machines. Note that we can’t force processes from executing
+ // x86_64 in aarch64 environments or vice versa since they can
+ // always exec with their own binary preferences.
+ if (pathExists("/Library/Apple/System/Library/LaunchDaemons/com.apple.oahd.plist")) {
+ if (std::string{SYSTEM} == "x86_64-darwin")
+ extraPlatforms.insert("aarch64-darwin");
+ else if (std::string{SYSTEM} == "aarch64-darwin")
+ extraPlatforms.insert("x86_64-darwin");
+ }
+#endif
+
+ return extraPlatforms;
+}
+
bool Settings::isExperimentalFeatureEnabled(const std::string & name)
{
auto & f = experimentalFeatures.get();
return std::find(f.begin(), f.end(), name) != f.end();
}
+MissingExperimentalFeature::MissingExperimentalFeature(std::string feature)
+ : Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", feature)
+ , missingFeature(feature)
+ {}
+
void Settings::requireExperimentalFeature(const std::string & name)
{
if (!isExperimentalFeatureEnabled(name))
- throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name);
+ throw MissingExperimentalFeature(name);
}
bool Settings::isWSL1()
@@ -154,7 +193,7 @@ NLOHMANN_JSON_SERIALIZE_ENUM(SandboxMode, {
{SandboxMode::smDisabled, false},
});
-template<> void BaseSetting<SandboxMode>::set(const std::string & str)
+template<> void BaseSetting<SandboxMode>::set(const std::string & str, bool append)
{
if (str == "true") value = smEnabled;
else if (str == "relaxed") value = smRelaxed;
@@ -162,6 +201,11 @@ template<> void BaseSetting<SandboxMode>::set(const std::string & str)
else throw UsageError("option '%s' has invalid value '%s'", name, str);
}
+template<> bool BaseSetting<SandboxMode>::isAppendable()
+{
+ return false;
+}
+
template<> std::string BaseSetting<SandboxMode>::to_string() const
{
if (value == smEnabled) return "true";
@@ -192,16 +236,29 @@ template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::s
});
}
-void MaxBuildJobsSetting::set(const std::string & str)
+void MaxBuildJobsSetting::set(const std::string & str, bool append)
{
if (str == "auto") value = std::max(1U, std::thread::hardware_concurrency());
- else if (!string2Int(str, value))
- throw UsageError("configuration setting '%s' should be 'auto' or an integer", name);
+ else {
+ if (auto n = string2Int<decltype(value)>(str))
+ value = *n;
+ else
+ throw UsageError("configuration setting '%s' should be 'auto' or an integer", name);
+ }
+}
+
+
+void PluginFilesSetting::set(const std::string & str, bool append)
+{
+ if (pluginsLoaded)
+ throw UsageError("plugin-files set after plugins were loaded, you may need to move the flag before the subcommand");
+ BaseSetting<Paths>::set(str, append);
}
void initPlugins()
{
+ assert(!settings.pluginFiles.pluginsLoaded);
for (const auto & pluginFile : settings.pluginFiles.get()) {
Paths pluginFiles;
try {
@@ -227,6 +284,9 @@ void initPlugins()
unknown settings. */
globalConfig.reapplyUnknownSettings();
globalConfig.warnUnknownSettings();
+
+ /* Tell the user if they try to set plugin-files after we've already loaded */
+ settings.pluginFiles.pluginsLoaded = true;
}
}
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index aa1fbdeb7..fa8e74b59 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -25,7 +25,33 @@ struct MaxBuildJobsSetting : public BaseSetting<unsigned int>
options->addSetting(this);
}
- void set(const std::string & str) override;
+ void set(const std::string & str, bool append = false) override;
+};
+
+struct PluginFilesSetting : public BaseSetting<Paths>
+{
+ bool pluginsLoaded = false;
+
+ PluginFilesSetting(Config * options,
+ const Paths & def,
+ const std::string & name,
+ const std::string & description,
+ const std::set<std::string> & aliases = {})
+ : BaseSetting<Paths>(def, name, description, aliases)
+ {
+ options->addSetting(this);
+ }
+
+ void set(const std::string & str, bool append = false) override;
+};
+
+class MissingExperimentalFeature: public Error
+{
+public:
+ std::string missingFeature;
+
+ MissingExperimentalFeature(std::string feature);
+ virtual const char* sname() const override { return "MissingExperimentalFeature"; }
};
class Settings : public Config {
@@ -34,6 +60,8 @@ class Settings : public Config {
StringSet getDefaultSystemFeatures();
+ StringSet getDefaultExtraPlatforms();
+
bool isWSL1();
public:
@@ -178,7 +206,10 @@ public:
Setting<std::string> builders{
this, "@" + nixConfDir + "/machines", "builders",
- "A semicolon-separated list of build machines, in the format of `nix.machines`."};
+ R"(
+ A semicolon-separated list of build machines.
+ For the exact format and examples, see [the manual chapter on remote builds](../advanced-topics/distributed-builds.md)
+ )"};
Setting<bool> buildersUseSubstitutes{
this, false, "builders-use-substitutes",
@@ -426,14 +457,6 @@ public:
Setting<bool> sandboxFallback{this, true, "sandbox-fallback",
"Whether to disable sandboxing when the kernel doesn't allow it."};
- Setting<PathSet> extraSandboxPaths{
- this, {}, "extra-sandbox-paths",
- R"(
- A list of additional paths appended to `sandbox-paths`. Useful if
- you want to extend its default value.
- )",
- {"build-extra-chroot-dirs", "build-extra-sandbox-paths"}};
-
Setting<size_t> buildRepeat{
this, 0, "repeat",
R"(
@@ -566,7 +589,7 @@ public:
Setting<StringSet> extraPlatforms{
this,
- std::string{SYSTEM} == "x86_64-linux" && !isWSL1() ? StringSet{"i686-linux"} : StringSet{},
+ getDefaultExtraPlatforms(),
"extra-platforms",
R"(
Platforms other than the native one which this machine is capable of
@@ -604,7 +627,7 @@ public:
Setting<Strings> substituters{
this,
- nixStore == "/nix/store" ? Strings{"https://cache.nixos.org/"} : Strings(),
+ Strings{"https://cache.nixos.org/"},
"substituters",
R"(
A list of URLs of substituters, separated by whitespace. The default
@@ -612,17 +635,6 @@ public:
)",
{"binary-caches"}};
- // FIXME: provide a way to add to option values.
- Setting<Strings> extraSubstituters{
- this, {}, "extra-substituters",
- R"(
- Additional binary caches appended to those specified in
- `substituters`. When used by unprivileged users, untrusted
- substituters (i.e. those not listed in `trusted-substituters`) are
- silently ignored.
- )",
- {"extra-binary-caches"}};
-
Setting<StringSet> trustedSubstituters{
this, {}, "trusted-substituters",
R"(
@@ -849,7 +861,7 @@ public:
Setting<uint64_t> minFreeCheckInterval{this, 5, "min-free-check-interval",
"Number of seconds between checking free disk space."};
- Setting<Paths> pluginFiles{
+ PluginFilesSetting pluginFiles{
this, {}, "plugin-files",
R"(
A list of plugin files to be loaded by Nix. Each of these files will
@@ -861,6 +873,9 @@ public:
command, and RegisterSetting to add new nix config settings. See the
constructors for those types for more details.
+ Warning! These APIs are inherently unstable and may change from
+ release to release.
+
Since these files are loaded into the same address space as Nix
itself, they must be DSOs compatible with the instance of Nix
running at the time (i.e. compiled against the same headers, not
@@ -899,7 +914,7 @@ public:
Example `~/.config/nix/nix.conf`:
```
- access-tokens = "github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk"
+ access-tokens = github.com=23ac...b289 gitlab.mycompany.com=PAT:A123Bp_Cd..EfG gitlab.com=OAuth2:1jklw3jk
```
Example `~/code/flake.nix`:
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 9d2a89f96..0a3afcd51 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -15,7 +15,7 @@ struct HttpBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
const std::string name() override { return "Http Binary Cache Store"; }
};
-class HttpBinaryCacheStore : public BinaryCacheStore, public HttpBinaryCacheStoreConfig
+class HttpBinaryCacheStore : public virtual HttpBinaryCacheStoreConfig, public virtual BinaryCacheStore
{
private:
@@ -36,6 +36,9 @@ public:
const Path & _cacheUri,
const Params & params)
: StoreConfig(params)
+ , BinaryCacheStoreConfig(params)
+ , HttpBinaryCacheStoreConfig(params)
+ , Store(params)
, BinaryCacheStore(params)
, cacheUri(scheme + "://" + _cacheUri)
{
diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc
index 467169ce8..edaf75136 100644
--- a/src/libstore/legacy-ssh-store.cc
+++ b/src/libstore/legacy-ssh-store.cc
@@ -3,6 +3,7 @@
#include "remote-store.hh"
#include "serve-protocol.hh"
#include "store-api.hh"
+#include "path-with-outputs.hh"
#include "worker-protocol.hh"
#include "ssh.hh"
#include "derivations.hh"
@@ -15,6 +16,7 @@ struct LegacySSHStoreConfig : virtual StoreConfig
using StoreConfig::StoreConfig;
const Setting<int> maxConnections{(StoreConfig*) this, 1, "max-connections", "maximum number of concurrent SSH connections"};
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
+ const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"};
const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-store", "remote-program", "path to the nix-store executable on the remote system"};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
@@ -22,7 +24,7 @@ struct LegacySSHStoreConfig : virtual StoreConfig
const std::string name() override { return "Legacy SSH Store"; }
};
-struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
+struct LegacySSHStore : public virtual LegacySSHStoreConfig, public virtual Store
{
// Hack for getting remote build log output.
// Intentionally not in `LegacySSHStoreConfig` so that it doesn't appear in
@@ -48,6 +50,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
LegacySSHStore(const string & scheme, const string & host, const Params & params)
: StoreConfig(params)
+ , LegacySSHStoreConfig(params)
, Store(params)
, host(host)
, connections(make_ref<Pool<Connection>>(
@@ -58,6 +61,7 @@ struct LegacySSHStore : public Store, public virtual LegacySSHStoreConfig
, master(
host,
sshKey,
+ sshPublicHostKey,
// Use SSH master only if using more than 1 connection.
connections->capacity() > 1,
compress,
@@ -257,18 +261,29 @@ public:
if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 3)
conn->from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime;
-
+ if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 6) {
+ status.builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
+ }
return status;
}
- void buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode) override
+ void buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode) override
{
auto conn(connections->get());
conn->to << cmdBuildPaths;
Strings ss;
- for (auto & p : drvPaths)
- ss.push_back(p.to_string(*this));
+ for (auto & p : drvPaths) {
+ auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p);
+ std::visit(overloaded {
+ [&](StorePathWithOutputs s) {
+ ss.push_back(s.to_string(*this));
+ },
+ [&](StorePath drvPath) {
+ throw Error("wanted to fetch '%s' but the legacy ssh protocol doesn't support merely substituting drv files via the build paths command. It would build them instead. Try using ssh-ng://", printStorePath(drvPath));
+ },
+ }, sOrDrvPath);
+ }
conn->to << ss;
putBuildSettings(*conn);
@@ -333,6 +348,10 @@ public:
auto conn(connections->get());
return conn->remoteVersion;
}
+
+ std::optional<const Realisation> queryRealisation(const DrvOutput&) override
+ // TODO: Implement
+ { unsupported("queryRealisation"); }
};
static RegisterStoreImplementation<LegacySSHStore, LegacySSHStoreConfig> regLegacySSHStore;
diff --git a/src/libstore/local-binary-cache-store.cc b/src/libstore/local-binary-cache-store.cc
index 7d979c5c2..964c4017e 100644
--- a/src/libstore/local-binary-cache-store.cc
+++ b/src/libstore/local-binary-cache-store.cc
@@ -2,6 +2,8 @@
#include "globals.hh"
#include "nar-info-disk-cache.hh"
+#include <atomic>
+
namespace nix {
struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
@@ -11,7 +13,7 @@ struct LocalBinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
const std::string name() override { return "Local Binary Cache Store"; }
};
-class LocalBinaryCacheStore : public BinaryCacheStore, public virtual LocalBinaryCacheStoreConfig
+class LocalBinaryCacheStore : public virtual LocalBinaryCacheStoreConfig, public virtual BinaryCacheStore
{
private:
@@ -24,6 +26,9 @@ public:
const Path & binaryCacheDir,
const Params & params)
: StoreConfig(params)
+ , BinaryCacheStoreConfig(params)
+ , LocalBinaryCacheStoreConfig(params)
+ , Store(params)
, BinaryCacheStore(params)
, binaryCacheDir(binaryCacheDir)
{
@@ -47,7 +52,8 @@ protected:
const std::string & mimeType) override
{
auto path2 = binaryCacheDir + "/" + path;
- Path tmp = path2 + ".tmp." + std::to_string(getpid());
+ static std::atomic<int> counter{0};
+ Path tmp = fmt("%s.tmp.%d.%d", path2, getpid(), ++counter);
AutoDelete del(tmp, false);
StreamToSourceAdapter source(istream);
writeFile(tmp, source);
@@ -87,6 +93,7 @@ protected:
void LocalBinaryCacheStore::init()
{
createDirs(binaryCacheDir + "/nar");
+ createDirs(binaryCacheDir + realisationsPrefix);
if (writeDebugInfo)
createDirs(binaryCacheDir + "/debuginfo");
BinaryCacheStore::init();
diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index e7c3dae92..6de13c73a 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -19,10 +19,10 @@ struct LocalStoreAccessor : public FSAccessor
LocalStoreAccessor(ref<LocalFSStore> store) : store(store) { }
- Path toRealPath(const Path & path)
+ Path toRealPath(const Path & path, bool requireValidPath = true)
{
auto storePath = store->toStorePath(path).first;
- if (!store->isValidPath(storePath))
+ if (requireValidPath && !store->isValidPath(storePath))
throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
return store->getRealStoreDir() + std::string(path, store->storeDir.size());
}
@@ -61,9 +61,9 @@ struct LocalStoreAccessor : public FSAccessor
return res;
}
- std::string readFile(const Path & path) override
+ std::string readFile(const Path & path, bool requireValidPath = true) override
{
- return nix::readFile(toRealPath(path));
+ return nix::readFile(toRealPath(path, requireValidPath));
}
std::string readLink(const Path & path) override
diff --git a/src/libstore/local-fs-store.hh b/src/libstore/local-fs-store.hh
index 8eccd8236..55941b771 100644
--- a/src/libstore/local-fs-store.hh
+++ b/src/libstore/local-fs-store.hh
@@ -20,7 +20,7 @@ struct LocalFSStoreConfig : virtual StoreConfig
"log", "directory where Nix will store state"};
};
-class LocalFSStore : public virtual Store, public virtual LocalFSStoreConfig
+class LocalFSStore : public virtual LocalFSStoreConfig, public virtual Store
{
public:
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 6a5f13996..1a7fa897a 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -7,6 +7,7 @@
#include "nar-info.hh"
#include "references.hh"
#include "callback.hh"
+#include "topo-sort.hh"
#include <iostream>
#include <algorithm>
@@ -41,9 +42,68 @@
namespace nix {
+struct LocalStore::State::Stmts {
+ /* Some precompiled SQLite statements. */
+ SQLiteStmt RegisterValidPath;
+ SQLiteStmt UpdatePathInfo;
+ SQLiteStmt AddReference;
+ SQLiteStmt QueryPathInfo;
+ SQLiteStmt QueryReferences;
+ SQLiteStmt QueryReferrers;
+ SQLiteStmt InvalidatePath;
+ SQLiteStmt AddDerivationOutput;
+ SQLiteStmt RegisterRealisedOutput;
+ SQLiteStmt QueryValidDerivers;
+ SQLiteStmt QueryDerivationOutputs;
+ SQLiteStmt QueryRealisedOutput;
+ SQLiteStmt QueryAllRealisedOutputs;
+ SQLiteStmt QueryPathFromHashPart;
+ SQLiteStmt QueryValidPaths;
+};
+
+int getSchema(Path schemaPath)
+{
+ int curSchema = 0;
+ if (pathExists(schemaPath)) {
+ string s = readFile(schemaPath);
+ auto n = string2Int<int>(s);
+ if (!n)
+ throw Error("'%1%' is corrupt", schemaPath);
+ curSchema = *n;
+ }
+ return curSchema;
+}
+
+void migrateCASchema(SQLite& db, Path schemaPath, AutoCloseFD& lockFd)
+{
+ const int nixCASchemaVersion = 1;
+ int curCASchema = getSchema(schemaPath);
+ if (curCASchema != nixCASchemaVersion) {
+ if (curCASchema > nixCASchemaVersion) {
+ throw Error("current Nix store ca-schema is version %1%, but I only support %2%",
+ curCASchema, nixCASchemaVersion);
+ }
+
+ if (!lockFile(lockFd.get(), ltWrite, false)) {
+ printInfo("waiting for exclusive access to the Nix store for ca drvs...");
+ lockFile(lockFd.get(), ltWrite, true);
+ }
+
+ if (curCASchema == 0) {
+ static const char schema[] =
+ #include "ca-specific-schema.sql.gen.hh"
+ ;
+ db.exec(schema);
+ }
+ writeFile(schemaPath, fmt("%d", nixCASchemaVersion));
+ lockFile(lockFd.get(), ltRead, true);
+ }
+}
LocalStore::LocalStore(const Params & params)
: StoreConfig(params)
+ , LocalFSStoreConfig(params)
+ , LocalStoreConfig(params)
, Store(params)
, LocalFSStore(params)
, realStoreDir_{this, false, rootDir != "" ? rootDir + "/nix/store" : storeDir, "real",
@@ -59,6 +119,7 @@ LocalStore::LocalStore(const Params & params)
, locksHeld(tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS").value_or("")))
{
auto state(_state.lock());
+ state->stmts = std::make_unique<State::Stmts>();
/* Create missing state directories if they don't already exist. */
createDirs(realStoreDir);
@@ -89,12 +150,7 @@ LocalStore::LocalStore(const Params & params)
struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
if (!gr)
- logError({
- .name = "'build-users-group' not found",
- .hint = hintfmt(
- "warning: the group '%1%' specified in 'build-users-group' does not exist",
- settings.buildUsersGroup)
- });
+ printError("warning: the group '%1%' specified in 'build-users-group' does not exist", settings.buildUsersGroup);
else {
struct stat st;
if (stat(realStoreDir.c_str(), &st))
@@ -221,32 +277,58 @@ LocalStore::LocalStore(const Params & params)
else openDB(*state, false);
+ if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+ migrateCASchema(state->db, dbDir + "/ca-schema", globalLock);
+ }
+
/* Prepare SQL statements. */
- state->stmtRegisterValidPath.create(state->db,
+ state->stmts->RegisterValidPath.create(state->db,
"insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);");
- state->stmtUpdatePathInfo.create(state->db,
+ state->stmts->UpdatePathInfo.create(state->db,
"update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;");
- state->stmtAddReference.create(state->db,
+ state->stmts->AddReference.create(state->db,
"insert or replace into Refs (referrer, reference) values (?, ?);");
- state->stmtQueryPathInfo.create(state->db,
+ state->stmts->QueryPathInfo.create(state->db,
"select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;");
- state->stmtQueryReferences.create(state->db,
+ state->stmts->QueryReferences.create(state->db,
"select path from Refs join ValidPaths on reference = id where referrer = ?;");
- state->stmtQueryReferrers.create(state->db,
+ state->stmts->QueryReferrers.create(state->db,
"select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);");
- state->stmtInvalidatePath.create(state->db,
+ state->stmts->InvalidatePath.create(state->db,
"delete from ValidPaths where path = ?;");
- state->stmtAddDerivationOutput.create(state->db,
+ state->stmts->AddDerivationOutput.create(state->db,
"insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);");
- state->stmtQueryValidDerivers.create(state->db,
+ state->stmts->QueryValidDerivers.create(state->db,
"select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;");
- state->stmtQueryDerivationOutputs.create(state->db,
+ state->stmts->QueryDerivationOutputs.create(state->db,
"select id, path from DerivationOutputs where drv = ?;");
// Use "path >= ?" with limit 1 rather than "path like '?%'" to
// ensure efficient lookup.
- state->stmtQueryPathFromHashPart.create(state->db,
+ state->stmts->QueryPathFromHashPart.create(state->db,
"select path from ValidPaths where path >= ? limit 1;");
- state->stmtQueryValidPaths.create(state->db, "select path from ValidPaths");
+ state->stmts->QueryValidPaths.create(state->db, "select path from ValidPaths");
+ if (settings.isExperimentalFeatureEnabled("ca-derivations")) {
+ state->stmts->RegisterRealisedOutput.create(state->db,
+ R"(
+ insert or replace into Realisations (drvPath, outputName, outputPath, signatures)
+ values (?, ?, (select id from ValidPaths where path = ?), ?)
+ ;
+ )");
+ state->stmts->QueryRealisedOutput.create(state->db,
+ R"(
+ select Output.path, Realisations.signatures from Realisations
+ inner join ValidPaths as Output on Output.id = Realisations.outputPath
+ where drvPath = ? and outputName = ?
+ ;
+ )");
+ state->stmts->QueryAllRealisedOutputs.create(state->db,
+ R"(
+ select outputName, Output.path from Realisations
+ inner join ValidPaths as Output on Output.id = Realisations.outputPath
+ where drvPath = ?
+ ;
+ )");
+ }
}
@@ -284,16 +366,7 @@ std::string LocalStore::getUri()
int LocalStore::getSchema()
-{
- int curSchema = 0;
- if (pathExists(schemaPath)) {
- string s = readFile(schemaPath);
- if (!string2Int(s, curSchema))
- throw Error("'%1%' is corrupt", schemaPath);
- }
- return curSchema;
-}
-
+{ return nix::getSchema(schemaPath); }
void LocalStore::openDB(State & state, bool create)
{
@@ -580,21 +653,39 @@ void LocalStore::checkDerivationOutputs(const StorePath & drvPath, const Derivat
[&](DerivationOutputCAFloating _) {
/* Nothing to check */
},
+ [&](DerivationOutputDeferred) {
+ },
}, i.second.output);
}
}
+void LocalStore::registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs)
+{
+ settings.requireExperimentalFeature("ca-derivations");
+ if (checkSigs == NoCheckSigs || !realisationIsUntrusted(info))
+ registerDrvOutput(info);
+ else
+ throw Error("cannot register realisation '%s' because it lacks a valid signature", info.outPath.to_string());
+}
-void LocalStore::linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output)
+void LocalStore::registerDrvOutput(const Realisation & info)
{
+ settings.requireExperimentalFeature("ca-derivations");
auto state(_state.lock());
- return linkDeriverToPath(*state, queryValidPathId(*state, deriver), outputName, output);
+ retrySQLite<void>([&]() {
+ state->stmts->RegisterRealisedOutput.use()
+ (info.id.strHash())
+ (info.id.outputName)
+ (printStorePath(info.outPath))
+ (concatStringsSep(" ", info.signatures))
+ .exec();
+ });
}
-void LocalStore::linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output)
+void LocalStore::cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output)
{
retrySQLite<void>([&]() {
- state.stmtAddDerivationOutput.use()
+ state.stmts->AddDerivationOutput.use()
(deriver)
(outputName)
(printStorePath(output))
@@ -611,7 +702,7 @@ uint64_t LocalStore::addValidPath(State & state,
throw Error("cannot add path '%s' to the Nix store because it claims to be content-addressed but isn't",
printStorePath(info.path));
- state.stmtRegisterValidPath.use()
+ state.stmts->RegisterValidPath.use()
(printStorePath(info.path))
(info.narHash.to_string(Base16, true))
(info.registrationTime == 0 ? time(0) : info.registrationTime)
@@ -628,7 +719,7 @@ uint64_t LocalStore::addValidPath(State & state,
efficiently query whether a path is an output of some
derivation. */
if (info.path.isDerivation()) {
- auto drv = readDerivation(info.path);
+ auto drv = readInvalidDerivation(info.path);
/* Verify that the output paths in the derivation are correct
(i.e., follow the scheme for computing output paths from
@@ -641,7 +732,7 @@ uint64_t LocalStore::addValidPath(State & state,
/* Floating CA derivations have indeterminate output paths until
they are built, so don't register anything in that case */
if (i.second.second)
- linkDeriverToPath(state, id, i.first, *i.second.second);
+ cacheDrvOutputMapping(state, id, i.first, *i.second.second);
}
}
@@ -659,11 +750,19 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
Callback<std::shared_ptr<const ValidPathInfo>> callback) noexcept
{
try {
- callback(retrySQLite<std::shared_ptr<ValidPathInfo>>([&]() {
+ callback(retrySQLite<std::shared_ptr<const ValidPathInfo>>([&]() {
auto state(_state.lock());
+ return queryPathInfoInternal(*state, path);
+ }));
+ } catch (...) { callback.rethrow(); }
+}
+
+
+std::shared_ptr<const ValidPathInfo> LocalStore::queryPathInfoInternal(State & state, const StorePath & path)
+{
/* Get the path info. */
- auto useQueryPathInfo(state->stmtQueryPathInfo.use()(printStorePath(path)));
+ auto useQueryPathInfo(state.stmts->QueryPathInfo.use()(printStorePath(path)));
if (!useQueryPathInfo.next())
return std::shared_ptr<ValidPathInfo>();
@@ -683,7 +782,7 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
info->registrationTime = useQueryPathInfo.getInt(2);
- auto s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 3);
+ auto s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 3);
if (s) info->deriver = parseStorePath(s);
/* Note that narSize = NULL yields 0. */
@@ -691,29 +790,26 @@ void LocalStore::queryPathInfoUncached(const StorePath & path,
info->ultimate = useQueryPathInfo.getInt(5) == 1;
- s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6);
+ s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 6);
if (s) info->sigs = tokenizeString<StringSet>(s, " ");
- s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7);
+ s = (const char *) sqlite3_column_text(state.stmts->QueryPathInfo, 7);
if (s) info->ca = parseContentAddressOpt(s);
/* Get the references. */
- auto useQueryReferences(state->stmtQueryReferences.use()(info->id));
+ auto useQueryReferences(state.stmts->QueryReferences.use()(info->id));
while (useQueryReferences.next())
info->references.insert(parseStorePath(useQueryReferences.getStr(0)));
return info;
- }));
-
- } catch (...) { callback.rethrow(); }
}
/* Update path info in the database. */
void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
{
- state.stmtUpdatePathInfo.use()
+ state.stmts->UpdatePathInfo.use()
(info.narSize, info.narSize != 0)
(info.narHash.to_string(Base16, true))
(info.ultimate ? 1 : 0, info.ultimate)
@@ -726,7 +822,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path)
{
- auto use(state.stmtQueryPathInfo.use()(printStorePath(path)));
+ auto use(state.stmts->QueryPathInfo.use()(printStorePath(path)));
if (!use.next())
throw InvalidPath("path '%s' is not valid", printStorePath(path));
return use.getInt(0);
@@ -735,7 +831,7 @@ uint64_t LocalStore::queryValidPathId(State & state, const StorePath & path)
bool LocalStore::isValidPath_(State & state, const StorePath & path)
{
- return state.stmtQueryPathInfo.use()(printStorePath(path)).next();
+ return state.stmts->QueryPathInfo.use()(printStorePath(path)).next();
}
@@ -761,7 +857,7 @@ StorePathSet LocalStore::queryAllValidPaths()
{
return retrySQLite<StorePathSet>([&]() {
auto state(_state.lock());
- auto use(state->stmtQueryValidPaths.use());
+ auto use(state->stmts->QueryValidPaths.use());
StorePathSet res;
while (use.next()) res.insert(parseStorePath(use.getStr(0)));
return res;
@@ -771,7 +867,7 @@ StorePathSet LocalStore::queryAllValidPaths()
void LocalStore::queryReferrers(State & state, const StorePath & path, StorePathSet & referrers)
{
- auto useQueryReferrers(state.stmtQueryReferrers.use()(printStorePath(path)));
+ auto useQueryReferrers(state.stmts->QueryReferrers.use()(printStorePath(path)));
while (useQueryReferrers.next())
referrers.insert(parseStorePath(useQueryReferrers.getStr(0)));
@@ -792,7 +888,7 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
return retrySQLite<StorePathSet>([&]() {
auto state(_state.lock());
- auto useQueryValidDerivers(state->stmtQueryValidDerivers.use()(printStorePath(path)));
+ auto useQueryValidDerivers(state->stmts->QueryValidDerivers.use()(printStorePath(path)));
StorePathSet derivers;
while (useQueryValidDerivers.next())
@@ -803,69 +899,38 @@ StorePathSet LocalStore::queryValidDerivers(const StorePath & path)
}
-std::map<std::string, std::optional<StorePath>> LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
+std::map<std::string, std::optional<StorePath>>
+LocalStore::queryPartialDerivationOutputMap(const StorePath & path_)
{
auto path = path_;
- std::map<std::string, std::optional<StorePath>> outputs;
- Derivation drv = readDerivation(path);
- for (auto & [outName, _] : drv.outputs) {
- outputs.insert_or_assign(outName, std::nullopt);
- }
- bool haveCached = false;
- {
- auto resolutions = drvPathResolutions.lock();
- auto resolvedPathOptIter = resolutions->find(path);
- if (resolvedPathOptIter != resolutions->end()) {
- auto & [_, resolvedPathOpt] = *resolvedPathOptIter;
- if (resolvedPathOpt)
- path = *resolvedPathOpt;
- haveCached = true;
- }
- }
- /* can't just use else-if instead of `!haveCached` because we need to unlock
- `drvPathResolutions` before it is locked in `Derivation::resolve`. */
- if (!haveCached && drv.type() == DerivationType::CAFloating) {
- /* Try resolve drv and use that path instead. */
- auto attempt = drv.tryResolve(*this);
- if (!attempt)
- /* If we cannot resolve the derivation, we cannot have any path
- assigned so we return the map of all std::nullopts. */
- return outputs;
- /* Just compute store path */
- auto pathResolved = writeDerivation(*this, *std::move(attempt), NoRepair, true);
- /* Store in memo table. */
- /* FIXME: memo logic should not be local-store specific, should have
- wrapper-method instead. */
- drvPathResolutions.lock()->insert_or_assign(path, pathResolved);
- path = std::move(pathResolved);
- }
- return retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
+ auto outputs = retrySQLite<std::map<std::string, std::optional<StorePath>>>([&]() {
auto state(_state.lock());
-
+ std::map<std::string, std::optional<StorePath>> outputs;
uint64_t drvId;
- try {
drvId = queryValidPathId(*state, path);
- } catch (InvalidPath &) {
- /* FIXME? if the derivation doesn't exist, we cannot have a mapping
- for it. */
- return outputs;
- }
-
- auto useQueryDerivationOutputs {
- state->stmtQueryDerivationOutputs.use()
- (drvId)
- };
-
- while (useQueryDerivationOutputs.next())
+ auto use(state->stmts->QueryDerivationOutputs.use()(drvId));
+ while (use.next())
outputs.insert_or_assign(
- useQueryDerivationOutputs.getStr(0),
- parseStorePath(useQueryDerivationOutputs.getStr(1))
- );
+ use.getStr(0), parseStorePath(use.getStr(1)));
return outputs;
});
-}
+ if (!settings.isExperimentalFeatureEnabled("ca-derivations"))
+ return outputs;
+
+ auto drv = readInvalidDerivation(path);
+ auto drvHashes = staticOutputHashes(*this, drv);
+ for (auto& [outputName, hash] : drvHashes) {
+ auto realisation = queryRealisation(DrvOutput{hash, outputName});
+ if (realisation)
+ outputs.insert_or_assign(outputName, realisation->outPath);
+ else
+ outputs.insert({outputName, std::nullopt});
+ }
+
+ return outputs;
+}
std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & hashPart)
{
@@ -876,11 +941,11 @@ std::optional<StorePath> LocalStore::queryPathFromHashPart(const std::string & h
return retrySQLite<std::optional<StorePath>>([&]() -> std::optional<StorePath> {
auto state(_state.lock());
- auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix));
+ auto useQueryPathFromHashPart(state->stmts->QueryPathFromHashPart.use()(prefix));
if (!useQueryPathFromHashPart.next()) return {};
- const char * s = (const char *) sqlite3_column_text(state->stmtQueryPathFromHashPart, 0);
+ const char * s = (const char *) sqlite3_column_text(state->stmts->QueryPathFromHashPart, 0);
if (s && prefix.compare(0, prefix.size(), s, prefix.size()) == 0)
return parseStorePath(s);
return {};
@@ -964,9 +1029,7 @@ void LocalStore::querySubstitutablePathInfos(const StorePathCAMap & paths, Subst
void LocalStore::registerValidPath(const ValidPathInfo & info)
{
- ValidPathInfos infos;
- infos.push_back(info);
- registerValidPaths(infos);
+ registerValidPaths({{info.path, info}});
}
@@ -984,7 +1047,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
SQLiteTxn txn(state->db);
StorePathSet paths;
- for (auto & i : infos) {
+ for (auto & [_, i] : infos) {
assert(i.narHash.type == htSHA256);
if (isValidPath_(*state, i.path))
updatePathInfo(*state, i);
@@ -993,26 +1056,37 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos)
paths.insert(i.path);
}
- for (auto & i : infos) {
+ for (auto & [_, i] : infos) {
auto referrer = queryValidPathId(*state, i.path);
for (auto & j : i.references)
- state->stmtAddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
+ state->stmts->AddReference.use()(referrer)(queryValidPathId(*state, j)).exec();
}
/* Check that the derivation outputs are correct. We can't do
this in addValidPath() above, because the references might
not be valid yet. */
- for (auto & i : infos)
+ for (auto & [_, i] : infos)
if (i.path.isDerivation()) {
// FIXME: inefficient; we already loaded the derivation in addValidPath().
- checkDerivationOutputs(i.path, readDerivation(i.path));
+ checkDerivationOutputs(i.path,
+ readInvalidDerivation(i.path));
}
/* Do a topological sort of the paths. This will throw an
error if a cycle is detected and roll back the
transaction. Cycles can only occur when a derivation
has multiple outputs. */
- topoSortPaths(paths);
+ topoSort(paths,
+ {[&](const StorePath & path) {
+ auto i = infos.find(path);
+ return i == infos.end() ? StorePathSet() : i->second.references;
+ }},
+ {[&](const StorePath & path, const StorePath & parent) {
+ return BuildError(
+ "cycle detected in the references of '%s' from '%s'",
+ printStorePath(path),
+ printStorePath(parent));
+ }});
txn.commit();
});
@@ -1025,7 +1099,7 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
{
debug("invalidating path '%s'", printStorePath(path));
- state.stmtInvalidatePath.use()(printStorePath(path)).exec();
+ state.stmts->InvalidatePath.use()(printStorePath(path)).exec();
/* Note that the foreign key constraints on the Refs table take
care of deleting the references entries for `path'. */
@@ -1036,7 +1110,6 @@ void LocalStore::invalidatePath(State & state, const StorePath & path)
}
}
-
const PublicKeys & LocalStore::getPublicKeys()
{
auto state(_state.lock());
@@ -1045,11 +1118,20 @@ const PublicKeys & LocalStore::getPublicKeys()
return *state->publicKeys;
}
+bool LocalStore::pathInfoIsUntrusted(const ValidPathInfo & info)
+{
+ return requireSigs && !info.checkSignatures(*this, getPublicKeys());
+}
+
+bool LocalStore::realisationIsUntrusted(const Realisation & realisation)
+{
+ return requireSigs && !realisation.checkSignatures(getPublicKeys());
+}
void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs)
{
- if (requireSigs && checkSigs && !info.checkSignatures(*this, getPublicKeys()))
+ if (checkSigs && pathInfoIsUntrusted(info))
throw Error("cannot add path '%s' because it lacks a valid signature", printStorePath(info.path));
addTempRoot(info.path);
@@ -1090,11 +1172,11 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source,
auto hashResult = hashSink->finish();
if (hashResult.first != info.narHash)
- throw Error("hash mismatch importing path '%s';\n wanted: %s\n got: %s",
+ throw Error("hash mismatch importing path '%s';\n specified: %s\n got: %s",
printStorePath(info.path), info.narHash.to_string(Base32, true), hashResult.first.to_string(Base32, true));
if (hashResult.second != info.narSize)
- throw Error("size mismatch importing path '%s';\n wanted: %s\n got: %s",
+ throw Error("size mismatch importing path '%s';\n specified: %s\n got: %s",
printStorePath(info.path), info.narSize, hashResult.second);
autoGC();
@@ -1138,7 +1220,7 @@ StorePath LocalStore::addToStoreFromDump(Source & source0, const string & name,
dump.resize(oldSize + want);
auto got = 0;
try {
- got = source.read((uint8_t *) dump.data() + oldSize, want);
+ got = source.read(dump.data() + oldSize, want);
} catch (EndOfFile &) {
inMemory = true;
break;
@@ -1341,12 +1423,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
Path linkPath = linksDir + "/" + link.name;
string hash = hashPath(htSHA256, linkPath).first.to_string(Base32, false);
if (hash != link.name) {
- logError({
- .name = "Invalid hash",
- .hint = hintfmt(
- "link '%s' was modified! expected hash '%s', got '%s'",
- linkPath, link.name, hash)
- });
+ printError("link '%s' was modified! expected hash '%s', got '%s'",
+ linkPath, link.name, hash);
if (repair) {
if (unlink(linkPath.c_str()) == 0)
printInfo("removed link '%s'", linkPath);
@@ -1379,11 +1457,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair)
auto current = hashSink->finish();
if (info->narHash != nullHash && info->narHash != current.first) {
- logError({
- .name = "Invalid hash - path modified",
- .hint = hintfmt("path '%s' was modified! expected hash '%s', got '%s'",
- printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true))
- });
+ printError("path '%s' was modified! expected hash '%s', got '%s'",
+ printStorePath(i), info->narHash.to_string(Base32, true), current.first.to_string(Base32, true));
if (repair) repairPath(i); else errors = true;
} else {
@@ -1434,10 +1509,7 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
if (!done.insert(pathS).second) return;
if (!isStorePath(pathS)) {
- logError({
- .name = "Nix path not found",
- .hint = hintfmt("path '%s' is not in the Nix store", pathS)
- });
+ printError("path '%s' is not in the Nix store", pathS);
return;
}
@@ -1460,10 +1532,7 @@ void LocalStore::verifyPath(const Path & pathS, const StringSet & store,
auto state(_state.lock());
invalidatePath(*state, path);
} else {
- logError({
- .name = "Missing path with referrers",
- .hint = hintfmt("path '%s' disappeared, but it still has valid referrers!", pathS)
- });
+ printError("path '%s' disappeared, but it still has valid referrers!", pathS);
if (repair)
try {
repairPath(path);
@@ -1553,7 +1622,7 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si
SQLiteTxn txn(state->db);
- auto info = std::const_pointer_cast<ValidPathInfo>(std::shared_ptr<const ValidPathInfo>(queryPathInfo(storePath)));
+ auto info = std::const_pointer_cast<ValidPathInfo>(queryPathInfoInternal(*state, storePath));
info->sigs.insert(sigs.begin(), sigs.end());
@@ -1564,6 +1633,18 @@ void LocalStore::addSignatures(const StorePath & storePath, const StringSet & si
}
+void LocalStore::signRealisation(Realisation & realisation)
+{
+ // FIXME: keep secret keys in memory.
+
+ auto secretKeyFiles = settings.secretKeyFiles;
+
+ for (auto & secretKeyFile : secretKeyFiles.get()) {
+ SecretKey secretKey(readFile(secretKeyFile));
+ realisation.sign(secretKey);
+ }
+}
+
void LocalStore::signPathInfo(ValidPathInfo & info)
{
// FIXME: keep secret keys in memory.
@@ -1591,5 +1672,19 @@ void LocalStore::createUser(const std::string & userName, uid_t userId)
}
}
-
+std::optional<const Realisation> LocalStore::queryRealisation(
+ const DrvOutput& id) {
+ typedef std::optional<const Realisation> Ret;
+ return retrySQLite<Ret>([&]() -> Ret {
+ auto state(_state.lock());
+ auto use(state->stmts->QueryRealisedOutput.use()(id.strHash())(
+ id.outputName));
+ if (!use.next())
+ return std::nullopt;
+ auto outputPath = parseStorePath(use.getStr(0));
+ auto signatures = tokenizeString<StringSet>(use.getStr(1));
+ return Ret{Realisation{
+ .id = id, .outPath = outputPath, .signatures = signatures}};
+ });
}
+} // namespace nix
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index dfb3aa49b..9778e5f2f 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -43,7 +43,7 @@ struct LocalStoreConfig : virtual LocalFSStoreConfig
};
-class LocalStore : public LocalFSStore, public virtual LocalStoreConfig
+class LocalStore : public virtual LocalStoreConfig, public virtual LocalFSStore
{
private:
@@ -55,19 +55,8 @@ private:
/* The SQLite database object. */
SQLite db;
- /* Some precompiled SQLite statements. */
- SQLiteStmt stmtRegisterValidPath;
- SQLiteStmt stmtUpdatePathInfo;
- SQLiteStmt stmtAddReference;
- SQLiteStmt stmtQueryPathInfo;
- SQLiteStmt stmtQueryReferences;
- SQLiteStmt stmtQueryReferrers;
- SQLiteStmt stmtInvalidatePath;
- SQLiteStmt stmtAddDerivationOutput;
- SQLiteStmt stmtQueryValidDerivers;
- SQLiteStmt stmtQueryDerivationOutputs;
- SQLiteStmt stmtQueryPathFromHashPart;
- SQLiteStmt stmtQueryValidPaths;
+ struct Stmts;
+ std::unique_ptr<Stmts> stmts;
/* The file to which we write our temporary roots. */
AutoCloseFD fdTempRoots;
@@ -90,7 +79,7 @@ private:
std::unique_ptr<PublicKeys> publicKeys;
};
- Sync<State, std::recursive_mutex> _state;
+ Sync<State> _state;
public:
@@ -147,6 +136,9 @@ public:
void querySubstitutablePathInfos(const StorePathCAMap & paths,
SubstitutablePathInfos & infos) override;
+ bool pathInfoIsUntrusted(const ValidPathInfo &) override;
+ bool realisationIsUntrusted(const Realisation & ) override;
+
void addToStore(const ValidPathInfo & info, Source & source,
RepairFlag repair, CheckSigsFlag checkSigs) override;
@@ -156,15 +148,6 @@ public:
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
- void buildPaths(
- const std::vector<StorePathWithOutputs> & paths,
- BuildMode buildMode) override;
-
- BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
- BuildMode buildMode) override;
-
- void ensurePath(const StorePath & path) override;
-
void addTempRoot(const StorePath & path) override;
void addIndirectRoot(const Path & path) override;
@@ -209,9 +192,7 @@ public:
void vacuumDB();
- /* Repair the contents of the given path by redownloading it using
- a substituter (if available). */
- void repairPath(const StorePath & path);
+ void repairPath(const StorePath & path) override;
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
@@ -219,6 +200,14 @@ public:
garbage until it exceeds maxFree. */
void autoGC(bool sync = true);
+ /* Register the store path 'output' as the output named 'outputName' of
+ derivation 'deriver'. */
+ void registerDrvOutput(const Realisation & info) override;
+ void registerDrvOutput(const Realisation & info, CheckSigsFlag checkSigs) override;
+ void cacheDrvOutputMapping(State & state, const uint64_t deriver, const string & outputName, const StorePath & output);
+
+ std::optional<const Realisation> queryRealisation(const DrvOutput&) override;
+
private:
int getSchema();
@@ -239,6 +228,8 @@ private:
void verifyPath(const Path & path, const StringSet & store,
PathSet & done, StorePathSet & validPaths, RepairFlag repair, bool & errors);
+ std::shared_ptr<const ValidPathInfo> queryPathInfoInternal(State & state, const StorePath & path);
+
void updatePathInfo(State & state, const ValidPathInfo & info);
void upgradeStore6();
@@ -283,21 +274,19 @@ private:
bool isValidPath_(State & state, const StorePath & path);
void queryReferrers(State & state, const StorePath & path, StorePathSet & referrers);
- /* Add signatures to a ValidPathInfo using the secret keys
+ /* Add signatures to a ValidPathInfo or Realisation using the secret keys
specified by the ‘secret-key-files’ option. */
void signPathInfo(ValidPathInfo & info);
-
- /* Register the store path 'output' as the output named 'outputName' of
- derivation 'deriver'. */
- void linkDeriverToPath(const StorePath & deriver, const string & outputName, const StorePath & output);
- void linkDeriverToPath(State & state, uint64_t deriver, const string & outputName, const StorePath & output);
+ void signRealisation(Realisation &);
Path getRealStoreDir() override { return realStoreDir; }
void createUser(const std::string & userName, uid_t userId) override;
- friend class DerivationGoal;
- friend class SubstitutionGoal;
+ friend struct LocalDerivationGoal;
+ friend struct PathSubstitutionGoal;
+ friend struct SubstitutionGoal;
+ friend struct DerivationGoal;
};
diff --git a/src/libstore/local.mk b/src/libstore/local.mk
index dfe1e2cc4..cf0933705 100644
--- a/src/libstore/local.mk
+++ b/src/libstore/local.mk
@@ -28,7 +28,7 @@ ifeq ($(OS), SunOS)
endif
ifeq ($(HAVE_SECCOMP), 1)
- libstore_LDFLAGS += -lseccomp
+ libstore_LDFLAGS += $(LIBSECCOMP_LIBS)
endif
libstore_CXXFLAGS += \
@@ -48,7 +48,7 @@ ifneq ($(sandbox_shell),)
libstore_CXXFLAGS += -DSANDBOX_SHELL="\"$(sandbox_shell)\""
endif
-$(d)/local-store.cc: $(d)/schema.sql.gen.hh
+$(d)/local-store.cc: $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
$(d)/build.cc:
@@ -58,7 +58,7 @@ $(d)/build.cc:
@echo ')foo"' >> $@.tmp
@mv $@.tmp $@
-clean-files += $(d)/schema.sql.gen.hh
+clean-files += $(d)/schema.sql.gen.hh $(d)/ca-specific-schema.sql.gen.hh
$(eval $(call install-file-in, $(d)/nix-store.pc, $(prefix)/lib/pkgconfig, 0644))
diff --git a/src/libstore/machines.cc b/src/libstore/machines.cc
index 7db2556f4..b42e5e434 100644
--- a/src/libstore/machines.cc
+++ b/src/libstore/machines.cc
@@ -54,9 +54,15 @@ ref<Store> Machine::openStore() const {
if (hasPrefix(storeUri, "ssh://")) {
storeParams["max-connections"] = "1";
storeParams["log-fd"] = "4";
+ }
+
+ if (hasPrefix(storeUri, "ssh://") || hasPrefix(storeUri, "ssh-ng://")) {
if (sshKey != "")
storeParams["ssh-key"] = sshKey;
+ if (sshPublicHostKey != "")
+ storeParams["base64-ssh-public-host-key"] = sshPublicHostKey;
}
+
{
auto & fs = storeParams["system-features"];
auto append = [&](auto feats) {
diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc
index ad4dccef9..a99a2fc78 100644
--- a/src/libstore/misc.cc
+++ b/src/libstore/misc.cc
@@ -22,55 +22,53 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
Sync<State> state_(State{0, paths_, 0});
- std::function<void(const Path &)> enqueue;
+ std::function<void(const StorePath &)> enqueue;
std::condition_variable done;
- enqueue = [&](const Path & path) -> void {
+ enqueue = [&](const StorePath & path) -> void {
{
auto state(state_.lock());
if (state->exc) return;
- if (!state->paths.insert(parseStorePath(path)).second) return;
+ if (!state->paths.insert(path).second) return;
state->pending++;
}
- queryPathInfo(parseStorePath(path), {[&, pathS(path)](std::future<ref<const ValidPathInfo>> fut) {
+ queryPathInfo(path, {[&](std::future<ref<const ValidPathInfo>> fut) {
// FIXME: calls to isValidPath() should be async
try {
auto info = fut.get();
- auto path = parseStorePath(pathS);
-
if (flipDirection) {
StorePathSet referrers;
queryReferrers(path, referrers);
for (auto & ref : referrers)
if (ref != path)
- enqueue(printStorePath(ref));
+ enqueue(ref);
if (includeOutputs)
for (auto & i : queryValidDerivers(path))
- enqueue(printStorePath(i));
+ enqueue(i);
if (includeDerivers && path.isDerivation())
for (auto & i : queryDerivationOutputs(path))
if (isValidPath(i) && queryPathInfo(i)->deriver == path)
- enqueue(printStorePath(i));
+ enqueue(i);
} else {
for (auto & ref : info->references)
if (ref != path)
- enqueue(printStorePath(ref));
+ enqueue(ref);
if (includeOutputs && path.isDerivation())
for (auto & i : queryDerivationOutputs(path))
- if (isValidPath(i)) enqueue(printStorePath(i));
+ if (isValidPath(i)) enqueue(i);
if (includeDerivers && info->deriver && isValidPath(*info->deriver))
- enqueue(printStorePath(*info->deriver));
+ enqueue(*info->deriver);
}
@@ -90,7 +88,7 @@ void Store::computeFSClosure(const StorePathSet & startPaths,
};
for (auto & startPath : startPaths)
- enqueue(printStorePath(startPath));
+ enqueue(startPath);
{
auto state(state_.lock());
@@ -119,7 +117,7 @@ std::optional<ContentAddress> getDerivationCA(const BasicDerivation & drv)
return std::nullopt;
}
-void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
+void Store::queryMissing(const std::vector<DerivedPath> & targets,
StorePathSet & willBuild_, StorePathSet & willSubstitute_, StorePathSet & unknown_,
uint64_t & downloadSize_, uint64_t & narSize_)
{
@@ -147,7 +145,7 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
Sync<State> state_(State{{}, unknown_, willSubstitute_, willBuild_, downloadSize_, narSize_});
- std::function<void(StorePathWithOutputs)> doPath;
+ std::function<void(DerivedPath)> doPath;
auto mustBuildDrv = [&](const StorePath & drvPath, const Derivation & drv) {
{
@@ -156,17 +154,14 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
}
for (auto & i : drv.inputDrvs)
- pool.enqueue(std::bind(doPath, StorePathWithOutputs { i.first, i.second }));
+ pool.enqueue(std::bind(doPath, DerivedPath::Built { i.first, i.second }));
};
auto checkOutput = [&](
- const Path & drvPathS, ref<Derivation> drv, const Path & outPathS, ref<Sync<DrvState>> drvState_)
+ const StorePath & drvPath, ref<Derivation> drv, const StorePath & outPath, ref<Sync<DrvState>> drvState_)
{
if (drvState_->lock()->done) return;
- auto drvPath = parseStorePath(drvPathS);
- auto outPath = parseStorePath(outPathS);
-
SubstitutablePathInfos infos;
querySubstitutablePathInfos({{outPath, getDerivationCA(*drv)}}, infos);
@@ -182,77 +177,80 @@ void Store::queryMissing(const std::vector<StorePathWithOutputs> & targets,
drvState->outPaths.insert(outPath);
if (!drvState->left) {
for (auto & path : drvState->outPaths)
- pool.enqueue(std::bind(doPath, StorePathWithOutputs { path } ));
+ pool.enqueue(std::bind(doPath, DerivedPath::Opaque { path } ));
}
}
}
};
- doPath = [&](const StorePathWithOutputs & path) {
+ doPath = [&](const DerivedPath & req) {
{
auto state(state_.lock());
- if (!state->done.insert(path.to_string(*this)).second) return;
+ if (!state->done.insert(req.to_string(*this)).second) return;
}
- if (path.path.isDerivation()) {
- if (!isValidPath(path.path)) {
+ std::visit(overloaded {
+ [&](DerivedPath::Built bfd) {
+ if (!isValidPath(bfd.drvPath)) {
// FIXME: we could try to substitute the derivation.
auto state(state_.lock());
- state->unknown.insert(path.path);
+ state->unknown.insert(bfd.drvPath);
return;
}
- PathSet invalid;
+ StorePathSet invalid;
/* true for regular derivations, and CA derivations for which we
have a trust mapping for all wanted outputs. */
auto knownOutputPaths = true;
- for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(path.path)) {
+ for (auto & [outputName, pathOpt] : queryPartialDerivationOutputMap(bfd.drvPath)) {
if (!pathOpt) {
knownOutputPaths = false;
break;
}
- if (wantOutput(outputName, path.outputs) && !isValidPath(*pathOpt))
- invalid.insert(printStorePath(*pathOpt));
+ if (wantOutput(outputName, bfd.outputs) && !isValidPath(*pathOpt))
+ invalid.insert(*pathOpt);
}
if (knownOutputPaths && invalid.empty()) return;
- auto drv = make_ref<Derivation>(derivationFromPath(path.path));
- ParsedDerivation parsedDrv(StorePath(path.path), *drv);
+ auto drv = make_ref<Derivation>(derivationFromPath(bfd.drvPath));
+ ParsedDerivation parsedDrv(StorePath(bfd.drvPath), *drv);
if (knownOutputPaths && settings.useSubstitutes && parsedDrv.substitutesAllowed()) {
auto drvState = make_ref<Sync<DrvState>>(DrvState(invalid.size()));
for (auto & output : invalid)
- pool.enqueue(std::bind(checkOutput, printStorePath(path.path), drv, output, drvState));
+ pool.enqueue(std::bind(checkOutput, bfd.drvPath, drv, output, drvState));
} else
- mustBuildDrv(path.path, *drv);
+ mustBuildDrv(bfd.drvPath, *drv);
- } else {
+ },
+ [&](DerivedPath::Opaque bo) {
- if (isValidPath(path.path)) return;
+ if (isValidPath(bo.path)) return;
SubstitutablePathInfos infos;
- querySubstitutablePathInfos({{path.path, std::nullopt}}, infos);
+ querySubstitutablePathInfos({{bo.path, std::nullopt}}, infos);
if (infos.empty()) {
auto state(state_.lock());
- state->unknown.insert(path.path);
+ state->unknown.insert(bo.path);
return;
}
- auto info = infos.find(path.path);
+ auto info = infos.find(bo.path);
assert(info != infos.end());
{
auto state(state_.lock());
- state->willSubstitute.insert(path.path);
+ state->willSubstitute.insert(bo.path);
state->downloadSize += info->second.downloadSize;
state->narSize += info->second.narSize;
}
for (auto & ref : info->second.references)
- pool.enqueue(std::bind(doPath, StorePathWithOutputs { ref }));
- }
+ pool.enqueue(std::bind(doPath, DerivedPath::Opaque { ref }));
+ },
+ }, req.raw());
};
for (auto & path : targets)
diff --git a/src/libstore/names.cc b/src/libstore/names.cc
index 41e28dc99..ce808accc 100644
--- a/src/libstore/names.cc
+++ b/src/libstore/names.cc
@@ -80,16 +80,16 @@ string nextComponent(string::const_iterator & p,
static bool componentsLT(const string & c1, const string & c2)
{
- int n1, n2;
- bool c1Num = string2Int(c1, n1), c2Num = string2Int(c2, n2);
+ auto n1 = string2Int<int>(c1);
+ auto n2 = string2Int<int>(c2);
- if (c1Num && c2Num) return n1 < n2;
- else if (c1 == "" && c2Num) return true;
+ if (n1 && n2) return *n1 < *n2;
+ else if (c1 == "" && n2) return true;
else if (c1 == "pre" && c2 != "pre") return true;
else if (c2 == "pre") return false;
/* Assume that `2.3a' < `2.3.1'. */
- else if (c2Num) return true;
- else if (c1Num) return false;
+ else if (n2) return true;
+ else if (n1) return false;
else return c1 < c2;
}
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index a9efdd0b6..784ebb719 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -87,7 +87,7 @@ struct NarAccessor : public FSAccessor
parents.top()->start = pos;
}
- void receiveContents(unsigned char * data, size_t len) override
+ void receiveContents(std::string_view data) override
{ }
void createSymlink(const Path & path, const string & target) override
@@ -96,7 +96,7 @@ struct NarAccessor : public FSAccessor
NarMember{FSAccessor::Type::tSymlink, false, 0, 0, target});
}
- size_t read(unsigned char * data, size_t len) override
+ size_t read(char * data, size_t len) override
{
auto n = source.read(data, len);
pos += n;
@@ -203,7 +203,7 @@ struct NarAccessor : public FSAccessor
return res;
}
- std::string readFile(const Path & path) override
+ std::string readFile(const Path & path, bool requireValidPath = true) override
{
auto i = get(path);
if (i.type != FSAccessor::Type::tRegular)
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 8541cc51f..1d8d2d57e 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -109,8 +109,10 @@ public:
SQLiteStmt(state->db,
"delete from NARs where ((present = 0 and timestamp < ?) or (present = 1 and timestamp < ?))")
.use()
- (now - settings.ttlNegativeNarInfoCache)
- (now - settings.ttlPositiveNarInfoCache)
+ // Use a minimum TTL to prevent --refresh from
+ // nuking the entire disk cache.
+ (now - std::max(settings.ttlNegativeNarInfoCache.get(), 3600U))
+ (now - std::max(settings.ttlPositiveNarInfoCache.get(), 30 * 24 * 3600U))
.exec();
debug("deleted %d entries from the NAR info disk cache", sqlite3_changes(state->db));
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index 3454f34bb..49079388a 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -46,14 +46,18 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
else if (name == "FileHash")
fileHash = parseHashField(value);
else if (name == "FileSize") {
- if (!string2Int(value, fileSize)) throw corrupt();
+ auto n = string2Int<decltype(fileSize)>(value);
+ if (!n) throw corrupt();
+ fileSize = *n;
}
else if (name == "NarHash") {
narHash = parseHashField(value);
haveNarHash = true;
}
else if (name == "NarSize") {
- if (!string2Int(value, narSize)) throw corrupt();
+ auto n = string2Int<decltype(narSize)>(value);
+ if (!n) throw corrupt();
+ narSize = *n;
}
else if (name == "References") {
auto refs = tokenizeString<Strings>(value, " ");
diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc
index a0d482ddf..78d587139 100644
--- a/src/libstore/optimise-store.cc
+++ b/src/libstore/optimise-store.cc
@@ -126,16 +126,13 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
NixOS (example: $fontconfig/var/cache being modified). Skip
those files. FIXME: check the modification time. */
if (S_ISREG(st.st_mode) && (st.st_mode & S_IWUSR)) {
- logWarning({
- .name = "Suspicious file",
- .hint = hintfmt("skipping suspicious writable file '%1%'", path)
- });
+ warn("skipping suspicious writable file '%1%'", path);
return;
}
/* This can still happen on top-level files. */
if (st.st_nlink > 1 && inodeHash.count(st.st_ino)) {
- debug(format("'%1%' is already linked, with %2% other file(s)") % path % (st.st_nlink - 2));
+ debug("'%s' is already linked, with %d other file(s)", path, st.st_nlink - 2);
return;
}
@@ -191,10 +188,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
}
if (st.st_size != stLink.st_size) {
- logWarning({
- .name = "Corrupted link",
- .hint = hintfmt("removing corrupted link '%1%'", linkPath)
- });
+ warn("removing corrupted link '%s'", linkPath);
unlink(linkPath.c_str());
goto retry;
}
@@ -229,10 +223,7 @@ void LocalStore::optimisePath_(Activity * act, OptimiseStats & stats,
/* Atomically replace the old file with the new hard link. */
if (rename(tempLink.c_str(), path.c_str()) == -1) {
if (unlink(tempLink.c_str()) == -1)
- logError({
- .name = "Unlink error",
- .hint = hintfmt("unable to unlink '%1%'", tempLink)
- });
+ printError("unable to unlink '%1%'", tempLink);
if (errno == EMLINK) {
/* Some filesystems generate too many links on the rename,
rather than on the original link. (Probably it
diff --git a/src/libstore/parsed-derivations.cc b/src/libstore/parsed-derivations.cc
index e7b7202d4..c5c3ae3dc 100644
--- a/src/libstore/parsed-derivations.cc
+++ b/src/libstore/parsed-derivations.cc
@@ -101,6 +101,10 @@ bool ParsedDerivation::canBuildLocally(Store & localStore) const
&& !drv.isBuiltin())
return false;
+ if (settings.maxBuildJobs.get() == 0
+ && !drv.isBuiltin())
+ return false;
+
for (auto & feature : getRequiredSystemFeatures())
if (!localStore.systemFeatures.get().count(feature)) return false;
diff --git a/src/libstore/path-info.hh b/src/libstore/path-info.hh
index 8ff5c466e..de87f8b33 100644
--- a/src/libstore/path-info.hh
+++ b/src/libstore/path-info.hh
@@ -107,6 +107,6 @@ struct ValidPathInfo
virtual ~ValidPathInfo() { }
};
-typedef list<ValidPathInfo> ValidPathInfos;
+typedef std::map<StorePath, ValidPathInfo> ValidPathInfos;
}
diff --git a/src/libstore/path-with-outputs.cc b/src/libstore/path-with-outputs.cc
new file mode 100644
index 000000000..865d64cf2
--- /dev/null
+++ b/src/libstore/path-with-outputs.cc
@@ -0,0 +1,71 @@
+#include "path-with-outputs.hh"
+#include "store-api.hh"
+
+namespace nix {
+
+std::string StorePathWithOutputs::to_string(const Store & store) const
+{
+ return outputs.empty()
+ ? store.printStorePath(path)
+ : store.printStorePath(path) + "!" + concatStringsSep(",", outputs);
+}
+
+
+DerivedPath StorePathWithOutputs::toDerivedPath() const
+{
+ if (!outputs.empty() || path.isDerivation())
+ return DerivedPath::Built { path, outputs };
+ else
+ return DerivedPath::Opaque { path };
+}
+
+
+std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs> ss)
+{
+ std::vector<DerivedPath> reqs;
+ for (auto & s : ss) reqs.push_back(s.toDerivedPath());
+ return reqs;
+}
+
+
+std::variant<StorePathWithOutputs, StorePath> StorePathWithOutputs::tryFromDerivedPath(const DerivedPath & p)
+{
+ return std::visit(overloaded {
+ [&](DerivedPath::Opaque bo) -> std::variant<StorePathWithOutputs, StorePath> {
+ if (bo.path.isDerivation()) {
+ // drv path gets interpreted as "build", not "get drv file itself"
+ return bo.path;
+ }
+ return StorePathWithOutputs { bo.path };
+ },
+ [&](DerivedPath::Built bfd) -> std::variant<StorePathWithOutputs, StorePath> {
+ return StorePathWithOutputs { bfd.drvPath, bfd.outputs };
+ },
+ }, p.raw());
+}
+
+
+std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
+{
+ size_t n = s.find("!");
+ return n == s.npos
+ ? std::make_pair(s, std::set<string>())
+ : std::make_pair(((std::string_view) s).substr(0, n),
+ tokenizeString<std::set<string>>(((std::string_view) s).substr(n + 1), ","));
+}
+
+
+StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs)
+{
+ auto [path, outputs] = parsePathWithOutputs(pathWithOutputs);
+ return StorePathWithOutputs { store.parseStorePath(path), std::move(outputs) };
+}
+
+
+StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs)
+{
+ auto [path, outputs] = parsePathWithOutputs(pathWithOutputs);
+ return StorePathWithOutputs { store.followLinksToStorePath(path), std::move(outputs) };
+}
+
+}
diff --git a/src/libstore/path-with-outputs.hh b/src/libstore/path-with-outputs.hh
new file mode 100644
index 000000000..4c4023dcb
--- /dev/null
+++ b/src/libstore/path-with-outputs.hh
@@ -0,0 +1,35 @@
+#pragma once
+
+#include <variant>
+
+#include "path.hh"
+#include "derived-path.hh"
+
+namespace nix {
+
+struct StorePathWithOutputs
+{
+ StorePath path;
+ std::set<std::string> outputs;
+
+ std::string to_string(const Store & store) const;
+
+ DerivedPath toDerivedPath() const;
+
+ static std::variant<StorePathWithOutputs, StorePath> tryFromDerivedPath(const DerivedPath &);
+};
+
+std::vector<DerivedPath> toDerivedPaths(const std::vector<StorePathWithOutputs>);
+
+std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
+
+class Store;
+
+/* Split a string specifying a derivation and a set of outputs
+ (/nix/store/hash-foo!out1,out2,...) into the derivation path
+ and the outputs. */
+StorePathWithOutputs parsePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
+
+StorePathWithOutputs followLinksToStorePathWithOutputs(const Store & store, std::string_view pathWithOutputs);
+
+}
diff --git a/src/libstore/path.cc b/src/libstore/path.cc
index dc9dc3897..e642abcd5 100644
--- a/src/libstore/path.cc
+++ b/src/libstore/path.cc
@@ -82,19 +82,4 @@ PathSet Store::printStorePathSet(const StorePathSet & paths) const
return res;
}
-std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s)
-{
- size_t n = s.find("!");
- return n == s.npos
- ? std::make_pair(s, std::set<string>())
- : std::make_pair(((std::string_view) s).substr(0, n),
- tokenizeString<std::set<string>>(((std::string_view) s).substr(n + 1), ","));
-}
-
-StorePathWithOutputs Store::parsePathWithOutputs(const std::string & s)
-{
- auto [path, outputs] = nix::parsePathWithOutputs(s);
- return {parseStorePath(path), std::move(outputs)};
-}
-
}
diff --git a/src/libstore/path.hh b/src/libstore/path.hh
index b03a0f69d..06ba0663b 100644
--- a/src/libstore/path.hh
+++ b/src/libstore/path.hh
@@ -69,16 +69,6 @@ typedef std::map<StorePath, std::optional<ContentAddress>> StorePathCAMap;
/* Extension of derivations in the Nix store. */
const std::string drvExtension = ".drv";
-struct StorePathWithOutputs
-{
- StorePath path;
- std::set<std::string> outputs;
-
- std::string to_string(const Store & store) const;
-};
-
-std::pair<std::string_view, StringSet> parsePathWithOutputs(std::string_view s);
-
}
namespace std {
diff --git a/src/libstore/profiles.cc b/src/libstore/profiles.cc
index ed10dd519..5d1723886 100644
--- a/src/libstore/profiles.cc
+++ b/src/libstore/profiles.cc
@@ -21,9 +21,8 @@ static std::optional<GenerationNumber> parseName(const string & profileName, con
string s = string(name, profileName.size() + 1);
string::size_type p = s.find("-link");
if (p == string::npos) return {};
- unsigned int n;
- if (string2Int(string(s, 0, p), n) && n >= 0)
- return n;
+ if (auto n = string2Int<unsigned int>(s.substr(0, p)))
+ return *n;
else
return {};
}
@@ -214,12 +213,12 @@ void deleteGenerationsOlderThan(const Path & profile, const string & timeSpec, b
{
time_t curTime = time(0);
string strDays = string(timeSpec, 0, timeSpec.size() - 1);
- int days;
+ auto days = string2Int<int>(strDays);
- if (!string2Int(strDays, days) || days < 1)
+ if (!days || *days < 1)
throw Error("invalid number of days specifier '%1%'", timeSpec);
- time_t oldTime = curTime - days * 24 * 3600;
+ time_t oldTime = curTime - *days * 24 * 3600;
deleteGenerationsOlderThan(profile, oldTime, dryRun);
}
diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc
new file mode 100644
index 000000000..638065547
--- /dev/null
+++ b/src/libstore/realisation.cc
@@ -0,0 +1,122 @@
+#include "realisation.hh"
+#include "store-api.hh"
+#include <nlohmann/json.hpp>
+
+namespace nix {
+
+MakeError(InvalidDerivationOutputId, Error);
+
+DrvOutput DrvOutput::parse(const std::string &strRep) {
+ size_t n = strRep.find("!");
+ if (n == strRep.npos)
+ throw InvalidDerivationOutputId("Invalid derivation output id %s", strRep);
+
+ return DrvOutput{
+ .drvHash = Hash::parseAnyPrefixed(strRep.substr(0, n)),
+ .outputName = strRep.substr(n+1),
+ };
+}
+
+std::string DrvOutput::to_string() const {
+ return strHash() + "!" + outputName;
+}
+
+nlohmann::json Realisation::toJSON() const {
+ return nlohmann::json{
+ {"id", id.to_string()},
+ {"outPath", outPath.to_string()},
+ {"signatures", signatures},
+ };
+}
+
+Realisation Realisation::fromJSON(
+ const nlohmann::json& json,
+ const std::string& whence) {
+ auto getOptionalField = [&](std::string fieldName) -> std::optional<std::string> {
+ auto fieldIterator = json.find(fieldName);
+ if (fieldIterator == json.end())
+ return std::nullopt;
+ return *fieldIterator;
+ };
+ auto getField = [&](std::string fieldName) -> std::string {
+ if (auto field = getOptionalField(fieldName))
+ return *field;
+ else
+ throw Error(
+ "Drv output info file '%1%' is corrupt, missing field %2%",
+ whence, fieldName);
+ };
+
+ StringSet signatures;
+ if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end())
+ signatures.insert(signaturesIterator->begin(), signaturesIterator->end());
+
+ return Realisation{
+ .id = DrvOutput::parse(getField("id")),
+ .outPath = StorePath(getField("outPath")),
+ .signatures = signatures,
+ };
+}
+
+std::string Realisation::fingerprint() const
+{
+ auto serialized = toJSON();
+ serialized.erase("signatures");
+ return serialized.dump();
+}
+
+void Realisation::sign(const SecretKey & secretKey)
+{
+ signatures.insert(secretKey.signDetached(fingerprint()));
+}
+
+bool Realisation::checkSignature(const PublicKeys & publicKeys, const std::string & sig) const
+{
+ return verifyDetached(fingerprint(), sig, publicKeys);
+}
+
+size_t Realisation::checkSignatures(const PublicKeys & publicKeys) const
+{
+ // FIXME: Maybe we should return `maxSigs` if the realisation corresponds to
+ // an input-addressed one − because in that case the drv is enough to check
+ // it − but we can't know that here.
+
+ size_t good = 0;
+ for (auto & sig : signatures)
+ if (checkSignature(publicKeys, sig))
+ good++;
+ return good;
+}
+
+StorePath RealisedPath::path() const {
+ return std::visit([](auto && arg) { return arg.getPath(); }, raw);
+}
+
+void RealisedPath::closure(
+ Store& store,
+ const RealisedPath::Set& startPaths,
+ RealisedPath::Set& ret)
+{
+ // FIXME: This only builds the store-path closure, not the real realisation
+ // closure
+ StorePathSet initialStorePaths, pathsClosure;
+ for (auto& path : startPaths)
+ initialStorePaths.insert(path.path());
+ store.computeFSClosure(initialStorePaths, pathsClosure);
+ ret.insert(startPaths.begin(), startPaths.end());
+ ret.insert(pathsClosure.begin(), pathsClosure.end());
+}
+
+void RealisedPath::closure(Store& store, RealisedPath::Set & ret) const
+{
+ RealisedPath::closure(store, {*this}, ret);
+}
+
+RealisedPath::Set RealisedPath::closure(Store& store) const
+{
+ RealisedPath::Set ret;
+ closure(store, ret);
+ return ret;
+}
+
+} // namespace nix
diff --git a/src/libstore/realisation.hh b/src/libstore/realisation.hh
new file mode 100644
index 000000000..f5049c9e9
--- /dev/null
+++ b/src/libstore/realisation.hh
@@ -0,0 +1,83 @@
+#pragma once
+
+#include "path.hh"
+#include <nlohmann/json_fwd.hpp>
+#include "comparator.hh"
+#include "crypto.hh"
+
+namespace nix {
+
+struct DrvOutput {
+ // The hash modulo of the derivation
+ Hash drvHash;
+ std::string outputName;
+
+ std::string to_string() const;
+
+ std::string strHash() const
+ { return drvHash.to_string(Base16, true); }
+
+ static DrvOutput parse(const std::string &);
+
+ GENERATE_CMP(DrvOutput, me->drvHash, me->outputName);
+};
+
+struct Realisation {
+ DrvOutput id;
+ StorePath outPath;
+
+ StringSet signatures;
+
+ nlohmann::json toJSON() const;
+ static Realisation fromJSON(const nlohmann::json& json, const std::string& whence);
+
+ std::string fingerprint() const;
+ void sign(const SecretKey &);
+ bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
+ size_t checkSignatures(const PublicKeys & publicKeys) const;
+
+ StorePath getPath() const { return outPath; }
+
+ GENERATE_CMP(Realisation, me->id, me->outPath);
+};
+
+typedef std::map<DrvOutput, Realisation> DrvOutputs;
+
+struct OpaquePath {
+ StorePath path;
+
+ StorePath getPath() const { return path; }
+
+ GENERATE_CMP(OpaquePath, me->path);
+};
+
+
+/**
+ * A store path with all the history of how it went into the store
+ */
+struct RealisedPath {
+ /*
+ * A path is either the result of the realisation of a derivation or
+ * an opaque blob that has been directly added to the store
+ */
+ using Raw = std::variant<Realisation, OpaquePath>;
+ Raw raw;
+
+ using Set = std::set<RealisedPath>;
+
+ RealisedPath(StorePath path) : raw(OpaquePath{path}) {}
+ RealisedPath(Realisation r) : raw(r) {}
+
+ /**
+ * Get the raw store path associated to this
+ */
+ StorePath path() const;
+
+ void closure(Store& store, Set& ret) const;
+ static void closure(Store& store, const Set& startPaths, Set& ret);
+ Set closure(Store& store) const;
+
+ GENERATE_CMP(RealisedPath, me->raw);
+};
+
+}
diff --git a/src/libstore/references.cc b/src/libstore/references.cc
index d2096cb49..39c4970c6 100644
--- a/src/libstore/references.cc
+++ b/src/libstore/references.cc
@@ -55,27 +55,23 @@ struct RefScanSink : Sink
RefScanSink() { }
- void operator () (const unsigned char * data, size_t len);
+ void operator () (std::string_view data) override
+ {
+ /* It's possible that a reference spans the previous and current
+ fragment, so search in the concatenation of the tail of the
+ previous fragment and the start of the current fragment. */
+ string s = tail + std::string(data, 0, refLength);
+ search((const unsigned char *) s.data(), s.size(), hashes, seen);
+
+ search((const unsigned char *) data.data(), data.size(), hashes, seen);
+
+ size_t tailLen = data.size() <= refLength ? data.size() : refLength;
+ tail = std::string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen));
+ tail.append({data.data() + data.size() - tailLen, tailLen});
+ }
};
-void RefScanSink::operator () (const unsigned char * data, size_t len)
-{
- /* It's possible that a reference spans the previous and current
- fragment, so search in the concatenation of the tail of the
- previous fragment and the start of the current fragment. */
- string s = tail + string((const char *) data, len > refLength ? refLength : len);
- search((const unsigned char *) s.data(), s.size(), hashes, seen);
-
- search(data, len, hashes, seen);
-
- size_t tailLen = len <= refLength ? len : refLength;
- tail =
- string(tail, tail.size() < refLength - tailLen ? 0 : tail.size() - (refLength - tailLen)) +
- string((const char *) data + len - tailLen, tailLen);
-}
-
-
std::pair<PathSet, HashResult> scanForReferences(const string & path,
const PathSet & refs)
{
@@ -92,9 +88,6 @@ PathSet scanForReferences(Sink & toTee,
TeeSink sink { refsSink, toTee };
std::map<string, Path> backMap;
- /* For efficiency (and a higher hit rate), just search for the
- hash part of the file name. (This assumes that all references
- have the form `HASH-bla'). */
for (auto & i : refs) {
auto baseName = std::string(baseNameOf(i));
string::size_type pos = baseName.find('-');
@@ -129,10 +122,10 @@ RewritingSink::RewritingSink(const std::string & from, const std::string & to, S
assert(from.size() == to.size());
}
-void RewritingSink::operator () (const unsigned char * data, size_t len)
+void RewritingSink::operator () (std::string_view data)
{
std::string s(prev);
- s.append((const char *) data, len);
+ s.append(data);
size_t j = 0;
while ((j = s.find(from, j)) != string::npos) {
@@ -146,14 +139,14 @@ void RewritingSink::operator () (const unsigned char * data, size_t len)
pos += consumed;
- if (consumed) nextSink((unsigned char *) s.data(), consumed);
+ if (consumed) nextSink(s.substr(0, consumed));
}
void RewritingSink::flush()
{
if (prev.empty()) return;
pos += prev.size();
- nextSink((unsigned char *) prev.data(), prev.size());
+ nextSink(prev);
prev.clear();
}
@@ -163,9 +156,9 @@ HashModuloSink::HashModuloSink(HashType ht, const std::string & modulus)
{
}
-void HashModuloSink::operator () (const unsigned char * data, size_t len)
+void HashModuloSink::operator () (std::string_view data)
{
- rewritingSink(data, len);
+ rewritingSink(data);
}
HashResult HashModuloSink::finish()
@@ -176,10 +169,8 @@ HashResult HashModuloSink::finish()
NAR with self-references and a NAR with some of the
self-references already zeroed out do not produce a hash
collision. FIXME: proof. */
- for (auto & pos : rewritingSink.matches) {
- auto s = fmt("|%d", pos);
- hashSink((unsigned char *) s.data(), s.size());
- }
+ for (auto & pos : rewritingSink.matches)
+ hashSink(fmt("|%d", pos));
auto h = hashSink.finish();
return {h.first, rewritingSink.pos};
diff --git a/src/libstore/references.hh b/src/libstore/references.hh
index c2efd095c..4f12e6b21 100644
--- a/src/libstore/references.hh
+++ b/src/libstore/references.hh
@@ -19,7 +19,7 @@ struct RewritingSink : Sink
RewritingSink(const std::string & from, const std::string & to, Sink & nextSink);
- void operator () (const unsigned char * data, size_t len) override;
+ void operator () (std::string_view data) override;
void flush();
};
@@ -31,7 +31,7 @@ struct HashModuloSink : AbstractHashSink
HashModuloSink(HashType ht, const std::string & modulus);
- void operator () (const unsigned char * data, size_t len) override;
+ void operator () (std::string_view data) override;
HashResult finish() override;
};
diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc
index 2d02a181b..f43456f0b 100644
--- a/src/libstore/remote-fs-accessor.cc
+++ b/src/libstore/remote-fs-accessor.cc
@@ -43,13 +43,13 @@ void RemoteFSAccessor::addToCache(std::string_view hashPart, const std::string &
}
}
-std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
+std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_, bool requireValidPath)
{
auto path = canonPath(path_);
auto [storePath, restPath] = store->toStorePath(path);
- if (!store->isValidPath(storePath))
+ if (requireValidPath && !store->isValidPath(storePath))
throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath));
auto i = nars.find(std::string(storePath.hashPart()));
@@ -75,7 +75,7 @@ std::pair<ref<FSAccessor>, Path> RemoteFSAccessor::fetch(const Path & path_)
throw SysError("seeking in '%s'", cacheFile);
std::string buf(length, 0);
- readFull(fd.get(), (unsigned char *) buf.data(), length);
+ readFull(fd.get(), buf.data(), length);
return buf;
});
@@ -113,9 +113,9 @@ StringSet RemoteFSAccessor::readDirectory(const Path & path)
return res.first->readDirectory(res.second);
}
-std::string RemoteFSAccessor::readFile(const Path & path)
+std::string RemoteFSAccessor::readFile(const Path & path, bool requireValidPath)
{
- auto res = fetch(path);
+ auto res = fetch(path, requireValidPath);
return res.first->readFile(res.second);
}
diff --git a/src/libstore/remote-fs-accessor.hh b/src/libstore/remote-fs-accessor.hh
index 347cf5764..594852d0e 100644
--- a/src/libstore/remote-fs-accessor.hh
+++ b/src/libstore/remote-fs-accessor.hh
@@ -14,7 +14,7 @@ class RemoteFSAccessor : public FSAccessor
Path cacheDir;
- std::pair<ref<FSAccessor>, Path> fetch(const Path & path_);
+ std::pair<ref<FSAccessor>, Path> fetch(const Path & path_, bool requireValidPath = true);
friend class BinaryCacheStore;
@@ -32,7 +32,7 @@ public:
StringSet readDirectory(const Path & path) override;
- std::string readFile(const Path & path) override;
+ std::string readFile(const Path & path, bool requireValidPath = true) override;
std::string readLink(const Path & path) override;
};
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index 488270f48..761b4a087 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -1,5 +1,6 @@
#include "serialise.hh"
#include "util.hh"
+#include "path-with-outputs.hh"
#include "remote-fs-accessor.hh"
#include "remote-store.hh"
#include "worker-protocol.hh"
@@ -11,6 +12,8 @@
#include "finally.hh"
#include "logging.hh"
#include "callback.hh"
+#include "filetransfer.hh"
+#include <nlohmann/json.hpp>
namespace nix {
@@ -49,6 +52,44 @@ void write(const Store & store, Sink & out, const ContentAddress & ca)
}
+DerivedPath read(const Store & store, Source & from, Phantom<DerivedPath> _)
+{
+ auto s = readString(from);
+ return DerivedPath::parse(store, s);
+}
+
+void write(const Store & store, Sink & out, const DerivedPath & req)
+{
+ out << req.to_string(store);
+}
+
+
+Realisation read(const Store & store, Source & from, Phantom<Realisation> _)
+{
+ std::string rawInput = readString(from);
+ return Realisation::fromJSON(
+ nlohmann::json::parse(rawInput),
+ "remote-protocol"
+ );
+}
+
+void write(const Store & store, Sink & out, const Realisation & realisation)
+{
+ out << realisation.toJSON().dump();
+}
+
+
+DrvOutput read(const Store & store, Source & from, Phantom<DrvOutput> _)
+{
+ return DrvOutput::parse(readString(from));
+}
+
+void write(const Store & store, Sink & out, const DrvOutput & drvOutput)
+{
+ out << drvOutput.to_string();
+}
+
+
std::optional<StorePath> read(const Store & store, Source & from, Phantom<std::optional<StorePath>> _)
{
auto s = readString(from);
@@ -76,8 +117,8 @@ void write(const Store & store, Sink & out, const std::optional<ContentAddress>
/* TODO: Separate these store impls into different files, give them better names */
RemoteStore::RemoteStore(const Params & params)
- : Store(params)
- , RemoteStoreConfig(params)
+ : RemoteStoreConfig(params)
+ , Store(params)
, connections(make_ref<Pool<Connection>>(
std::max(1, (int) maxConnections),
[this]() {
@@ -171,7 +212,8 @@ void RemoteStore::setOptions(Connection & conn)
if (GET_PROTOCOL_MINOR(conn.daemonVersion) >= 12) {
std::map<std::string, Config::SettingInfo> overrides;
- globalConfig.getSettings(overrides, true);
+ settings.getSettings(overrides, true); // libstore settings
+ fileTransferSettings.getSettings(overrides, true);
overrides.erase(settings.keepFailed.name);
overrides.erase(settings.keepGoing.name);
overrides.erase(settings.tryFallback.name);
@@ -257,6 +299,9 @@ StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, Substitute
} else {
conn->to << wopQueryValidPaths;
worker_proto::write(*this, conn->to, paths);
+ if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 27) {
+ conn->to << (settings.buildersUseSubstitutes ? 1 : 0);
+ }
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
}
@@ -407,10 +452,10 @@ StorePathSet RemoteStore::queryValidDerivers(const StorePath & path)
StorePathSet RemoteStore::queryDerivationOutputs(const StorePath & path)
{
- auto conn(getConnection());
- if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 0x16) {
+ if (GET_PROTOCOL_MINOR(getProtocol()) >= 0x16) {
return Store::queryDerivationOutputs(path);
}
+ auto conn(getConnection());
conn->to << wopQueryDerivationOutputs << printStorePath(path);
conn.processStderr();
return worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
@@ -471,9 +516,14 @@ ref<const ValidPathInfo> RemoteStore::addCAToStore(
worker_proto::write(*this, conn->to, references);
conn->to << repair;
- conn.withFramedSink([&](Sink & sink) {
- dump.drainInto(sink);
- });
+ // The dump source may invoke the store, so we need to make some room.
+ connections->incCapacity();
+ {
+ Finally cleanup([&]() { connections->decCapacity(); });
+ conn.withFramedSink([&](Sink & sink) {
+ dump.drainInto(sink);
+ });
+ }
auto path = parseStorePath(readString(conn->from));
return readValidPathInfo(conn, path);
@@ -599,16 +649,57 @@ StorePath RemoteStore::addTextToStore(const string & name, const string & s,
return addCAToStore(source, name, TextHashMethod{}, references, repair)->path;
}
+void RemoteStore::registerDrvOutput(const Realisation & info)
+{
+ auto conn(getConnection());
+ conn->to << wopRegisterDrvOutput;
+ conn->to << info.id.to_string();
+ conn->to << std::string(info.outPath.to_string());
+ conn.processStderr();
+}
+
+std::optional<const Realisation> RemoteStore::queryRealisation(const DrvOutput & id)
+{
+ auto conn(getConnection());
+ conn->to << wopQueryRealisation;
+ conn->to << id.to_string();
+ conn.processStderr();
+ auto outPaths = worker_proto::read(*this, conn->from, Phantom<std::set<StorePath>>{});
+ if (outPaths.empty())
+ return std::nullopt;
+ return {Realisation{.id = id, .outPath = *outPaths.begin()}};
+}
+
+static void writeDerivedPaths(RemoteStore & store, ConnectionHandle & conn, const std::vector<DerivedPath> & reqs)
+{
+ if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {
+ worker_proto::write(store, conn->to, reqs);
+ } else {
+ Strings ss;
+ for (auto & p : reqs) {
+ auto sOrDrvPath = StorePathWithOutputs::tryFromDerivedPath(p);
+ std::visit(overloaded {
+ [&](StorePathWithOutputs s) {
+ ss.push_back(s.to_string(store));
+ },
+ [&](StorePath drvPath) {
+ throw Error("trying to request '%s', but daemon protocol %d.%d is too old (< 1.29) to request a derivation file",
+ store.printStorePath(drvPath),
+ GET_PROTOCOL_MAJOR(conn->daemonVersion),
+ GET_PROTOCOL_MINOR(conn->daemonVersion));
+ },
+ }, sOrDrvPath);
+ }
+ conn->to << ss;
+ }
+}
-void RemoteStore::buildPaths(const std::vector<StorePathWithOutputs> & drvPaths, BuildMode buildMode)
+void RemoteStore::buildPaths(const std::vector<DerivedPath> & drvPaths, BuildMode buildMode)
{
auto conn(getConnection());
conn->to << wopBuildPaths;
assert(GET_PROTOCOL_MINOR(conn->daemonVersion) >= 13);
- Strings ss;
- for (auto & p : drvPaths)
- ss.push_back(p.to_string(*this));
- conn->to << ss;
+ writeDerivedPaths(*this, conn, drvPaths);
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 15)
conn->to << buildMode;
else
@@ -630,9 +721,15 @@ BuildResult RemoteStore::buildDerivation(const StorePath & drvPath, const BasicD
conn->to << buildMode;
conn.processStderr();
BuildResult res;
- unsigned int status;
- conn->from >> status >> res.errorMsg;
- res.status = (BuildResult::Status) status;
+ res.status = (BuildResult::Status) readInt(conn->from);
+ conn->from >> res.errorMsg;
+ if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 29) {
+ conn->from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime;
+ }
+ if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 28) {
+ auto builtOutputs = worker_proto::read(*this, conn->from, Phantom<DrvOutputs> {});
+ res.builtOutputs = builtOutputs;
+ }
return res;
}
@@ -741,7 +838,7 @@ void RemoteStore::addSignatures(const StorePath & storePath, const StringSet & s
}
-void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets,
+void RemoteStore::queryMissing(const std::vector<DerivedPath> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
uint64_t & downloadSize, uint64_t & narSize)
{
@@ -752,10 +849,7 @@ void RemoteStore::queryMissing(const std::vector<StorePathWithOutputs> & targets
// to prevent a deadlock.
goto fallback;
conn->to << wopQueryMissing;
- Strings ss;
- for (auto & p : targets)
- ss.push_back(p.to_string(*this));
- conn->to << ss;
+ writeDerivedPaths(*this, conn, targets);
conn.processStderr();
willBuild = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
willSubstitute = worker_proto::read(*this, conn->from, Phantom<StorePathSet> {});
@@ -846,8 +940,8 @@ std::exception_ptr RemoteStore::Connection::processStderr(Sink * sink, Source *
else if (msg == STDERR_READ) {
if (!source) throw Error("no source");
size_t len = readNum<size_t>(from);
- auto buf = std::make_unique<unsigned char[]>(len);
- writeString(buf.get(), source->read(buf.get(), len), to);
+ auto buf = std::make_unique<char[]>(len);
+ writeString({(const char *) buf.get(), source->read(buf.get(), len)}, to);
to.flush();
}
diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh
index 9f78fcb02..6cf76a46d 100644
--- a/src/libstore/remote-store.hh
+++ b/src/libstore/remote-store.hh
@@ -29,7 +29,7 @@ struct RemoteStoreConfig : virtual StoreConfig
/* FIXME: RemoteStore is a misnomer - should be something like
DaemonStore. */
-class RemoteStore : public virtual Store, public virtual RemoteStoreConfig
+class RemoteStore : public virtual RemoteStoreConfig, public virtual Store
{
public:
@@ -81,7 +81,11 @@ public:
StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair) override;
- void buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode) override;
+ void registerDrvOutput(const Realisation & info) override;
+
+ std::optional<const Realisation> queryRealisation(const DrvOutput &) override;
+
+ void buildPaths(const std::vector<DerivedPath> & paths, BuildMode buildMode) override;
BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
BuildMode buildMode) override;
@@ -104,7 +108,7 @@ public:
void addSignatures(const StorePath & storePath, const StringSet & sigs) override;
- void queryMissing(const std::vector<StorePathWithOutputs> & targets,
+ void queryMissing(const std::vector<DerivedPath> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
uint64_t & downloadSize, uint64_t & narSize) override;
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index 552c4aac7..6bfbee044 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -57,6 +57,10 @@ class AwsLogger : public Aws::Utils::Logging::FormattedLogSystem
{
debug("AWS: %s", chomp(statement));
}
+
+#if !(AWS_VERSION_MAJOR <= 1 && AWS_VERSION_MINOR <= 7 && AWS_VERSION_PATCH <= 115)
+ void Flush() override {}
+#endif
};
static void initAWS()
@@ -162,7 +166,8 @@ S3Helper::FileTransferResult S3Helper::getObject(
dynamic_cast<std::stringstream &>(result.GetBody()).str());
} catch (S3Error & e) {
- if (e.err != Aws::S3::S3Errors::NO_SUCH_KEY) throw;
+ if ((e.err != Aws::S3::S3Errors::NO_SUCH_KEY) &&
+ (e.err != Aws::S3::S3Errors::ACCESS_DENIED)) throw;
}
auto now2 = std::chrono::steady_clock::now();
@@ -172,6 +177,11 @@ S3Helper::FileTransferResult S3Helper::getObject(
return res;
}
+S3BinaryCacheStore::S3BinaryCacheStore(const Params & params)
+ : BinaryCacheStoreConfig(params)
+ , BinaryCacheStore(params)
+{ }
+
struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
{
using BinaryCacheStoreConfig::BinaryCacheStoreConfig;
@@ -190,7 +200,7 @@ struct S3BinaryCacheStoreConfig : virtual BinaryCacheStoreConfig
const std::string name() override { return "S3 Binary Cache Store"; }
};
-struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCacheStoreConfig
+struct S3BinaryCacheStoreImpl : virtual S3BinaryCacheStoreConfig, public virtual S3BinaryCacheStore
{
std::string bucketName;
@@ -203,6 +213,10 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache
const std::string & bucketName,
const Params & params)
: StoreConfig(params)
+ , BinaryCacheStoreConfig(params)
+ , S3BinaryCacheStoreConfig(params)
+ , Store(params)
+ , BinaryCacheStore(params)
, S3BinaryCacheStore(params)
, bucketName(bucketName)
, s3Helper(profile, region, scheme, endpoint)
@@ -398,7 +412,7 @@ struct S3BinaryCacheStoreImpl : public S3BinaryCacheStore, virtual S3BinaryCache
printTalkative("downloaded 's3://%s/%s' (%d bytes) in %d ms",
bucketName, path, res.data->size(), res.durationMs);
- sink((unsigned char *) res.data->data(), res.data->size());
+ sink(*res.data);
} else
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
}
diff --git a/src/libstore/s3-binary-cache-store.hh b/src/libstore/s3-binary-cache-store.hh
index 4d43fe4d2..bce828b11 100644
--- a/src/libstore/s3-binary-cache-store.hh
+++ b/src/libstore/s3-binary-cache-store.hh
@@ -6,13 +6,11 @@
namespace nix {
-class S3BinaryCacheStore : public BinaryCacheStore
+class S3BinaryCacheStore : public virtual BinaryCacheStore
{
protected:
- S3BinaryCacheStore(const Params & params)
- : BinaryCacheStore(params)
- { }
+ S3BinaryCacheStore(const Params & params);
public:
diff --git a/src/libstore/serve-protocol.hh b/src/libstore/serve-protocol.hh
index 9fae6d534..02d0810cc 100644
--- a/src/libstore/serve-protocol.hh
+++ b/src/libstore/serve-protocol.hh
@@ -5,7 +5,7 @@ namespace nix {
#define SERVE_MAGIC_1 0x390c9deb
#define SERVE_MAGIC_2 0x5452eecb
-#define SERVE_PROTOCOL_VERSION 0x205
+#define SERVE_PROTOCOL_VERSION (2 << 8 | 6)
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index 31a1f0cac..447b4179b 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -147,14 +147,14 @@ void SQLiteStmt::Use::exec()
int r = step();
assert(r != SQLITE_ROW);
if (r != SQLITE_DONE)
- throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", stmt.sql));
+ throwSQLiteError(stmt.db, fmt("executing SQLite statement '%s'", sqlite3_expanded_sql(stmt.stmt)));
}
bool SQLiteStmt::Use::next()
{
int r = step();
if (r != SQLITE_DONE && r != SQLITE_ROW)
- throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", stmt.sql));
+ throwSQLiteError(stmt.db, fmt("executing SQLite query '%s'", sqlite3_expanded_sql(stmt.stmt)));
return r == SQLITE_ROW;
}
@@ -211,7 +211,7 @@ void handleSQLiteBusy(const SQLiteBusy & e)
lastWarned = now;
logWarning({
.name = "Sqlite busy",
- .hint = hintfmt(e.what())
+ .msg = hintfmt(e.what())
});
}
diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc
index 08d0bd565..f2caf2aeb 100644
--- a/src/libstore/ssh-store.cc
+++ b/src/libstore/ssh-store.cc
@@ -13,6 +13,7 @@ struct SSHStoreConfig : virtual RemoteStoreConfig
using RemoteStoreConfig::RemoteStoreConfig;
const Setting<Path> sshKey{(StoreConfig*) this, "", "ssh-key", "path to an SSH private key"};
+ const Setting<std::string> sshPublicHostKey{(StoreConfig*) this, "", "base64-ssh-public-host-key", "The public half of the host's SSH key"};
const Setting<bool> compress{(StoreConfig*) this, false, "compress", "whether to compress the connection"};
const Setting<Path> remoteProgram{(StoreConfig*) this, "nix-daemon", "remote-program", "path to the nix-daemon executable on the remote system"};
const Setting<std::string> remoteStore{(StoreConfig*) this, "", "remote-store", "URI of the store on the remote system"};
@@ -20,18 +21,21 @@ struct SSHStoreConfig : virtual RemoteStoreConfig
const std::string name() override { return "SSH Store"; }
};
-class SSHStore : public virtual RemoteStore, public virtual SSHStoreConfig
+class SSHStore : public virtual SSHStoreConfig, public virtual RemoteStore
{
public:
SSHStore(const std::string & scheme, const std::string & host, const Params & params)
: StoreConfig(params)
+ , RemoteStoreConfig(params)
+ , SSHStoreConfig(params)
, Store(params)
, RemoteStore(params)
, host(host)
, master(
host,
sshKey,
+ sshPublicHostKey,
// Use SSH master only if using more than 1 connection.
connections->capacity() > 1,
compress)
diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc
index 84548a6e4..93f72675d 100644
--- a/src/libstore/ssh.cc
+++ b/src/libstore/ssh.cc
@@ -2,24 +2,37 @@
namespace nix {
-SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD)
+SSHMaster::SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD)
: host(host)
, fakeSSH(host == "localhost")
, keyFile(keyFile)
+ , sshPublicHostKey(sshPublicHostKey)
, useMaster(useMaster && !fakeSSH)
, compress(compress)
, logFD(logFD)
{
if (host == "" || hasPrefix(host, "-"))
throw Error("invalid SSH host name '%s'", host);
+
+ auto state(state_.lock());
+ state->tmpDir = std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700));
}
void SSHMaster::addCommonSSHOpts(Strings & args)
{
+ auto state(state_.lock());
+
for (auto & i : tokenizeString<Strings>(getEnv("NIX_SSHOPTS").value_or("")))
args.push_back(i);
if (!keyFile.empty())
args.insert(args.end(), {"-i", keyFile});
+ if (!sshPublicHostKey.empty()) {
+ Path fileName = (Path) *state->tmpDir + "/host-key";
+ auto p = host.rfind("@");
+ string thost = p != string::npos ? string(host, p + 1) : host;
+ writeFile(fileName, thost + " " + base64Decode(sshPublicHostKey) + "\n");
+ args.insert(args.end(), {"-oUserKnownHostsFile=" + fileName});
+ }
if (compress)
args.push_back("-C");
}
@@ -37,7 +50,7 @@ std::unique_ptr<SSHMaster::Connection> SSHMaster::startCommand(const std::string
options.dieWithParent = false;
conn->sshPid = startProcess([&]() {
- restoreSignals();
+ restoreProcessContext();
close(in.writeSide.get());
close(out.readSide.get());
@@ -87,7 +100,6 @@ Path SSHMaster::startMaster()
if (state->sshMaster != -1) return state->socketPath;
- state->tmpDir = std::make_unique<AutoDelete>(createTempDir("", "nix", true, true, 0700));
state->socketPath = (Path) *state->tmpDir + "/ssh.sock";
@@ -98,7 +110,7 @@ Path SSHMaster::startMaster()
options.dieWithParent = false;
state->sshMaster = startProcess([&]() {
- restoreSignals();
+ restoreProcessContext();
close(out.readSide.get());
diff --git a/src/libstore/ssh.hh b/src/libstore/ssh.hh
index 4f0f0bd29..dabbcedda 100644
--- a/src/libstore/ssh.hh
+++ b/src/libstore/ssh.hh
@@ -12,6 +12,7 @@ private:
const std::string host;
bool fakeSSH;
const std::string keyFile;
+ const std::string sshPublicHostKey;
const bool useMaster;
const bool compress;
const int logFD;
@@ -29,7 +30,7 @@ private:
public:
- SSHMaster(const std::string & host, const std::string & keyFile, bool useMaster, bool compress, int logFD = -1);
+ SSHMaster(const std::string & host, const std::string & keyFile, const std::string & sshPublicHostKey, bool useMaster, bool compress, int logFD = -1);
struct Connection
{
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 9f21f0434..93fcb068f 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -10,6 +10,8 @@
#include "archive.hh"
#include "callback.hh"
+#include <regex>
+
namespace nix {
@@ -51,13 +53,6 @@ StorePath Store::followLinksToStorePath(std::string_view path) const
}
-StorePathWithOutputs Store::followLinksToStorePathWithOutputs(std::string_view path) const
-{
- auto [path2, outputs] = nix::parsePathWithOutputs(path);
- return StorePathWithOutputs { followLinksToStorePath(path2), std::move(outputs) };
-}
-
-
/* Store paths have the following form:
<realized-path> = <store>/<h>-<name>
@@ -364,12 +359,22 @@ bool Store::PathInfoCacheValue::isKnownNow()
return std::chrono::steady_clock::now() < time_point + ttl;
}
+std::map<std::string, std::optional<StorePath>> Store::queryPartialDerivationOutputMap(const StorePath & path)
+{
+ std::map<std::string, std::optional<StorePath>> outputs;
+ auto drv = readInvalidDerivation(path);
+ for (auto& [outputName, output] : drv.outputsAndOptPaths(*this)) {
+ outputs.emplace(outputName, output.second);
+ }
+ return outputs;
+}
+
OutputPathMap Store::queryDerivationOutputMap(const StorePath & path) {
auto resp = queryPartialDerivationOutputMap(path);
OutputPathMap result;
for (auto & [outName, optOutPath] : resp) {
if (!optOutPath)
- throw Error("output '%s' has no store path mapped to it", outName);
+ throw Error("output '%s' of derivation '%s' has no store path mapped to it", outName, printStorePath(path));
result.insert_or_assign(outName, *optOutPath);
}
return result;
@@ -522,6 +527,28 @@ void Store::queryPathInfo(const StorePath & storePath,
}
+void Store::substitutePaths(const StorePathSet & paths)
+{
+ std::vector<DerivedPath> paths2;
+ for (auto & path : paths)
+ if (!path.isDerivation())
+ paths2.push_back(DerivedPath::Opaque{path});
+ uint64_t downloadSize, narSize;
+ StorePathSet willBuild, willSubstitute, unknown;
+ queryMissing(paths2,
+ willBuild, willSubstitute, unknown, downloadSize, narSize);
+
+ if (!willSubstitute.empty())
+ try {
+ std::vector<DerivedPath> subs;
+ for (auto & p : willSubstitute) subs.push_back(DerivedPath::Opaque{p});
+ buildPaths(subs);
+ } catch (Error & e) {
+ logWarning(e.info());
+ }
+}
+
+
StorePathSet Store::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute)
{
struct State
@@ -700,21 +727,6 @@ const Store::Stats & Store::getStats()
}
-void Store::buildPaths(const std::vector<StorePathWithOutputs> & paths, BuildMode buildMode)
-{
- StorePathSet paths2;
-
- for (auto & path : paths) {
- if (path.path.isDerivation())
- unsupported("buildPaths");
- paths2.insert(path.path);
- }
-
- if (queryValidPaths(paths2).size() != paths2.size())
- unsupported("buildPaths");
-}
-
-
void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
const StorePath & storePath, RepairFlag repair, CheckSigsFlag checkSigs)
{
@@ -750,8 +762,8 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
}
auto source = sinkToSource([&](Sink & sink) {
- LambdaSink progressSink([&](const unsigned char * data, size_t len) {
- total += len;
+ LambdaSink progressSink([&](std::string_view data) {
+ total += data.size();
act.progress(total, info->narSize);
});
TeeSink tee { sink, progressSink };
@@ -764,6 +776,36 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
}
+std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const RealisedPath::Set & paths,
+ RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
+{
+ StorePathSet storePaths;
+ std::set<Realisation> realisations;
+ for (auto & path : paths) {
+ storePaths.insert(path.path());
+ if (auto realisation = std::get_if<Realisation>(&path.raw)) {
+ settings.requireExperimentalFeature("ca-derivations");
+ realisations.insert(*realisation);
+ }
+ }
+ auto pathsMap = copyPaths(srcStore, dstStore, storePaths, repair, checkSigs, substitute);
+ try {
+ for (auto & realisation : realisations) {
+ dstStore->registerDrvOutput(realisation, checkSigs);
+ }
+ } catch (MissingExperimentalFeature & e) {
+ // Don't fail if the remote doesn't support CA derivations is it might
+ // not be within our control to change that, and we might still want
+ // to at least copy the output paths.
+ if (e.missingFeature == "ca-derivations")
+ ignoreException();
+ else
+ throw;
+ }
+
+ return pathsMap;
+}
+
std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore, const StorePathSet & storePaths,
RepairFlag repair, CheckSigsFlag checkSigs, SubstituteFlag substitute)
{
@@ -777,7 +819,6 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
for (auto & path : storePaths)
pathsMap.insert_or_assign(path, path);
- if (missing.empty()) return pathsMap;
Activity act(*logger, lvlInfo, actCopyPaths, fmt("copying %d paths", missing.size()));
@@ -852,21 +893,9 @@ std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStor
nrDone++;
showProgress();
});
-
return pathsMap;
}
-
-void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
- const StorePathSet & storePaths, RepairFlag repair, CheckSigsFlag checkSigs,
- SubstituteFlag substitute)
-{
- StorePathSet closure;
- srcStore->computeFSClosure(storePaths, closure);
- copyPaths(srcStore, dstStore, closure, repair, checkSigs, substitute);
-}
-
-
std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istream & str, std::optional<HashResult> hashGiven)
{
std::string path;
@@ -877,19 +906,20 @@ std::optional<ValidPathInfo> decodeValidPathInfo(const Store & store, std::istre
getline(str, s);
auto narHash = Hash::parseAny(s, htSHA256);
getline(str, s);
- uint64_t narSize;
- if (!string2Int(s, narSize)) throw Error("number expected");
- hashGiven = { narHash, narSize };
+ auto narSize = string2Int<uint64_t>(s);
+ if (!narSize) throw Error("number expected");
+ hashGiven = { narHash, *narSize };
}
ValidPathInfo info(store.parseStorePath(path), hashGiven->first);
info.narSize = hashGiven->second;
std::string deriver;
getline(str, deriver);
if (deriver != "") info.deriver = store.parseStorePath(deriver);
- string s; int n;
+ string s;
getline(str, s);
- if (!string2Int(s, n)) throw Error("number expected");
- while (n--) {
+ auto n = string2Int<int>(s);
+ if (!n) throw Error("number expected");
+ while ((*n)--) {
getline(str, s);
info.references.insert(store.parseStorePath(s));
}
@@ -994,19 +1024,24 @@ Derivation Store::derivationFromPath(const StorePath & drvPath)
return readDerivation(drvPath);
}
-
-Derivation Store::readDerivation(const StorePath & drvPath)
+Derivation readDerivationCommon(Store& store, const StorePath& drvPath, bool requireValidPath)
{
- auto accessor = getFSAccessor();
+ auto accessor = store.getFSAccessor();
try {
- return parseDerivation(*this,
- accessor->readFile(printStorePath(drvPath)),
+ return parseDerivation(store,
+ accessor->readFile(store.printStorePath(drvPath), requireValidPath),
Derivation::nameFromPath(drvPath));
} catch (FormatError & e) {
- throw Error("error parsing derivation '%s': %s", printStorePath(drvPath), e.msg());
+ throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg());
}
}
+Derivation Store::readDerivation(const StorePath & drvPath)
+{ return readDerivationCommon(*this, drvPath, true); }
+
+Derivation Store::readInvalidDerivation(const StorePath & drvPath)
+{ return readDerivationCommon(*this, drvPath, false); }
+
}
@@ -1061,6 +1096,34 @@ std::shared_ptr<Store> openFromNonUri(const std::string & uri, const Store::Para
}
}
+// The `parseURL` function supports both IPv6 URIs as defined in
+// RFC2732, but also pure addresses. The latter one is needed here to
+// connect to a remote store via SSH (it's possible to do e.g. `ssh root@::1`).
+//
+// This function now ensures that a usable connection string is available:
+// * If the store to be opened is not an SSH store, nothing will be done.
+// * If the URL looks like `root@[::1]` (which is allowed by the URL parser and probably
+// needed to pass further flags), it
+// will be transformed into `root@::1` for SSH (same for `[::1]` -> `::1`).
+// * If the URL looks like `root@::1` it will be left as-is.
+// * In any other case, the string will be left as-is.
+static std::string extractConnStr(const std::string &proto, const std::string &connStr)
+{
+ if (proto.rfind("ssh") != std::string::npos) {
+ std::smatch result;
+ std::regex v6AddrRegex("^((.*)@)?\\[(.*)\\]$");
+
+ if (std::regex_match(connStr, result, v6AddrRegex)) {
+ if (result[1].matched) {
+ return result.str(1) + result.str(3);
+ }
+ return result.str(3);
+ }
+ }
+
+ return connStr;
+}
+
ref<Store> openStore(const std::string & uri_,
const Store::Params & extraParams)
{
@@ -1069,7 +1132,10 @@ ref<Store> openStore(const std::string & uri_,
auto parsedUri = parseURL(uri_);
params.insert(parsedUri.query.begin(), parsedUri.query.end());
- auto baseURI = parsedUri.authority.value_or("") + parsedUri.path;
+ auto baseURI = extractConnStr(
+ parsedUri.scheme,
+ parsedUri.authority.value_or("") + parsedUri.path
+ );
for (auto implem : *Implementations::registered) {
if (implem.uriSchemes.count(parsedUri.scheme)) {
@@ -1114,9 +1180,6 @@ std::list<ref<Store>> getDefaultSubstituters()
for (auto uri : settings.substituters.get())
addStore(uri);
- for (auto uri : settings.extraSubstituters.get())
- addStore(uri);
-
stores.sort([](ref<Store> & a, ref<Store> & b) {
return a->priority < b->priority;
});
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index f77bc21d1..f66298991 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -1,6 +1,8 @@
#pragma once
+#include "realisation.hh"
#include "path.hh"
+#include "derived-path.hh"
#include "hash.hh"
#include "content-address.hh"
#include "serialise.hh"
@@ -161,6 +163,8 @@ struct BuildResult
non-determinism.) */
bool isNonDeterministic = false;
+ DrvOutputs builtOutputs;
+
/* The start/stop times of the build (or one of the rounds, if it
was repeated). */
time_t startTime = 0, stopTime = 0;
@@ -174,25 +178,7 @@ struct StoreConfig : public Config
{
using Config::Config;
- /**
- * When constructing a store implementation, we pass in a map `params` of
- * parameters that's supposed to initialize the associated config.
- * To do that, we must use the `StoreConfig(StringMap & params)`
- * constructor, so we'd like to `delete` its default constructor to enforce
- * it.
- *
- * However, actually deleting it means that all the subclasses of
- * `StoreConfig` will have their default constructor deleted (because it's
- * supposed to call the deleted default constructor of `StoreConfig`). But
- * because we're always using virtual inheritance, the constructors of
- * child classes will never implicitely call this one, so deleting it will
- * be more painful than anything else.
- *
- * So we `assert(false)` here to ensure at runtime that the right
- * constructor is always called without having to redefine a custom
- * constructor for each `*Config` class.
- */
- StoreConfig() { assert(false); }
+ StoreConfig() = delete;
virtual ~StoreConfig() { }
@@ -276,11 +262,6 @@ public:
PathSet printStorePathSet(const StorePathSet & path) const;
- /* Split a string specifying a derivation and a set of outputs
- (/nix/store/hash-foo!out1,out2,...) into the derivation path
- and the outputs. */
- StorePathWithOutputs parsePathWithOutputs(const string & s);
-
/* Display a set of paths in human-readable form (i.e., between quotes
and separated by commas). */
std::string showPaths(const StorePathSet & paths);
@@ -304,8 +285,6 @@ public:
result. */
StorePath followLinksToStorePath(std::string_view path) const;
- StorePathWithOutputs followLinksToStorePathWithOutputs(std::string_view path) const;
-
/* Constructs a unique store path name. */
StorePath makeStorePath(std::string_view type,
std::string_view hash, std::string_view name) const;
@@ -360,6 +339,11 @@ protected:
public:
+ /* If requested, substitute missing paths. This
+ implements nix-copy-closure's --use-substitutes
+ flag. */
+ void substitutePaths(const StorePathSet & paths);
+
/* Query which of the given paths is valid. Optionally, try to
substitute missing paths. */
virtual StorePathSet queryValidPaths(const StorePathSet & paths,
@@ -384,6 +368,26 @@ public:
void queryPathInfo(const StorePath & path,
Callback<ref<const ValidPathInfo>> callback) noexcept;
+ /* Check whether the given valid path info is sufficiently attested, by
+ either being signed by a trusted public key or content-addressed, in
+ order to be included in the given store.
+
+ These same checks would be performed in addToStore, but this allows an
+ earlier failure in the case where dependencies need to be added too, but
+ the addToStore wouldn't fail until those dependencies are added. Also,
+ we don't really want to add the dependencies listed in a nar info we
+ don't trust anyyways.
+ */
+ virtual bool pathInfoIsUntrusted(const ValidPathInfo &)
+ {
+ return true;
+ }
+
+ virtual bool realisationIsUntrusted(const Realisation & )
+ {
+ return true;
+ }
+
protected:
virtual void queryPathInfoUncached(const StorePath & path,
@@ -391,6 +395,8 @@ protected:
public:
+ virtual std::optional<const Realisation> queryRealisation(const DrvOutput &) = 0;
+
/* Queries the set of incoming FS references for a store path.
The result is not cleared. */
virtual void queryReferrers(const StorePath & path, StorePathSet & referrers)
@@ -408,8 +414,7 @@ public:
/* Query the mapping outputName => outputPath for the given derivation. All
outputs are mentioned so ones mising the mapping are mapped to
`std::nullopt`. */
- virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path)
- { unsupported("queryPartialDerivationOutputMap"); }
+ virtual std::map<std::string, std::optional<StorePath>> queryPartialDerivationOutputMap(const StorePath & path);
/* Query the mapping outputName=>outputPath for the given derivation.
Assume every output has a mapping and throw an exception otherwise. */
@@ -463,6 +468,20 @@ public:
virtual StorePath addTextToStore(const string & name, const string & s,
const StorePathSet & references, RepairFlag repair = NoRepair) = 0;
+ /**
+ * Add a mapping indicating that `deriver!outputName` maps to the output path
+ * `output`.
+ *
+ * This is redundant for known-input-addressed and fixed-output derivations
+ * as this information is already present in the drv file, but necessary for
+ * floating-ca derivations and their dependencies as there's no way to
+ * retrieve this information otherwise.
+ */
+ virtual void registerDrvOutput(const Realisation & output)
+ { unsupported("registerDrvOutput"); }
+ virtual void registerDrvOutput(const Realisation & output, CheckSigsFlag checkSigs)
+ { return registerDrvOutput(output); }
+
/* Write a NAR dump of a store path. */
virtual void narFromPath(const StorePath & path, Sink & sink) = 0;
@@ -475,7 +494,7 @@ public:
recursively building any sub-derivations. For inputs that are
not derivations, substitute them. */
virtual void buildPaths(
- const std::vector<StorePathWithOutputs> & paths,
+ const std::vector<DerivedPath> & paths,
BuildMode buildMode = bmNormal);
/* Build a single non-materialized derivation (i.e. not from an
@@ -512,17 +531,17 @@ public:
explicitly choosing to allow it).
*/
virtual BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv,
- BuildMode buildMode = bmNormal) = 0;
+ BuildMode buildMode = bmNormal);
/* Ensure that a path is valid. If it is not currently valid, it
may be made valid by running a substitute (if defined for the
path). */
- virtual void ensurePath(const StorePath & path) = 0;
+ virtual void ensurePath(const StorePath & path);
/* Add a store path as a temporary root of the garbage collector.
The root disappears as soon as we exit. */
virtual void addTempRoot(const StorePath & path)
- { unsupported("addTempRoot"); }
+ { warn("not creating temp root, store doesn't support GC"); }
/* Add an indirect root, which is merely a symlink to `path' from
/nix/var/nix/gcroots/auto/<hash of `path'>. `path' is supposed
@@ -597,6 +616,11 @@ public:
virtual ref<FSAccessor> getFSAccessor()
{ unsupported("getFSAccessor"); }
+ /* Repair the contents of the given path by redownloading it using
+ a substituter (if available). */
+ virtual void repairPath(const StorePath & path)
+ { unsupported("repairPath"); }
+
/* Add signatures to the specified store path. The signatures are
not verified. */
virtual void addSignatures(const StorePath & storePath, const StringSet & sigs)
@@ -611,6 +635,9 @@ public:
/* Read a derivation (which must already be valid). */
Derivation readDerivation(const StorePath & drvPath);
+ /* Read a derivation from a potentially invalid path. */
+ Derivation readInvalidDerivation(const StorePath & drvPath);
+
/* Place in `out' the set of all store paths in the file system
closure of `storePath'; that is, all paths than can be directly
or indirectly reached from it. `out' is not cleared. If
@@ -629,7 +656,7 @@ public:
/* Given a set of paths that are to be built, return the set of
derivations that will be built, and the set of output paths
that will be substituted. */
- virtual void queryMissing(const std::vector<StorePathWithOutputs> & targets,
+ virtual void queryMissing(const std::vector<DerivedPath> & targets,
StorePathSet & willBuild, StorePathSet & willSubstitute, StorePathSet & unknown,
uint64_t & downloadSize, uint64_t & narSize);
@@ -728,15 +755,12 @@ void copyStorePath(ref<Store> srcStore, ref<Store> dstStore,
that. Returns a map of what each path was copied to the dstStore
as. */
std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore,
- const StorePathSet & storePaths,
+ const RealisedPath::Set &,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
-
-
-/* Copy the closure of the specified paths from one store to another. */
-void copyClosure(ref<Store> srcStore, ref<Store> dstStore,
- const StorePathSet & storePaths,
+std::map<StorePath, StorePath> copyPaths(ref<Store> srcStore, ref<Store> dstStore,
+ const StorePathSet& paths,
RepairFlag repair = NoRepair,
CheckSigsFlag checkSigs = CheckSigs,
SubstituteFlag substitute = NoSubstitute);
diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc
index 24f3e9c6d..cac4fa036 100644
--- a/src/libstore/uds-remote-store.cc
+++ b/src/libstore/uds-remote-store.cc
@@ -15,6 +15,9 @@ namespace nix {
UDSRemoteStore::UDSRemoteStore(const Params & params)
: StoreConfig(params)
+ , LocalFSStoreConfig(params)
+ , RemoteStoreConfig(params)
+ , UDSRemoteStoreConfig(params)
, Store(params)
, LocalFSStore(params)
, RemoteStore(params)
diff --git a/src/libstore/uds-remote-store.hh b/src/libstore/uds-remote-store.hh
index e5de104c9..ddc7716cd 100644
--- a/src/libstore/uds-remote-store.hh
+++ b/src/libstore/uds-remote-store.hh
@@ -14,15 +14,10 @@ struct UDSRemoteStoreConfig : virtual LocalFSStoreConfig, virtual RemoteStoreCon
{
}
- UDSRemoteStoreConfig()
- : UDSRemoteStoreConfig(Store::Params({}))
- {
- }
-
const std::string name() override { return "Local Daemon Store"; }
};
-class UDSRemoteStore : public LocalFSStore, public RemoteStore, public virtual UDSRemoteStoreConfig
+class UDSRemoteStore : public virtual UDSRemoteStoreConfig, public virtual LocalFSStore, public virtual RemoteStore
{
public:
diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh
index b3705578e..001ed25e3 100644
--- a/src/libstore/worker-protocol.hh
+++ b/src/libstore/worker-protocol.hh
@@ -1,12 +1,15 @@
#pragma once
+#include "store-api.hh"
+#include "serialise.hh"
+
namespace nix {
#define WORKER_MAGIC_1 0x6e697863
#define WORKER_MAGIC_2 0x6478696f
-#define PROTOCOL_VERSION 0x11a
+#define PROTOCOL_VERSION (1 << 8 | 29)
#define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00)
#define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff)
@@ -50,6 +53,8 @@ typedef enum {
wopAddToStoreNar = 39,
wopQueryMissing = 40,
wopQueryDerivationOutputMap = 41,
+ wopRegisterDrvOutput = 42,
+ wopQueryRealisation = 43,
} WorkerOp;
@@ -81,7 +86,11 @@ namespace worker_proto {
MAKE_WORKER_PROTO(, std::string);
MAKE_WORKER_PROTO(, StorePath);
MAKE_WORKER_PROTO(, ContentAddress);
+MAKE_WORKER_PROTO(, DerivedPath);
+MAKE_WORKER_PROTO(, Realisation);
+MAKE_WORKER_PROTO(, DrvOutput);
+MAKE_WORKER_PROTO(template<typename T>, std::vector<T>);
MAKE_WORKER_PROTO(template<typename T>, std::set<T>);
#define X_ template<typename K, typename V>
@@ -107,6 +116,26 @@ MAKE_WORKER_PROTO(, std::optional<StorePath>);
MAKE_WORKER_PROTO(, std::optional<ContentAddress>);
template<typename T>
+std::vector<T> read(const Store & store, Source & from, Phantom<std::vector<T>> _)
+{
+ std::vector<T> resSet;
+ auto size = readNum<size_t>(from);
+ while (size--) {
+ resSet.push_back(read(store, from, Phantom<T> {}));
+ }
+ return resSet;
+}
+
+template<typename T>
+void write(const Store & store, Sink & out, const std::vector<T> & resSet)
+{
+ out << resSet.size();
+ for (auto & key : resSet) {
+ write(store, out, key);
+ }
+}
+
+template<typename T>
std::set<T> read(const Store & store, Source & from, Phantom<std::set<T>> _)
{
std::set<T> resSet;
diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc
index f1479329f..ed0eb2fb5 100644
--- a/src/libutil/archive.cc
+++ b/src/libutil/archive.cc
@@ -27,7 +27,7 @@ struct ArchiveSettings : Config
#endif
"use-case-hack",
"Whether to enable a Darwin-specific hack for dealing with file name collisions."};
- Setting<bool> preallocateContents{this, true, "preallocate-contents",
+ Setting<bool> preallocateContents{this, false, "preallocate-contents",
"Whether to preallocate files when writing objects with known size."};
};
@@ -50,14 +50,14 @@ static void dumpContents(const Path & path, size_t size,
AutoCloseFD fd = open(path.c_str(), O_RDONLY | O_CLOEXEC);
if (!fd) throw SysError("opening file '%1%'", path);
- std::vector<unsigned char> buf(65536);
+ std::vector<char> buf(65536);
size_t left = size;
while (left > 0) {
auto n = std::min(left, buf.size());
readFull(fd.get(), buf.data(), n);
left -= n;
- sink(buf.data(), n);
+ sink({buf.data(), n});
}
writePadding(size, sink);
@@ -155,14 +155,14 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path)
sink.preallocateContents(size);
uint64_t left = size;
- std::vector<unsigned char> buf(65536);
+ std::vector<char> buf(65536);
while (left) {
checkInterrupt();
auto n = buf.size();
if ((uint64_t)n > left) n = left;
source(buf.data(), n);
- sink.receiveContents(buf.data(), n);
+ sink.receiveContents({buf.data(), n});
left -= n;
}
@@ -300,21 +300,21 @@ struct RestoreSink : ParseSink
Path dstPath;
AutoCloseFD fd;
- void createDirectory(const Path & path)
+ void createDirectory(const Path & path) override
{
Path p = dstPath + path;
if (mkdir(p.c_str(), 0777) == -1)
throw SysError("creating directory '%1%'", p);
};
- void createRegularFile(const Path & path)
+ void createRegularFile(const Path & path) override
{
Path p = dstPath + path;
fd = open(p.c_str(), O_CREAT | O_EXCL | O_WRONLY | O_CLOEXEC, 0666);
if (!fd) throw SysError("creating file '%1%'", p);
}
- void isExecutable()
+ void isExecutable() override
{
struct stat st;
if (fstat(fd.get(), &st) == -1)
@@ -323,7 +323,7 @@ struct RestoreSink : ParseSink
throw SysError("fchmod");
}
- void preallocateContents(uint64_t len)
+ void preallocateContents(uint64_t len) override
{
if (!archiveSettings.preallocateContents)
return;
@@ -341,12 +341,12 @@ struct RestoreSink : ParseSink
#endif
}
- void receiveContents(unsigned char * data, size_t len)
+ void receiveContents(std::string_view data) override
{
- writeFull(fd.get(), data, len);
+ writeFull(fd.get(), data);
}
- void createSymlink(const Path & path, const string & target)
+ void createSymlink(const Path & path, const string & target) override
{
Path p = dstPath + path;
nix::createSymlink(target, p);
diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh
index 5665732d2..9e9e11b1a 100644
--- a/src/libutil/archive.hh
+++ b/src/libutil/archive.hh
@@ -58,7 +58,7 @@ struct ParseSink
virtual void createRegularFile(const Path & path) { };
virtual void isExecutable() { };
virtual void preallocateContents(uint64_t size) { };
- virtual void receiveContents(unsigned char * data, size_t len) { };
+ virtual void receiveContents(std::string_view data) { };
virtual void createSymlink(const Path & path, const string & target) { };
};
@@ -72,17 +72,17 @@ struct RetrieveRegularNARSink : ParseSink
RetrieveRegularNARSink(Sink & sink) : sink(sink) { }
- void createDirectory(const Path & path)
+ void createDirectory(const Path & path) override
{
regular = false;
}
- void receiveContents(unsigned char * data, size_t len)
+ void receiveContents(std::string_view data) override
{
- sink(data, len);
+ sink(data);
}
- void createSymlink(const Path & path, const string & target)
+ void createSymlink(const Path & path, const string & target) override
{
regular = false;
}
diff --git a/src/libutil/args.cc b/src/libutil/args.cc
index 8bd9c8aeb..afed0670f 100644
--- a/src/libutil/args.cc
+++ b/src/libutil/args.cc
@@ -14,9 +14,19 @@ void Args::addFlag(Flag && flag_)
assert(flag->handler.arity == flag->labels.size());
assert(flag->longName != "");
longFlags[flag->longName] = flag;
+ for (auto & alias : flag->aliases)
+ longFlags[alias] = flag;
if (flag->shortName) shortFlags[flag->shortName] = flag;
}
+void Args::removeFlag(const std::string & longName)
+{
+ auto flag = longFlags.find(longName);
+ assert(flag != longFlags.end());
+ if (flag->second->shortName) shortFlags.erase(flag->second->shortName);
+ longFlags.erase(flag);
+}
+
void Completions::add(std::string completion, std::string description)
{
assert(description.find('\n') == std::string::npos);
@@ -58,6 +68,7 @@ void Args::parseCmdline(const Strings & _cmdline)
verbosity = lvlError;
}
+ bool argsSeen = false;
for (auto pos = cmdline.begin(); pos != cmdline.end(); ) {
auto arg = *pos;
@@ -86,6 +97,11 @@ void Args::parseCmdline(const Strings & _cmdline)
throw UsageError("unrecognised flag '%1%'", arg);
}
else {
+ if (!argsSeen) {
+ argsSeen = true;
+ initialFlagsProcessed();
+ }
+ pos = rewriteArgs(cmdline, pos);
pendingArgs.push_back(*pos++);
if (processArgs(pendingArgs, false))
pendingArgs.clear();
@@ -93,41 +109,9 @@ void Args::parseCmdline(const Strings & _cmdline)
}
processArgs(pendingArgs, true);
-}
-
-void Args::printHelp(const string & programName, std::ostream & out)
-{
- std::cout << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "FLAGS..." ANSI_NORMAL, programName);
- for (auto & exp : expectedArgs) {
- std::cout << renderLabels({exp.label});
- // FIXME: handle arity > 1
- if (exp.handler.arity == ArityAny) std::cout << "...";
- if (exp.optional) std::cout << "?";
- }
- std::cout << "\n";
-
- auto s = description();
- if (s != "")
- std::cout << "\n" ANSI_BOLD "Summary:" ANSI_NORMAL " " << s << ".\n";
- if (longFlags.size()) {
- std::cout << "\n";
- std::cout << ANSI_BOLD "Flags:" ANSI_NORMAL "\n";
- printFlags(out);
- }
-}
-
-void Args::printFlags(std::ostream & out)
-{
- Table2 table;
- for (auto & flag : longFlags) {
- if (hiddenCategories.count(flag.second->category)) continue;
- table.push_back(std::make_pair(
- (flag.second->shortName ? std::string("-") + flag.second->shortName + ", " : " ")
- + "--" + flag.first + renderLabels(flag.second->labels),
- flag.second->description));
- }
- printTable(out, table);
+ if (!argsSeen)
+ initialFlagsProcessed();
}
bool Args::processFlag(Strings::iterator & pos, Strings::iterator end)
@@ -225,12 +209,12 @@ nlohmann::json Args::toJSON()
for (auto & [name, flag] : longFlags) {
auto j = nlohmann::json::object();
+ if (flag->aliases.count(name)) continue;
if (flag->shortName)
j["shortName"] = std::string(1, flag->shortName);
if (flag->description != "")
j["description"] = flag->description;
- if (flag->category != "")
- j["category"] = flag->category;
+ j["category"] = flag->category;
if (flag->handler.arity != ArityAny)
j["arity"] = flag->handler.arity;
if (!flag->labels.empty())
@@ -253,6 +237,8 @@ nlohmann::json Args::toJSON()
res["description"] = description();
res["flags"] = std::move(flags);
res["args"] = std::move(args);
+ auto s = doc();
+ if (s != "") res.emplace("doc", stripIndentation(s));
return res;
}
@@ -328,72 +314,14 @@ Strings argvToStrings(int argc, char * * argv)
return args;
}
-std::string renderLabels(const Strings & labels)
-{
- std::string res;
- for (auto label : labels) {
- for (auto & c : label) c = std::toupper(c);
- res += " " ANSI_ITALIC + label + ANSI_NORMAL;
- }
- return res;
-}
-
-void printTable(std::ostream & out, const Table2 & table)
-{
- size_t max = 0;
- for (auto & row : table)
- max = std::max(max, filterANSIEscapes(row.first, true).size());
- for (auto & row : table) {
- out << " " << row.first
- << std::string(max - filterANSIEscapes(row.first, true).size() + 2, ' ')
- << row.second << "\n";
- }
-}
-
-void Command::printHelp(const string & programName, std::ostream & out)
-{
- Args::printHelp(programName, out);
-
- auto exs = examples();
- if (!exs.empty()) {
- out << "\n" ANSI_BOLD "Examples:" ANSI_NORMAL "\n";
- for (auto & ex : exs)
- out << "\n"
- << " " << ex.description << "\n" // FIXME: wrap
- << " $ " << ex.command << "\n";
- }
-}
-
-nlohmann::json Command::toJSON()
-{
- auto exs = nlohmann::json::array();
-
- for (auto & example : examples()) {
- auto ex = nlohmann::json::object();
- ex["description"] = example.description;
- ex["command"] = chomp(stripIndentation(example.command));
- exs.push_back(std::move(ex));
- }
-
- auto res = Args::toJSON();
- res["examples"] = std::move(exs);
- auto s = doc();
- if (s != "") res.emplace("doc", stripIndentation(s));
- return res;
-}
-
-MultiCommand::MultiCommand(const Commands & commands)
- : commands(commands)
+MultiCommand::MultiCommand(const Commands & commands_)
+ : commands(commands_)
{
expectArgs({
.label = "subcommand",
.optional = true,
.handler = {[=](std::string s) {
assert(!command);
- if (auto alias = get(deprecatedAliases, s)) {
- warn("'%s' is a deprecated alias for '%s'", s, *alias);
- s = *alias;
- }
if (auto prefix = needsCompletion(s)) {
for (auto & [name, command] : commands)
if (hasPrefix(name, *prefix))
@@ -409,38 +337,6 @@ MultiCommand::MultiCommand(const Commands & commands)
categories[Command::catDefault] = "Available commands";
}
-void MultiCommand::printHelp(const string & programName, std::ostream & out)
-{
- if (command) {
- command->second->printHelp(programName + " " + command->first, out);
- return;
- }
-
- out << fmt(ANSI_BOLD "Usage:" ANSI_NORMAL " %s " ANSI_ITALIC "COMMAND FLAGS... ARGS..." ANSI_NORMAL "\n", programName);
-
- out << "\n" ANSI_BOLD "Common flags:" ANSI_NORMAL "\n";
- printFlags(out);
-
- std::map<Command::Category, std::map<std::string, ref<Command>>> commandsByCategory;
-
- for (auto & [name, commandFun] : commands) {
- auto command = commandFun();
- commandsByCategory[command->category()].insert_or_assign(name, command);
- }
-
- for (auto & [category, commands] : commandsByCategory) {
- out << fmt("\n" ANSI_BOLD "%s:" ANSI_NORMAL "\n", categories[category]);
-
- Table2 table;
- for (auto & [name, command] : commands) {
- auto descr = command->description();
- if (!descr.empty())
- table.push_back(std::make_pair(name, descr));
- }
- printTable(out, table);
- }
-}
-
bool MultiCommand::processFlag(Strings::iterator & pos, Strings::iterator end)
{
if (Args::processFlag(pos, end)) return true;
@@ -463,7 +359,10 @@ nlohmann::json MultiCommand::toJSON()
for (auto & [name, commandFun] : commands) {
auto command = commandFun();
auto j = command->toJSON();
- j["category"] = categories[command->category()];
+ auto cat = nlohmann::json::object();
+ cat["id"] = command->category();
+ cat["description"] = categories[command->category()];
+ j["category"] = std::move(cat);
cmds[name] = std::move(j);
}
diff --git a/src/libutil/args.hh b/src/libutil/args.hh
index 26f1bc11b..c08ba8abd 100644
--- a/src/libutil/args.hh
+++ b/src/libutil/args.hh
@@ -20,11 +20,12 @@ public:
wrong. */
void parseCmdline(const Strings & cmdline);
- virtual void printHelp(const string & programName, std::ostream & out);
-
/* Return a short one-line description of the command. */
virtual std::string description() { return ""; }
+ /* Return documentation about this command, in Markdown format. */
+ virtual std::string doc() { return ""; }
+
protected:
static const size_t ArityAny = std::numeric_limits<size_t>::max();
@@ -65,8 +66,12 @@ protected:
, arity(ArityAny)
{ }
- template<class T>
- Handler(T * dest)
+ Handler(std::string * dest)
+ : fun([=](std::vector<std::string> ss) { *dest = ss[0]; })
+ , arity(1)
+ { }
+
+ Handler(std::optional<std::string> * dest)
: fun([=](std::vector<std::string> ss) { *dest = ss[0]; })
, arity(1)
{ }
@@ -76,14 +81,23 @@ protected:
: fun([=](std::vector<std::string> ss) { *dest = val; })
, arity(0)
{ }
+
+ template<class I>
+ Handler(I * dest)
+ : fun([=](std::vector<std::string> ss) {
+ *dest = string2IntWithUnitPrefix<I>(ss[0]);
+ })
+ , arity(1)
+ { }
};
- /* Flags. */
+ /* Options. */
struct Flag
{
typedef std::shared_ptr<Flag> ptr;
std::string longName;
+ std::set<std::string> aliases;
char shortName = 0;
std::string description;
std::string category;
@@ -100,8 +114,6 @@ protected:
virtual bool processFlag(Strings::iterator & pos, Strings::iterator end);
- virtual void printFlags(std::ostream & out);
-
/* Positional arguments. */
struct ExpectedArg
{
@@ -115,72 +127,20 @@ protected:
virtual bool processArgs(const Strings & args, bool finish);
+ virtual Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos)
+ { return pos; }
+
std::set<std::string> hiddenCategories;
+ /* Called after all command line flags before the first non-flag
+ argument (if any) have been processed. */
+ virtual void initialFlagsProcessed() {}
+
public:
void addFlag(Flag && flag);
- /* Helper functions for constructing flags / positional
- arguments. */
-
- void mkFlag1(char shortName, const std::string & longName,
- const std::string & label, const std::string & description,
- std::function<void(std::string)> fun)
- {
- addFlag({
- .longName = longName,
- .shortName = shortName,
- .description = description,
- .labels = {label},
- .handler = {[=](std::string s) { fun(s); }}
- });
- }
-
- void mkFlag(char shortName, const std::string & name,
- const std::string & description, bool * dest)
- {
- mkFlag(shortName, name, description, dest, true);
- }
-
- template<class T>
- void mkFlag(char shortName, const std::string & longName, const std::string & description,
- T * dest, const T & value)
- {
- addFlag({
- .longName = longName,
- .shortName = shortName,
- .description = description,
- .handler = {[=]() { *dest = value; }}
- });
- }
-
- template<class I>
- void mkIntFlag(char shortName, const std::string & longName,
- const std::string & description, I * dest)
- {
- mkFlag<I>(shortName, longName, description, [=](I n) {
- *dest = n;
- });
- }
-
- template<class I>
- void mkFlag(char shortName, const std::string & longName,
- const std::string & description, std::function<void(I)> fun)
- {
- addFlag({
- .longName = longName,
- .shortName = shortName,
- .description = description,
- .labels = {"N"},
- .handler = {[=](std::string s) {
- I n;
- if (!string2Int(s, n))
- throw UsageError("flag '--%s' requires a integer argument", longName);
- fun(n);
- }}
- });
- }
+ void removeFlag(const std::string & longName);
void expectArgs(ExpectedArg && arg)
{
@@ -222,28 +182,11 @@ struct Command : virtual Args
virtual void prepare() { };
virtual void run() = 0;
- /* Return documentation about this command, in Markdown format. */
- virtual std::string doc() { return ""; }
-
- struct Example
- {
- std::string description;
- std::string command;
- };
-
- typedef std::list<Example> Examples;
-
- virtual Examples examples() { return Examples(); }
-
typedef int Category;
static constexpr Category catDefault = 0;
virtual Category category() { return catDefault; }
-
- void printHelp(const string & programName, std::ostream & out) override;
-
- nlohmann::json toJSON() override;
};
typedef std::map<std::string, std::function<ref<Command>()>> Commands;
@@ -257,15 +200,11 @@ public:
std::map<Command::Category, std::string> categories;
- std::map<std::string, std::string> deprecatedAliases;
-
// Selected command, if any.
std::optional<std::pair<std::string, ref<Command>>> command;
MultiCommand(const Commands & commands);
- void printHelp(const string & programName, std::ostream & out) override;
-
bool processFlag(Strings::iterator & pos, Strings::iterator end) override;
bool processArgs(const Strings & args, bool finish) override;
@@ -275,14 +214,6 @@ public:
Strings argvToStrings(int argc, char * * argv);
-/* Helper function for rendering argument labels. */
-std::string renderLabels(const Strings & labels);
-
-/* Helper function for printing 2-column tables. */
-typedef std::vector<std::pair<std::string, std::string>> Table2;
-
-void printTable(std::ostream & out, const Table2 & table);
-
struct Completion {
std::string completion;
std::string description;
diff --git a/src/libutil/comparator.hh b/src/libutil/comparator.hh
new file mode 100644
index 000000000..0315dc506
--- /dev/null
+++ b/src/libutil/comparator.hh
@@ -0,0 +1,30 @@
+#pragma once
+
+/* Awfull hacky generation of the comparison operators by doing a lexicographic
+ * comparison between the choosen fields.
+ *
+ * ```
+ * GENERATE_CMP(ClassName, me->field1, me->field2, ...)
+ * ```
+ *
+ * will generate comparison operators semantically equivalent to:
+ *
+ * ```
+ * bool operator<(const ClassName& other) {
+ * return field1 < other.field1 && field2 < other.field2 && ...;
+ * }
+ * ```
+ */
+#define GENERATE_ONE_CMP(COMPARATOR, MY_TYPE, FIELDS...) \
+ bool operator COMPARATOR(const MY_TYPE& other) const { \
+ const MY_TYPE* me = this; \
+ auto fields1 = std::make_tuple( FIELDS ); \
+ me = &other; \
+ auto fields2 = std::make_tuple( FIELDS ); \
+ return fields1 COMPARATOR fields2; \
+ }
+#define GENERATE_EQUAL(args...) GENERATE_ONE_CMP(==, args)
+#define GENERATE_LEQ(args...) GENERATE_ONE_CMP(<, args)
+#define GENERATE_CMP(args...) \
+ GENERATE_EQUAL(args) \
+ GENERATE_LEQ(args)
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index a117ddc72..986ba2976 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -22,18 +22,17 @@ struct ChunkedCompressionSink : CompressionSink
{
uint8_t outbuf[32 * 1024];
- void write(const unsigned char * data, size_t len) override
+ void write(std::string_view data) override
{
const size_t CHUNK_SIZE = sizeof(outbuf) << 2;
- while (len) {
- size_t n = std::min(CHUNK_SIZE, len);
- writeInternal(data, n);
- data += n;
- len -= n;
+ while (!data.empty()) {
+ size_t n = std::min(CHUNK_SIZE, data.size());
+ writeInternal(data);
+ data.remove_prefix(n);
}
}
- virtual void writeInternal(const unsigned char * data, size_t len) = 0;
+ virtual void writeInternal(std::string_view data) = 0;
};
struct NoneSink : CompressionSink
@@ -41,7 +40,7 @@ struct NoneSink : CompressionSink
Sink & nextSink;
NoneSink(Sink & nextSink) : nextSink(nextSink) { }
void finish() override { flush(); }
- void write(const unsigned char * data, size_t len) override { nextSink(data, len); }
+ void write(std::string_view data) override { nextSink(data); }
};
struct GzipDecompressionSink : CompressionSink
@@ -75,28 +74,28 @@ struct GzipDecompressionSink : CompressionSink
void finish() override
{
CompressionSink::flush();
- write(nullptr, 0);
+ write({});
}
- void write(const unsigned char * data, size_t len) override
+ void write(std::string_view data) override
{
- assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
+ assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
- strm.next_in = (Bytef *) data;
- strm.avail_in = len;
+ strm.next_in = (Bytef *) data.data();
+ strm.avail_in = data.size();
- while (!finished && (!data || strm.avail_in)) {
+ while (!finished && (!data.data() || strm.avail_in)) {
checkInterrupt();
int ret = inflate(&strm,Z_SYNC_FLUSH);
if (ret != Z_OK && ret != Z_STREAM_END)
throw CompressionError("error while decompressing gzip file: %d (%d, %d)",
- zError(ret), len, strm.avail_in);
+ zError(ret), data.size(), strm.avail_in);
finished = ret == Z_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
- nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+ nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = (Bytef *) outbuf;
strm.avail_out = sizeof(outbuf);
}
@@ -130,25 +129,25 @@ struct XzDecompressionSink : CompressionSink
void finish() override
{
CompressionSink::flush();
- write(nullptr, 0);
+ write({});
}
- void write(const unsigned char * data, size_t len) override
+ void write(std::string_view data) override
{
- strm.next_in = data;
- strm.avail_in = len;
+ strm.next_in = (const unsigned char *) data.data();
+ strm.avail_in = data.size();
- while (!finished && (!data || strm.avail_in)) {
+ while (!finished && (!data.data() || strm.avail_in)) {
checkInterrupt();
- lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH);
+ lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH);
if (ret != LZMA_OK && ret != LZMA_STREAM_END)
throw CompressionError("error %d while decompressing xz file", ret);
finished = ret == LZMA_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
- nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+ nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
@@ -181,15 +180,15 @@ struct BzipDecompressionSink : ChunkedCompressionSink
void finish() override
{
flush();
- write(nullptr, 0);
+ write({});
}
- void writeInternal(const unsigned char * data, size_t len) override
+ void writeInternal(std::string_view data) override
{
- assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
+ assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
- strm.next_in = (char *) data;
- strm.avail_in = len;
+ strm.next_in = (char *) data.data();
+ strm.avail_in = data.size();
while (strm.avail_in) {
checkInterrupt();
@@ -201,7 +200,7 @@ struct BzipDecompressionSink : ChunkedCompressionSink
finished = ret == BZ_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
- nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+ nextSink({(char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = (char *) outbuf;
strm.avail_out = sizeof(outbuf);
}
@@ -230,17 +229,17 @@ struct BrotliDecompressionSink : ChunkedCompressionSink
void finish() override
{
flush();
- writeInternal(nullptr, 0);
+ writeInternal({});
}
- void writeInternal(const unsigned char * data, size_t len) override
+ void writeInternal(std::string_view data) override
{
- const uint8_t * next_in = data;
- size_t avail_in = len;
+ auto next_in = (const uint8_t *) data.data();
+ size_t avail_in = data.size();
uint8_t * next_out = outbuf;
size_t avail_out = sizeof(outbuf);
- while (!finished && (!data || avail_in)) {
+ while (!finished && (!data.data() || avail_in)) {
checkInterrupt();
if (!BrotliDecoderDecompressStream(state,
@@ -250,7 +249,7 @@ struct BrotliDecompressionSink : ChunkedCompressionSink
throw CompressionError("error while decompressing brotli file");
if (avail_out < sizeof(outbuf) || avail_in == 0) {
- nextSink(outbuf, sizeof(outbuf) - avail_out);
+ nextSink({(char *) outbuf, sizeof(outbuf) - avail_out});
next_out = outbuf;
avail_out = sizeof(outbuf);
}
@@ -338,25 +337,25 @@ struct XzCompressionSink : CompressionSink
void finish() override
{
CompressionSink::flush();
- write(nullptr, 0);
+ write({});
}
- void write(const unsigned char * data, size_t len) override
+ void write(std::string_view data) override
{
- strm.next_in = data;
- strm.avail_in = len;
+ strm.next_in = (const unsigned char *) data.data();
+ strm.avail_in = data.size();
- while (!finished && (!data || strm.avail_in)) {
+ while (!finished && (!data.data() || strm.avail_in)) {
checkInterrupt();
- lzma_ret ret = lzma_code(&strm, data ? LZMA_RUN : LZMA_FINISH);
+ lzma_ret ret = lzma_code(&strm, data.data() ? LZMA_RUN : LZMA_FINISH);
if (ret != LZMA_OK && ret != LZMA_STREAM_END)
throw CompressionError("error %d while compressing xz file", ret);
finished = ret == LZMA_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
- nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+ nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = outbuf;
strm.avail_out = sizeof(outbuf);
}
@@ -389,27 +388,27 @@ struct BzipCompressionSink : ChunkedCompressionSink
void finish() override
{
flush();
- writeInternal(nullptr, 0);
+ writeInternal({});
}
- void writeInternal(const unsigned char * data, size_t len) override
+ void writeInternal(std::string_view data) override
{
- assert(len <= std::numeric_limits<decltype(strm.avail_in)>::max());
+ assert(data.size() <= std::numeric_limits<decltype(strm.avail_in)>::max());
- strm.next_in = (char *) data;
- strm.avail_in = len;
+ strm.next_in = (char *) data.data();
+ strm.avail_in = data.size();
- while (!finished && (!data || strm.avail_in)) {
+ while (!finished && (!data.data() || strm.avail_in)) {
checkInterrupt();
- int ret = BZ2_bzCompress(&strm, data ? BZ_RUN : BZ_FINISH);
+ int ret = BZ2_bzCompress(&strm, data.data() ? BZ_RUN : BZ_FINISH);
if (ret != BZ_RUN_OK && ret != BZ_FINISH_OK && ret != BZ_STREAM_END)
throw CompressionError("error %d while compressing bzip2 file", ret);
finished = ret == BZ_STREAM_END;
if (strm.avail_out < sizeof(outbuf) || strm.avail_in == 0) {
- nextSink(outbuf, sizeof(outbuf) - strm.avail_out);
+ nextSink({(const char *) outbuf, sizeof(outbuf) - strm.avail_out});
strm.next_out = (char *) outbuf;
strm.avail_out = sizeof(outbuf);
}
@@ -439,28 +438,28 @@ struct BrotliCompressionSink : ChunkedCompressionSink
void finish() override
{
flush();
- writeInternal(nullptr, 0);
+ writeInternal({});
}
- void writeInternal(const unsigned char * data, size_t len) override
+ void writeInternal(std::string_view data) override
{
- const uint8_t * next_in = data;
- size_t avail_in = len;
+ auto next_in = (const uint8_t *) data.data();
+ size_t avail_in = data.size();
uint8_t * next_out = outbuf;
size_t avail_out = sizeof(outbuf);
- while (!finished && (!data || avail_in)) {
+ while (!finished && (!data.data() || avail_in)) {
checkInterrupt();
if (!BrotliEncoderCompressStream(state,
- data ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH,
+ data.data() ? BROTLI_OPERATION_PROCESS : BROTLI_OPERATION_FINISH,
&avail_in, &next_in,
&avail_out, &next_out,
nullptr))
throw CompressionError("error while compressing brotli compression");
if (avail_out < sizeof(outbuf) || avail_in == 0) {
- nextSink(outbuf, sizeof(outbuf) - avail_out);
+ nextSink({(const char *) outbuf, sizeof(outbuf) - avail_out});
next_out = outbuf;
avail_out = sizeof(outbuf);
}
diff --git a/src/libutil/compute-levels.cc b/src/libutil/compute-levels.cc
new file mode 100644
index 000000000..19eaedfa8
--- /dev/null
+++ b/src/libutil/compute-levels.cc
@@ -0,0 +1,80 @@
+#include "types.hh"
+
+#if HAVE_LIBCPUID
+#include <libcpuid/libcpuid.h>
+#endif
+
+namespace nix {
+
+#if HAVE_LIBCPUID
+
+StringSet computeLevels() {
+ StringSet levels;
+
+ if (!cpuid_present())
+ return levels;
+
+ cpu_raw_data_t raw;
+ cpu_id_t data;
+
+ if (cpuid_get_raw_data(&raw) < 0)
+ return levels;
+
+ if (cpu_identify(&raw, &data) < 0)
+ return levels;
+
+ if (!(data.flags[CPU_FEATURE_CMOV] &&
+ data.flags[CPU_FEATURE_CX8] &&
+ data.flags[CPU_FEATURE_FPU] &&
+ data.flags[CPU_FEATURE_FXSR] &&
+ data.flags[CPU_FEATURE_MMX] &&
+ data.flags[CPU_FEATURE_SSE] &&
+ data.flags[CPU_FEATURE_SSE2]))
+ return levels;
+
+ levels.insert("x86_64-v1");
+
+ if (!(data.flags[CPU_FEATURE_CX16] &&
+ data.flags[CPU_FEATURE_LAHF_LM] &&
+ data.flags[CPU_FEATURE_POPCNT] &&
+ // SSE3
+ data.flags[CPU_FEATURE_PNI] &&
+ data.flags[CPU_FEATURE_SSSE3] &&
+ data.flags[CPU_FEATURE_SSE4_1] &&
+ data.flags[CPU_FEATURE_SSE4_2]))
+ return levels;
+
+ levels.insert("x86_64-v2");
+
+ if (!(data.flags[CPU_FEATURE_AVX] &&
+ data.flags[CPU_FEATURE_AVX2] &&
+ data.flags[CPU_FEATURE_F16C] &&
+ data.flags[CPU_FEATURE_FMA3] &&
+ // LZCNT
+ data.flags[CPU_FEATURE_ABM] &&
+ data.flags[CPU_FEATURE_MOVBE]))
+ return levels;
+
+ levels.insert("x86_64-v3");
+
+ if (!(data.flags[CPU_FEATURE_AVX512F] &&
+ data.flags[CPU_FEATURE_AVX512BW] &&
+ data.flags[CPU_FEATURE_AVX512CD] &&
+ data.flags[CPU_FEATURE_AVX512DQ] &&
+ data.flags[CPU_FEATURE_AVX512VL]))
+ return levels;
+
+ levels.insert("x86_64-v4");
+
+ return levels;
+}
+
+#else
+
+StringSet computeLevels() {
+ return StringSet{};
+}
+
+#endif // HAVE_LIBCPUID
+
+}
diff --git a/src/libutil/compute-levels.hh b/src/libutil/compute-levels.hh
new file mode 100644
index 000000000..8ded295f9
--- /dev/null
+++ b/src/libutil/compute-levels.hh
@@ -0,0 +1,7 @@
+#include "types.hh"
+
+namespace nix {
+
+StringSet computeLevels();
+
+}
diff --git a/src/libutil/config.cc b/src/libutil/config.cc
index 521733025..bda07cd55 100644
--- a/src/libutil/config.cc
+++ b/src/libutil/config.cc
@@ -8,10 +8,19 @@ namespace nix {
bool Config::set(const std::string & name, const std::string & value)
{
+ bool append = false;
auto i = _settings.find(name);
- if (i == _settings.end()) return false;
- i->second.setting->set(value);
- i->second.setting->overriden = true;
+ if (i == _settings.end()) {
+ if (hasPrefix(name, "extra-")) {
+ i = _settings.find(std::string(name, 6));
+ if (i == _settings.end() || !i->second.setting->isAppendable())
+ return false;
+ append = true;
+ } else
+ return false;
+ }
+ i->second.setting->set(value, append);
+ i->second.setting->overridden = true;
return true;
}
@@ -26,7 +35,7 @@ void Config::addSetting(AbstractSetting * setting)
auto i = unknownSettings.find(setting->name);
if (i != unknownSettings.end()) {
setting->set(i->second);
- setting->overriden = true;
+ setting->overridden = true;
unknownSettings.erase(i);
set = true;
}
@@ -39,7 +48,7 @@ void Config::addSetting(AbstractSetting * setting)
alias, setting->name);
else {
setting->set(i->second);
- setting->overriden = true;
+ setting->overridden = true;
unknownSettings.erase(i);
set = true;
}
@@ -60,10 +69,10 @@ void AbstractConfig::reapplyUnknownSettings()
set(s.first, s.second);
}
-void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly)
+void Config::getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly)
{
for (auto & opt : _settings)
- if (!opt.second.isAlias && (!overridenOnly || opt.second.setting->overriden))
+ if (!opt.second.isAlias && (!overriddenOnly || opt.second.setting->overridden))
res.emplace(opt.first, SettingInfo{opt.second.setting->to_string(), opt.second.setting->description});
}
@@ -127,10 +136,10 @@ void AbstractConfig::applyConfigFile(const Path & path)
} catch (SysError &) { }
}
-void Config::resetOverriden()
+void Config::resetOverridden()
{
for (auto & s : _settings)
- s.second.setting->overriden = false;
+ s.second.setting->overridden = false;
}
nlohmann::json Config::toJSON()
@@ -160,7 +169,7 @@ AbstractSetting::AbstractSetting(
void AbstractSetting::setDefault(const std::string & str)
{
- if (!overriden) set(str);
+ if (!overridden) set(str);
}
nlohmann::json AbstractSetting::toJSON()
@@ -181,18 +190,33 @@ void AbstractSetting::convertToArg(Args & args, const std::string & category)
}
template<typename T>
+bool BaseSetting<T>::isAppendable()
+{
+ return false;
+}
+
+template<typename T>
void BaseSetting<T>::convertToArg(Args & args, const std::string & category)
{
args.addFlag({
.longName = name,
- .description = description,
+ .description = fmt("Set the `%s` setting.", name),
.category = category,
.labels = {"value"},
- .handler = {[=](std::string s) { overriden = true; set(s); }},
+ .handler = {[=](std::string s) { overridden = true; set(s); }},
});
+
+ if (isAppendable())
+ args.addFlag({
+ .longName = "extra-" + name,
+ .description = fmt("Append to the `%s` setting.", name),
+ .category = category,
+ .labels = {"value"},
+ .handler = {[=](std::string s) { overridden = true; set(s, true); }},
+ });
}
-template<> void BaseSetting<std::string>::set(const std::string & str)
+template<> void BaseSetting<std::string>::set(const std::string & str, bool append)
{
value = str;
}
@@ -203,10 +227,12 @@ template<> std::string BaseSetting<std::string>::to_string() const
}
template<typename T>
-void BaseSetting<T>::set(const std::string & str)
+void BaseSetting<T>::set(const std::string & str, bool append)
{
static_assert(std::is_integral<T>::value, "Integer required.");
- if (!string2Int(str, value))
+ if (auto n = string2Int<T>(str))
+ value = *n;
+ else
throw UsageError("setting '%s' has invalid value '%s'", name, str);
}
@@ -217,7 +243,7 @@ std::string BaseSetting<T>::to_string() const
return std::to_string(value);
}
-template<> void BaseSetting<bool>::set(const std::string & str)
+template<> void BaseSetting<bool>::set(const std::string & str, bool append)
{
if (str == "true" || str == "yes" || str == "1")
value = true;
@@ -236,21 +262,28 @@ template<> void BaseSetting<bool>::convertToArg(Args & args, const std::string &
{
args.addFlag({
.longName = name,
- .description = description,
+ .description = fmt("Enable the `%s` setting.", name),
.category = category,
.handler = {[=]() { override(true); }}
});
args.addFlag({
.longName = "no-" + name,
- .description = description,
+ .description = fmt("Disable the `%s` setting.", name),
.category = category,
.handler = {[=]() { override(false); }}
});
}
-template<> void BaseSetting<Strings>::set(const std::string & str)
+template<> void BaseSetting<Strings>::set(const std::string & str, bool append)
+{
+ auto ss = tokenizeString<Strings>(str);
+ if (!append) value.clear();
+ for (auto & s : ss) value.push_back(std::move(s));
+}
+
+template<> bool BaseSetting<Strings>::isAppendable()
{
- value = tokenizeString<Strings>(str);
+ return true;
}
template<> std::string BaseSetting<Strings>::to_string() const
@@ -258,9 +291,16 @@ template<> std::string BaseSetting<Strings>::to_string() const
return concatStringsSep(" ", value);
}
-template<> void BaseSetting<StringSet>::set(const std::string & str)
+template<> void BaseSetting<StringSet>::set(const std::string & str, bool append)
+{
+ if (!append) value.clear();
+ for (auto & s : tokenizeString<StringSet>(str))
+ value.insert(s);
+}
+
+template<> bool BaseSetting<StringSet>::isAppendable()
{
- value = tokenizeString<StringSet>(str);
+ return true;
}
template<> std::string BaseSetting<StringSet>::to_string() const
@@ -268,11 +308,10 @@ template<> std::string BaseSetting<StringSet>::to_string() const
return concatStringsSep(" ", value);
}
-template<> void BaseSetting<StringMap>::set(const std::string & str)
+template<> void BaseSetting<StringMap>::set(const std::string & str, bool append)
{
- auto kvpairs = tokenizeString<Strings>(str);
- for (auto & s : kvpairs)
- {
+ if (!append) value.clear();
+ for (auto & s : tokenizeString<Strings>(str)) {
auto eq = s.find_first_of('=');
if (std::string::npos != eq)
value.emplace(std::string(s, 0, eq), std::string(s, eq + 1));
@@ -280,6 +319,11 @@ template<> void BaseSetting<StringMap>::set(const std::string & str)
}
}
+template<> bool BaseSetting<StringMap>::isAppendable()
+{
+ return true;
+}
+
template<> std::string BaseSetting<StringMap>::to_string() const
{
Strings kvstrs;
@@ -300,7 +344,7 @@ template class BaseSetting<Strings>;
template class BaseSetting<StringSet>;
template class BaseSetting<StringMap>;
-void PathSetting::set(const std::string & str)
+void PathSetting::set(const std::string & str, bool append)
{
if (str == "") {
if (allowEmpty)
@@ -321,16 +365,16 @@ bool GlobalConfig::set(const std::string & name, const std::string & value)
return false;
}
-void GlobalConfig::getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly)
+void GlobalConfig::getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly)
{
for (auto & config : *configRegistrations)
- config->getSettings(res, overridenOnly);
+ config->getSettings(res, overriddenOnly);
}
-void GlobalConfig::resetOverriden()
+void GlobalConfig::resetOverridden()
{
for (auto & config : *configRegistrations)
- config->resetOverriden();
+ config->resetOverridden();
}
nlohmann::json GlobalConfig::toJSON()
diff --git a/src/libutil/config.hh b/src/libutil/config.hh
index 1f5f4e7b9..bf81b4892 100644
--- a/src/libutil/config.hh
+++ b/src/libutil/config.hh
@@ -71,9 +71,9 @@ public:
/**
* Adds the currently known settings to the given result map `res`.
* - res: map to store settings in
- * - overridenOnly: when set to true only overridden settings will be added to `res`
+ * - overriddenOnly: when set to true only overridden settings will be added to `res`
*/
- virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) = 0;
+ virtual void getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly = false) = 0;
/**
* Parses the configuration in `contents` and applies it
@@ -91,7 +91,7 @@ public:
/**
* Resets the `overridden` flag of all Settings
*/
- virtual void resetOverriden() = 0;
+ virtual void resetOverridden() = 0;
/**
* Outputs all settings to JSON
@@ -127,7 +127,7 @@ public:
MyClass() : Config(readConfigFile("/etc/my-app.conf"))
{
- std::cout << foo << "\n"; // will print 123 unless overriden
+ std::cout << foo << "\n"; // will print 123 unless overridden
}
};
*/
@@ -163,9 +163,9 @@ public:
void addSetting(AbstractSetting * setting);
- void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override;
+ void getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly = false) override;
- void resetOverriden() override;
+ void resetOverridden() override;
nlohmann::json toJSON() override;
@@ -184,7 +184,7 @@ public:
int created = 123;
- bool overriden = false;
+ bool overridden = false;
void setDefault(const std::string & str);
@@ -202,7 +202,10 @@ protected:
assert(created == 123);
}
- virtual void set(const std::string & value) = 0;
+ virtual void set(const std::string & value, bool append = false) = 0;
+
+ virtual bool isAppendable()
+ { return false; }
virtual std::string to_string() const = 0;
@@ -212,7 +215,7 @@ protected:
virtual void convertToArg(Args & args, const std::string & category);
- bool isOverriden() const { return overriden; }
+ bool isOverridden() const { return overridden; }
};
/* A setting of type T. */
@@ -243,11 +246,13 @@ public:
void operator =(const T & v) { assign(v); }
virtual void assign(const T & v) { value = v; }
- void set(const std::string & str) override;
+ void set(const std::string & str, bool append = false) override;
+
+ bool isAppendable() override;
virtual void override(const T & v)
{
- overriden = true;
+ overridden = true;
value = v;
}
@@ -305,7 +310,7 @@ public:
options->addSetting(this);
}
- void set(const std::string & str) override;
+ void set(const std::string & str, bool append = false) override;
Path operator +(const char * p) const { return value + p; }
@@ -319,9 +324,9 @@ struct GlobalConfig : public AbstractConfig
bool set(const std::string & name, const std::string & value) override;
- void getSettings(std::map<std::string, SettingInfo> & res, bool overridenOnly = false) override;
+ void getSettings(std::map<std::string, SettingInfo> & res, bool overriddenOnly = false) override;
- void resetOverriden() override;
+ void resetOverridden() override;
nlohmann::json toJSON() override;
diff --git a/src/libutil/error.cc b/src/libutil/error.cc
index 803a72953..0eea3455d 100644
--- a/src/libutil/error.cc
+++ b/src/libutil/error.cc
@@ -43,9 +43,9 @@ string showErrPos(const ErrPos & errPos)
{
if (errPos.line > 0) {
if (errPos.column > 0) {
- return fmt("(%1%:%2%)", errPos.line, errPos.column);
+ return fmt("%d:%d", errPos.line, errPos.column);
} else {
- return fmt("(%1%)", errPos.line);
+ return fmt("%d", errPos.line);
}
}
else {
@@ -61,36 +61,31 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos)
if (errPos.origin == foFile) {
LinesOfCode loc;
try {
+ // FIXME: when running as the daemon, make sure we don't
+ // open a file to which the client doesn't have access.
AutoCloseFD fd = open(errPos.file.c_str(), O_RDONLY | O_CLOEXEC);
- if (!fd) {
- logError(SysError("opening file '%1%'", errPos.file).info());
- return std::nullopt;
- }
- else
+ if (!fd) return {};
+
+ // count the newlines.
+ int count = 0;
+ string line;
+ int pl = errPos.line - 1;
+ do
{
- // count the newlines.
- int count = 0;
- string line;
- int pl = errPos.line - 1;
- do
- {
- line = readLine(fd.get());
- ++count;
- if (count < pl)
- {
- ;
- }
- else if (count == pl) {
- loc.prevLineOfCode = line;
- } else if (count == pl + 1) {
- loc.errLineOfCode = line;
- } else if (count == pl + 2) {
- loc.nextLineOfCode = line;
- break;
- }
- } while (true);
- return loc;
- }
+ line = readLine(fd.get());
+ ++count;
+ if (count < pl)
+ ;
+ else if (count == pl)
+ loc.prevLineOfCode = line;
+ else if (count == pl + 1)
+ loc.errLineOfCode = line;
+ else if (count == pl + 2) {
+ loc.nextLineOfCode = line;
+ break;
+ }
+ } while (true);
+ return loc;
}
catch (EndOfFile & eof) {
if (loc.errLineOfCode.has_value())
@@ -99,7 +94,6 @@ std::optional<LinesOfCode> getCodeLines(const ErrPos & errPos)
return std::nullopt;
}
catch (std::exception & e) {
- printError("error reading nix file: %s\n%s", errPos.file, e.what());
return std::nullopt;
}
} else {
@@ -186,24 +180,20 @@ void printCodeLines(std::ostream & out,
}
}
-void printAtPos(const string & prefix, const ErrPos & pos, std::ostream & out)
+void printAtPos(const ErrPos & pos, std::ostream & out)
{
- if (pos)
- {
+ if (pos) {
switch (pos.origin) {
case foFile: {
- out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
- ANSI_BLUE << " in file: " << ANSI_NORMAL << pos.file;
+ out << fmt(ANSI_BLUE "at " ANSI_YELLOW "%s:%s" ANSI_NORMAL ":", pos.file, showErrPos(pos));
break;
}
case foString: {
- out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
- ANSI_BLUE << " from string" << ANSI_NORMAL;
+ out << fmt(ANSI_BLUE "at " ANSI_YELLOW "«string»:%s" ANSI_NORMAL ":", showErrPos(pos));
break;
}
case foStdin: {
- out << prefix << ANSI_BLUE << "at: " << ANSI_YELLOW << showErrPos(pos) <<
- ANSI_BLUE << " from stdin" << ANSI_NORMAL;
+ out << fmt(ANSI_BLUE "at " ANSI_YELLOW "«stdin»:%s" ANSI_NORMAL ":", showErrPos(pos));
break;
}
default:
@@ -212,168 +202,108 @@ void printAtPos(const string & prefix, const ErrPos & pos, std::ostream & out)
}
}
-std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace)
+static std::string indent(std::string_view indentFirst, std::string_view indentRest, std::string_view s)
{
- auto errwidth = std::max<size_t>(getWindowSize().second, 20);
- string prefix = "";
+ std::string res;
+ bool first = true;
+
+ while (!s.empty()) {
+ auto end = s.find('\n');
+ if (!first) res += "\n";
+ res += chomp(std::string(first ? indentFirst : indentRest) + std::string(s.substr(0, end)));
+ first = false;
+ if (end == s.npos) break;
+ s = s.substr(end + 1);
+ }
- string levelString;
+ return res;
+}
+
+std::ostream & showErrorInfo(std::ostream & out, const ErrorInfo & einfo, bool showTrace)
+{
+ std::string prefix;
switch (einfo.level) {
case Verbosity::lvlError: {
- levelString = ANSI_RED;
- levelString += "error:";
- levelString += ANSI_NORMAL;
+ prefix = ANSI_RED "error";
+ break;
+ }
+ case Verbosity::lvlNotice: {
+ prefix = ANSI_RED "note";
break;
}
case Verbosity::lvlWarn: {
- levelString = ANSI_YELLOW;
- levelString += "warning:";
- levelString += ANSI_NORMAL;
+ prefix = ANSI_YELLOW "warning";
break;
}
case Verbosity::lvlInfo: {
- levelString = ANSI_GREEN;
- levelString += "info:";
- levelString += ANSI_NORMAL;
+ prefix = ANSI_GREEN "info";
break;
}
case Verbosity::lvlTalkative: {
- levelString = ANSI_GREEN;
- levelString += "talk:";
- levelString += ANSI_NORMAL;
+ prefix = ANSI_GREEN "talk";
break;
}
case Verbosity::lvlChatty: {
- levelString = ANSI_GREEN;
- levelString += "chat:";
- levelString += ANSI_NORMAL;
+ prefix = ANSI_GREEN "chat";
break;
}
case Verbosity::lvlVomit: {
- levelString = ANSI_GREEN;
- levelString += "vomit:";
- levelString += ANSI_NORMAL;
+ prefix = ANSI_GREEN "vomit";
break;
}
case Verbosity::lvlDebug: {
- levelString = ANSI_YELLOW;
- levelString += "debug:";
- levelString += ANSI_NORMAL;
- break;
- }
- default: {
- levelString = fmt("invalid error level: %1%", einfo.level);
+ prefix = ANSI_YELLOW "debug";
break;
}
+ default:
+ assert(false);
}
- auto ndl = prefix.length()
- + filterANSIEscapes(levelString, true).length()
- + 7
- + einfo.name.length()
- + einfo.programName.value_or("").length();
- auto dashwidth = std::max<int>(errwidth - ndl, 3);
-
- std::string dashes(dashwidth, '-');
-
- // divider.
- if (einfo.name != "")
- out << fmt("%1%%2%" ANSI_BLUE " --- %3% %4% %5%" ANSI_NORMAL,
- prefix,
- levelString,
- einfo.name,
- dashes,
- einfo.programName.value_or(""));
+ // FIXME: show the program name as part of the trace?
+ if (einfo.programName && einfo.programName != ErrorInfo::programName)
+ prefix += fmt(" [%s]:" ANSI_NORMAL " ", einfo.programName.value_or(""));
else
- out << fmt("%1%%2%" ANSI_BLUE " -----%3% %4%" ANSI_NORMAL,
- prefix,
- levelString,
- dashes,
- einfo.programName.value_or(""));
-
- bool nl = false; // intersperse newline between sections.
- if (einfo.errPos.has_value() && (*einfo.errPos)) {
- out << prefix << std::endl;
- printAtPos(prefix, *einfo.errPos, out);
- nl = true;
- }
+ prefix += ":" ANSI_NORMAL " ";
- // description
- if (einfo.description != "") {
- if (nl)
- out << std::endl << prefix;
- out << std::endl << prefix << einfo.description;
- nl = true;
- }
+ std::ostringstream oss;
+ oss << einfo.msg << "\n";
+
+ if (einfo.errPos.has_value() && *einfo.errPos) {
+ oss << "\n";
+ printAtPos(*einfo.errPos, oss);
- if (einfo.errPos.has_value() && (*einfo.errPos)) {
auto loc = getCodeLines(*einfo.errPos);
// lines of code.
if (loc.has_value()) {
- if (nl)
- out << std::endl << prefix;
- printCodeLines(out, prefix, *einfo.errPos, *loc);
- nl = true;
+ oss << "\n";
+ printCodeLines(oss, "", *einfo.errPos, *loc);
+ oss << "\n";
}
}
- // hint
- if (einfo.hint.has_value()) {
- if (nl)
- out << std::endl << prefix;
- out << std::endl << prefix << *einfo.hint;
- nl = true;
- }
-
// traces
- if (showTrace && !einfo.traces.empty())
- {
- const string tracetitle(" show-trace ");
-
- int fill = errwidth - tracetitle.length();
- int lw = 0;
- int rw = 0;
- const int min_dashes = 3;
- if (fill > min_dashes * 2) {
- if (fill % 2 != 0) {
- lw = fill / 2;
- rw = lw + 1;
- }
- else
- {
- lw = rw = fill / 2;
- }
- }
- else
- lw = rw = min_dashes;
-
- if (nl)
- out << std::endl << prefix;
-
- out << ANSI_BLUE << std::string(lw, '-') << tracetitle << std::string(rw, '-') << ANSI_NORMAL;
-
- for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter)
- {
- out << std::endl << prefix;
- out << ANSI_BLUE << "trace: " << ANSI_NORMAL << iter->hint.str();
+ if (showTrace && !einfo.traces.empty()) {
+ for (auto iter = einfo.traces.rbegin(); iter != einfo.traces.rend(); ++iter) {
+ oss << "\n" << "… " << iter->hint.str() << "\n";
if (iter->pos.has_value() && (*iter->pos)) {
auto pos = iter->pos.value();
- out << std::endl << prefix;
- printAtPos(prefix, pos, out);
+ oss << "\n";
+ printAtPos(pos, oss);
auto loc = getCodeLines(pos);
- if (loc.has_value())
- {
- out << std::endl << prefix;
- printCodeLines(out, prefix, pos, *loc);
- out << std::endl << prefix;
+ if (loc.has_value()) {
+ oss << "\n";
+ printCodeLines(oss, "", pos, *loc);
+ oss << "\n";
}
}
}
}
+ out << indent(prefix, std::string(filterANSIEscapes(prefix, true).size(), ' '), chomp(oss.str()));
+
return out;
}
}
diff --git a/src/libutil/error.hh b/src/libutil/error.hh
index d1b6d82bb..ff58d3e00 100644
--- a/src/libutil/error.hh
+++ b/src/libutil/error.hh
@@ -38,13 +38,14 @@ namespace nix {
ErrorInfo structs are sent to the logger as part of an exception, or directly with the
logError or logWarning macros.
- See the error-demo.cc program for usage examples.
+ See libutil/tests/logging.cc for usage examples.
*/
typedef enum {
lvlError = 0,
lvlWarn,
+ lvlNotice,
lvlInfo,
lvlTalkative,
lvlChatty,
@@ -106,9 +107,8 @@ struct Trace {
struct ErrorInfo {
Verbosity level;
- string name;
- string description; // FIXME: remove? it seems to be barely used
- std::optional<hintformat> hint;
+ string name; // FIXME: rename
+ hintformat msg;
std::optional<ErrPos> errPos;
std::list<Trace> traces;
@@ -132,23 +132,17 @@ public:
template<typename... Args>
BaseError(unsigned int status, const Args & ... args)
- : err {.level = lvlError,
- .hint = hintfmt(args...)
- }
+ : err { .level = lvlError, .msg = hintfmt(args...) }
, status(status)
{ }
template<typename... Args>
BaseError(const std::string & fs, const Args & ... args)
- : err {.level = lvlError,
- .hint = hintfmt(fs, args...)
- }
+ : err { .level = lvlError, .msg = hintfmt(fs, args...) }
{ }
BaseError(hintformat hint)
- : err {.level = lvlError,
- .hint = hint
- }
+ : err { .level = lvlError, .msg = hint }
{ }
BaseError(ErrorInfo && e)
@@ -205,7 +199,7 @@ public:
{
errNo = errno;
auto hf = hintfmt(args...);
- err.hint = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
+ err.msg = hintfmt("%1%: %2%", normaltxt(hf.str()), strerror(errNo));
}
virtual const char* sname() const override { return "SysError"; }
diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc
index 4a94f0dfd..4df8b4ecb 100644
--- a/src/libutil/hash.cc
+++ b/src/libutil/hash.cc
@@ -192,7 +192,7 @@ Hash Hash::parseAny(std::string_view original, std::optional<HashType> optType)
// Either the string or user must provide the type, if they both do they
// must agree.
if (!optParsedType && !optType)
- throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context.", rest);
+ throw BadHash("hash '%s' does not include a type, nor is the type otherwise known from context", rest);
else if (optParsedType && optType && *optParsedType != *optType)
throw BadHash("hash '%s' should have type '%s'", original, printHashType(*optType));
@@ -291,12 +291,12 @@ static void start(HashType ht, Ctx & ctx)
static void update(HashType ht, Ctx & ctx,
- const unsigned char * bytes, size_t len)
+ std::string_view data)
{
- if (ht == htMD5) MD5_Update(&ctx.md5, bytes, len);
- else if (ht == htSHA1) SHA1_Update(&ctx.sha1, bytes, len);
- else if (ht == htSHA256) SHA256_Update(&ctx.sha256, bytes, len);
- else if (ht == htSHA512) SHA512_Update(&ctx.sha512, bytes, len);
+ if (ht == htMD5) MD5_Update(&ctx.md5, data.data(), data.size());
+ else if (ht == htSHA1) SHA1_Update(&ctx.sha1, data.data(), data.size());
+ else if (ht == htSHA256) SHA256_Update(&ctx.sha256, data.data(), data.size());
+ else if (ht == htSHA512) SHA512_Update(&ctx.sha512, data.data(), data.size());
}
@@ -314,7 +314,7 @@ Hash hashString(HashType ht, std::string_view s)
Ctx ctx;
Hash hash(ht);
start(ht, ctx);
- update(ht, ctx, (const unsigned char *) s.data(), s.length());
+ update(ht, ctx, s);
finish(ht, ctx, hash.hash);
return hash;
}
@@ -341,10 +341,10 @@ HashSink::~HashSink()
delete ctx;
}
-void HashSink::write(const unsigned char * data, size_t len)
+void HashSink::write(std::string_view data)
{
- bytes += len;
- update(ht, *ctx, data, len);
+ bytes += data.size();
+ update(ht, *ctx, data);
}
HashResult HashSink::finish()
diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh
index 6d6eb70ca..1b626dd85 100644
--- a/src/libutil/hash.hh
+++ b/src/libutil/hash.hh
@@ -156,7 +156,7 @@ public:
HashSink(HashType ht);
HashSink(const HashSink & h);
~HashSink();
- void write(const unsigned char * data, size_t len) override;
+ void write(std::string_view data) override;
HashResult finish() override;
HashResult currentHash();
};
diff --git a/src/libutil/local.mk b/src/libutil/local.mk
index ae7eb67ad..5341c58e6 100644
--- a/src/libutil/local.mk
+++ b/src/libutil/local.mk
@@ -7,3 +7,7 @@ libutil_DIR := $(d)
libutil_SOURCES := $(wildcard $(d)/*.cc)
libutil_LDFLAGS = $(LIBLZMA_LIBS) -lbz2 -pthread $(OPENSSL_LIBS) $(LIBBROTLI_LIBS) $(LIBARCHIVE_LIBS) $(BOOST_LDFLAGS) -lboost_context
+
+ifeq ($(HAVE_LIBCPUID), 1)
+ libutil_LDFLAGS += -lcpuid
+endif
diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc
index 8a6752e22..d2e801175 100644
--- a/src/libutil/logging.cc
+++ b/src/libutil/logging.cc
@@ -184,7 +184,7 @@ struct JSONLogger : Logger {
json["action"] = "msg";
json["level"] = ei.level;
json["msg"] = oss.str();
- json["raw_msg"] = ei.hint->str();
+ json["raw_msg"] = ei.msg.str();
if (ei.errPos.has_value() && (*ei.errPos)) {
json["line"] = ei.errPos->line;
@@ -305,10 +305,7 @@ bool handleJSONLogMessage(const std::string & msg,
}
} catch (std::exception & e) {
- logError({
- .name = "Json log message",
- .hint = hintfmt("bad log message from builder: %s", e.what())
- });
+ printError("bad JSON log message from builder: %s", e.what());
}
return true;
diff --git a/src/libutil/logging.hh b/src/libutil/logging.hh
index 63cb2b268..96ad69790 100644
--- a/src/libutil/logging.hh
+++ b/src/libutil/logging.hh
@@ -100,12 +100,15 @@ public:
virtual void writeToStdout(std::string_view s);
template<typename... Args>
- inline void stdout(const std::string & fs, const Args & ... args)
+ inline void cout(const std::string & fs, const Args & ... args)
{
boost::format f(fs);
formatHelper(f, args...);
writeToStdout(f.str());
}
+
+ virtual std::optional<char> ask(std::string_view s)
+ { return {}; }
};
ActivityId getCurActivity();
@@ -175,8 +178,8 @@ extern Verbosity verbosity; /* suppress msgs > this */
lightweight status messages. */
#define logErrorInfo(level, errorInfo...) \
do { \
- if (level <= nix::verbosity) { \
- logger->logEI(level, errorInfo); \
+ if ((level) <= nix::verbosity) { \
+ logger->logEI((level), errorInfo); \
} \
} while (0)
@@ -188,12 +191,14 @@ extern Verbosity verbosity; /* suppress msgs > this */
arguments are evaluated lazily. */
#define printMsg(level, args...) \
do { \
- if (level <= nix::verbosity) { \
- logger->log(level, fmt(args)); \
+ auto __lvl = level; \
+ if (__lvl <= nix::verbosity) { \
+ logger->log(__lvl, fmt(args)); \
} \
} while (0)
#define printError(args...) printMsg(lvlError, args)
+#define notice(args...) printMsg(lvlNotice, args)
#define printInfo(args...) printMsg(lvlInfo, args)
#define printTalkative(args...) printMsg(lvlTalkative, args)
#define debug(args...) printMsg(lvlDebug, args)
diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc
index 5c9f6f901..d1a16b6ba 100644
--- a/src/libutil/serialise.cc
+++ b/src/libutil/serialise.cc
@@ -11,23 +11,23 @@
namespace nix {
-void BufferedSink::operator () (const unsigned char * data, size_t len)
+void BufferedSink::operator () (std::string_view data)
{
- if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]);
+ if (!buffer) buffer = decltype(buffer)(new char[bufSize]);
- while (len) {
+ while (!data.empty()) {
/* Optimisation: bypass the buffer if the data exceeds the
buffer size. */
- if (bufPos + len >= bufSize) {
+ if (bufPos + data.size() >= bufSize) {
flush();
- write(data, len);
+ write(data);
break;
}
/* Otherwise, copy the bytes to the buffer. Flush the buffer
when it's full. */
- size_t n = bufPos + len > bufSize ? bufSize - bufPos : len;
- memcpy(buffer.get() + bufPos, data, n);
- data += n; bufPos += n; len -= n;
+ size_t n = bufPos + data.size() > bufSize ? bufSize - bufPos : data.size();
+ memcpy(buffer.get() + bufPos, data.data(), n);
+ data.remove_prefix(n); bufPos += n;
if (bufPos == bufSize) flush();
}
}
@@ -38,7 +38,7 @@ void BufferedSink::flush()
if (bufPos == 0) return;
size_t n = bufPos;
bufPos = 0; // don't trigger the assert() in ~BufferedSink()
- write(buffer.get(), n);
+ write({buffer.get(), n});
}
@@ -52,16 +52,13 @@ size_t threshold = 256 * 1024 * 1024;
static void warnLargeDump()
{
- logWarning({
- .name = "Large path",
- .description = "dumping very large path (> 256 MiB); this may run out of memory"
- });
+ warn("dumping very large path (> 256 MiB); this may run out of memory");
}
-void FdSink::write(const unsigned char * data, size_t len)
+void FdSink::write(std::string_view data)
{
- written += len;
+ written += data.size();
static bool warned = false;
if (warn && !warned) {
if (written > threshold) {
@@ -70,7 +67,7 @@ void FdSink::write(const unsigned char * data, size_t len)
}
}
try {
- writeFull(fd, data, len);
+ writeFull(fd, data);
} catch (SysError & e) {
_good = false;
throw;
@@ -84,7 +81,7 @@ bool FdSink::good()
}
-void Source::operator () (unsigned char * data, size_t len)
+void Source::operator () (char * data, size_t len)
{
while (len) {
size_t n = read(data, len);
@@ -96,12 +93,12 @@ void Source::operator () (unsigned char * data, size_t len)
void Source::drainInto(Sink & sink)
{
std::string s;
- std::vector<unsigned char> buf(8192);
+ std::vector<char> buf(8192);
while (true) {
size_t n;
try {
n = read(buf.data(), buf.size());
- sink(buf.data(), n);
+ sink({buf.data(), n});
} catch (EndOfFile &) {
break;
}
@@ -117,9 +114,9 @@ std::string Source::drain()
}
-size_t BufferedSource::read(unsigned char * data, size_t len)
+size_t BufferedSource::read(char * data, size_t len)
{
- if (!buffer) buffer = decltype(buffer)(new unsigned char[bufSize]);
+ if (!buffer) buffer = decltype(buffer)(new char[bufSize]);
if (!bufPosIn) bufPosIn = readUnbuffered(buffer.get(), bufSize);
@@ -138,12 +135,12 @@ bool BufferedSource::hasData()
}
-size_t FdSource::readUnbuffered(unsigned char * data, size_t len)
+size_t FdSource::readUnbuffered(char * data, size_t len)
{
ssize_t n;
do {
checkInterrupt();
- n = ::read(fd, (char *) data, len);
+ n = ::read(fd, data, len);
} while (n == -1 && errno == EINTR);
if (n == -1) { _good = false; throw SysError("reading from file"); }
if (n == 0) { _good = false; throw EndOfFile("unexpected end-of-file"); }
@@ -158,10 +155,10 @@ bool FdSource::good()
}
-size_t StringSource::read(unsigned char * data, size_t len)
+size_t StringSource::read(char * data, size_t len)
{
if (pos == s.size()) throw EndOfFile("end of string reached");
- size_t n = s.copy((char *) data, len, pos);
+ size_t n = s.copy(data, len, pos);
pos += n;
return n;
}
@@ -171,6 +168,39 @@ size_t StringSource::read(unsigned char * data, size_t len)
#error Coroutines are broken in this version of Boost!
#endif
+/* A concrete datatype allow virtual dispatch of stack allocation methods. */
+struct VirtualStackAllocator {
+ StackAllocator *allocator = StackAllocator::defaultAllocator;
+
+ boost::context::stack_context allocate() {
+ return allocator->allocate();
+ }
+
+ void deallocate(boost::context::stack_context sctx) {
+ allocator->deallocate(sctx);
+ }
+};
+
+
+/* This class reifies the default boost coroutine stack allocation strategy with
+ a virtual interface. */
+class DefaultStackAllocator : public StackAllocator {
+ boost::coroutines2::default_stack stack;
+
+ boost::context::stack_context allocate() {
+ return stack.allocate();
+ }
+
+ void deallocate(boost::context::stack_context sctx) {
+ stack.deallocate(sctx);
+ }
+};
+
+static DefaultStackAllocator defaultAllocatorSingleton;
+
+StackAllocator *StackAllocator::defaultAllocator = &defaultAllocatorSingleton;
+
+
std::unique_ptr<Source> sinkToSource(
std::function<void(Sink &)> fun,
std::function<void()> eof)
@@ -192,13 +222,13 @@ std::unique_ptr<Source> sinkToSource(
std::string cur;
size_t pos = 0;
- size_t read(unsigned char * data, size_t len) override
+ size_t read(char * data, size_t len) override
{
if (!coro)
- coro = coro_t::pull_type([&](coro_t::push_type & yield) {
- LambdaSink sink([&](const unsigned char * data, size_t len) {
- if (len) yield(std::string((const char *) data, len));
- });
+ coro = coro_t::pull_type(VirtualStackAllocator{}, [&](coro_t::push_type & yield) {
+ LambdaSink sink([&](std::string_view data) {
+ if (!data.empty()) yield(std::string(data));
+ });
fun(sink);
});
@@ -211,7 +241,7 @@ std::unique_ptr<Source> sinkToSource(
}
auto n = std::min(cur.size() - pos, len);
- memcpy(data, (unsigned char *) cur.data() + pos, n);
+ memcpy(data, cur.data() + pos, n);
pos += n;
return n;
@@ -225,24 +255,24 @@ std::unique_ptr<Source> sinkToSource(
void writePadding(size_t len, Sink & sink)
{
if (len % 8) {
- unsigned char zero[8];
+ char zero[8];
memset(zero, 0, sizeof(zero));
- sink(zero, 8 - (len % 8));
+ sink({zero, 8 - (len % 8)});
}
}
-void writeString(const unsigned char * buf, size_t len, Sink & sink)
+void writeString(std::string_view data, Sink & sink)
{
- sink << len;
- sink(buf, len);
- writePadding(len, sink);
+ sink << data.size();
+ sink(data);
+ writePadding(data.size(), sink);
}
Sink & operator << (Sink & sink, const string & s)
{
- writeString((const unsigned char *) s.data(), s.size(), sink);
+ writeString(s, sink);
return sink;
}
@@ -273,8 +303,7 @@ Sink & operator << (Sink & sink, const Error & ex)
<< "Error"
<< info.level
<< info.name
- << info.description
- << (info.hint ? info.hint->str() : "")
+ << info.msg.str()
<< 0 // FIXME: info.errPos
<< info.traces.size();
for (auto & trace : info.traces) {
@@ -288,7 +317,7 @@ Sink & operator << (Sink & sink, const Error & ex)
void readPadding(size_t len, Source & source)
{
if (len % 8) {
- unsigned char zero[8];
+ char zero[8];
size_t n = 8 - (len % 8);
source(zero, n);
for (unsigned int i = 0; i < n; i++)
@@ -297,7 +326,7 @@ void readPadding(size_t len, Source & source)
}
-size_t readString(unsigned char * buf, size_t max, Source & source)
+size_t readString(char * buf, size_t max, Source & source)
{
auto len = readNum<size_t>(source);
if (len > max) throw SerialisationError("string is too long");
@@ -312,7 +341,7 @@ string readString(Source & source, size_t max)
auto len = readNum<size_t>(source);
if (len > max) throw SerialisationError("string is too long");
std::string res(len, 0);
- source((unsigned char*) res.data(), len);
+ source(res.data(), len);
readPadding(len, source);
return res;
}
@@ -341,12 +370,14 @@ Error readError(Source & source)
{
auto type = readString(source);
assert(type == "Error");
- ErrorInfo info;
- info.level = (Verbosity) readInt(source);
- info.name = readString(source);
- info.description = readString(source);
- auto hint = readString(source);
- if (hint != "") info.hint = hintformat(std::move(format("%s") % hint));
+ auto level = (Verbosity) readInt(source);
+ auto name = readString(source);
+ auto msg = readString(source);
+ ErrorInfo info {
+ .level = level,
+ .name = name,
+ .msg = hintformat(std::move(format("%s") % msg)),
+ };
auto havePos = readNum<size_t>(source);
assert(havePos == 0);
auto nrTraces = readNum<size_t>(source);
@@ -361,17 +392,17 @@ Error readError(Source & source)
}
-void StringSink::operator () (const unsigned char * data, size_t len)
+void StringSink::operator () (std::string_view data)
{
static bool warned = false;
if (!warned && s->size() > threshold) {
warnLargeDump();
warned = true;
}
- s->append((const char *) data, len);
+ s->append(data);
}
-size_t ChainSource::read(unsigned char * data, size_t len)
+size_t ChainSource::read(char * data, size_t len)
{
if (useSecond) {
return source2.read(data, len);
diff --git a/src/libutil/serialise.hh b/src/libutil/serialise.hh
index d7fe0b81e..5bbbc7ce3 100644
--- a/src/libutil/serialise.hh
+++ b/src/libutil/serialise.hh
@@ -5,6 +5,7 @@
#include "types.hh"
#include "util.hh"
+namespace boost::context { struct stack_context; }
namespace nix {
@@ -13,19 +14,14 @@ namespace nix {
struct Sink
{
virtual ~Sink() { }
- virtual void operator () (const unsigned char * data, size_t len) = 0;
+ virtual void operator () (std::string_view data) = 0;
virtual bool good() { return true; }
-
- void operator () (const std::string & s)
- {
- (*this)((const unsigned char *) s.data(), s.size());
- }
};
/* Just throws away data. */
struct NullSink : Sink
{
- void operator () (const unsigned char * data, size_t len) override
+ void operator () (std::string_view data) override
{ }
};
@@ -34,21 +30,16 @@ struct NullSink : Sink
struct BufferedSink : virtual Sink
{
size_t bufSize, bufPos;
- std::unique_ptr<unsigned char[]> buffer;
+ std::unique_ptr<char[]> buffer;
BufferedSink(size_t bufSize = 32 * 1024)
: bufSize(bufSize), bufPos(0), buffer(nullptr) { }
- void operator () (const unsigned char * data, size_t len) override;
-
- void operator () (const std::string & s)
- {
- Sink::operator()(s);
- }
+ void operator () (std::string_view data) override;
void flush();
- virtual void write(const unsigned char * data, size_t len) = 0;
+ virtual void write(std::string_view data) = 0;
};
@@ -60,12 +51,12 @@ struct Source
/* Store exactly ‘len’ bytes in the buffer pointed to by ‘data’.
It blocks until all the requested data is available, or throws
an error if it is not going to be available. */
- void operator () (unsigned char * data, size_t len);
+ void operator () (char * data, size_t len);
/* Store up to ‘len’ in the buffer pointed to by ‘data’, and
return the number of bytes stored. It blocks until at least
one byte is available. */
- virtual size_t read(unsigned char * data, size_t len) = 0;
+ virtual size_t read(char * data, size_t len) = 0;
virtual bool good() { return true; }
@@ -80,18 +71,18 @@ struct Source
struct BufferedSource : Source
{
size_t bufSize, bufPosIn, bufPosOut;
- std::unique_ptr<unsigned char[]> buffer;
+ std::unique_ptr<char[]> buffer;
BufferedSource(size_t bufSize = 32 * 1024)
: bufSize(bufSize), bufPosIn(0), bufPosOut(0), buffer(nullptr) { }
- size_t read(unsigned char * data, size_t len) override;
+ size_t read(char * data, size_t len) override;
bool hasData();
protected:
/* Underlying read call, to be overridden. */
- virtual size_t readUnbuffered(unsigned char * data, size_t len) = 0;
+ virtual size_t readUnbuffered(char * data, size_t len) = 0;
};
@@ -118,7 +109,7 @@ struct FdSink : BufferedSink
~FdSink();
- void write(const unsigned char * data, size_t len) override;
+ void write(std::string_view data) override;
bool good() override;
@@ -147,7 +138,7 @@ struct FdSource : BufferedSource
bool good() override;
protected:
- size_t readUnbuffered(unsigned char * data, size_t len) override;
+ size_t readUnbuffered(char * data, size_t len) override;
private:
bool _good = true;
};
@@ -162,7 +153,7 @@ struct StringSink : Sink
s->reserve(reservedSize);
};
StringSink(ref<std::string> s) : s(s) { };
- void operator () (const unsigned char * data, size_t len) override;
+ void operator () (std::string_view data) override;
};
@@ -172,7 +163,7 @@ struct StringSource : Source
const string & s;
size_t pos;
StringSource(const string & _s) : s(_s), pos(0) { }
- size_t read(unsigned char * data, size_t len) override;
+ size_t read(char * data, size_t len) override;
};
@@ -181,10 +172,10 @@ struct TeeSink : Sink
{
Sink & sink1, & sink2;
TeeSink(Sink & sink1, Sink & sink2) : sink1(sink1), sink2(sink2) { }
- virtual void operator () (const unsigned char * data, size_t len)
+ virtual void operator () (std::string_view data)
{
- sink1(data, len);
- sink2(data, len);
+ sink1(data);
+ sink2(data);
}
};
@@ -196,10 +187,10 @@ struct TeeSource : Source
Sink & sink;
TeeSource(Source & orig, Sink & sink)
: orig(orig), sink(sink) { }
- size_t read(unsigned char * data, size_t len)
+ size_t read(char * data, size_t len)
{
size_t n = orig.read(data, len);
- sink(data, n);
+ sink({data, n});
return n;
}
};
@@ -211,7 +202,7 @@ struct SizedSource : Source
size_t remain;
SizedSource(Source & orig, size_t size)
: orig(orig), remain(size) { }
- size_t read(unsigned char * data, size_t len)
+ size_t read(char * data, size_t len)
{
if (this->remain <= 0) {
throw EndOfFile("sized: unexpected end-of-file");
@@ -225,7 +216,7 @@ struct SizedSource : Source
/* Consume the original source until no remain data is left to consume. */
size_t drainAll()
{
- std::vector<unsigned char> buf(8192);
+ std::vector<char> buf(8192);
size_t sum = 0;
while (this->remain > 0) {
size_t n = read(buf.data(), buf.size());
@@ -240,24 +231,24 @@ struct LengthSink : Sink
{
uint64_t length = 0;
- virtual void operator () (const unsigned char * _, size_t len)
+ void operator () (std::string_view data) override
{
- length += len;
+ length += data.size();
}
};
/* Convert a function into a sink. */
struct LambdaSink : Sink
{
- typedef std::function<void(const unsigned char *, size_t)> lambda_t;
+ typedef std::function<void(std::string_view data)> lambda_t;
lambda_t lambda;
LambdaSink(const lambda_t & lambda) : lambda(lambda) { }
- virtual void operator () (const unsigned char * data, size_t len)
+ void operator () (std::string_view data) override
{
- lambda(data, len);
+ lambda(data);
}
};
@@ -265,13 +256,13 @@ struct LambdaSink : Sink
/* Convert a function into a source. */
struct LambdaSource : Source
{
- typedef std::function<size_t(unsigned char *, size_t)> lambda_t;
+ typedef std::function<size_t(char *, size_t)> lambda_t;
lambda_t lambda;
LambdaSource(const lambda_t & lambda) : lambda(lambda) { }
- size_t read(unsigned char * data, size_t len) override
+ size_t read(char * data, size_t len) override
{
return lambda(data, len);
}
@@ -287,7 +278,7 @@ struct ChainSource : Source
: source1(s1), source2(s2)
{ }
- size_t read(unsigned char * data, size_t len) override;
+ size_t read(char * data, size_t len) override;
};
@@ -301,7 +292,7 @@ std::unique_ptr<Source> sinkToSource(
void writePadding(size_t len, Sink & sink);
-void writeString(const unsigned char * buf, size_t len, Sink & sink);
+void writeString(std::string_view s, Sink & sink);
inline Sink & operator << (Sink & sink, uint64_t n)
{
@@ -314,7 +305,7 @@ inline Sink & operator << (Sink & sink, uint64_t n)
buf[5] = (n >> 40) & 0xff;
buf[6] = (n >> 48) & 0xff;
buf[7] = (unsigned char) (n >> 56) & 0xff;
- sink(buf, sizeof(buf));
+ sink({(char *) buf, sizeof(buf)});
return sink;
}
@@ -331,7 +322,7 @@ template<typename T>
T readNum(Source & source)
{
unsigned char buf[8];
- source(buf, sizeof(buf));
+ source((char *) buf, sizeof(buf));
uint64_t n =
((uint64_t) buf[0]) |
@@ -363,7 +354,7 @@ inline uint64_t readLongLong(Source & source)
void readPadding(size_t len, Source & source);
-size_t readString(unsigned char * buf, size_t max, Source & source);
+size_t readString(char * buf, size_t max, Source & source);
string readString(Source & source, size_t max = std::numeric_limits<size_t>::max());
template<class T> T readStrings(Source & source);
@@ -395,9 +386,9 @@ struct StreamToSourceAdapter : Source
: istream(istream)
{ }
- size_t read(unsigned char * data, size_t len) override
+ size_t read(char * data, size_t len) override
{
- if (!istream->read((char *) data, len)) {
+ if (!istream->read(data, len)) {
if (istream->eof()) {
if (istream->gcount() == 0)
throw EndOfFile("end of file");
@@ -420,7 +411,7 @@ struct FramedSource : Source
{
Source & from;
bool eof = false;
- std::vector<unsigned char> pending;
+ std::vector<char> pending;
size_t pos = 0;
FramedSource(Source & from) : from(from)
@@ -432,13 +423,13 @@ struct FramedSource : Source
while (true) {
auto n = readInt(from);
if (!n) break;
- std::vector<unsigned char> data(n);
+ std::vector<char> data(n);
from(data.data(), n);
}
}
}
- size_t read(unsigned char * data, size_t len) override
+ size_t read(char * data, size_t len) override
{
if (eof) throw EndOfFile("reached end of FramedSource");
@@ -448,7 +439,7 @@ struct FramedSource : Source
eof = true;
return 0;
}
- pending = std::vector<unsigned char>(len);
+ pending = std::vector<char>(len);
pos = 0;
from(pending.data(), len);
}
@@ -483,7 +474,7 @@ struct FramedSink : nix::BufferedSink
}
}
- void write(const unsigned char * data, size_t len) override
+ void write(std::string_view data) override
{
/* Don't send more data if the remote has
encountered an error. */
@@ -492,10 +483,23 @@ struct FramedSink : nix::BufferedSink
ex = nullptr;
std::rethrow_exception(ex2);
}
- to << len;
- to(data, len);
+ to << data.size();
+ to(data);
};
};
+/* Stack allocation strategy for sinkToSource.
+ Mutable to avoid a boehm gc dependency in libutil.
+
+ boost::context doesn't provide a virtual class, so we define our own.
+ */
+struct StackAllocator {
+ virtual boost::context::stack_context allocate() = 0;
+ virtual void deallocate(boost::context::stack_context sctx) = 0;
+
+ /* The stack allocator to use in sinkToSource and potentially elsewhere.
+ It is reassigned by the initGC() method in libexpr. */
+ static StackAllocator *defaultAllocator;
+};
}
diff --git a/src/libutil/tarfile.cc b/src/libutil/tarfile.cc
index c4d8a4f91..2da169ba7 100644
--- a/src/libutil/tarfile.cc
+++ b/src/libutil/tarfile.cc
@@ -66,7 +66,7 @@ private:
*buffer = self->buffer.data();
try {
- return self->source->read(self->buffer.data(), 4096);
+ return self->source->read((char *) self->buffer.data(), 4096);
} catch (EndOfFile &) {
return 0;
} catch (std::exception & err) {
diff --git a/src/libutil/tests/config.cc b/src/libutil/tests/config.cc
index c7777a21f..0ebdaf3db 100644
--- a/src/libutil/tests/config.cc
+++ b/src/libutil/tests/config.cc
@@ -29,20 +29,20 @@ namespace nix {
std::map<std::string, Config::SettingInfo> settings;
Setting<std::string> foo{&config, value, "name-of-the-setting", "description"};
- config.getSettings(settings, /* overridenOnly = */ false);
+ config.getSettings(settings, /* overriddenOnly = */ false);
const auto iter = settings.find("name-of-the-setting");
ASSERT_NE(iter, settings.end());
ASSERT_EQ(iter->second.value, "");
ASSERT_EQ(iter->second.description, "description\n");
}
- TEST(Config, getDefinedOverridenSettingNotSet) {
+ TEST(Config, getDefinedOverriddenSettingNotSet) {
Config config;
std::string value;
std::map<std::string, Config::SettingInfo> settings;
Setting<std::string> foo{&config, value, "name-of-the-setting", "description"};
- config.getSettings(settings, /* overridenOnly = */ true);
+ config.getSettings(settings, /* overriddenOnly = */ true);
const auto e = settings.find("name-of-the-setting");
ASSERT_EQ(e, settings.end());
}
@@ -55,7 +55,7 @@ namespace nix {
setting.assign("value");
- config.getSettings(settings, /* overridenOnly = */ false);
+ config.getSettings(settings, /* overriddenOnly = */ false);
const auto iter = settings.find("name-of-the-setting");
ASSERT_NE(iter, settings.end());
ASSERT_EQ(iter->second.value, "value");
@@ -69,7 +69,7 @@ namespace nix {
ASSERT_TRUE(config.set("name-of-the-setting", "value"));
- config.getSettings(settings, /* overridenOnly = */ false);
+ config.getSettings(settings, /* overriddenOnly = */ false);
const auto e = settings.find("name-of-the-setting");
ASSERT_NE(e, settings.end());
ASSERT_EQ(e->second.value, "value");
@@ -80,8 +80,8 @@ namespace nix {
class TestSetting : public AbstractSetting {
public:
TestSetting() : AbstractSetting("test", "test", {}) {}
- void set(const std::string & value) {}
- std::string to_string() const { return {}; }
+ void set(const std::string & value, bool append) override {}
+ std::string to_string() const override { return {}; }
};
Config config;
@@ -100,7 +100,7 @@ namespace nix {
{
std::map<std::string, Config::SettingInfo> settings;
- config.getSettings(settings, /* overridenOnly = */ false);
+ config.getSettings(settings, /* overriddenOnly = */ false);
ASSERT_EQ(settings.find("key"), settings.end());
}
@@ -108,17 +108,17 @@ namespace nix {
{
std::map<std::string, Config::SettingInfo> settings;
- config.getSettings(settings, /* overridenOnly = */ false);
+ config.getSettings(settings, /* overriddenOnly = */ false);
ASSERT_EQ(settings["key"].value, "value");
}
}
- TEST(Config, resetOverriden) {
+ TEST(Config, resetOverridden) {
Config config;
- config.resetOverriden();
+ config.resetOverridden();
}
- TEST(Config, resetOverridenWithSetting) {
+ TEST(Config, resetOverriddenWithSetting) {
Config config;
Setting<std::string> setting{&config, "", "name-of-the-setting", "description"};
@@ -127,7 +127,7 @@ namespace nix {
setting.set("foo");
ASSERT_EQ(setting.get(), "foo");
- config.getSettings(settings, /* overridenOnly = */ true);
+ config.getSettings(settings, /* overriddenOnly = */ true);
ASSERT_TRUE(settings.empty());
}
@@ -135,18 +135,18 @@ namespace nix {
std::map<std::string, Config::SettingInfo> settings;
setting.override("bar");
- ASSERT_TRUE(setting.overriden);
+ ASSERT_TRUE(setting.overridden);
ASSERT_EQ(setting.get(), "bar");
- config.getSettings(settings, /* overridenOnly = */ true);
+ config.getSettings(settings, /* overriddenOnly = */ true);
ASSERT_FALSE(settings.empty());
}
{
std::map<std::string, Config::SettingInfo> settings;
- config.resetOverriden();
- ASSERT_FALSE(setting.overriden);
- config.getSettings(settings, /* overridenOnly = */ true);
+ config.resetOverridden();
+ ASSERT_FALSE(setting.overridden);
+ config.getSettings(settings, /* overriddenOnly = */ true);
ASSERT_TRUE(settings.empty());
}
}
diff --git a/src/libutil/tests/logging.cc b/src/libutil/tests/logging.cc
index 7e53f17c6..d990e5499 100644
--- a/src/libutil/tests/logging.cc
+++ b/src/libutil/tests/logging.cc
@@ -1,3 +1,5 @@
+#if 0
+
#include "logging.hh"
#include "nixexpr.hh"
#include "util.hh"
@@ -41,15 +43,14 @@ namespace nix {
makeJSONLogger(*logger)->logEI({
.name = "error name",
- .description = "error without any code lines.",
- .hint = hintfmt("this hint has %1% templated %2%!!",
+ .msg = hintfmt("this hint has %1% templated %2%!!",
"yellow",
"values"),
.errPos = Pos(foFile, problem_file, 02, 13)
});
auto str = testing::internal::GetCapturedStderr();
- ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1mrandom.nix\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n");
+ ASSERT_STREQ(str.c_str(), "@nix {\"action\":\"msg\",\"column\":13,\"file\":\"random.nix\",\"level\":0,\"line\":2,\"msg\":\"\\u001b[31;1merror:\\u001b[0m\\u001b[34;1m --- error name --- error-unit-test\\u001b[0m\\n\\u001b[34;1mat: \\u001b[33;1m(2:13)\\u001b[34;1m in file: \\u001b[0mrandom.nix\\n\\nerror without any code lines.\\n\\nthis hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\",\"raw_msg\":\"this hint has \\u001b[33;1myellow\\u001b[0m templated \\u001b[33;1mvalues\\u001b[0m!!\"}\n");
}
TEST(logEI, appendingHintsToPreviousError) {
@@ -62,7 +63,7 @@ namespace nix {
throw TestError(e.info());
} catch (Error &e) {
ErrorInfo ei = e.info();
- ei.hint = hintfmt("%s; subsequent error message.", normaltxt(e.info().hint ? e.info().hint->str() : ""));
+ ei.msg = hintfmt("%s; subsequent error message.", normaltxt(e.info().msg.str()));
testing::internal::CaptureStderr();
logger->logEI(ei);
@@ -95,7 +96,6 @@ namespace nix {
logger->logEI({ .level = lvlInfo,
.name = "Info name",
- .description = "Info description",
});
auto str = testing::internal::GetCapturedStderr();
@@ -109,7 +109,6 @@ namespace nix {
logger->logEI({ .level = lvlTalkative,
.name = "Talkative name",
- .description = "Talkative description",
});
auto str = testing::internal::GetCapturedStderr();
@@ -123,7 +122,6 @@ namespace nix {
logger->logEI({ .level = lvlChatty,
.name = "Chatty name",
- .description = "Talkative description",
});
auto str = testing::internal::GetCapturedStderr();
@@ -137,7 +135,6 @@ namespace nix {
logger->logEI({ .level = lvlDebug,
.name = "Debug name",
- .description = "Debug description",
});
auto str = testing::internal::GetCapturedStderr();
@@ -151,7 +148,6 @@ namespace nix {
logger->logEI({ .level = lvlVomit,
.name = "Vomit name",
- .description = "Vomit description",
});
auto str = testing::internal::GetCapturedStderr();
@@ -167,7 +163,6 @@ namespace nix {
logError({
.name = "name",
- .description = "error description",
});
auto str = testing::internal::GetCapturedStderr();
@@ -182,8 +177,7 @@ namespace nix {
logError({
.name = "error name",
- .description = "error with code lines",
- .hint = hintfmt("this hint has %1% templated %2%!!",
+ .msg = hintfmt("this hint has %1% templated %2%!!",
"yellow",
"values"),
.errPos = Pos(foString, problem_file, 02, 13),
@@ -200,15 +194,14 @@ namespace nix {
logError({
.name = "error name",
- .description = "error without any code lines.",
- .hint = hintfmt("this hint has %1% templated %2%!!",
+ .msg = hintfmt("this hint has %1% templated %2%!!",
"yellow",
"values"),
.errPos = Pos(foFile, problem_file, 02, 13)
});
auto str = testing::internal::GetCapturedStderr();
- ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
+ ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- error name --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m in file: \x1B[0minvalid filename\n\nerror without any code lines.\n\nthis hint has \x1B[33;1myellow\x1B[0m templated \x1B[33;1mvalues\x1B[0m!!\n");
}
TEST(logError, logErrorWithOnlyHintAndName) {
@@ -216,7 +209,7 @@ namespace nix {
logError({
.name = "error name",
- .hint = hintfmt("hint %1%", "only"),
+ .msg = hintfmt("hint %1%", "only"),
});
auto str = testing::internal::GetCapturedStderr();
@@ -233,8 +226,7 @@ namespace nix {
logWarning({
.name = "name",
- .description = "warning description",
- .hint = hintfmt("there was a %1%", "warning"),
+ .msg = hintfmt("there was a %1%", "warning"),
});
auto str = testing::internal::GetCapturedStderr();
@@ -250,8 +242,7 @@ namespace nix {
logWarning({
.name = "warning name",
- .description = "warning description",
- .hint = hintfmt("this hint has %1% templated %2%!!",
+ .msg = hintfmt("this hint has %1% templated %2%!!",
"yellow",
"values"),
.errPos = Pos(foStdin, problem_file, 2, 13),
@@ -274,8 +265,7 @@ namespace nix {
auto e = AssertionError(ErrorInfo {
.name = "wat",
- .description = "show-traces",
- .hint = hintfmt("it has been %1% days since our last error", "zero"),
+ .msg = hintfmt("it has been %1% days since our last error", "zero"),
.errPos = Pos(foString, problem_file, 2, 13),
});
@@ -290,7 +280,7 @@ namespace nix {
logError(e.info());
auto str = testing::internal::GetCapturedStderr();
- ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- SysError --- error-unit-test\x1B[0m\nopening file '\x1B[33;1minvalid filename\x1B[0m': \x1B[33;1mNo such file or directory\x1B[0m\n\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n 1| this is the other problem line of code\n | \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n");
+ ASSERT_STREQ(str.c_str(), "\x1B[31;1merror:\x1B[0m\x1B[34;1m --- AssertionError --- error-unit-test\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(2:13)\x1B[34;1m from string\x1B[0m\n\nshow-traces\n\n 1| previous line of code\n 2| this is the problem line of code\n | \x1B[31;1m^\x1B[0m\n 3| next line of code\n\nit has been \x1B[33;1mzero\x1B[0m days since our last error\n\x1B[34;1m---- show-trace ----\x1B[0m\n\x1B[34;1mtrace: \x1B[0mwhile trying to compute \x1B[33;1m42\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(1:19)\x1B[34;1m from stdin\x1B[0m\n\n 1| this is the other problem line of code\n | \x1B[31;1m^\x1B[0m\n\n\x1B[34;1mtrace: \x1B[0mwhile doing something without a \x1B[33;1mpos\x1B[0m\n\x1B[34;1mtrace: \x1B[0mmissing \x1B[33;1mnix file\x1B[0m\n\x1B[34;1mat: \x1B[33;1m(100:1)\x1B[34;1m in file: \x1B[0minvalid filename\n");
}
TEST(addTrace, hideTracesWithoutShowTrace) {
@@ -301,8 +291,7 @@ namespace nix {
auto e = AssertionError(ErrorInfo {
.name = "wat",
- .description = "hide traces",
- .hint = hintfmt("it has been %1% days since our last error", "zero"),
+ .msg = hintfmt("it has been %1% days since our last error", "zero"),
.errPos = Pos(foString, problem_file, 2, 13),
});
@@ -377,3 +366,5 @@ namespace nix {
}
}
+
+#endif
diff --git a/src/libutil/tests/tests.cc b/src/libutil/tests/tests.cc
index 8e77ccbe1..58df9c5ac 100644
--- a/src/libutil/tests/tests.cc
+++ b/src/libutil/tests/tests.cc
@@ -1,6 +1,7 @@
#include "util.hh"
#include "types.hh"
+#include <limits.h>
#include <gtest/gtest.h>
namespace nix {
@@ -319,20 +320,15 @@ namespace nix {
* --------------------------------------------------------------------------*/
TEST(string2Float, emptyString) {
- double n;
- ASSERT_EQ(string2Float("", n), false);
+ ASSERT_EQ(string2Float<double>(""), std::nullopt);
}
TEST(string2Float, trivialConversions) {
- double n;
- ASSERT_EQ(string2Float("1.0", n), true);
- ASSERT_EQ(n, 1.0);
+ ASSERT_EQ(string2Float<double>("1.0"), 1.0);
- ASSERT_EQ(string2Float("0.0", n), true);
- ASSERT_EQ(n, 0.0);
+ ASSERT_EQ(string2Float<double>("0.0"), 0.0);
- ASSERT_EQ(string2Float("-100.25", n), true);
- ASSERT_EQ(n, (-100.25));
+ ASSERT_EQ(string2Float<double>("-100.25"), -100.25);
}
/* ----------------------------------------------------------------------------
@@ -340,20 +336,15 @@ namespace nix {
* --------------------------------------------------------------------------*/
TEST(string2Int, emptyString) {
- double n;
- ASSERT_EQ(string2Int("", n), false);
+ ASSERT_EQ(string2Int<int>(""), std::nullopt);
}
TEST(string2Int, trivialConversions) {
- double n;
- ASSERT_EQ(string2Int("1", n), true);
- ASSERT_EQ(n, 1);
+ ASSERT_EQ(string2Int<int>("1"), 1);
- ASSERT_EQ(string2Int("0", n), true);
- ASSERT_EQ(n, 0);
+ ASSERT_EQ(string2Int<int>("0"), 0);
- ASSERT_EQ(string2Int("-100", n), true);
- ASSERT_EQ(n, (-100));
+ ASSERT_EQ(string2Int<int>("-100"), -100);
}
/* ----------------------------------------------------------------------------
@@ -586,4 +577,14 @@ namespace nix {
ASSERT_EQ(filterANSIEscapes(s, true), "foo bar baz" );
}
+
+ TEST(filterANSIEscapes, utf8) {
+ ASSERT_EQ(filterANSIEscapes("foobar", true, 5), "fooba");
+ ASSERT_EQ(filterANSIEscapes("fóóbär", true, 6), "fóóbär");
+ ASSERT_EQ(filterANSIEscapes("fóóbär", true, 5), "fóóbä");
+ ASSERT_EQ(filterANSIEscapes("fóóbär", true, 3), "fóó");
+ ASSERT_EQ(filterANSIEscapes("f€€bär", true, 4), "f€€b");
+ ASSERT_EQ(filterANSIEscapes("f𐍈𐍈bär", true, 4), "f𐍈𐍈b");
+ }
+
}
diff --git a/src/libutil/tests/url.cc b/src/libutil/tests/url.cc
index 80646ad3e..aff58e9ee 100644
--- a/src/libutil/tests/url.cc
+++ b/src/libutil/tests/url.cc
@@ -117,6 +117,24 @@ namespace nix {
ASSERT_EQ(parsed, expected);
}
+ TEST(parseURL, parseScopedRFC4007IPv6Address) {
+ auto s = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080";
+ auto parsed = parseURL(s);
+
+ ParsedURL expected {
+ .url = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080",
+ .base = "http://[fe80::818c:da4d:8975:415c\%enp0s25]:8080",
+ .scheme = "http",
+ .authority = "[fe80::818c:da4d:8975:415c\%enp0s25]:8080",
+ .path = "",
+ .query = (StringMap) { },
+ .fragment = "",
+ };
+
+ ASSERT_EQ(parsed, expected);
+
+ }
+
TEST(parseURL, parseIPv6Address) {
auto s = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080";
auto parsed = parseURL(s);
diff --git a/src/libutil/types.hh b/src/libutil/types.hh
index 55d02bcf9..9c85fef62 100644
--- a/src/libutil/types.hh
+++ b/src/libutil/types.hh
@@ -4,6 +4,7 @@
#include <list>
#include <set>
+#include <string>
#include <map>
#include <vector>
@@ -33,4 +34,16 @@ struct OnStartup
OnStartup(T && t) { t(); }
};
+/* Wrap bools to prevent string literals (i.e. 'char *') from being
+ cast to a bool in Attr. */
+template<typename T>
+struct Explicit {
+ T t;
+
+ bool operator ==(const Explicit<T> & other) const
+ {
+ return t == other.t;
+ }
+};
+
}
diff --git a/src/libutil/url-parts.hh b/src/libutil/url-parts.hh
index 68be15cb0..da10a6bbc 100644
--- a/src/libutil/url-parts.hh
+++ b/src/libutil/url-parts.hh
@@ -8,7 +8,8 @@ namespace nix {
// URI stuff.
const static std::string pctEncoded = "(?:%[0-9a-fA-F][0-9a-fA-F])";
const static std::string schemeRegex = "(?:[a-z][a-z0-9+.-]*)";
-const static std::string ipv6AddressRegex = "(?:\\[[0-9a-fA-F:]+\\])";
+const static std::string ipv6AddressSegmentRegex = "[0-9a-fA-F:]+(?:%\\w+)?";
+const static std::string ipv6AddressRegex = "(?:\\[" + ipv6AddressSegmentRegex + "\\]|" + ipv6AddressSegmentRegex + ")";
const static std::string unreservedRegex = "(?:[a-zA-Z0-9-._~])";
const static std::string subdelimsRegex = "(?:[!$&'\"()*+,;=])";
const static std::string hostnameRegex = "(?:(?:" + unreservedRegex + "|" + pctEncoded + "|" + subdelimsRegex + ")*)";
@@ -22,7 +23,7 @@ const static std::string absPathRegex = "(?:(?:/" + segmentRegex + ")*/?)";
const static std::string pathRegex = "(?:" + segmentRegex + "(?:/" + segmentRegex + ")*/?)";
// A Git ref (i.e. branch or tag name).
-const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.-]*"; // FIXME: check
+const static std::string refRegexS = "[a-zA-Z0-9][a-zA-Z0-9_.\\/-]*"; // FIXME: check
extern std::regex refRegex;
// Instead of defining what a good Git Ref is, we define what a bad Git Ref is
diff --git a/src/libutil/util.cc b/src/libutil/util.cc
index 53342b5cb..60b318559 100644
--- a/src/libutil/util.cc
+++ b/src/libutil/util.cc
@@ -32,6 +32,7 @@
#ifdef __linux__
#include <sys/prctl.h>
+#include <sys/resource.h>
#endif
@@ -320,7 +321,7 @@ void readFile(const Path & path, Sink & sink)
}
-void writeFile(const Path & path, const string & s, mode_t mode)
+void writeFile(const Path & path, std::string_view s, mode_t mode)
{
AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT | O_CLOEXEC, mode);
if (!fd)
@@ -340,13 +341,13 @@ void writeFile(const Path & path, Source & source, mode_t mode)
if (!fd)
throw SysError("opening file '%1%'", path);
- std::vector<unsigned char> buf(64 * 1024);
+ std::vector<char> buf(64 * 1024);
try {
while (true) {
try {
auto n = source.read(buf.data(), buf.size());
- writeFull(fd.get(), (unsigned char *) buf.data(), n);
+ writeFull(fd.get(), {buf.data(), n});
} catch (EndOfFile &) { break; }
}
} catch (Error & e) {
@@ -632,11 +633,11 @@ void replaceSymlink(const Path & target, const Path & link,
}
-void readFull(int fd, unsigned char * buf, size_t count)
+void readFull(int fd, char * buf, size_t count)
{
while (count) {
checkInterrupt();
- ssize_t res = read(fd, (char *) buf, count);
+ ssize_t res = read(fd, buf, count);
if (res == -1) {
if (errno == EINTR) continue;
throw SysError("reading from file");
@@ -648,27 +649,19 @@ void readFull(int fd, unsigned char * buf, size_t count)
}
-void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts)
+void writeFull(int fd, std::string_view s, bool allowInterrupts)
{
- while (count) {
+ while (!s.empty()) {
if (allowInterrupts) checkInterrupt();
- ssize_t res = write(fd, (char *) buf, count);
+ ssize_t res = write(fd, s.data(), s.size());
if (res == -1 && errno != EINTR)
throw SysError("writing to file");
- if (res > 0) {
- count -= res;
- buf += res;
- }
+ if (res > 0)
+ s.remove_prefix(res);
}
}
-void writeFull(int fd, const string & s, bool allowInterrupts)
-{
- writeFull(fd, (const unsigned char *) s.data(), s.size(), allowInterrupts);
-}
-
-
string drainFD(int fd, bool block, const size_t reserveSize)
{
StringSink sink(reserveSize);
@@ -705,7 +698,7 @@ void drainFD(int fd, Sink & sink, bool block)
throw SysError("reading from file");
}
else if (rd == 0) break;
- else sink(buf.data(), rd);
+ else sink({(char *) buf.data(), (size_t) rd});
}
}
@@ -760,13 +753,13 @@ AutoCloseFD::AutoCloseFD() : fd{-1} {}
AutoCloseFD::AutoCloseFD(int fd) : fd{fd} {}
-AutoCloseFD::AutoCloseFD(AutoCloseFD&& that) : fd{that.fd}
+AutoCloseFD::AutoCloseFD(AutoCloseFD && that) : fd{that.fd}
{
that.fd = -1;
}
-AutoCloseFD& AutoCloseFD::operator =(AutoCloseFD&& that)
+AutoCloseFD & AutoCloseFD::operator =(AutoCloseFD && that)
{
close();
fd = that.fd;
@@ -797,6 +790,7 @@ void AutoCloseFD::close()
if (::close(fd) == -1)
/* This should never happen. */
throw SysError("closing file descriptor %1%", fd);
+ fd = -1;
}
}
@@ -830,6 +824,12 @@ void Pipe::create()
}
+void Pipe::close()
+{
+ readSide.close();
+ writeSide.close();
+}
+
//////////////////////////////////////////////////////////////////////
@@ -954,7 +954,7 @@ void killUser(uid_t uid)
#else
if (kill(-1, SIGKILL) == 0) break;
#endif
- if (errno == ESRCH) break; /* no more processes */
+ if (errno == ESRCH || errno == EPERM) break; /* no more processes */
if (errno != EINTR)
throw SysError("cannot kill processes for uid '%1%'", uid);
}
@@ -1117,7 +1117,7 @@ void runProgram2(const RunOptions & options)
Strings args_(options.args);
args_.push_front(options.program);
- restoreSignals();
+ restoreProcessContext();
if (options.searchPath)
execvp(options.program.c_str(), stringsToCharPtrs(args_).data());
@@ -1129,7 +1129,7 @@ void runProgram2(const RunOptions & options)
throw SysError("executing '%1%'", options.program);
}, processOptions);
- out.writeSide = -1;
+ out.writeSide.close();
std::thread writerThread;
@@ -1142,10 +1142,10 @@ void runProgram2(const RunOptions & options)
if (source) {
- in.readSide = -1;
+ in.readSide.close();
writerThread = std::thread([&]() {
try {
- std::vector<unsigned char> buf(8 * 1024);
+ std::vector<char> buf(8 * 1024);
while (true) {
size_t n;
try {
@@ -1153,13 +1153,13 @@ void runProgram2(const RunOptions & options)
} catch (EndOfFile &) {
break;
}
- writeFull(in.writeSide.get(), buf.data(), n);
+ writeFull(in.writeSide.get(), {buf.data(), n});
}
promise.set_value();
} catch (...) {
promise.set_exception(std::current_exception());
}
- in.writeSide = -1;
+ in.writeSide.close();
});
}
@@ -1257,7 +1257,7 @@ template StringSet tokenizeString(std::string_view s, const string & separators)
template vector<string> tokenizeString(std::string_view s, const string & separators);
-string chomp(const string & s)
+string chomp(std::string_view s)
{
size_t i = s.find_last_not_of(" \n\r\t");
return i == string::npos ? "" : string(s, 0, i + 1);
@@ -1273,11 +1273,11 @@ string trim(const string & s, const string & whitespace)
}
-string replaceStrings(const std::string & s,
+string replaceStrings(std::string_view s,
const std::string & from, const std::string & to)
{
- if (from.empty()) return s;
- string res = s;
+ string res(s);
+ if (from.empty()) return res;
size_t pos = 0;
while ((pos = res.find(from, pos)) != std::string::npos) {
res.replace(pos, from.size(), to);
@@ -1409,7 +1409,28 @@ std::string filterANSIEscapes(const std::string & s, bool filterAll, unsigned in
i++;
else {
- t += *i++; w++;
+ w++;
+ // Copy one UTF-8 character.
+ if ((*i & 0xe0) == 0xc0) {
+ t += *i++;
+ if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
+ } else if ((*i & 0xf0) == 0xe0) {
+ t += *i++;
+ if (i != s.end() && ((*i & 0xc0) == 0x80)) {
+ t += *i++;
+ if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
+ }
+ } else if ((*i & 0xf8) == 0xf0) {
+ t += *i++;
+ if (i != s.end() && ((*i & 0xc0) == 0x80)) {
+ t += *i++;
+ if (i != s.end() && ((*i & 0xc0) == 0x80)) {
+ t += *i++;
+ if (i != s.end() && ((*i & 0xc0) == 0x80)) t += *i++;
+ }
+ }
+ } else
+ t += *i++;
}
}
@@ -1577,7 +1598,7 @@ void startSignalHandlerThread()
updateWindowSize();
if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask))
- throw SysError("quering signal mask");
+ throw SysError("querying signal mask");
sigset_t set;
sigemptyset(&set);
@@ -1592,12 +1613,45 @@ void startSignalHandlerThread()
std::thread(signalHandlerThread, set).detach();
}
-void restoreSignals()
+static void restoreSignals()
{
if (sigprocmask(SIG_SETMASK, &savedSignalMask, nullptr))
throw SysError("restoring signals");
}
+#if __linux__
+rlim_t savedStackSize = 0;
+#endif
+
+void setStackSize(size_t stackSize)
+{
+ #if __linux__
+ struct rlimit limit;
+ if (getrlimit(RLIMIT_STACK, &limit) == 0 && limit.rlim_cur < stackSize) {
+ savedStackSize = limit.rlim_cur;
+ limit.rlim_cur = stackSize;
+ setrlimit(RLIMIT_STACK, &limit);
+ }
+ #endif
+}
+
+void restoreProcessContext()
+{
+ restoreSignals();
+
+ restoreAffinity();
+
+ #if __linux__
+ if (savedStackSize) {
+ struct rlimit limit;
+ if (getrlimit(RLIMIT_STACK, &limit) == 0) {
+ limit.rlim_cur = savedStackSize;
+ setrlimit(RLIMIT_STACK, &limit);
+ }
+ }
+ #endif
+}
+
/* RAII helper to automatically deregister a callback. */
struct InterruptCallbackImpl : InterruptCallback
{
@@ -1660,10 +1714,11 @@ string showBytes(uint64_t bytes)
}
+// FIXME: move to libstore/build
void commonChildInit(Pipe & logPipe)
{
const static string pathNullDevice = "/dev/null";
- restoreSignals();
+ restoreProcessContext();
/* Put the child in a separate session (and thus a separate
process group) so that it has no controlling terminal (meaning
diff --git a/src/libutil/util.hh b/src/libutil/util.hh
index cafe93702..f84d0fb31 100644
--- a/src/libutil/util.hh
+++ b/src/libutil/util.hh
@@ -106,7 +106,7 @@ string readFile(const Path & path);
void readFile(const Path & path, Sink & sink);
/* Write a string to a file. */
-void writeFile(const Path & path, const string & s, mode_t mode = 0666);
+void writeFile(const Path & path, std::string_view s, mode_t mode = 0666);
void writeFile(const Path & path, Source & source, mode_t mode = 0666);
@@ -155,9 +155,8 @@ void replaceSymlink(const Path & target, const Path & link,
/* Wrappers arount read()/write() that read/write exactly the
requested number of bytes. */
-void readFull(int fd, unsigned char * buf, size_t count);
-void writeFull(int fd, const unsigned char * buf, size_t count, bool allowInterrupts = true);
-void writeFull(int fd, const string & s, bool allowInterrupts = true);
+void readFull(int fd, char * buf, size_t count);
+void writeFull(int fd, std::string_view s, bool allowInterrupts = true);
MakeError(EndOfFile, Error);
@@ -189,7 +188,6 @@ public:
class AutoCloseFD
{
int fd;
- void close();
public:
AutoCloseFD();
AutoCloseFD(int fd);
@@ -201,6 +199,7 @@ public:
int get() const;
explicit operator bool() const;
int release();
+ void close();
};
@@ -217,6 +216,7 @@ class Pipe
public:
AutoCloseFD readSide, writeSide;
void create();
+ void close();
};
@@ -300,6 +300,15 @@ std::pair<int, std::string> runProgram(const RunOptions & options);
void runProgram2(const RunOptions & options);
+/* Change the stack size. */
+void setStackSize(size_t stackSize);
+
+
+/* Restore the original inherited Unix process context (such as signal
+ masks, stack size, CPU affinity). */
+void restoreProcessContext();
+
+
class ExecError : public Error
{
public:
@@ -374,8 +383,9 @@ template<class C> Strings quoteStrings(const C & c)
}
-/* Remove trailing whitespace from a string. */
-string chomp(const string & s);
+/* Remove trailing whitespace from a string. FIXME: return
+ std::string_view. */
+string chomp(std::string_view s);
/* Remove whitespace from the start and end of a string. */
@@ -383,7 +393,7 @@ string trim(const string & s, const string & whitespace = " \n\r\t");
/* Replace all occurrences of a string inside another string. */
-string replaceStrings(const std::string & s,
+string replaceStrings(std::string_view s,
const std::string & from, const std::string & to);
@@ -398,21 +408,49 @@ bool statusOk(int status);
/* Parse a string into an integer. */
-template<class N> bool string2Int(const string & s, N & n)
+template<class N>
+std::optional<N> string2Int(const std::string & s)
{
- if (string(s, 0, 1) == "-" && !std::numeric_limits<N>::is_signed)
- return false;
+ if (s.substr(0, 1) == "-" && !std::numeric_limits<N>::is_signed)
+ return std::nullopt;
std::istringstream str(s);
+ N n;
str >> n;
- return str && str.get() == EOF;
+ if (str && str.get() == EOF) return n;
+ return std::nullopt;
+}
+
+/* Like string2Int(), but support an optional suffix 'K', 'M', 'G' or
+ 'T' denoting a binary unit prefix. */
+template<class N>
+N string2IntWithUnitPrefix(std::string s)
+{
+ N multiplier = 1;
+ if (!s.empty()) {
+ char u = std::toupper(*s.rbegin());
+ if (std::isalpha(u)) {
+ if (u == 'K') multiplier = 1ULL << 10;
+ else if (u == 'M') multiplier = 1ULL << 20;
+ else if (u == 'G') multiplier = 1ULL << 30;
+ else if (u == 'T') multiplier = 1ULL << 40;
+ else throw UsageError("invalid unit specifier '%1%'", u);
+ s.resize(s.size() - 1);
+ }
+ }
+ if (auto n = string2Int<N>(s))
+ return *n * multiplier;
+ throw UsageError("'%s' is not an integer", s);
}
/* Parse a string into a float. */
-template<class N> bool string2Float(const string & s, N & n)
+template<class N>
+std::optional<N> string2Float(const string & s)
{
std::istringstream str(s);
+ N n;
str >> n;
- return str && str.get() == EOF;
+ if (str && str.get() == EOF) return n;
+ return std::nullopt;
}
@@ -484,9 +522,6 @@ class Callback;
on the current thread (and thus any threads created by it). */
void startSignalHandlerThread();
-/* Restore default signal handling. */
-void restoreSignals();
-
struct InterruptCallback
{
virtual ~InterruptCallback() { };
diff --git a/src/nix-build/nix-build.cc b/src/nix-build/nix-build.cc
index f60e0706c..9acbedda2 100755
--- a/src/nix-build/nix-build.cc
+++ b/src/nix-build/nix-build.cc
@@ -12,12 +12,13 @@
#include "affinity.hh"
#include "util.hh"
#include "shared.hh"
+#include "path-with-outputs.hh"
#include "eval.hh"
#include "eval-inline.hh"
#include "get-drvs.hh"
#include "common-eval-args.hh"
#include "attr-path.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
using namespace nix;
using namespace std::string_literals;
@@ -217,9 +218,9 @@ static void main_nix_build(int argc, char * * argv)
// read the shebang to understand which packages to read from. Since
// this is handled via nix-shell -p, we wrap our ruby script execution
// in ruby -e 'load' which ignores the shebangs.
- envCommand = (format("exec %1% %2% -e 'load(\"%3%\")' -- %4%") % execArgs % interpreter % script % joined.str()).str();
+ envCommand = (format("exec %1% %2% -e 'load(ARGV.shift)' -- %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str();
} else {
- envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % script % joined.str()).str();
+ envCommand = (format("exec %1% %2% %3% %4%") % execArgs % interpreter % shellEscape(script) % joined.str()).str();
}
}
@@ -240,8 +241,6 @@ static void main_nix_build(int argc, char * * argv)
myArgs.parseCmdline(args);
- initPlugins();
-
if (packages && fromArgs)
throw UsageError("'-p' and '-E' are mutually exclusive");
@@ -323,7 +322,8 @@ static void main_nix_build(int argc, char * * argv)
state->printStats();
- auto buildPaths = [&](const std::vector<StorePathWithOutputs> & paths) {
+ auto buildPaths = [&](const std::vector<StorePathWithOutputs> & paths0) {
+ auto paths = toDerivedPaths(paths0);
/* Note: we do this even when !printMissing to efficiently
fetch binary cache data. */
uint64_t downloadSize, narSize;
@@ -369,11 +369,8 @@ static void main_nix_build(int argc, char * * argv)
shell = drv->queryOutPath() + "/bin/bash";
} catch (Error & e) {
- logWarning({
- .name = "bashInteractive",
- .hint = hintfmt("%s; will use bash from your environment",
- (e.info().hint ? e.info().hint->str() : ""))
- });
+ logError(e.info());
+ notice("will use bash from your environment");
shell = "bash";
}
}
@@ -425,8 +422,6 @@ static void main_nix_build(int argc, char * * argv)
} else
env[var.first] = var.second;
- restoreAffinity();
-
/* Run a shell using the derivation's environment. For
convenience, source $stdenv/setup to setup additional
environment variables and shell functions. Also don't
@@ -452,6 +447,7 @@ static void main_nix_build(int argc, char * * argv)
"unset NIX_ENFORCE_PURITY; "
"shopt -u nullglob; "
"unset TZ; %6%"
+ "shopt -s execfail;"
"%7%",
shellEscape(tmpDir),
(pure ? "" : "p=$PATH; "),
@@ -475,7 +471,7 @@ static void main_nix_build(int argc, char * * argv)
auto argPtrs = stringsToCharPtrs(args);
- restoreSignals();
+ restoreProcessContext();
logger->stop();
@@ -487,6 +483,7 @@ static void main_nix_build(int argc, char * * argv)
else {
std::vector<StorePathWithOutputs> pathsToBuild;
+ std::vector<std::pair<StorePath, std::string>> pathsToBuildOrdered;
std::map<StorePath, std::pair<size_t, StringSet>> drvMap;
@@ -498,6 +495,7 @@ static void main_nix_build(int argc, char * * argv)
throw Error("derivation '%s' lacks an 'outputName' attribute", store->printStorePath(drvPath));
pathsToBuild.push_back({drvPath, {outputName}});
+ pathsToBuildOrdered.push_back({drvPath, {outputName}});
auto i = drvMap.find(drvPath);
if (i != drvMap.end())
@@ -513,25 +511,25 @@ static void main_nix_build(int argc, char * * argv)
std::vector<StorePath> outPaths;
- for (auto & [drvPath, info] : drvMap) {
- auto & [counter, wantedOutputs] = info;
+ for (auto & [drvPath, outputName] : pathsToBuildOrdered) {
+ auto & [counter, _wantedOutputs] = drvMap.at({drvPath});
std::string drvPrefix = outLink;
if (counter)
drvPrefix += fmt("-%d", counter + 1);
- auto builtOutputs = store->queryDerivationOutputMap(drvPath);
+ auto builtOutputs = store->queryPartialDerivationOutputMap(drvPath);
- for (auto & outputName : wantedOutputs) {
- auto outputPath = builtOutputs.at(outputName);
+ auto maybeOutputPath = builtOutputs.at(outputName);
+ assert(maybeOutputPath);
+ auto outputPath = *maybeOutputPath;
- if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
- std::string symlink = drvPrefix;
- if (outputName != "out") symlink += "-" + outputName;
- store2->addPermRoot(outputPath, absPath(symlink));
- }
-
- outPaths.push_back(outputPath);
+ if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>()) {
+ std::string symlink = drvPrefix;
+ if (outputName != "out") symlink += "-" + outputName;
+ store2->addPermRoot(outputPath, absPath(symlink));
}
+
+ outPaths.push_back(outputPath);
}
logger->stop();
diff --git a/src/nix-channel/nix-channel.cc b/src/nix-channel/nix-channel.cc
index 309970df6..3272c6125 100755
--- a/src/nix-channel/nix-channel.cc
+++ b/src/nix-channel/nix-channel.cc
@@ -2,7 +2,7 @@
#include "globals.hh"
#include "filetransfer.hh"
#include "store-api.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
#include "fetchers.hh"
#include <fcntl.h>
@@ -196,8 +196,6 @@ static int main_nix_channel(int argc, char ** argv)
return true;
});
- initPlugins();
-
switch (cmd) {
case cNone:
throw UsageError("no command specified");
diff --git a/src/nix-collect-garbage/nix-collect-garbage.cc b/src/nix-collect-garbage/nix-collect-garbage.cc
index 57092b887..4f953fab4 100644
--- a/src/nix-collect-garbage/nix-collect-garbage.cc
+++ b/src/nix-collect-garbage/nix-collect-garbage.cc
@@ -2,7 +2,7 @@
#include "profiles.hh"
#include "shared.hh"
#include "globals.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
#include <iostream>
#include <cerrno>
@@ -74,8 +74,6 @@ static int main_nix_collect_garbage(int argc, char * * argv)
return true;
});
- initPlugins();
-
auto profilesDir = settings.nixStateDir + "/profiles";
if (removeOld) removeOldGenerations(profilesDir);
diff --git a/src/nix-copy-closure/nix-copy-closure.cc b/src/nix-copy-closure/nix-copy-closure.cc
index 10990f7b5..02ccbe541 100755
--- a/src/nix-copy-closure/nix-copy-closure.cc
+++ b/src/nix-copy-closure/nix-copy-closure.cc
@@ -1,6 +1,6 @@
#include "shared.hh"
#include "store-api.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
using namespace nix;
@@ -43,8 +43,6 @@ static int main_nix_copy_closure(int argc, char ** argv)
return true;
});
- initPlugins();
-
if (sshHost.empty())
throw UsageError("no host name specified");
@@ -52,12 +50,12 @@ static int main_nix_copy_closure(int argc, char ** argv)
auto to = toMode ? openStore(remoteUri) : openStore();
auto from = toMode ? openStore() : openStore(remoteUri);
- StorePathSet storePaths2;
+ RealisedPath::Set storePaths2;
for (auto & path : storePaths)
storePaths2.insert(from->followLinksToStorePath(path));
- StorePathSet closure;
- from->computeFSClosure(storePaths2, closure, false, includeOutputs);
+ RealisedPath::Set closure;
+ RealisedPath::closure(*from, storePaths2, closure);
copyPaths(from, to, closure, NoRepair, NoCheckSigs, useSubstitutes);
diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc
index a4b5c9e2c..e04954d45 100644
--- a/src/nix-env/nix-env.cc
+++ b/src/nix-env/nix-env.cc
@@ -6,6 +6,7 @@
#include "globals.hh"
#include "names.hh"
#include "profiles.hh"
+#include "path-with-outputs.hh"
#include "shared.hh"
#include "store-api.hh"
#include "local-fs-store.hh"
@@ -14,7 +15,7 @@
#include "json.hh"
#include "value-to-json.hh"
#include "xml-writer.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
#include <cerrno>
#include <ctime>
@@ -124,10 +125,7 @@ static void getAllExprs(EvalState & state,
if (hasSuffix(attrName, ".nix"))
attrName = string(attrName, 0, attrName.size() - 4);
if (!attrs.insert(attrName).second) {
- logError({
- .name = "Name collision",
- .hint = hintfmt("warning: name collision in input Nix expressions, skipping '%1%'", path2)
- });
+ printError("warning: name collision in input Nix expressions, skipping '%1%'", path2);
continue;
}
/* Load the expression on demand. */
@@ -421,13 +419,13 @@ static void queryInstSources(EvalState & state,
static void printMissing(EvalState & state, DrvInfos & elems)
{
- std::vector<StorePathWithOutputs> targets;
+ std::vector<DerivedPath> targets;
for (auto & i : elems) {
Path drvPath = i.queryDrvPath();
if (drvPath != "")
- targets.push_back({state.store->parseStorePath(drvPath)});
+ targets.push_back(DerivedPath::Built{state.store->parseStorePath(drvPath)});
else
- targets.push_back({state.store->parseStorePath(i.queryOutPath())});
+ targets.push_back(DerivedPath::Opaque{state.store->parseStorePath(i.queryOutPath())});
}
printMissing(state.store, targets);
@@ -696,17 +694,18 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs)
if (globals.forceName != "")
drv.setName(globals.forceName);
- if (drv.queryDrvPath() != "") {
- std::vector<StorePathWithOutputs> paths{{globals.state->store->parseStorePath(drv.queryDrvPath())}};
- printMissing(globals.state->store, paths);
- if (globals.dryRun) return;
- globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal);
- } else {
- printMissing(globals.state->store,
- {{globals.state->store->parseStorePath(drv.queryOutPath())}});
- if (globals.dryRun) return;
- globals.state->store->ensurePath(globals.state->store->parseStorePath(drv.queryOutPath()));
- }
+ std::vector<DerivedPath> paths {
+ (drv.queryDrvPath() != "")
+ ? (DerivedPath) (DerivedPath::Built {
+ globals.state->store->parseStorePath(drv.queryDrvPath())
+ })
+ : (DerivedPath) (DerivedPath::Opaque {
+ globals.state->store->parseStorePath(drv.queryOutPath())
+ }),
+ };
+ printMissing(globals.state->store, paths);
+ if (globals.dryRun) return;
+ globals.state->store->buildPaths(paths, globals.state->repair ? bmRepair : bmNormal);
debug(format("switching to new user environment"));
Path generation = createGeneration(
@@ -876,11 +875,7 @@ static void queryJSON(Globals & globals, vector<DrvInfo> & elems)
auto placeholder = metaObj.placeholder(j);
Value * v = i.queryMeta(j);
if (!v) {
- logError({
- .name = "Invalid meta attribute",
- .hint = hintfmt("derivation '%s' has invalid meta attribute '%s'",
- i.queryName(), j)
- });
+ printError("derivation '%s' has invalid meta attribute '%s'", i.queryName(), j);
placeholder.write(nullptr);
} else {
PathSet context;
@@ -1131,45 +1126,42 @@ static void opQuery(Globals & globals, Strings opFlags, Strings opArgs)
attrs2["name"] = j;
Value * v = i.queryMeta(j);
if (!v)
- logError({
- .name = "Invalid meta attribute",
- .hint = hintfmt(
- "derivation '%s' has invalid meta attribute '%s'",
- i.queryName(), j)
- });
+ printError(
+ "derivation '%s' has invalid meta attribute '%s'",
+ i.queryName(), j);
else {
- if (v->type == tString) {
+ if (v->type() == nString) {
attrs2["type"] = "string";
attrs2["value"] = v->string.s;
xml.writeEmptyElement("meta", attrs2);
- } else if (v->type == tInt) {
+ } else if (v->type() == nInt) {
attrs2["type"] = "int";
attrs2["value"] = (format("%1%") % v->integer).str();
xml.writeEmptyElement("meta", attrs2);
- } else if (v->type == tFloat) {
+ } else if (v->type() == nFloat) {
attrs2["type"] = "float";
attrs2["value"] = (format("%1%") % v->fpoint).str();
xml.writeEmptyElement("meta", attrs2);
- } else if (v->type == tBool) {
+ } else if (v->type() == nBool) {
attrs2["type"] = "bool";
attrs2["value"] = v->boolean ? "true" : "false";
xml.writeEmptyElement("meta", attrs2);
- } else if (v->isList()) {
+ } else if (v->type() == nList) {
attrs2["type"] = "strings";
XMLOpenElement m(xml, "meta", attrs2);
for (unsigned int j = 0; j < v->listSize(); ++j) {
- if (v->listElems()[j]->type != tString) continue;
+ if (v->listElems()[j]->type() != nString) continue;
XMLAttrs attrs3;
attrs3["value"] = v->listElems()[j]->string.s;
xml.writeEmptyElement("string", attrs3);
}
- } else if (v->type == tAttrs) {
+ } else if (v->type() == nAttrs) {
attrs2["type"] = "strings";
XMLOpenElement m(xml, "meta", attrs2);
Bindings & attrs = *v->attrs;
for (auto &i : attrs) {
Attr & a(*attrs.find(i.name));
- if(a.value->type != tString) continue;
+ if(a.value->type() != nString) continue;
XMLAttrs attrs3;
attrs3["type"] = i.name;
attrs3["value"] = a.value->string.s;
@@ -1250,11 +1242,10 @@ static void opSwitchGeneration(Globals & globals, Strings opFlags, Strings opArg
if (opArgs.size() != 1)
throw UsageError("exactly one argument expected");
- GenerationNumber dstGen;
- if (!string2Int(opArgs.front(), dstGen))
+ if (auto dstGen = string2Int<GenerationNumber>(opArgs.front()))
+ switchGeneration(globals, *dstGen);
+ else
throw UsageError("expected a generation number");
-
- switchGeneration(globals, dstGen);
}
@@ -1308,17 +1299,17 @@ static void opDeleteGenerations(Globals & globals, Strings opFlags, Strings opAr
if(opArgs.front().size() < 2)
throw Error("invalid number of generations ‘%1%’", opArgs.front());
string str_max = string(opArgs.front(), 1, opArgs.front().size());
- GenerationNumber max;
- if (!string2Int(str_max, max) || max == 0)
+ auto max = string2Int<GenerationNumber>(str_max);
+ if (!max || *max == 0)
throw Error("invalid number of generations to keep ‘%1%’", opArgs.front());
- deleteGenerationsGreaterThan(globals.profile, max, globals.dryRun);
+ deleteGenerationsGreaterThan(globals.profile, *max, globals.dryRun);
} else {
std::set<GenerationNumber> gens;
for (auto & i : opArgs) {
- GenerationNumber n;
- if (!string2Int(i, n))
+ if (auto n = string2Int<GenerationNumber>(i))
+ gens.insert(*n);
+ else
throw UsageError("invalid generation number '%1%'", i);
- gens.insert(n);
}
deleteGenerations(globals.profile, gens, globals.dryRun);
}
@@ -1431,8 +1422,6 @@ static int main_nix_env(int argc, char * * argv)
myArgs.parseCmdline(argvToStrings(argc, argv));
- initPlugins();
-
if (!op) throw UsageError("no operation specified");
auto store = openStore();
diff --git a/src/nix-env/user-env.cc b/src/nix-env/user-env.cc
index 87387e794..5ceb2ae67 100644
--- a/src/nix-env/user-env.cc
+++ b/src/nix-env/user-env.cc
@@ -2,6 +2,7 @@
#include "util.hh"
#include "derivations.hh"
#include "store-api.hh"
+#include "path-with-outputs.hh"
#include "local-fs-store.hh"
#include "globals.hh"
#include "shared.hh"
@@ -41,7 +42,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
drvsToBuild.push_back({state.store->parseStorePath(i.queryDrvPath())});
debug(format("building user environment dependencies"));
- state.store->buildPaths(drvsToBuild, state.repair ? bmRepair : bmNormal);
+ state.store->buildPaths(
+ toDerivedPaths(drvsToBuild),
+ state.repair ? bmRepair : bmNormal);
/* Construct the whole top level derivation. */
StorePathSet references;
@@ -53,10 +56,12 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
output paths, and optionally the derivation path, as well
as the meta attributes. */
Path drvPath = keepDerivations ? i.queryDrvPath() : "";
+ DrvInfo::Outputs outputs = i.queryOutputs(true);
+ StringSet metaNames = i.queryMetaNames();
Value & v(*state.allocValue());
manifest.listElems()[n++] = &v;
- state.mkAttrs(v, 16);
+ state.mkAttrs(v, 7 + outputs.size());
mkString(*state.allocAttr(v, state.sType), "derivation");
mkString(*state.allocAttr(v, state.sName), i.queryName());
@@ -68,7 +73,6 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
mkString(*state.allocAttr(v, state.sDrvPath), i.queryDrvPath());
// Copy each output meant for installation.
- DrvInfo::Outputs outputs = i.queryOutputs(true);
Value & vOutputs = *state.allocAttr(v, state.sOutputs);
state.mkList(vOutputs, outputs.size());
unsigned int m = 0;
@@ -88,8 +92,7 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
// Copy the meta attributes.
Value & vMeta = *state.allocAttr(v, state.sMeta);
- state.mkAttrs(vMeta, 16);
- StringSet metaNames = i.queryMetaNames();
+ state.mkAttrs(vMeta, metaNames.size());
for (auto & j : metaNames) {
Value * v = i.queryMeta(j);
if (!v) continue;
@@ -136,7 +139,9 @@ bool createUserEnv(EvalState & state, DrvInfos & elems,
debug("building user environment");
std::vector<StorePathWithOutputs> topLevelDrvs;
topLevelDrvs.push_back({topLevelDrv});
- state.store->buildPaths(topLevelDrvs, state.repair ? bmRepair : bmNormal);
+ state.store->buildPaths(
+ toDerivedPaths(topLevelDrvs),
+ state.repair ? bmRepair : bmNormal);
/* Switch the current user environment to the output path. */
auto store2 = state.store.dynamic_pointer_cast<LocalFSStore>();
diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc
index 3956fef6d..95903d882 100644
--- a/src/nix-instantiate/nix-instantiate.cc
+++ b/src/nix-instantiate/nix-instantiate.cc
@@ -10,7 +10,7 @@
#include "store-api.hh"
#include "local-fs-store.hh"
#include "common-eval-args.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
#include <map>
#include <iostream>
@@ -149,8 +149,6 @@ static int main_nix_instantiate(int argc, char * * argv)
myArgs.parseCmdline(argvToStrings(argc, argv));
- initPlugins();
-
if (evalOnly && !wantsReadWrite)
settings.readOnlyMode = true;
diff --git a/src/nix-prefetch-url/nix-prefetch-url.cc b/src/nix-prefetch-url/nix-prefetch-url.cc
deleted file mode 100644
index 3bdee55a7..000000000
--- a/src/nix-prefetch-url/nix-prefetch-url.cc
+++ /dev/null
@@ -1,232 +0,0 @@
-#include "hash.hh"
-#include "shared.hh"
-#include "filetransfer.hh"
-#include "store-api.hh"
-#include "eval.hh"
-#include "eval-inline.hh"
-#include "common-eval-args.hh"
-#include "attr-path.hh"
-#include "finally.hh"
-#include "../nix/legacy.hh"
-#include "progress-bar.hh"
-#include "tarfile.hh"
-
-#include <iostream>
-
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-
-using namespace nix;
-
-
-/* If ‘uri’ starts with ‘mirror://’, then resolve it using the list of
- mirrors defined in Nixpkgs. */
-string resolveMirrorUri(EvalState & state, string uri)
-{
- if (string(uri, 0, 9) != "mirror://") return uri;
-
- string s(uri, 9);
- auto p = s.find('/');
- if (p == string::npos) throw Error("invalid mirror URI");
- string mirrorName(s, 0, p);
-
- Value vMirrors;
- state.eval(state.parseExprFromString("import <nixpkgs/pkgs/build-support/fetchurl/mirrors.nix>", "."), vMirrors);
- state.forceAttrs(vMirrors);
-
- auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName));
- if (mirrorList == vMirrors.attrs->end())
- throw Error("unknown mirror name '%1%'", mirrorName);
- state.forceList(*mirrorList->value);
-
- if (mirrorList->value->listSize() < 1)
- throw Error("mirror URI '%1%' did not expand to anything", uri);
-
- string mirror = state.forceString(*mirrorList->value->listElems()[0]);
- return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1);
-}
-
-
-static int main_nix_prefetch_url(int argc, char * * argv)
-{
- {
- HashType ht = htSHA256;
- std::vector<string> args;
- bool printPath = getEnv("PRINT_PATH") == "1";
- bool fromExpr = false;
- string attrPath;
- bool unpack = false;
- bool executable = false;
- string name;
-
- struct MyArgs : LegacyArgs, MixEvalArgs
- {
- using LegacyArgs::LegacyArgs;
- };
-
- MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) {
- if (*arg == "--help")
- showManPage("nix-prefetch-url");
- else if (*arg == "--version")
- printVersion("nix-prefetch-url");
- else if (*arg == "--type") {
- string s = getArg(*arg, arg, end);
- ht = parseHashType(s);
- }
- else if (*arg == "--print-path")
- printPath = true;
- else if (*arg == "--attr" || *arg == "-A") {
- fromExpr = true;
- attrPath = getArg(*arg, arg, end);
- }
- else if (*arg == "--unpack")
- unpack = true;
- else if (*arg == "--executable")
- executable = true;
- else if (*arg == "--name")
- name = getArg(*arg, arg, end);
- else if (*arg != "" && arg->at(0) == '-')
- return false;
- else
- args.push_back(*arg);
- return true;
- });
-
- myArgs.parseCmdline(argvToStrings(argc, argv));
-
- initPlugins();
-
- if (args.size() > 2)
- throw UsageError("too many arguments");
-
- Finally f([]() { stopProgressBar(); });
-
- if (isatty(STDERR_FILENO))
- startProgressBar();
-
- auto store = openStore();
- auto state = std::make_unique<EvalState>(myArgs.searchPath, store);
-
- Bindings & autoArgs = *myArgs.getAutoArgs(*state);
-
- /* If -A is given, get the URI from the specified Nix
- expression. */
- string uri;
- if (!fromExpr) {
- if (args.empty())
- throw UsageError("you must specify a URI");
- uri = args[0];
- } else {
- Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0]));
- Value vRoot;
- state->evalFile(path, vRoot);
- Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first);
- state->forceAttrs(v);
-
- /* Extract the URI. */
- auto attr = v.attrs->find(state->symbols.create("urls"));
- if (attr == v.attrs->end())
- throw Error("attribute set does not contain a 'urls' attribute");
- state->forceList(*attr->value);
- if (attr->value->listSize() < 1)
- throw Error("'urls' list is empty");
- uri = state->forceString(*attr->value->listElems()[0]);
-
- /* Extract the hash mode. */
- attr = v.attrs->find(state->symbols.create("outputHashMode"));
- if (attr == v.attrs->end())
- printInfo("warning: this does not look like a fetchurl call");
- else
- unpack = state->forceString(*attr->value) == "recursive";
-
- /* Extract the name. */
- if (name.empty()) {
- attr = v.attrs->find(state->symbols.create("name"));
- if (attr != v.attrs->end())
- name = state->forceString(*attr->value);
- }
- }
-
- /* Figure out a name in the Nix store. */
- if (name.empty())
- name = baseNameOf(uri);
- if (name.empty())
- throw Error("cannot figure out file name for '%1%'", uri);
-
- /* If an expected hash is given, the file may already exist in
- the store. */
- std::optional<Hash> expectedHash;
- Hash hash(ht);
- std::optional<StorePath> storePath;
- if (args.size() == 2) {
- expectedHash = Hash::parseAny(args[1], ht);
- const auto recursive = unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
- storePath = store->makeFixedOutputPath(recursive, *expectedHash, name);
- if (store->isValidPath(*storePath))
- hash = *expectedHash;
- else
- storePath.reset();
- }
-
- if (!storePath) {
-
- auto actualUri = resolveMirrorUri(*state, uri);
-
- AutoDelete tmpDir(createTempDir(), true);
- Path tmpFile = (Path) tmpDir + "/tmp";
-
- /* Download the file. */
- {
- auto mode = 0600;
- if (executable)
- mode = 0700;
-
- AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode);
- if (!fd) throw SysError("creating temporary file '%s'", tmpFile);
-
- FdSink sink(fd.get());
-
- FileTransferRequest req(actualUri);
- req.decompress = false;
- getFileTransfer()->download(std::move(req), sink);
- }
-
- /* Optionally unpack the file. */
- if (unpack) {
- printInfo("unpacking...");
- Path unpacked = (Path) tmpDir + "/unpacked";
- createDirs(unpacked);
- unpackTarfile(tmpFile, unpacked);
-
- /* If the archive unpacks to a single file/directory, then use
- that as the top-level. */
- auto entries = readDirectory(unpacked);
- if (entries.size() == 1)
- tmpFile = unpacked + "/" + entries[0].name;
- else
- tmpFile = unpacked;
- }
-
- const auto method = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
-
- auto info = store->addToStoreSlow(name, tmpFile, method, ht, expectedHash);
- storePath = info.path;
- assert(info.ca);
- hash = getContentAddressHash(*info.ca);
- }
-
- stopProgressBar();
-
- if (!printPath)
- printInfo("path is '%s'", store->printStorePath(*storePath));
-
- std::cout << printHash16or32(hash) << std::endl;
- if (printPath)
- std::cout << store->printStorePath(*storePath) << std::endl;
-
- return 0;
- }
-}
-
-static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url);
diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc
index 70927925d..c3e2b57c9 100644
--- a/src/nix-store/nix-store.cc
+++ b/src/nix-store/nix-store.cc
@@ -9,7 +9,8 @@
#include "util.hh"
#include "worker-protocol.hh"
#include "graphml.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
+#include "path-with-outputs.hh"
#include <iostream>
#include <algorithm>
@@ -19,10 +20,6 @@
#include <sys/stat.h>
#include <fcntl.h>
-#if HAVE_SODIUM
-#include <sodium.h>
-#endif
-
namespace nix_store {
@@ -66,7 +63,7 @@ static PathSet realisePath(StorePathWithOutputs path, bool build = true)
auto store2 = std::dynamic_pointer_cast<LocalFSStore>(store);
if (path.path.isDerivation()) {
- if (build) store->buildPaths({path});
+ if (build) store->buildPaths({path.toDerivedPath()});
auto outputPaths = store->queryDerivationOutputMap(path.path);
Derivation drv = store->derivationFromPath(path.path);
rootNr++;
@@ -132,11 +129,13 @@ static void opRealise(Strings opFlags, Strings opArgs)
std::vector<StorePathWithOutputs> paths;
for (auto & i : opArgs)
- paths.push_back(store->followLinksToStorePathWithOutputs(i));
+ paths.push_back(followLinksToStorePathWithOutputs(*store, i));
uint64_t downloadSize, narSize;
StorePathSet willBuild, willSubstitute, unknown;
- store->queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize);
+ store->queryMissing(
+ toDerivedPaths(paths),
+ willBuild, willSubstitute, unknown, downloadSize, narSize);
if (ignoreUnknown) {
std::vector<StorePathWithOutputs> paths2;
@@ -152,7 +151,7 @@ static void opRealise(Strings opFlags, Strings opArgs)
if (dryRun) return;
/* Build all paths at the same time to exploit parallelism. */
- store->buildPaths(paths, buildMode);
+ store->buildPaths(toDerivedPaths(paths), buildMode);
if (!ignoreUnknown)
for (auto & i : paths) {
@@ -516,7 +515,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise)
info->narHash = hash.first;
info->narSize = hash.second;
}
- infos.push_back(std::move(*info));
+ infos.insert_or_assign(info->path, *info);
}
}
@@ -712,10 +711,7 @@ static void opVerify(Strings opFlags, Strings opArgs)
else throw UsageError("unknown flag '%1%'", i);
if (store->verifyStore(checkContents, repair)) {
- logWarning({
- .name = "Store consistency",
- .description = "not all errors were fixed"
- });
+ warn("not all store errors were fixed");
throw Exit(1);
}
}
@@ -737,14 +733,10 @@ static void opVerifyPath(Strings opFlags, Strings opArgs)
store->narFromPath(path, sink);
auto current = sink.finish();
if (current.first != info->narHash) {
- logError({
- .name = "Hash mismatch",
- .hint = hintfmt(
- "path '%s' was modified! expected hash '%s', got '%s'",
+ printError("path '%s' was modified! expected hash '%s', got '%s'",
store->printStorePath(path),
info->narHash.to_string(Base32, true),
- current.first.to_string(Base32, true))
- });
+ current.first.to_string(Base32, true));
status = 1;
}
}
@@ -761,7 +753,7 @@ static void opRepairPath(Strings opFlags, Strings opArgs)
throw UsageError("no flags expected");
for (auto & i : opArgs)
- ensureLocalStore()->repairPath(store->followLinksToStorePath(i));
+ store->repairPath(store->followLinksToStorePath(i));
}
/* Optimise the disk space usage of the Nix store by hard-linking
@@ -830,29 +822,8 @@ static void opServe(Strings opFlags, Strings opArgs)
for (auto & path : paths)
store->addTempRoot(path);
- /* If requested, substitute missing paths. This
- implements nix-copy-closure's --use-substitutes
- flag. */
if (substitute && writeAllowed) {
- /* Filter out .drv files (we don't want to build anything). */
- std::vector<StorePathWithOutputs> paths2;
- for (auto & path : paths)
- if (!path.isDerivation())
- paths2.push_back({path});
- uint64_t downloadSize, narSize;
- StorePathSet willBuild, willSubstitute, unknown;
- store->queryMissing(paths2,
- willBuild, willSubstitute, unknown, downloadSize, narSize);
- /* FIXME: should use ensurePath(), but it only
- does one path at a time. */
- if (!willSubstitute.empty())
- try {
- std::vector<StorePathWithOutputs> subs;
- for (auto & p : willSubstitute) subs.push_back({p});
- store->buildPaths(subs);
- } catch (Error & e) {
- logWarning(e.info());
- }
+ store->substitutePaths(paths);
}
worker_proto::write(*store, out, store->queryValidPaths(paths));
@@ -905,13 +876,13 @@ static void opServe(Strings opFlags, Strings opArgs)
std::vector<StorePathWithOutputs> paths;
for (auto & s : readStrings<Strings>(in))
- paths.push_back(store->parsePathWithOutputs(s));
+ paths.push_back(parsePathWithOutputs(*store, s));
getBuildSettings();
try {
MonitorFdHup monitor(in.fd);
- store->buildPaths(paths);
+ store->buildPaths(toDerivedPaths(paths));
out << 0;
} catch (Error & e) {
assert(e.status);
@@ -937,6 +908,10 @@ static void opServe(Strings opFlags, Strings opArgs)
if (GET_PROTOCOL_MINOR(clientVersion) >= 3)
out << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime;
+ if (GET_PROTOCOL_MINOR(clientVersion >= 6)) {
+ worker_proto::write(*store, out, status.builtOutputs);
+ }
+
break;
}
@@ -1001,21 +976,11 @@ static void opGenerateBinaryCacheKey(Strings opFlags, Strings opArgs)
string secretKeyFile = *i++;
string publicKeyFile = *i++;
-#if HAVE_SODIUM
- if (sodium_init() == -1)
- throw Error("could not initialise libsodium");
-
- unsigned char pk[crypto_sign_PUBLICKEYBYTES];
- unsigned char sk[crypto_sign_SECRETKEYBYTES];
- if (crypto_sign_keypair(pk, sk) != 0)
- throw Error("key generation failed");
+ auto secretKey = SecretKey::generate(keyName);
- writeFile(publicKeyFile, keyName + ":" + base64Encode(string((char *) pk, crypto_sign_PUBLICKEYBYTES)));
+ writeFile(publicKeyFile, secretKey.toPublicKey().to_string());
umask(0077);
- writeFile(secretKeyFile, keyName + ":" + base64Encode(string((char *) sk, crypto_sign_SECRETKEYBYTES)));
-#else
- throw Error("Nix was not compiled with libsodium, required for signed binary cache support");
-#endif
+ writeFile(secretKeyFile, secretKey.to_string());
}
@@ -1109,8 +1074,6 @@ static int main_nix_store(int argc, char * * argv)
return true;
});
- initPlugins();
-
if (!op) throw UsageError("no operation specified");
if (op != opDump && op != opRestore) /* !!! hack */
diff --git a/src/nix/add-file.md b/src/nix/add-file.md
new file mode 100644
index 000000000..ed237a035
--- /dev/null
+++ b/src/nix/add-file.md
@@ -0,0 +1,28 @@
+R""(
+
+# Description
+
+Copy the regular file *path* to the Nix store, and print the resulting
+store path on standard output.
+
+> **Warning**
+>
+> The resulting store path is not registered as a garbage
+> collector root, so it could be deleted before you have a
+> chance to register it.
+
+# Examples
+
+Add a regular file to the store:
+
+```console
+# echo foo > bar
+
+# nix store add-file ./bar
+/nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar
+
+# cat /nix/store/cbv2s4bsvzjri77s2gb8g8bpcb6dpa8w-bar
+foo
+```
+
+)""
diff --git a/src/nix/add-path.md b/src/nix/add-path.md
new file mode 100644
index 000000000..87473611d
--- /dev/null
+++ b/src/nix/add-path.md
@@ -0,0 +1,29 @@
+R""(
+
+# Description
+
+Copy *path* to the Nix store, and print the resulting store path on
+standard output.
+
+> **Warning**
+>
+> The resulting store path is not registered as a garbage
+> collector root, so it could be deleted before you have a
+> chance to register it.
+
+# Examples
+
+Add a directory to the store:
+
+```console
+# mkdir dir
+# echo foo > dir/bar
+
+# nix store add-path ./dir
+/nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir
+
+# cat /nix/store/6pmjx56pm94n66n4qw1nff0y1crm8nqg-dir/bar
+foo
+```
+
+)""
diff --git a/src/nix/add-to-store.cc b/src/nix/add-to-store.cc
index 7fe87d757..2ae042789 100644
--- a/src/nix/add-to-store.cc
+++ b/src/nix/add-to-store.cc
@@ -9,49 +9,22 @@ struct CmdAddToStore : MixDryRun, StoreCommand
{
Path path;
std::optional<std::string> namePart;
- FileIngestionMethod ingestionMethod = FileIngestionMethod::Recursive;
+ FileIngestionMethod ingestionMethod;
CmdAddToStore()
{
+ // FIXME: completion
expectArg("path", &path);
addFlag({
.longName = "name",
.shortName = 'n',
- .description = "name component of the store path",
+ .description = "Override the name component of the store path. It defaults to the base name of *path*.",
.labels = {"name"},
.handler = {&namePart},
});
-
- addFlag({
- .longName = "flat",
- .shortName = 0,
- .description = "add flat file to the Nix store",
- .handler = {&ingestionMethod, FileIngestionMethod::Flat},
- });
- }
-
- std::string description() override
- {
- return "add a path to the Nix store";
- }
-
- std::string doc() override
- {
- return R"(
- Copy the file or directory *path* to the Nix store, and
- print the resulting store path on standard output.
- )";
- }
-
- Examples examples() override
- {
- return {
- };
}
- Category category() override { return catUtility; }
-
void run(ref<Store> store) override
{
if (!namePart) namePart = baseNameOf(path);
@@ -83,8 +56,49 @@ struct CmdAddToStore : MixDryRun, StoreCommand
store->addToStore(info, source);
}
- logger->stdout("%s", store->printStorePath(info.path));
+ logger->cout("%s", store->printStorePath(info.path));
+ }
+};
+
+struct CmdAddFile : CmdAddToStore
+{
+ CmdAddFile()
+ {
+ ingestionMethod = FileIngestionMethod::Flat;
+ }
+
+ std::string description() override
+ {
+ return "add a regular file to the Nix store";
+ }
+
+ std::string doc() override
+ {
+ return
+ #include "add-file.md"
+ ;
+ }
+};
+
+struct CmdAddPath : CmdAddToStore
+{
+ CmdAddPath()
+ {
+ ingestionMethod = FileIngestionMethod::Recursive;
+ }
+
+ std::string description() override
+ {
+ return "add a path to the Nix store";
+ }
+
+ std::string doc() override
+ {
+ return
+ #include "add-path.md"
+ ;
}
};
-static auto rCmdAddToStore = registerCommand<CmdAddToStore>("add-to-store");
+static auto rCmdAddFile = registerCommand2<CmdAddFile>({"store", "add-file"});
+static auto rCmdAddPath = registerCommand2<CmdAddPath>({"store", "add-path"});
diff --git a/src/nix/app.cc b/src/nix/app.cc
index 80acbf658..cf147c631 100644
--- a/src/nix/app.cc
+++ b/src/nix/app.cc
@@ -12,11 +12,16 @@ App Installable::toApp(EvalState & state)
auto type = cursor->getAttr("type")->getString();
+ auto checkProgram = [&](const Path & program)
+ {
+ if (!state.store->isInStore(program))
+ throw Error("app program '%s' is not in the Nix store", program);
+ };
+
if (type == "app") {
auto [program, context] = cursor->getAttr("program")->getStringWithContext();
- if (!state.store->isInStore(program))
- throw Error("app program '%s' is not in the Nix store", program);
+ checkProgram(program);
std::vector<StorePathWithOutputs> context2;
for (auto & [path, name] : context)
@@ -33,9 +38,17 @@ App Installable::toApp(EvalState & state)
auto outPath = cursor->getAttr(state.sOutPath)->getString();
auto outputName = cursor->getAttr(state.sOutputName)->getString();
auto name = cursor->getAttr(state.sName)->getString();
+ auto aMeta = cursor->maybeGetAttr("meta");
+ auto aMainProgram = aMeta ? aMeta->maybeGetAttr("mainProgram") : nullptr;
+ auto mainProgram =
+ aMainProgram
+ ? aMainProgram->getString()
+ : DrvName(name).name;
+ auto program = outPath + "/bin/" + mainProgram;
+ checkProgram(program);
return App {
.context = { { drvPath, {outputName} } },
- .program = outPath + "/bin/" + DrvName(name).name,
+ .program = program,
};
}
diff --git a/src/nix/build.cc b/src/nix/build.cc
index 65708e98b..03159b6cc 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -5,9 +5,11 @@
#include "store-api.hh"
#include "local-fs-store.hh"
+#include <nlohmann/json.hpp>
+
using namespace nix;
-struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
+struct CmdBuild : InstallablesCommand, MixDryRun, MixJSON, MixProfile
{
Path outLink = "result";
BuildMode buildMode = bmNormal;
@@ -17,7 +19,7 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
addFlag({
.longName = "out-link",
.shortName = 'o',
- .description = "path of the symlink to the build result",
+ .description = "Use *path* as prefix for the symlinks to the build results. It defaults to `result`.",
.labels = {"path"},
.handler = {&outLink},
.completer = completePath
@@ -25,13 +27,13 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
addFlag({
.longName = "no-link",
- .description = "do not create a symlink to the build result",
+ .description = "Do not create symlinks to the build results.",
.handler = {&outLink, Path("")},
});
addFlag({
.longName = "rebuild",
- .description = "rebuild an already built package and compare the result to the existing store paths",
+ .description = "Rebuild an already built package and compare the result to the existing store paths.",
.handler = {&buildMode, bmCheck},
});
}
@@ -41,22 +43,11 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
return "build a derivation or fetch a store path";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To build and run GNU Hello from NixOS 17.03:",
- "nix build -f channel:nixos-17.03 hello; ./result/bin/hello"
- },
- Example{
- "To build the build.x86_64-linux attribute from release.nix:",
- "nix build -f release.nix build.x86_64-linux"
- },
- Example{
- "To make a profile point at GNU Hello:",
- "nix build --profile /tmp/profile nixpkgs#hello"
- },
- };
+ return
+ #include "build.md"
+ ;
}
void run(ref<Store> store) override
@@ -67,14 +58,15 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
if (outLink != "")
if (auto store2 = store.dynamic_pointer_cast<LocalFSStore>())
- for (size_t i = 0; i < buildables.size(); ++i)
+ for (const auto & [_i, buildable] : enumerate(buildables)) {
+ auto i = _i;
std::visit(overloaded {
- [&](BuildableOpaque bo) {
+ [&](DerivedPathWithHints::Opaque bo) {
std::string symlink = outLink;
if (i) symlink += fmt("-%d", i);
store2->addPermRoot(bo.path, absPath(symlink));
},
- [&](BuildableFromDrv bfd) {
+ [&](DerivedPathWithHints::Built bfd) {
auto builtOutputs = store->queryDerivationOutputMap(bfd.drvPath);
for (auto & output : builtOutputs) {
std::string symlink = outLink;
@@ -83,9 +75,12 @@ struct CmdBuild : InstallablesCommand, MixDryRun, MixProfile
store2->addPermRoot(output.second, absPath(symlink));
}
},
- }, buildables[i]);
+ }, buildable.raw());
+ }
updateProfile(buildables);
+
+ if (json) logger->cout("%s", derivedPathsWithHintsToJSON(buildables, store).dump());
}
};
diff --git a/src/nix/build.md b/src/nix/build.md
new file mode 100644
index 000000000..20138b7e0
--- /dev/null
+++ b/src/nix/build.md
@@ -0,0 +1,92 @@
+R""(
+
+# Examples
+
+* Build the default package from the flake in the current directory:
+
+ ```console
+ # nix build
+ ```
+
+* Build and run GNU Hello from the `nixpkgs` flake:
+
+ ```console
+ # nix build nixpkgs#hello
+ # ./result/bin/hello
+ Hello, world!
+ ```
+
+* Build GNU Hello and Cowsay, leaving two result symlinks:
+
+ ```console
+ # nix build nixpkgs#hello nixpkgs#cowsay
+ # ls -l result*
+ lrwxrwxrwx 1 … result -> /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10
+ lrwxrwxrwx 1 … result-1 -> /nix/store/rkfrm0z6x6jmi7d3gsmma4j53h15mg33-cowsay-3.03+dfsg2
+ ```
+
+* Build a specific output:
+
+ ```console
+ # nix build nixpkgs#glibc.dev
+ # ls -ld ./result-dev
+ lrwxrwxrwx 1 … ./result-dev -> /nix/store/dkm3gwl0xrx0wrw6zi5x3px3lpgjhlw4-glibc-2.32-dev
+ ```
+
+* Build attribute `build.x86_64-linux` from (non-flake) Nix expression
+ `release.nix`:
+
+ ```console
+ # nix build -f release.nix build.x86_64-linux
+ ```
+
+* Build a NixOS system configuration from a flake, and make a profile
+ point to the result:
+
+ ```console
+ # nix build --profile /nix/var/nix/profiles/system \
+ ~/my-configurations#nixosConfigurations.machine.config.system.build.toplevel
+ ```
+
+ (This is essentially what `nixos-rebuild` does.)
+
+* Build an expression specified on the command line:
+
+ ```console
+ # nix build --impure --expr \
+ 'with import <nixpkgs> {};
+ runCommand "foo" {
+ buildInputs = [ hello ];
+ }
+ "hello > $out"'
+ # cat ./result
+ Hello, world!
+ ```
+
+ Note that `--impure` is needed because we're using `<nixpkgs>`,
+ which relies on the `$NIX_PATH` environment variable.
+
+* Fetch a store path from the configured substituters, if it doesn't
+ already exist:
+
+ ```console
+ # nix build /nix/store/rkfrm0z6x6jmi7d3gsmma4j53h15mg33-cowsay-3.03+dfsg2
+ ```
+
+# Description
+
+`nix build` builds the specified *installables*. Installables that
+resolve to derivations are built (or substituted if possible). Store
+path installables are substituted.
+
+Unless `--no-link` is specified, after a successful build, it creates
+symlinks to the store paths of the installables. These symlinks have
+the prefix `./result` by default; this can be overridden using the
+`--out-link` option. Each symlink has a suffix `-<N>-<outname>`, where
+*N* is the index of the installable (with the left-most installable
+having index 0), and *outname* is the symbolic derivation output name
+(e.g. `bin`, `dev` or `lib`). `-<N>` is omitted if *N* = 0, and
+`-<outname>` is omitted if *outname* = `out` (denoting the default
+output).
+
+)""
diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc
index 2d0a0b6ea..53dccc63a 100644
--- a/src/nix/bundle.cc
+++ b/src/nix/bundle.cc
@@ -16,7 +16,7 @@ struct CmdBundle : InstallableCommand
{
addFlag({
.longName = "bundler",
- .description = "use custom bundler",
+ .description = fmt("Use a custom bundler instead of the default (`%s`).", bundler),
.labels = {"flake-url"},
.handler = {&bundler},
.completer = {[&](size_t, std::string_view prefix) {
@@ -27,11 +27,12 @@ struct CmdBundle : InstallableCommand
addFlag({
.longName = "out-link",
.shortName = 'o',
- .description = "path of the symlink to the build result",
+ .description = "Override the name of the symlink to the build result. It defaults to the base name of the app.",
.labels = {"path"},
.handler = {&outLink},
.completer = completePath
});
+
}
std::string description() override
@@ -39,14 +40,11 @@ struct CmdBundle : InstallableCommand
return "bundle an application so that it works outside of the Nix store";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To bundle Hello:",
- "nix bundle hello"
- },
- };
+ return
+ #include "bundle.md"
+ ;
}
Category category() override { return catSecondary; }
@@ -72,11 +70,11 @@ struct CmdBundle : InstallableCommand
auto evalState = getEvalState();
auto app = installable->toApp(*evalState);
- store->buildPaths(app.context);
+ store->buildPaths(toDerivedPaths(app.context));
auto [bundlerFlakeRef, bundlerName] = parseFlakeRefWithFragment(bundler, absPath("."));
const flake::LockFlags lockFlags{ .writeLockFile = false };
- auto bundler = InstallableFlake(
+ auto bundler = InstallableFlake(this,
evalState, std::move(bundlerFlakeRef),
Strings{bundlerName == "" ? "defaultBundler" : bundlerName},
Strings({"bundlers."}), lockFlags);
@@ -92,7 +90,7 @@ struct CmdBundle : InstallableCommand
mkString(*evalState->allocAttr(*arg, evalState->symbols.create("system")), settings.thisSystem.get());
arg->attrs->sort();
-
+
auto vRes = evalState->allocValue();
evalState->callFunction(*bundler.toValue(*evalState).first, *arg, *vRes, noPos);
@@ -112,14 +110,10 @@ struct CmdBundle : InstallableCommand
StorePath outPath = store->parseStorePath(evalState->coerceToPath(*attr2->pos, *attr2->value, context2));
- store->buildPaths({{drvPath}});
+ store->buildPaths({ DerivedPath::Built { drvPath } });
auto outPathS = store->printStorePath(outPath);
- auto info = store->queryPathInfo(outPath);
- if (!info->references.empty())
- throw Error("'%s' has references; a bundler must not leave any references", outPathS);
-
if (!outLink)
outLink = baseNameOf(app.program);
diff --git a/src/nix/bundle.md b/src/nix/bundle.md
new file mode 100644
index 000000000..5e2298376
--- /dev/null
+++ b/src/nix/bundle.md
@@ -0,0 +1,36 @@
+R""(
+
+# Examples
+
+* Bundle Hello:
+
+ ```console
+ # nix bundle nixpkgs#hello
+ # ./hello
+ Hello, world!
+ ```
+
+* Bundle a specific version of Nix:
+
+ ```console
+ # nix bundle github:NixOS/nix/e3ddffb27e5fc37a209cfd843c6f7f6a9460a8ec
+ # ./nix --version
+ nix (Nix) 2.4pre20201215_e3ddffb
+ ```
+
+# Description
+
+`nix bundle` packs the closure of the [Nix app](./nix3-run.md)
+*installable* into a single self-extracting executable. See the
+[`nix-bundle` homepage](https://github.com/matthewbauer/nix-bundle)
+for more details.
+
+> **Note**
+>
+> This command only works on Linux.
+
+# Bundler definitions
+
+TODO
+
+)""
diff --git a/src/nix/cat.cc b/src/nix/cat.cc
index eef172cfc..e28ee3c50 100644
--- a/src/nix/cat.cc
+++ b/src/nix/cat.cc
@@ -37,7 +37,12 @@ struct CmdCatStore : StoreCommand, MixCat
return "print the contents of a file in the Nix store on stdout";
}
- Category category() override { return catUtility; }
+ std::string doc() override
+ {
+ return
+ #include "store-cat.md"
+ ;
+ }
void run(ref<Store> store) override
{
@@ -64,7 +69,12 @@ struct CmdCatNar : StoreCommand, MixCat
return "print the contents of a file inside a NAR file on stdout";
}
- Category category() override { return catUtility; }
+ std::string doc() override
+ {
+ return
+ #include "nar-cat.md"
+ ;
+ }
void run(ref<Store> store) override
{
@@ -72,5 +82,5 @@ struct CmdCatNar : StoreCommand, MixCat
}
};
-static auto rCmdCatStore = registerCommand<CmdCatStore>("cat-store");
-static auto rCmdCatNar = registerCommand<CmdCatNar>("cat-nar");
+static auto rCmdCatStore = registerCommand2<CmdCatStore>({"store", "cat"});
+static auto rCmdCatNar = registerCommand2<CmdCatNar>({"nar", "cat"});
diff --git a/src/nix/copy.cc b/src/nix/copy.cc
index cb31aac8f..f59f7c76b 100644
--- a/src/nix/copy.cc
+++ b/src/nix/copy.cc
@@ -8,7 +8,7 @@
using namespace nix;
-struct CmdCopy : StorePathsCommand
+struct CmdCopy : RealisedPathsCommand
{
std::string srcUri, dstUri;
@@ -16,33 +16,35 @@ struct CmdCopy : StorePathsCommand
SubstituteFlag substitute = NoSubstitute;
+ using RealisedPathsCommand::run;
+
CmdCopy()
- : StorePathsCommand(true)
+ : RealisedPathsCommand(true)
{
addFlag({
.longName = "from",
- .description = "URI of the source Nix store",
+ .description = "URL of the source Nix store.",
.labels = {"store-uri"},
.handler = {&srcUri},
});
addFlag({
.longName = "to",
- .description = "URI of the destination Nix store",
+ .description = "URL of the destination Nix store.",
.labels = {"store-uri"},
.handler = {&dstUri},
});
addFlag({
.longName = "no-check-sigs",
- .description = "do not require that paths are signed by trusted keys",
+ .description = "Do not require that paths are signed by trusted keys.",
.handler = {&checkSigs, NoCheckSigs},
});
addFlag({
.longName = "substitute-on-destination",
.shortName = 's',
- .description = "whether to try substitutes on the destination store (only supported by SSH)",
+ .description = "Whether to try substitutes on the destination store (only supported by SSH stores).",
.handler = {&substitute, Substitute},
});
@@ -54,32 +56,11 @@ struct CmdCopy : StorePathsCommand
return "copy paths between Nix stores";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To copy Firefox from the local store to a binary cache in file:///tmp/cache:",
- "nix copy --to file:///tmp/cache $(type -p firefox)"
- },
- Example{
- "To copy the entire current NixOS system closure to another machine via SSH:",
- "nix copy --to ssh://server /run/current-system"
- },
- Example{
- "To copy a closure from another machine via SSH:",
- "nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2"
- },
-#ifdef ENABLE_S3
- Example{
- "To copy Hello to an S3 binary cache:",
- "nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs#hello"
- },
- Example{
- "To copy Hello to an S3-compatible binary cache:",
- "nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs#hello"
- },
-#endif
- };
+ return
+ #include "copy.md"
+ ;
}
Category category() override { return catSecondary; }
@@ -94,14 +75,15 @@ struct CmdCopy : StorePathsCommand
if (srcUri.empty() && dstUri.empty())
throw UsageError("you must pass '--from' and/or '--to'");
- StorePathsCommand::run(store);
+ RealisedPathsCommand::run(store);
}
- void run(ref<Store> srcStore, StorePaths storePaths) override
+ void run(ref<Store> srcStore, std::vector<RealisedPath> paths) override
{
ref<Store> dstStore = dstUri.empty() ? openStore() : openStore(dstUri);
- copyPaths(srcStore, dstStore, StorePathSet(storePaths.begin(), storePaths.end()),
+ copyPaths(
+ srcStore, dstStore, RealisedPath::Set(paths.begin(), paths.end()),
NoRepair, checkSigs, substitute);
}
};
diff --git a/src/nix/copy.md b/src/nix/copy.md
new file mode 100644
index 000000000..25e0ddadc
--- /dev/null
+++ b/src/nix/copy.md
@@ -0,0 +1,58 @@
+R""(
+
+# Examples
+
+* Copy Firefox from the local store to a binary cache in `/tmp/cache`:
+
+ ```console
+ # nix copy --to file:///tmp/cache $(type -p firefox)
+ ```
+
+ Note the `file://` - without this, the destination is a chroot
+ store, not a binary cache.
+
+* Copy the entire current NixOS system closure to another machine via
+ SSH:
+
+ ```console
+ # nix copy -s --to ssh://server /run/current-system
+ ```
+
+ The `-s` flag causes the remote machine to try to substitute missing
+ store paths, which may be faster if the link between the local and
+ remote machines is slower than the link between the remote machine
+ and its substituters (e.g. `https://cache.nixos.org`).
+
+* Copy a closure from another machine via SSH:
+
+ ```console
+ # nix copy --from ssh://server /nix/store/a6cnl93nk1wxnq84brbbwr6hxw9gp2w9-blender-2.79-rc2
+ ```
+
+* Copy Hello to a binary cache in an Amazon S3 bucket:
+
+ ```console
+ # nix copy --to s3://my-bucket?region=eu-west-1 nixpkgs#hello
+ ```
+
+ or to an S3-compatible storage system:
+
+ ```console
+ # nix copy --to s3://my-bucket?region=eu-west-1&endpoint=example.com nixpkgs#hello
+ ```
+
+ Note that this only works if Nix is built with AWS support.
+
+* Copy a closure from `/nix/store` to the chroot store `/tmp/nix/nix/store`:
+
+ ```console
+ # nix copy --to /tmp/nix nixpkgs#hello --no-check-sigs
+ ```
+
+# Description
+
+`nix copy` copies store path closures between two Nix stores. The
+source store is specified using `--from` and the destination using
+`--to`. If one of these is omitted, it defaults to the local store.
+
+)""
diff --git a/src/nix-daemon/nix-daemon.cc b/src/nix/daemon.cc
index fc6195cf0..2cf2a04c9 100644
--- a/src/nix-daemon/nix-daemon.cc
+++ b/src/nix/daemon.cc
@@ -1,3 +1,4 @@
+#include "command.hh"
#include "shared.hh"
#include "local-store.hh"
#include "remote-store.hh"
@@ -7,7 +8,7 @@
#include "globals.hh"
#include "derivations.hh"
#include "finally.hh"
-#include "../nix/legacy.hh"
+#include "legacy.hh"
#include "daemon.hh"
#include <algorithm>
@@ -150,7 +151,7 @@ static ref<Store> openUncachedStore()
}
-static void daemonLoop(char * * argv)
+static void daemonLoop()
{
if (chdir("/") == -1)
throw SysError("cannot change current directory");
@@ -232,9 +233,9 @@ static void daemonLoop(char * * argv)
setSigChldAction(false);
// For debugging, stuff the pid into argv[1].
- if (peer.pidKnown && argv[1]) {
+ if (peer.pidKnown && savedArgv[1]) {
string processName = std::to_string(peer.pid);
- strncpy(argv[1], processName.c_str(), strlen(argv[1]));
+ strncpy(savedArgv[1], processName.c_str(), strlen(savedArgv[1]));
}
// Handle the connection.
@@ -257,13 +258,55 @@ static void daemonLoop(char * * argv)
return;
} catch (Error & error) {
ErrorInfo ei = error.info();
- ei.hint = std::optional(hintfmt("error processing connection: %1%",
- (error.info().hint.has_value() ? error.info().hint->str() : "")));
+ // FIXME: add to trace?
+ ei.msg = hintfmt("error processing connection: %1%", ei.msg.str());
logError(ei);
}
}
}
+static void runDaemon(bool stdio)
+{
+ if (stdio) {
+ if (auto store = openUncachedStore().dynamic_pointer_cast<RemoteStore>()) {
+ auto conn = store->openConnectionWrapper();
+ int from = conn->from.fd;
+ int to = conn->to.fd;
+
+ auto nfds = std::max(from, STDIN_FILENO) + 1;
+ while (true) {
+ fd_set fds;
+ FD_ZERO(&fds);
+ FD_SET(from, &fds);
+ FD_SET(STDIN_FILENO, &fds);
+ if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1)
+ throw SysError("waiting for data from client or server");
+ if (FD_ISSET(from, &fds)) {
+ auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
+ if (res == -1)
+ throw SysError("splicing data from daemon socket to stdout");
+ else if (res == 0)
+ throw EndOfFile("unexpected EOF from daemon socket");
+ }
+ if (FD_ISSET(STDIN_FILENO, &fds)) {
+ auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
+ if (res == -1)
+ throw SysError("splicing data from stdin to daemon socket");
+ else if (res == 0)
+ return;
+ }
+ }
+ } else {
+ FdSource from(STDIN_FILENO);
+ FdSink to(STDOUT_FILENO);
+ /* Auth hook is empty because in this mode we blindly trust the
+ standard streams. Limiting access to those is explicitly
+ not `nix-daemon`'s responsibility. */
+ processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){});
+ }
+ } else
+ daemonLoop();
+}
static int main_nix_daemon(int argc, char * * argv)
{
@@ -283,51 +326,34 @@ static int main_nix_daemon(int argc, char * * argv)
return true;
});
- initPlugins();
-
- if (stdio) {
- if (auto store = openUncachedStore().dynamic_pointer_cast<RemoteStore>()) {
- auto conn = store->openConnectionWrapper();
- int from = conn->from.fd;
- int to = conn->to.fd;
-
- auto nfds = std::max(from, STDIN_FILENO) + 1;
- while (true) {
- fd_set fds;
- FD_ZERO(&fds);
- FD_SET(from, &fds);
- FD_SET(STDIN_FILENO, &fds);
- if (select(nfds, &fds, nullptr, nullptr, nullptr) == -1)
- throw SysError("waiting for data from client or server");
- if (FD_ISSET(from, &fds)) {
- auto res = splice(from, nullptr, STDOUT_FILENO, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
- if (res == -1)
- throw SysError("splicing data from daemon socket to stdout");
- else if (res == 0)
- throw EndOfFile("unexpected EOF from daemon socket");
- }
- if (FD_ISSET(STDIN_FILENO, &fds)) {
- auto res = splice(STDIN_FILENO, nullptr, to, nullptr, SSIZE_MAX, SPLICE_F_MOVE);
- if (res == -1)
- throw SysError("splicing data from stdin to daemon socket");
- else if (res == 0)
- return 0;
- }
- }
- } else {
- FdSource from(STDIN_FILENO);
- FdSink to(STDOUT_FILENO);
- /* Auth hook is empty because in this mode we blindly trust the
- standard streams. Limitting access to thoses is explicitly
- not `nix-daemon`'s responsibility. */
- processConnection(openUncachedStore(), from, to, Trusted, NotRecursive, [&](Store & _){});
- }
- } else {
- daemonLoop(argv);
- }
+ runDaemon(stdio);
return 0;
}
}
static RegisterLegacyCommand r_nix_daemon("nix-daemon", main_nix_daemon);
+
+struct CmdDaemon : StoreCommand
+{
+ std::string description() override
+ {
+ return "daemon to perform store operations on behalf of non-root clients";
+ }
+
+ Category category() override { return catUtility; }
+
+ std::string doc() override
+ {
+ return
+ #include "daemon.md"
+ ;
+ }
+
+ void run(ref<Store> store) override
+ {
+ runDaemon(false);
+ }
+};
+
+static auto rCmdDaemon = registerCommand2<CmdDaemon>({"daemon"});
diff --git a/src/nix/daemon.md b/src/nix/daemon.md
new file mode 100644
index 000000000..e97016a94
--- /dev/null
+++ b/src/nix/daemon.md
@@ -0,0 +1,21 @@
+R""(
+
+# Example
+
+* Run the daemon in the foreground:
+
+ ```console
+ # nix daemon
+ ```
+
+# Description
+
+This command runs the Nix daemon, which is a required component in
+multi-user Nix installations. It performs build actions and other
+operations on the Nix store on behalf of non-root users. Usually you
+don't run the daemon directly; instead it's managed by a service
+management framework such as `systemd`.
+
+Note that this daemon does not fork into the background.
+
+)""
diff --git a/src/nix/develop.cc b/src/nix/develop.cc
index 9372f43de..498a7b45c 100644
--- a/src/nix/develop.cc
+++ b/src/nix/develop.cc
@@ -3,6 +3,7 @@
#include "common-args.hh"
#include "shared.hh"
#include "store-api.hh"
+#include "path-with-outputs.hh"
#include "derivations.hh"
#include "affinity.hh"
#include "progress-bar.hh"
@@ -11,6 +12,19 @@
using namespace nix;
+struct DevelopSettings : Config
+{
+ Setting<std::string> bashPrompt{this, "", "bash-prompt",
+ "The bash prompt (`PS1`) in `nix develop` shells."};
+
+ Setting<std::string> bashPromptSuffix{this, "", "bash-prompt-suffix",
+ "Suffix appended to the `PS1` environment variable in `nix develop` shells."};
+};
+
+static DevelopSettings developSettings;
+
+static GlobalConfig::Register rDevelopSettings(&developSettings);
+
struct Var
{
bool exported = true;
@@ -39,21 +53,24 @@ BuildEnvironment readEnvironment(const Path & path)
static std::string varNameRegex =
R"re((?:[a-zA-Z_][a-zA-Z0-9_]*))re";
- static std::regex declareRegex(
- "^declare -x (" + varNameRegex + ")" +
- R"re((?:="((?:[^"\\]|\\.)*)")?\n)re");
-
static std::string simpleStringRegex =
R"re((?:[a-zA-Z0-9_/:\.\-\+=]*))re";
- static std::string quotedStringRegex =
- R"re((?:\$?'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'))re";
+ static std::string dquotedStringRegex =
+ R"re((?:\$?"(?:[^"\\]|\\[$`"\\\n])*"))re";
+
+ static std::string squotedStringRegex =
+ R"re((?:\$?(?:'(?:[^'\\]|\\[abeEfnrtv\\'"?])*'|\\')+))re";
static std::string indexedArrayRegex =
R"re((?:\(( *\[[0-9]+\]="(?:[^"\\]|\\.)*")*\)))re";
+ static std::regex declareRegex(
+ "^declare -a?x (" + varNameRegex + ")(=(" +
+ dquotedStringRegex + "|" + indexedArrayRegex + "))?\n");
+
static std::regex varRegex(
- "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + quotedStringRegex + "|" + indexedArrayRegex + ")\n");
+ "^(" + varNameRegex + ")=(" + simpleStringRegex + "|" + squotedStringRegex + "|" + indexedArrayRegex + ")\n");
/* Note: we distinguish between an indexed and associative array
using the space before the closing parenthesis. Will
@@ -143,7 +160,7 @@ StorePath getDerivationEnvironment(ref<Store> store, const StorePath & drvPath)
auto shellDrvPath = writeDerivation(*store, drv);
/* Build the derivation. */
- store->buildPaths({{shellDrvPath}});
+ store->buildPaths({DerivedPath::Built{shellDrvPath}});
for (auto & [_0, outputAndOptPath] : drv.outputsAndOptPaths(*store)) {
auto & [_1, optPath] = outputAndOptPath;
@@ -182,7 +199,22 @@ struct Common : InstallableCommand, MixProfile
"UID",
};
+ std::vector<std::pair<std::string, std::string>> redirects;
+
+ Common()
+ {
+ addFlag({
+ .longName = "redirect",
+ .description = "Redirect a store path to a mutable location.",
+ .labels = {"installable", "outputs-dir"},
+ .handler = {[&](std::string installable, std::string outputsDir) {
+ redirects.push_back({installable, outputsDir});
+ }}
+ });
+ }
+
std::string makeRcScript(
+ ref<Store> store,
const BuildEnvironment & buildEnvironment,
const Path & outputsDir = absPath(".") + "/outputs")
{
@@ -208,12 +240,14 @@ struct Common : InstallableCommand, MixProfile
out << buildEnvironment.bashFunctions << "\n";
- out << "export NIX_BUILD_TOP=\"$(mktemp -d --tmpdir nix-shell.XXXXXX)\"\n";
+ out << "export NIX_BUILD_TOP=\"$(mktemp -d -t nix-shell.XXXXXX)\"\n";
for (auto & i : {"TMP", "TMPDIR", "TEMP", "TEMPDIR"})
out << fmt("export %s=\"$NIX_BUILD_TOP\"\n", i);
out << "eval \"$shellHook\"\n";
+ auto script = out.str();
+
/* Substitute occurrences of output paths. */
auto outputs = buildEnvironment.env.find("outputs");
assert(outputs != buildEnvironment.env.end());
@@ -227,7 +261,33 @@ struct Common : InstallableCommand, MixProfile
rewrites.insert({from->second.quoted, outputsDir + "/" + outputName});
}
- return rewriteStrings(out.str(), rewrites);
+ /* Substitute redirects. */
+ for (auto & [installable_, dir_] : redirects) {
+ auto dir = absPath(dir_);
+ auto installable = parseInstallable(store, installable_);
+ auto buildable = installable->toDerivedPathWithHints();
+ auto doRedirect = [&](const StorePath & path)
+ {
+ auto from = store->printStorePath(path);
+ if (script.find(from) == std::string::npos)
+ warn("'%s' (path '%s') is not used by this build environment", installable->what(), from);
+ else {
+ printInfo("redirecting '%s' to '%s'", from, dir);
+ rewrites.insert({from, dir});
+ }
+ };
+ std::visit(overloaded {
+ [&](const DerivedPathWithHints::Opaque & bo) {
+ doRedirect(bo.path);
+ },
+ [&](const DerivedPathWithHints::Built & bfd) {
+ for (auto & [outputName, path] : bfd.outputs)
+ if (path) doRedirect(*path);
+ },
+ }, buildable.raw());
+ }
+
+ return rewriteStrings(script, rewrites);
}
Strings getDefaultFlakeAttrPaths() override
@@ -275,7 +335,7 @@ struct CmdDevelop : Common, MixEnvironment
addFlag({
.longName = "command",
.shortName = 'c',
- .description = "command and arguments to be executed instead of an interactive shell",
+ .description = "Instead of starting an interactive shell, start the specified command and arguments.",
.labels = {"command", "args"},
.handler = {[&](std::vector<std::string> ss) {
if (ss.empty()) throw UsageError("--command requires at least one argument");
@@ -285,38 +345,38 @@ struct CmdDevelop : Common, MixEnvironment
addFlag({
.longName = "phase",
- .description = "phase to run (e.g. `build` or `configure`)",
+ .description = "The stdenv phase to run (e.g. `build` or `configure`).",
.labels = {"phase-name"},
.handler = {&phase},
});
addFlag({
.longName = "configure",
- .description = "run the configure phase",
+ .description = "Run the `configure` phase.",
.handler = {&phase, {"configure"}},
});
addFlag({
.longName = "build",
- .description = "run the build phase",
+ .description = "Run the `build` phase.",
.handler = {&phase, {"build"}},
});
addFlag({
.longName = "check",
- .description = "run the check phase",
+ .description = "Run the `check` phase.",
.handler = {&phase, {"check"}},
});
addFlag({
.longName = "install",
- .description = "run the install phase",
+ .description = "Run the `install` phase.",
.handler = {&phase, {"install"}},
});
addFlag({
.longName = "installcheck",
- .description = "run the installcheck phase",
+ .description = "Run the `installcheck` phase.",
.handler = {&phase, {"installCheck"}},
});
}
@@ -326,26 +386,11 @@ struct CmdDevelop : Common, MixEnvironment
return "run a bash shell that provides the build environment of a derivation";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To get the build environment of GNU hello:",
- "nix develop nixpkgs#hello"
- },
- Example{
- "To get the build environment of the default package of flake in the current directory:",
- "nix develop"
- },
- Example{
- "To store the build environment in a profile:",
- "nix develop --profile /tmp/my-shell nixpkgs#hello"
- },
- Example{
- "To use a build environment previously recorded in a profile:",
- "nix develop /tmp/my-shell"
- },
- };
+ return
+ #include "develop.md"
+ ;
}
void run(ref<Store> store) override
@@ -354,7 +399,7 @@ struct CmdDevelop : Common, MixEnvironment
auto [rcFileFd, rcFilePath] = createTempFile("nix-shell");
- auto script = makeRcScript(buildEnvironment);
+ auto script = makeRcScript(store, buildEnvironment);
if (verbosity >= lvlDebug)
script += "set -x\n";
@@ -368,7 +413,6 @@ struct CmdDevelop : Common, MixEnvironment
// rid of that.
script += fmt("foundMakefile=1\n");
script += fmt("runHook %1%Phase\n", *phase);
- script += fmt("exit 0\n", *phase);
}
else if (!command.empty()) {
@@ -380,6 +424,10 @@ struct CmdDevelop : Common, MixEnvironment
else {
script += "[ -n \"$PS1\" ] && [ -e ~/.bashrc ] && source ~/.bashrc;\n";
+ if (developSettings.bashPrompt != "")
+ script += fmt("[ -n \"$PS1\" ] && PS1=%s;\n", shellEscape(developSettings.bashPrompt));
+ if (developSettings.bashPromptSuffix != "")
+ script += fmt("[ -n \"$PS1\" ] && PS1+=%s;\n", shellEscape(developSettings.bashPromptSuffix));
}
writeFull(rcFileFd.get(), script);
@@ -396,6 +444,7 @@ struct CmdDevelop : Common, MixEnvironment
auto state = getEvalState();
auto bashInstallable = std::make_shared<InstallableFlake>(
+ this,
state,
installable->nixpkgsFlakeRef(),
Strings{"bashInteractive"},
@@ -408,10 +457,12 @@ struct CmdDevelop : Common, MixEnvironment
ignoreException();
}
- auto args = Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath};
+ // If running a phase or single command, don't want an interactive shell running after
+ // Ctrl-C, so don't pass --rcfile
+ auto args = phase || !command.empty() ? Strings{std::string(baseNameOf(shell)), rcFilePath}
+ : Strings{std::string(baseNameOf(shell)), "--rcfile", rcFilePath};
- restoreAffinity();
- restoreSignals();
+ restoreProcessContext();
execvp(shell.c_str(), stringsToCharPtrs(args).data());
@@ -426,14 +477,11 @@ struct CmdPrintDevEnv : Common
return "print shell code that can be sourced by bash to reproduce the build environment of a derivation";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To apply the build environment of GNU hello to the current shell:",
- ". <(nix print-dev-env nixpkgs#hello)"
- },
- };
+ return
+ #include "print-dev-env.md"
+ ;
}
Category category() override { return catUtility; }
@@ -444,7 +492,7 @@ struct CmdPrintDevEnv : Common
stopProgressBar();
- std::cout << makeRcScript(buildEnvironment);
+ std::cout << makeRcScript(store, buildEnvironment);
}
};
diff --git a/src/nix/develop.md b/src/nix/develop.md
new file mode 100644
index 000000000..e71d9f8aa
--- /dev/null
+++ b/src/nix/develop.md
@@ -0,0 +1,94 @@
+R""(
+
+# Examples
+
+* Start a shell with the build environment of the default package of
+ the flake in the current directory:
+
+ ```console
+ # nix develop
+ ```
+
+ Typical commands to run inside this shell are:
+
+ ```console
+ # configurePhase
+ # buildPhase
+ # installPhase
+ ```
+
+ Alternatively, you can run whatever build tools your project uses
+ directly, e.g. for a typical Unix project:
+
+ ```console
+ # ./configure --prefix=$out
+ # make
+ # make install
+ ```
+
+* Run a particular build phase directly:
+
+ ```console
+ # nix develop --configure
+ # nix develop --build
+ # nix develop --check
+ # nix develop --install
+ # nix develop --installcheck
+ ```
+
+* Start a shell with the build environment of GNU Hello:
+
+ ```console
+ # nix develop nixpkgs#hello
+ ```
+
+* Record a build environment in a profile:
+
+ ```console
+ # nix develop --profile /tmp/my-build-env nixpkgs#hello
+ ```
+
+* Use a build environment previously recorded in a profile:
+
+ ```console
+ # nix develop /tmp/my-build-env
+ ```
+
+* Replace all occurences of the store path corresponding to
+ `glibc.dev` with a writable directory:
+
+ ```console
+ # nix develop --redirect nixpkgs#glibc.dev ~/my-glibc/outputs/dev
+ ```
+
+ Note that this is useful if you're running a `nix develop` shell for
+ `nixpkgs#glibc` in `~/my-glibc` and want to compile another package
+ against it.
+
+# Description
+
+`nix develop` starts a `bash` shell that provides an interactive build
+environment nearly identical to what Nix would use to build
+*installable*. Inside this shell, environment variables and shell
+functions are set up so that you can interactively and incrementally
+build your package.
+
+Nix determines the build environment by building a modified version of
+the derivation *installable* that just records the environment
+initialised by `stdenv` and exits. This build environment can be
+recorded into a profile using `--profile`.
+
+The prompt used by the `bash` shell can be customised by setting the
+`bash-prompt` and `bash-prompt-suffix` settings in `nix.conf` or in
+the flake's `nixConfig` attribute.
+
+# Flake output attributes
+
+If no flake output attribute is given, `nix run` tries the following
+flake output attributes:
+
+* `devShell.<system>`
+
+* `defaultPackage.<system>`
+
+)""
diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc
index 30e7b20e1..0c7d531c1 100644
--- a/src/nix/diff-closures.cc
+++ b/src/nix/diff-closures.cc
@@ -121,16 +121,11 @@ struct CmdDiffClosures : SourceExprCommand
return "show what packages and versions were added and removed between two closures";
}
- Category category() override { return catSecondary; }
-
- Examples examples() override
+ std::string doc() override
{
- return {
- {
- "To show what got added and removed between two versions of the NixOS system profile:",
- "nix diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link",
- },
- };
+ return
+ #include "diff-closures.md"
+ ;
}
void run(ref<Store> store) override
@@ -143,4 +138,4 @@ struct CmdDiffClosures : SourceExprCommand
}
};
-static auto rCmdDiffClosures = registerCommand<CmdDiffClosures>("diff-closures");
+static auto rCmdDiffClosures = registerCommand2<CmdDiffClosures>({"store", "diff-closures"});
diff --git a/src/nix/diff-closures.md b/src/nix/diff-closures.md
new file mode 100644
index 000000000..0294c0d8d
--- /dev/null
+++ b/src/nix/diff-closures.md
@@ -0,0 +1,51 @@
+R""(
+
+# Examples
+
+* Show what got added and removed between two versions of the NixOS
+ system profile:
+
+ ```console
+ # nix store diff-closures /nix/var/nix/profiles/system-655-link /nix/var/nix/profiles/system-658-link
+ acpi-call: 2020-04-07-5.8.16 → 2020-04-07-5.8.18
+ baloo-widgets: 20.08.1 → 20.08.2
+ bluez-qt: +12.6 KiB
+ dolphin: 20.08.1 → 20.08.2, +13.9 KiB
+ kdeconnect: 20.08.2 → ∅, -6597.8 KiB
+ kdeconnect-kde: ∅ → 20.08.2, +6599.7 KiB
+ …
+ ```
+
+# Description
+
+This command shows the differences between the two closures *before*
+and *after* with respect to the addition, removal, or version change
+of packages, as well as changes in store path sizes.
+
+For each package name in the two closures (where a package name is
+defined as the name component of a store path excluding the version),
+if there is a change in the set of versions of the package, or a
+change in the size of the store paths of more than 8 KiB, it prints a
+line like this:
+
+```console
+dolphin: 20.08.1 → 20.08.2, +13.9 KiB
+```
+
+No size change is shown if it's below the threshold. If the package
+does not exist in either the *before* or *after* closures, it is
+represented using `∅` (empty set) on the appropriate side of the
+arrow. If a package has an empty version string, the version is
+rendered as `ε` (epsilon).
+
+There may be multiple versions of a package in each closure. In that
+case, only the changed versions are shown. Thus,
+
+```console
+libfoo: 1.2, 1.3 → 1.4
+```
+
+leaves open the possibility that there are other versions (e.g. `1.1`)
+that exist in both closures.
+
+)""
diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc
index 6fd197531..c4edc894b 100644
--- a/src/nix/dump-path.cc
+++ b/src/nix/dump-path.cc
@@ -1,5 +1,6 @@
#include "command.hh"
#include "store-api.hh"
+#include "archive.hh"
using namespace nix;
@@ -7,21 +8,16 @@ struct CmdDumpPath : StorePathCommand
{
std::string description() override
{
- return "dump a store path to stdout (in NAR format)";
+ return "serialise a store path to stdout in NAR format";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To get a NAR from the binary cache https://cache.nixos.org/:",
- "nix dump-path --store https://cache.nixos.org/ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25"
- },
- };
+ return
+ #include "store-dump-path.md"
+ ;
}
- Category category() override { return catUtility; }
-
void run(ref<Store> store, const StorePath & storePath) override
{
FdSink sink(STDOUT_FILENO);
@@ -30,4 +26,39 @@ struct CmdDumpPath : StorePathCommand
}
};
-static auto rDumpPath = registerCommand<CmdDumpPath>("dump-path");
+static auto rDumpPath = registerCommand2<CmdDumpPath>({"store", "dump-path"});
+
+struct CmdDumpPath2 : Command
+{
+ Path path;
+
+ CmdDumpPath2()
+ {
+ expectArgs({
+ .label = "path",
+ .handler = {&path},
+ .completer = completePath
+ });
+ }
+
+ std::string description() override
+ {
+ return "serialise a path to stdout in NAR format";
+ }
+
+ std::string doc() override
+ {
+ return
+ #include "nar-dump-path.md"
+ ;
+ }
+
+ void run() override
+ {
+ FdSink sink(STDOUT_FILENO);
+ dumpPath(path, sink);
+ sink.flush();
+ }
+};
+
+static auto rDumpPath2 = registerCommand2<CmdDumpPath2>({"nar", "dump-path"});
diff --git a/src/nix/edit.cc b/src/nix/edit.cc
index 51c16f5a9..b26417b18 100644
--- a/src/nix/edit.cc
+++ b/src/nix/edit.cc
@@ -15,14 +15,11 @@ struct CmdEdit : InstallableCommand
return "open the Nix expression of a Nix package in $EDITOR";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To open the Nix expression of the GNU Hello package:",
- "nix edit nixpkgs#hello"
- },
- };
+ return
+ #include "edit.md"
+ ;
}
Category category() override { return catSecondary; }
@@ -45,7 +42,8 @@ struct CmdEdit : InstallableCommand
auto args = editorFor(pos);
- restoreSignals();
+ restoreProcessContext();
+
execvp(args.front().c_str(), stringsToCharPtrs(args).data());
std::string command;
diff --git a/src/nix/edit.md b/src/nix/edit.md
new file mode 100644
index 000000000..80563d06b
--- /dev/null
+++ b/src/nix/edit.md
@@ -0,0 +1,31 @@
+R""(
+
+# Examples
+
+* Open the Nix expression of the GNU Hello package:
+
+ ```console
+ # nix edit nixpkgs#hello
+ ```
+
+* Get the filename and line number used by `nix edit`:
+
+ ```console
+ # nix eval --raw nixpkgs#hello.meta.position
+ /nix/store/fvafw0gvwayzdan642wrv84pzm5bgpmy-source/pkgs/applications/misc/hello/default.nix:15
+ ```
+
+# Description
+
+This command opens the Nix expression of a derivation in an
+editor. The filename and line number of the derivation are taken from
+its `meta.position` attribute. Nixpkgs' `stdenv.mkDerivation` sets
+this attribute to the location of the definition of the
+`meta.description`, `version` or `name` derivation attributes.
+
+The editor to invoke is specified by the `EDITOR` environment
+variable. It defaults to `cat`. If the editor is `emacs`, `nano` or
+`vim`, it is passed the line number of the derivation using the
+argument `+<lineno>`.
+
+)""
diff --git a/src/nix/eval.cc b/src/nix/eval.cc
index 43ce46546..65d61e005 100644
--- a/src/nix/eval.cc
+++ b/src/nix/eval.cc
@@ -3,6 +3,7 @@
#include "shared.hh"
#include "store-api.hh"
#include "eval.hh"
+#include "eval-inline.hh"
#include "json.hh"
#include "value-to-json.hh"
#include "progress-bar.hh"
@@ -13,17 +14,29 @@ struct CmdEval : MixJSON, InstallableCommand
{
bool raw = false;
std::optional<std::string> apply;
+ std::optional<Path> writeTo;
CmdEval()
{
- mkFlag(0, "raw", "print strings unquoted", &raw);
+ addFlag({
+ .longName = "raw",
+ .description = "Print strings without quotes or escaping.",
+ .handler = {&raw, true},
+ });
addFlag({
.longName = "apply",
- .description = "apply a function to each argument",
+ .description = "Apply the function *expr* to each argument.",
.labels = {"expr"},
.handler = {&apply},
});
+
+ addFlag({
+ .longName = "write-to",
+ .description = "Write a string or attrset of strings to *path*.",
+ .labels = {"path"},
+ .handler = {&writeTo},
+ });
}
std::string description() override
@@ -31,30 +44,11 @@ struct CmdEval : MixJSON, InstallableCommand
return "evaluate a Nix expression";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- {
- "To evaluate a Nix expression given on the command line:",
- "nix eval --expr '1 + 2'"
- },
- {
- "To evaluate a Nix expression from a file or URI:",
- "nix eval -f ./my-nixpkgs hello.name"
- },
- {
- "To get the current version of Nixpkgs:",
- "nix eval --raw nixpkgs#lib.version"
- },
- {
- "To print the store path of the Hello package:",
- "nix eval --raw nixpkgs#hello"
- },
- {
- "To get a list of checks in the 'nix' flake:",
- "nix eval nix#checks.x86_64-linux --apply builtins.attrNames"
- },
- };
+ return
+ #include "eval.md"
+ ;
}
Category category() override { return catSecondary; }
@@ -66,7 +60,7 @@ struct CmdEval : MixJSON, InstallableCommand
auto state = getEvalState();
- auto v = installable->toValue(*state).first;
+ auto [v, pos] = installable->toValue(*state);
PathSet context;
if (apply) {
@@ -77,15 +71,53 @@ struct CmdEval : MixJSON, InstallableCommand
v = vRes;
}
- if (raw) {
+ if (writeTo) {
+ stopProgressBar();
+
+ if (pathExists(*writeTo))
+ throw Error("path '%s' already exists", *writeTo);
+
+ std::function<void(Value & v, const Pos & pos, const Path & path)> recurse;
+
+ recurse = [&](Value & v, const Pos & pos, const Path & path)
+ {
+ state->forceValue(v);
+ if (v.type() == nString)
+ // FIXME: disallow strings with contexts?
+ writeFile(path, v.string.s);
+ else if (v.type() == nAttrs) {
+ if (mkdir(path.c_str(), 0777) == -1)
+ throw SysError("creating directory '%s'", path);
+ for (auto & attr : *v.attrs)
+ try {
+ if (attr.name == "." || attr.name == "..")
+ throw Error("invalid file name '%s'", attr.name);
+ recurse(*attr.value, *attr.pos, path + "/" + std::string(attr.name));
+ } catch (Error & e) {
+ e.addTrace(*attr.pos, hintfmt("while evaluating the attribute '%s'", attr.name));
+ throw;
+ }
+ }
+ else
+ throw TypeError("value at '%s' is not a string or an attribute set", pos);
+ };
+
+ recurse(*v, pos, *writeTo);
+ }
+
+ else if (raw) {
stopProgressBar();
std::cout << state->coerceToString(noPos, *v, context);
- } else if (json) {
+ }
+
+ else if (json) {
JSONPlaceholder jsonOut(std::cout);
printValueAsJSON(*state, true, *v, jsonOut, context);
- } else {
+ }
+
+ else {
state->forceValueDeep(*v);
- logger->stdout("%s", *v);
+ logger->cout("%s", *v);
}
}
};
diff --git a/src/nix/eval.md b/src/nix/eval.md
new file mode 100644
index 000000000..61334cde1
--- /dev/null
+++ b/src/nix/eval.md
@@ -0,0 +1,74 @@
+R""(
+
+# Examples
+
+* Evaluate a Nix expression given on the command line:
+
+ ```console
+ # nix eval --expr '1 + 2'
+ ```
+
+* Evaluate a Nix expression to JSON:
+
+ ```console
+ # nix eval --json --expr '{ x = 1; }'
+ {"x":1}
+ ```
+
+* Evaluate a Nix expression from a file:
+
+ ```console
+ # nix eval -f ./my-nixpkgs hello.name
+ ```
+
+* Get the current version of the `nixpkgs` flake:
+
+ ```console
+ # nix eval --raw nixpkgs#lib.version
+ ```
+
+* Print the store path of the Hello package:
+
+ ```console
+ # nix eval --raw nixpkgs#hello
+ ```
+
+* Get a list of checks in the `nix` flake:
+
+ ```console
+ # nix eval nix#checks.x86_64-linux --apply builtins.attrNames
+ ```
+
+* Generate a directory with the specified contents:
+
+ ```console
+ # nix eval --write-to ./out --expr '{ foo = "bar"; subdir.bla = "123"; }'
+ # cat ./out/foo
+ bar
+ # cat ./out/subdir/bla
+ 123
+
+# Description
+
+This command evaluates the Nix expression *installable* and prints the
+result on standard output.
+
+# Output format
+
+`nix eval` can produce output in several formats:
+
+* By default, the evaluation result is printed as a Nix expression.
+
+* With `--json`, the evaluation result is printed in JSON format. Note
+ that this fails if the result contains values that are not
+ representable as JSON, such as functions.
+
+* With `--raw`, the evaluation result must be a string, which is
+ printed verbatim, without any quoting.
+
+* With `--write-to` *path*, the evaluation result must be a string or
+ a nested attribute set whose leaf values are strings. These strings
+ are written to files named *path*/*attrpath*. *path* must not
+ already exist.
+
+)""
diff --git a/src/nix/flake-archive.md b/src/nix/flake-archive.md
new file mode 100644
index 000000000..85bbeeb16
--- /dev/null
+++ b/src/nix/flake-archive.md
@@ -0,0 +1,29 @@
+R""(
+
+# Examples
+
+* Copy the `dwarffs` flake and its dependencies to a binary cache:
+
+ ```console
+ # nix flake archive --to file:///tmp/my-cache dwarffs
+ ```
+
+* Fetch the `dwarffs` flake and its dependencies to the local Nix
+ store:
+
+ ```console
+ # nix flake archive dwarffs
+ ```
+
+* Print the store paths of the flake sources of NixOps without
+ fetching them:
+
+ ```console
+ # nix flake archive --json --dry-run nixops
+ ```
+
+# Description
+
+FIXME
+
+)""
diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md
new file mode 100644
index 000000000..dc079ba0c
--- /dev/null
+++ b/src/nix/flake-check.md
@@ -0,0 +1,68 @@
+R""(
+
+# Examples
+
+* Evaluate the flake in the current directory, and build its checks:
+
+ ```console
+ # nix flake check
+ ```
+
+* Verify that the `patchelf` flake evaluates, but don't build its
+ checks:
+
+ ```console
+ # nix flake check --no-build github:NixOS/patchelf
+ ```
+
+# Description
+
+This command verifies that the flake specified by flake reference
+*flake-url* can be evaluated successfully (as detailed below), and
+that the derivations specified by the flake's `checks` output can be
+built successfully.
+
+# Evaluation checks
+
+This following flake output attributes must be derivations:
+
+* `checks.`*system*`.`*name*
+* `defaultPackage.`*system*`
+* `devShell.`*system*`
+* `nixosConfigurations.`*name*`.config.system.build.toplevel
+* `packages.`*system*`.`*name*
+
+The following flake output attributes must be [app
+definitions](./nix3-run.md):
+
+* `apps.`*system*`.`*name*
+* `defaultApp.`*system*`
+
+The following flake output attributes must be [template
+definitions](./nix3-flake-init.md):
+
+* `defaultTemplate`
+* `templates`.`*name*
+
+The following flake output attributes must be *Nixpkgs overlays*:
+
+* `overlay`
+* `overlays`.`*name*
+
+The following flake output attributes must be *NixOS modules*:
+
+* `nixosModule`
+* `nixosModules`.`*name*
+
+The following flake output attributes must be
+[bundlers](./nix3-bundle.md):
+
+* `bundlers`.`*name*
+* `defaultBundler`
+
+In addition, the `hydraJobs` output is evaluated in the same way as
+Hydra's `hydra-eval-jobs` (i.e. as a arbitrarily deeply nested
+attribute set of derivations). Similarly, the
+`legacyPackages`.*system* output is evaluated like `nix-env -qa`.
+
+)""
diff --git a/src/nix/flake-clone.md b/src/nix/flake-clone.md
new file mode 100644
index 000000000..36cb96051
--- /dev/null
+++ b/src/nix/flake-clone.md
@@ -0,0 +1,18 @@
+R""(
+
+# Examples
+
+* Check out the source code of the `dwarffs` flake and build it:
+
+ ```console
+ # nix flake clone dwarffs --dest dwarffs
+ # cd dwarffs
+ # nix build
+ ```
+
+# Description
+
+This command performs a Git or Mercurial clone of the repository
+containing the source code of the flake *flake-url*.
+
+)""
diff --git a/src/nix/flake-init.md b/src/nix/flake-init.md
new file mode 100644
index 000000000..890038016
--- /dev/null
+++ b/src/nix/flake-init.md
@@ -0,0 +1,54 @@
+R""(
+
+# Examples
+
+* Create a flake using the default template:
+
+ ```console
+ # nix flake init
+ ```
+
+* List available templates:
+
+ ```console
+ # nix flake show templates
+ ```
+
+* Create a flake from a specific template:
+
+ ```console
+ # nix flake init -t templates#simpleContainer
+ ```
+
+# Description
+
+This command creates a flake in the current directory by copying the
+files of a template. It will not overwrite existing files. The default
+template is `templates#defaultTemplate`, but this can be overridden
+using `-t`.
+
+# Template definitions
+
+A flake can declare templates through its `templates` and
+`defaultTemplate` output attributes. A template has two attributes:
+
+* `description`: A one-line description of the template, in CommonMark
+ syntax.
+
+* `path`: The path of the directory to be copied.
+
+Here is an example:
+
+```
+outputs = { self }: {
+
+ templates.rust = {
+ path = ./rust;
+ description = "A simple Rust/Cargo project";
+ };
+
+ templates.defaultTemplate = self.templates.rust;
+}
+```
+
+)""
diff --git a/src/nix/flake-lock.md b/src/nix/flake-lock.md
new file mode 100644
index 000000000..2af0ad81e
--- /dev/null
+++ b/src/nix/flake-lock.md
@@ -0,0 +1,38 @@
+R""(
+
+# Examples
+
+* Update the `nixpkgs` and `nix` inputs of the flake in the current
+ directory:
+
+ ```console
+ # nix flake lock --update-input nixpkgs --update-input nix
+ * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c'
+ * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293'
+ ```
+
+# Description
+
+This command updates the lock file of a flake (`flake.lock`) so that
+it contains a lock for every flake input specified in
+`flake.nix`. Existing lock file entries are not updated unless
+required by a flag such as `--update-input`.
+
+Note that every command that operates on a flake will also update the
+lock file if needed, and supports the same flags. Therefore,
+
+```console
+# nix flake lock --update-input nixpkgs
+# nix build
+```
+
+is equivalent to:
+
+```console
+# nix build --update-input nixpkgs
+```
+
+Thus, this command is only useful if you want to update the lock file
+separately from any other action such as building.
+
+)""
diff --git a/src/nix/flake-metadata.md b/src/nix/flake-metadata.md
new file mode 100644
index 000000000..5a009409b
--- /dev/null
+++ b/src/nix/flake-metadata.md
@@ -0,0 +1,110 @@
+R""(
+
+# Examples
+
+* Show what `nixpkgs` resolves to:
+
+ ```console
+ # nix flake metadata nixpkgs
+ Resolved URL: github:edolstra/dwarffs
+ Locked URL: github:edolstra/dwarffs/f691e2c991e75edb22836f1dbe632c40324215c5
+ Description: A filesystem that fetches DWARF debug info from the Internet on demand
+ Path: /nix/store/769s05vjydmc2lcf6b02az28wsa9ixh1-source
+ Revision: f691e2c991e75edb22836f1dbe632c40324215c5
+ Last modified: 2021-01-21 15:41:26
+ Inputs:
+ ├───nix: github:NixOS/nix/6254b1f5d298ff73127d7b0f0da48f142bdc753c
+ │ ├───lowdown-src: github:kristapsdz/lowdown/1705b4a26fbf065d9574dce47a94e8c7c79e052f
+ │ └───nixpkgs: github:NixOS/nixpkgs/ad0d20345219790533ebe06571f82ed6b034db31
+ └───nixpkgs follows input 'nix/nixpkgs'
+ ```
+
+* Show information about `dwarffs` in JSON format:
+
+ ```console
+ # nix flake metadata dwarffs --json | jq .
+ {
+ "description": "A filesystem that fetches DWARF debug info from the Internet on demand",
+ "lastModified": 1597153508,
+ "locked": {
+ "lastModified": 1597153508,
+ "narHash": "sha256-VHg3MYVgQ12LeRSU2PSoDeKlSPD8PYYEFxxwkVVDRd0=",
+ "owner": "edolstra",
+ "repo": "dwarffs",
+ "rev": "d181d714fd36eb06f4992a1997cd5601e26db8f5",
+ "type": "github"
+ },
+ "locks": { ... },
+ "original": {
+ "id": "dwarffs",
+ "type": "indirect"
+ },
+ "originalUrl": "flake:dwarffs",
+ "path": "/nix/store/hang3792qwdmm2n0d9nsrs5n6bsws6kv-source",
+ "resolved": {
+ "owner": "edolstra",
+ "repo": "dwarffs",
+ "type": "github"
+ },
+ "resolvedUrl": "github:edolstra/dwarffs",
+ "revision": "d181d714fd36eb06f4992a1997cd5601e26db8f5",
+ "url": "github:edolstra/dwarffs/d181d714fd36eb06f4992a1997cd5601e26db8f5"
+ }
+ ```
+
+# Description
+
+This command shows information about the flake specified by the flake
+reference *flake-url*. It resolves the flake reference using the
+[flake registry](./nix3-registry.md), fetches it, and prints some meta
+data. This includes:
+
+* `Resolved URL`: If *flake-url* is a flake identifier, then this is
+ the flake reference that specifies its actual location, looked up in
+ the flake registry.
+
+* `Locked URL`: A flake reference that contains a commit or content
+ hash and thus uniquely identifies a specific flake version.
+
+* `Description`: A one-line description of the flake, taken from the
+ `description` field in `flake.nix`.
+
+* `Path`: The store path containing the source code of the flake.
+
+* `Revision`: The Git or Mercurial commit hash of the locked flake.
+
+* `Revisions`: The number of ancestors of the Git or Mercurial commit
+ of the locked flake. Note that this is not available for `github`
+ flakes.
+
+* `Last modified`: For Git or Mercurial flakes, this is the commit
+ time of the commit of the locked flake; for tarball flakes, it's the
+ most recent timestamp of any file inside the tarball.
+
+* `Inputs`: The flake inputs with their corresponding lock file
+ entries.
+
+With `--json`, the output is a JSON object with the following fields:
+
+* `original` and `originalUrl`: The flake reference specified by the
+ user (*flake-url*) in attribute set and URL representation.
+
+* `resolved` and `resolvedUrl`: The resolved flake reference (see
+ above) in attribute set and URL representation.
+
+* `locked` and `lockedUrl`: The locked flake reference (see above) in
+ attribute set and URL representation.
+
+* `description`: See `Description` above.
+
+* `path`: See `Path` above.
+
+* `revision`: See `Revision` above.
+
+* `revCount`: See `Revisions` above.
+
+* `lastModified`: See `Last modified` above.
+
+* `locks`: The contents of `flake.lock`.
+
+)""
diff --git a/src/nix/flake-new.md b/src/nix/flake-new.md
new file mode 100644
index 000000000..725695c01
--- /dev/null
+++ b/src/nix/flake-new.md
@@ -0,0 +1,34 @@
+R""(
+
+# Examples
+
+* Create a flake using the default template in the directory `hello`:
+
+ ```console
+ # nix flake new hello
+ ```
+
+* List available templates:
+
+ ```console
+ # nix flake show templates
+ ```
+
+* Create a flake from a specific template in the directory `hello`:
+
+ ```console
+ # nix flake new hello -t templates#trivial
+ ```
+
+# Description
+
+This command creates a flake in the directory `dest-dir`, which must
+not already exist. It's equivalent to:
+
+```console
+# mkdir dest-dir
+# cd dest-dir
+# nix flake init
+```
+
+)""
diff --git a/src/nix/flake-prefetch.md b/src/nix/flake-prefetch.md
new file mode 100644
index 000000000..a1cf0289a
--- /dev/null
+++ b/src/nix/flake-prefetch.md
@@ -0,0 +1,28 @@
+R""(
+
+# Examples
+
+* Download a tarball and unpack it:
+
+ ```console
+ # nix flake prefetch https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz
+ Downloaded 'https://cdn.kernel.org/pub/linux/kernel/v5.x/linux-5.10.5.tar.xz?narHash=sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY='
+ to '/nix/store/sl5vvk8mb4ma1sjyy03kwpvkz50hd22d-source' (hash
+ 'sha256-3XYHZANT6AFBV0BqegkAZHbba6oeDkIUCDwbATLMhAY=').
+ ```
+
+* Download the `dwarffs` flake (looked up in the flake registry):
+
+ ```console
+ # nix flake prefetch dwarffs --json
+ {"hash":"sha256-VHg3MYVgQ12LeRSU2PSoDeKlSPD8PYYEFxxwkVVDRd0="
+ ,"storePath":"/nix/store/hang3792qwdmm2n0d9nsrs5n6bsws6kv-source"}
+ ```
+
+# Description
+
+This command downloads the source tree denoted by flake reference
+*flake-url*. Note that this does not need to be a flake (i.e. it does
+not have to contain a `flake.nix` file).
+
+)""
diff --git a/src/nix/flake-show.md b/src/nix/flake-show.md
new file mode 100644
index 000000000..1a42c44a0
--- /dev/null
+++ b/src/nix/flake-show.md
@@ -0,0 +1,38 @@
+R""(
+
+# Examples
+
+* Show the output attributes provided by the `patchelf` flake:
+
+ ```console
+ github:NixOS/patchelf/f34751b88bd07d7f44f5cd3200fb4122bf916c7e
+ ├───checks
+ │ ├───aarch64-linux
+ │ │ └───build: derivation 'patchelf-0.12.20201207.f34751b'
+ │ ├───i686-linux
+ │ │ └───build: derivation 'patchelf-0.12.20201207.f34751b'
+ │ └───x86_64-linux
+ │ └───build: derivation 'patchelf-0.12.20201207.f34751b'
+ ├───defaultPackage
+ │ ├───aarch64-linux: package 'patchelf-0.12.20201207.f34751b'
+ │ ├───i686-linux: package 'patchelf-0.12.20201207.f34751b'
+ │ └───x86_64-linux: package 'patchelf-0.12.20201207.f34751b'
+ ├───hydraJobs
+ │ ├───build
+ │ │ ├───aarch64-linux: derivation 'patchelf-0.12.20201207.f34751b'
+ │ │ ├───i686-linux: derivation 'patchelf-0.12.20201207.f34751b'
+ │ │ └───x86_64-linux: derivation 'patchelf-0.12.20201207.f34751b'
+ │ ├───coverage: derivation 'patchelf-coverage-0.12.20201207.f34751b'
+ │ ├───release: derivation 'patchelf-0.12.20201207.f34751b'
+ │ └───tarball: derivation 'patchelf-tarball-0.12.20201207.f34751b'
+ └───overlay: Nixpkgs overlay
+ ```
+
+# Description
+
+This command shows the output attributes provided by the flake
+specified by flake reference *flake-url*. These are the top-level
+attributes in the `outputs` of the flake, as well as lower-level
+attributes for some standard outputs (e.g. `packages` or `checks`).
+
+)""
diff --git a/src/nix/flake-update.md b/src/nix/flake-update.md
new file mode 100644
index 000000000..03b50e38e
--- /dev/null
+++ b/src/nix/flake-update.md
@@ -0,0 +1,34 @@
+R""(
+
+# Examples
+
+* Recreate the lock file (i.e. update all inputs) and commit the new
+ lock file:
+
+ ```console
+ # nix flake update
+ * Updated 'nix': 'github:NixOS/nix/9fab14adbc3810d5cc1f88672fde1eee4358405c' -> 'github:NixOS/nix/8927cba62f5afb33b01016d5c4f7f8b7d0adde3c'
+ * Updated 'nixpkgs': 'github:NixOS/nixpkgs/3d2d8f281a27d466fa54b469b5993f7dde198375' -> 'github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293'
+ …
+ warning: committed new revision '158bcbd9d6cc08ab859c0810186c1beebc982aad'
+ ```
+
+# Description
+
+This command recreates the lock file of a flake (`flake.lock`), thus
+updating the lock for every mutable input (like `nixpkgs`) to its
+current version. This is equivalent to passing `--recreate-lock-file`
+to any command that operates on a flake. That is,
+
+```console
+# nix flake update
+# nix build
+```
+
+is equivalent to:
+
+```console
+# nix build --recreate-lock-file
+```
+
+)""
diff --git a/src/nix/flake.cc b/src/nix/flake.cc
index d45f13029..62a413e27 100644
--- a/src/nix/flake.cc
+++ b/src/nix/flake.cc
@@ -7,6 +7,7 @@
#include "get-drvs.hh"
#include "store-api.hh"
#include "derivations.hh"
+#include "path-with-outputs.hh"
#include "attr-path.hh"
#include "fetchers.hh"
#include "registry.hh"
@@ -43,12 +44,6 @@ public:
return parseFlakeRef(flakeUrl, absPath(".")); //FIXME
}
- Flake getFlake()
- {
- auto evalState = getEvalState();
- return flake::getFlake(*evalState, getFlakeRef(), lockFlags.useRegistries);
- }
-
LockedFlake lockFlake()
{
return flake::lockFlake(*getEvalState(), getFlakeRef(), lockFlags);
@@ -60,55 +55,66 @@ public:
}
};
-static void printFlakeInfo(const Store & store, const Flake & flake)
+struct CmdFlakeUpdate : FlakeCommand
{
- logger->stdout("Resolved URL: %s", flake.resolvedRef.to_string());
- logger->stdout("Locked URL: %s", flake.lockedRef.to_string());
- if (flake.description)
- logger->stdout("Description: %s", *flake.description);
- logger->stdout("Path: %s", store.printStorePath(flake.sourceInfo->storePath));
- if (auto rev = flake.lockedRef.input.getRev())
- logger->stdout("Revision: %s", rev->to_string(Base16, false));
- if (auto revCount = flake.lockedRef.input.getRevCount())
- logger->stdout("Revisions: %s", *revCount);
- if (auto lastModified = flake.lockedRef.input.getLastModified())
- logger->stdout("Last modified: %s",
- std::put_time(std::localtime(&*lastModified), "%F %T"));
-}
+ std::string description() override
+ {
+ return "update flake lock file";
+ }
-static nlohmann::json flakeToJson(const Store & store, const Flake & flake)
-{
- nlohmann::json j;
- if (flake.description)
- j["description"] = *flake.description;
- j["originalUrl"] = flake.originalRef.to_string();
- j["original"] = attrsToJson(flake.originalRef.toAttrs());
- j["resolvedUrl"] = flake.resolvedRef.to_string();
- j["resolved"] = attrsToJson(flake.resolvedRef.toAttrs());
- j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl
- j["locked"] = attrsToJson(flake.lockedRef.toAttrs());
- if (auto rev = flake.lockedRef.input.getRev())
- j["revision"] = rev->to_string(Base16, false);
- if (auto revCount = flake.lockedRef.input.getRevCount())
- j["revCount"] = *revCount;
- if (auto lastModified = flake.lockedRef.input.getLastModified())
- j["lastModified"] = *lastModified;
- j["path"] = store.printStorePath(flake.sourceInfo->storePath);
- return j;
-}
+ CmdFlakeUpdate()
+ {
+ /* Remove flags that don't make sense. */
+ removeFlag("recreate-lock-file");
+ removeFlag("update-input");
+ removeFlag("no-update-lock-file");
+ removeFlag("no-write-lock-file");
+ }
-struct CmdFlakeUpdate : FlakeCommand
+ std::string doc() override
+ {
+ return
+ #include "flake-update.md"
+ ;
+ }
+
+ void run(nix::ref<nix::Store> store) override
+ {
+ settings.tarballTtl = 0;
+
+ lockFlags.recreateLockFile = true;
+ lockFlags.writeLockFile = true;
+
+ lockFlake();
+ }
+};
+
+struct CmdFlakeLock : FlakeCommand
{
std::string description() override
{
- return "update flake lock file";
+ return "create missing lock file entries";
+ }
+
+ CmdFlakeLock()
+ {
+ /* Remove flags that don't make sense. */
+ removeFlag("no-write-lock-file");
+ }
+
+ std::string doc() override
+ {
+ return
+ #include "flake-lock.md"
+ ;
}
void run(nix::ref<nix::Store> store) override
{
- /* Use --refresh by default for 'nix flake update'. */
settings.tarballTtl = 0;
+ lockFlags.writeLockFile = true;
+
lockFlake();
}
};
@@ -127,40 +133,72 @@ static void enumerateOutputs(EvalState & state, Value & vFlake,
callback(attr.name, *attr.value, *attr.pos);
}
-struct CmdFlakeInfo : FlakeCommand, MixJSON
+struct CmdFlakeMetadata : FlakeCommand, MixJSON
{
std::string description() override
{
- return "list info about a given flake";
+ return "show flake metadata";
}
- void run(nix::ref<nix::Store> store) override
+ std::string doc() override
{
- auto flake = getFlake();
-
- if (json) {
- auto json = flakeToJson(*store, flake);
- logger->stdout("%s", json.dump());
- } else
- printFlakeInfo(*store, flake);
- }
-};
-
-struct CmdFlakeListInputs : FlakeCommand, MixJSON
-{
- std::string description() override
- {
- return "list flake inputs";
+ return
+ #include "flake-metadata.md"
+ ;
}
void run(nix::ref<nix::Store> store) override
{
- auto flake = lockFlake();
+ auto lockedFlake = lockFlake();
+ auto & flake = lockedFlake.flake;
- if (json)
- logger->stdout("%s", flake.lockFile.toJson());
- else {
- logger->stdout("%s", flake.flake.lockedRef);
+ if (json) {
+ nlohmann::json j;
+ if (flake.description)
+ j["description"] = *flake.description;
+ j["originalUrl"] = flake.originalRef.to_string();
+ j["original"] = fetchers::attrsToJSON(flake.originalRef.toAttrs());
+ j["resolvedUrl"] = flake.resolvedRef.to_string();
+ j["resolved"] = fetchers::attrsToJSON(flake.resolvedRef.toAttrs());
+ j["url"] = flake.lockedRef.to_string(); // FIXME: rename to lockedUrl
+ j["locked"] = fetchers::attrsToJSON(flake.lockedRef.toAttrs());
+ if (auto rev = flake.lockedRef.input.getRev())
+ j["revision"] = rev->to_string(Base16, false);
+ if (auto revCount = flake.lockedRef.input.getRevCount())
+ j["revCount"] = *revCount;
+ if (auto lastModified = flake.lockedRef.input.getLastModified())
+ j["lastModified"] = *lastModified;
+ j["path"] = store->printStorePath(flake.sourceInfo->storePath);
+ j["locks"] = lockedFlake.lockFile.toJSON();
+ logger->cout("%s", j.dump());
+ } else {
+ logger->cout(
+ ANSI_BOLD "Resolved URL:" ANSI_NORMAL " %s",
+ flake.resolvedRef.to_string());
+ logger->cout(
+ ANSI_BOLD "Locked URL:" ANSI_NORMAL " %s",
+ flake.lockedRef.to_string());
+ if (flake.description)
+ logger->cout(
+ ANSI_BOLD "Description:" ANSI_NORMAL " %s",
+ *flake.description);
+ logger->cout(
+ ANSI_BOLD "Path:" ANSI_NORMAL " %s",
+ store->printStorePath(flake.sourceInfo->storePath));
+ if (auto rev = flake.lockedRef.input.getRev())
+ logger->cout(
+ ANSI_BOLD "Revision:" ANSI_NORMAL " %s",
+ rev->to_string(Base16, false));
+ if (auto revCount = flake.lockedRef.input.getRevCount())
+ logger->cout(
+ ANSI_BOLD "Revisions:" ANSI_NORMAL " %s",
+ *revCount);
+ if (auto lastModified = flake.lockedRef.input.getLastModified())
+ logger->cout(
+ ANSI_BOLD "Last modified:" ANSI_NORMAL " %s",
+ std::put_time(std::localtime(&*lastModified), "%F %T"));
+
+ logger->cout(ANSI_BOLD "Inputs:" ANSI_NORMAL);
std::unordered_set<std::shared_ptr<Node>> visited;
@@ -172,27 +210,36 @@ struct CmdFlakeListInputs : FlakeCommand, MixJSON
bool last = i + 1 == node.inputs.size();
if (auto lockedNode = std::get_if<0>(&input.second)) {
- logger->stdout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s",
+ logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL ": %s",
prefix + (last ? treeLast : treeConn), input.first,
- *lockedNode ? (*lockedNode)->lockedRef : flake.flake.lockedRef);
+ *lockedNode ? (*lockedNode)->lockedRef : flake.lockedRef);
bool firstVisit = visited.insert(*lockedNode).second;
if (firstVisit) recurse(**lockedNode, prefix + (last ? treeNull : treeLine));
} else if (auto follows = std::get_if<1>(&input.second)) {
- logger->stdout("%s" ANSI_BOLD "%s" ANSI_NORMAL " follows input '%s'",
+ logger->cout("%s" ANSI_BOLD "%s" ANSI_NORMAL " follows input '%s'",
prefix + (last ? treeLast : treeConn), input.first,
printInputPath(*follows));
}
}
};
- visited.insert(flake.lockFile.root);
- recurse(*flake.lockFile.root, "");
+ visited.insert(lockedFlake.lockFile.root);
+ recurse(*lockedFlake.lockFile.root, "");
}
}
};
+struct CmdFlakeInfo : CmdFlakeMetadata
+{
+ void run(nix::ref<nix::Store> store) override
+ {
+ warn("'nix flake info' is a deprecated alias for 'nix flake metadata'");
+ CmdFlakeMetadata::run(store);
+ }
+};
+
struct CmdFlakeCheck : FlakeCommand
{
bool build = true;
@@ -201,7 +248,7 @@ struct CmdFlakeCheck : FlakeCommand
{
addFlag({
.longName = "no-build",
- .description = "do not build checks",
+ .description = "Do not build checks.",
.handler = {&build, false}
});
}
@@ -211,6 +258,13 @@ struct CmdFlakeCheck : FlakeCommand
return "check whether the flake evaluates and run its tests";
}
+ std::string doc() override
+ {
+ return
+ #include "flake-check.md"
+ ;
+ }
+
void run(nix::ref<nix::Store> store) override
{
settings.readOnlyMode = !build;
@@ -239,7 +293,7 @@ struct CmdFlakeCheck : FlakeCommand
}
};
- std::vector<StorePathWithOutputs> drvPaths;
+ std::vector<DerivedPath> drvPaths;
auto checkApp = [&](const std::string & attrPath, Value & v, const Pos & pos) {
try {
@@ -260,7 +314,7 @@ struct CmdFlakeCheck : FlakeCommand
auto checkOverlay = [&](const std::string & attrPath, Value & v, const Pos & pos) {
try {
state->forceValue(v, pos);
- if (v.type != tLambda || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final")
+ if (!v.isLambda() || v.lambda.fun->matchAttrs || std::string(v.lambda.fun->arg) != "final")
throw Error("overlay does not take an argument named 'final'");
auto body = dynamic_cast<ExprLambda *>(v.lambda.fun->body);
if (!body || body->matchAttrs || std::string(body->arg) != "prev")
@@ -276,10 +330,10 @@ struct CmdFlakeCheck : FlakeCommand
auto checkModule = [&](const std::string & attrPath, Value & v, const Pos & pos) {
try {
state->forceValue(v, pos);
- if (v.type == tLambda) {
+ if (v.isLambda()) {
if (!v.lambda.fun->matchAttrs || !v.lambda.fun->formals->ellipsis)
throw Error("module must match an open attribute set ('{ config, ... }')");
- } else if (v.type == tAttrs) {
+ } else if (v.type() == nAttrs) {
for (auto & attr : *v.attrs)
try {
state->forceValue(*attr.value, *attr.pos);
@@ -371,7 +425,7 @@ struct CmdFlakeCheck : FlakeCommand
auto checkBundler = [&](const std::string & attrPath, Value & v, const Pos & pos) {
try {
state->forceValue(v, pos);
- if (v.type != tLambda)
+ if (!v.isLambda())
throw Error("bundler must be a function");
if (!v.lambda.fun->formals ||
v.lambda.fun->formals->argNames.find(state->symbols.create("program")) == v.lambda.fun->formals->argNames.end() ||
@@ -408,7 +462,7 @@ struct CmdFlakeCheck : FlakeCommand
fmt("%s.%s.%s", name, attr.name, attr2.name),
*attr2.value, *attr2.pos);
if ((std::string) attr.name == settings.thisSystem.get())
- drvPaths.push_back({drvPath});
+ drvPaths.push_back(DerivedPath::Built{drvPath});
}
}
}
@@ -545,7 +599,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
addFlag({
.longName = "template",
.shortName = 't',
- .description = "the template to use",
+ .description = "The template to use.",
.labels = {"template"},
.handler = {&templateUrl},
.completer = {[&](size_t, std::string_view prefix) {
@@ -567,7 +621,7 @@ struct CmdFlakeInitCommon : virtual Args, EvalCommand
auto [templateFlakeRef, templateName] = parseFlakeRefWithFragment(templateUrl, absPath("."));
- auto installable = InstallableFlake(
+ auto installable = InstallableFlake(nullptr,
evalState, std::move(templateFlakeRef),
Strings{templateName == "" ? "defaultTemplate" : templateName},
Strings(attrsPathPrefixes), lockFlags);
@@ -631,22 +685,11 @@ struct CmdFlakeInit : CmdFlakeInitCommon
return "create a flake in the current directory from a template";
}
- Examples examples() override
- {
- return {
- Example{
- "To create a flake using the default template:",
- "nix flake init"
- },
- Example{
- "To see available templates:",
- "nix flake show templates"
- },
- Example{
- "To create a flake from a specific template:",
- "nix flake init -t templates#nixos-container"
- },
- };
+ std::string doc() override
+ {
+ return
+ #include "flake-init.md"
+ ;
}
CmdFlakeInit()
@@ -662,6 +705,13 @@ struct CmdFlakeNew : CmdFlakeInitCommon
return "create a flake in the specified directory from a template";
}
+ std::string doc() override
+ {
+ return
+ #include "flake-new.md"
+ ;
+ }
+
CmdFlakeNew()
{
expectArgs({
@@ -681,12 +731,19 @@ struct CmdFlakeClone : FlakeCommand
return "clone flake repository";
}
+ std::string doc() override
+ {
+ return
+ #include "flake-clone.md"
+ ;
+ }
+
CmdFlakeClone()
{
addFlag({
.longName = "dest",
.shortName = 'f',
- .description = "destination path",
+ .description = "Clone the flake to path *dest*.",
.labels = {"path"},
.handler = {&destDir}
});
@@ -720,22 +777,11 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun
return "copy a flake and all its inputs to a store";
}
- Examples examples() override
- {
- return {
- Example{
- "To copy the dwarffs flake and its dependencies to a binary cache:",
- "nix flake archive --to file:///tmp/my-cache dwarffs"
- },
- Example{
- "To fetch the dwarffs flake and its dependencies to the local Nix store:",
- "nix flake archive dwarffs"
- },
- Example{
- "To print the store paths of the flake sources of NixOps without fetching them:",
- "nix flake archive --json --dry-run nixops"
- },
- };
+ std::string doc() override
+ {
+ return
+ #include "flake-archive.md"
+ ;
}
void run(nix::ref<nix::Store> store) override
@@ -787,7 +833,7 @@ struct CmdFlakeShow : FlakeCommand
{
addFlag({
.longName = "legacy",
- .description = "show the contents of the 'legacyPackages' output",
+ .description = "Show the contents of the `legacyPackages` output.",
.handler = {&showLegacy, true}
});
}
@@ -797,6 +843,13 @@ struct CmdFlakeShow : FlakeCommand
return "show the outputs provided by a flake";
}
+ std::string doc() override
+ {
+ return
+ #include "flake-show.md"
+ ;
+ }
+
void run(nix::ref<nix::Store> store) override
{
auto state = getEvalState();
@@ -811,7 +864,7 @@ struct CmdFlakeShow : FlakeCommand
try {
auto recurse = [&]()
{
- logger->stdout("%s", headerPrefix);
+ logger->cout("%s", headerPrefix);
auto attrs = visitor.getAttrs();
for (const auto & [i, attr] : enumerate(attrs)) {
bool last = i + 1 == attrs.size();
@@ -837,7 +890,7 @@ struct CmdFlakeShow : FlakeCommand
}
*/
- logger->stdout("%s: %s '%s'",
+ logger->cout("%s: %s '%s'",
headerPrefix,
attrPath.size() == 2 && attrPath[0] == "devShell" ? "development environment" :
attrPath.size() == 3 && attrPath[0] == "checks" ? "derivation" :
@@ -853,7 +906,8 @@ struct CmdFlakeShow : FlakeCommand
|| attrPath[0] == "nixosConfigurations"
|| attrPath[0] == "nixosModules"
|| attrPath[0] == "defaultApp"
- || attrPath[0] == "templates"))
+ || attrPath[0] == "templates"
+ || attrPath[0] == "overlays"))
|| ((attrPath.size() == 1 || attrPath.size() == 2)
&& (attrPath[0] == "checks"
|| attrPath[0] == "packages"
@@ -885,7 +939,7 @@ struct CmdFlakeShow : FlakeCommand
if (attrPath.size() == 1)
recurse();
else if (!showLegacy)
- logger->stdout("%s: " ANSI_YELLOW "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix);
+ logger->cout("%s: " ANSI_YELLOW "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix);
else {
if (visitor.isDerivation())
showDerivation();
@@ -902,7 +956,7 @@ struct CmdFlakeShow : FlakeCommand
auto aType = visitor.maybeGetAttr("type");
if (!aType || aType->getString() != "app")
throw EvalError("not an app definition");
- logger->stdout("%s: app", headerPrefix);
+ logger->cout("%s: app", headerPrefix);
}
else if (
@@ -910,13 +964,14 @@ struct CmdFlakeShow : FlakeCommand
(attrPath.size() == 2 && attrPath[0] == "templates"))
{
auto description = visitor.getAttr("description")->getString();
- logger->stdout("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description);
+ logger->cout("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description);
}
else {
- logger->stdout("%s: %s",
+ logger->cout("%s: %s",
headerPrefix,
- attrPath.size() == 1 && attrPath[0] == "overlay" ? "Nixpkgs overlay" :
+ (attrPath.size() == 1 && attrPath[0] == "overlay")
+ || (attrPath.size() == 2 && attrPath[0] == "overlays") ? "Nixpkgs overlay" :
attrPath.size() == 2 && attrPath[0] == "nixosConfigurations" ? "NixOS configuration" :
attrPath.size() == 2 && attrPath[0] == "nixosModules" ? "NixOS module" :
ANSI_YELLOW "unknown" ANSI_NORMAL);
@@ -933,19 +988,60 @@ struct CmdFlakeShow : FlakeCommand
}
};
+struct CmdFlakePrefetch : FlakeCommand, MixJSON
+{
+ CmdFlakePrefetch()
+ {
+ }
+
+ std::string description() override
+ {
+ return "download the source tree denoted by a flake reference into the Nix store";
+ }
+
+ std::string doc() override
+ {
+ return
+ #include "flake-prefetch.md"
+ ;
+ }
+
+ void run(ref<Store> store) override
+ {
+ auto originalRef = getFlakeRef();
+ auto resolvedRef = originalRef.resolve(store);
+ auto [tree, lockedRef] = resolvedRef.fetchTree(store);
+ auto hash = store->queryPathInfo(tree.storePath)->narHash;
+
+ if (json) {
+ auto res = nlohmann::json::object();
+ res["storePath"] = store->printStorePath(tree.storePath);
+ res["hash"] = hash.to_string(SRI, true);
+ logger->cout(res.dump());
+ } else {
+ notice("Downloaded '%s' to '%s' (hash '%s').",
+ lockedRef.to_string(),
+ store->printStorePath(tree.storePath),
+ hash.to_string(SRI, true));
+ }
+ }
+};
+
struct CmdFlake : NixMultiCommand
{
CmdFlake()
: MultiCommand({
{"update", []() { return make_ref<CmdFlakeUpdate>(); }},
+ {"lock", []() { return make_ref<CmdFlakeLock>(); }},
+ {"metadata", []() { return make_ref<CmdFlakeMetadata>(); }},
{"info", []() { return make_ref<CmdFlakeInfo>(); }},
- {"list-inputs", []() { return make_ref<CmdFlakeListInputs>(); }},
{"check", []() { return make_ref<CmdFlakeCheck>(); }},
{"init", []() { return make_ref<CmdFlakeInit>(); }},
{"new", []() { return make_ref<CmdFlakeNew>(); }},
{"clone", []() { return make_ref<CmdFlakeClone>(); }},
{"archive", []() { return make_ref<CmdFlakeArchive>(); }},
{"show", []() { return make_ref<CmdFlakeShow>(); }},
+ {"prefetch", []() { return make_ref<CmdFlakePrefetch>(); }},
})
{
}
@@ -955,6 +1051,13 @@ struct CmdFlake : NixMultiCommand
return "manage Nix flakes";
}
+ std::string doc() override
+ {
+ return
+ #include "flake.md"
+ ;
+ }
+
void run() override
{
if (!command)
diff --git a/src/nix/flake.md b/src/nix/flake.md
new file mode 100644
index 000000000..0035195e5
--- /dev/null
+++ b/src/nix/flake.md
@@ -0,0 +1,566 @@
+R""(
+
+# Description
+
+`nix flake` provides subcommands for creating, modifying and querying
+*Nix flakes*. Flakes are the unit for packaging Nix code in a
+reproducible and discoverable way. They can have dependencies on other
+flakes, making it possible to have multi-repository Nix projects.
+
+A flake is a filesystem tree (typically fetched from a Git repository
+or a tarball) that contains a file named `flake.nix` in the root
+directory. `flake.nix` specifies some metadata about the flake such as
+dependencies (called *inputs*), as well as its *outputs* (the Nix
+values such as packages or NixOS modules provided by the flake).
+
+# Flake references
+
+Flake references (*flakerefs*) are a way to specify the location of a
+flake. These have two different forms:
+
+* An attribute set representation, e.g.
+
+ ```nix
+ {
+ type = "github";
+ owner = "NixOS";
+ repo = "nixpkgs";
+ }
+ ```
+
+ The only required attribute is `type`. The supported types are
+ listed below.
+
+* A URL-like syntax, e.g.
+
+ ```
+ github:NixOS/nixpkgs
+ ```
+
+ These are used on the command line as a more convenient alternative
+ to the attribute set representation. For instance, in the command
+
+ ```console
+ # nix build github:NixOS/nixpkgs#hello
+ ```
+
+ `github:NixOS/nixpkgs` is a flake reference (while `hello` is an
+ output attribute). They are also allowed in the `inputs` attribute
+ of a flake, e.g.
+
+ ```nix
+ inputs.nixpkgs.url = github:NixOS/nixpkgs;
+ ```
+
+ is equivalent to
+
+ ```nix
+ inputs.nixpkgs = {
+ type = "github";
+ owner = "NixOS";
+ repo = "nixpkgs";
+ };
+ ```
+
+## Examples
+
+Here are some examples of flake references in their URL-like representation:
+
+* `.`: The flake in the current directory.
+* `/home/alice/src/patchelf`: A flake in some other directory.
+* `nixpkgs`: The `nixpkgs` entry in the flake registry.
+* `nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: The `nixpkgs`
+ entry in the flake registry, with its Git revision overridden to a
+ specific value.
+* `github:NixOS/nixpkgs`: The `master` branch of the `NixOS/nixpkgs`
+ repository on GitHub.
+* `github:NixOS/nixpkgs/nixos-20.09`: The `nixos-20.09` branch of the
+ `nixpkgs` repository.
+* `github:NixOS/nixpkgs/a3a3dda3bacf61e8a39258a0ed9c924eeca8e293`: A
+ specific revision of the `nixpkgs` repository.
+* `github:edolstra/nix-warez?dir=blender`: A flake in a subdirectory
+ of a GitHub repository.
+* `git+https://github.com/NixOS/patchelf`: A Git repository.
+* `git+https://github.com/NixOS/patchelf?ref=master`: A specific
+ branch of a Git repository.
+* `git+https://github.com/NixOS/patchelf?ref=master&rev=f34751b88bd07d7f44f5cd3200fb4122bf916c7e`:
+ A specific branch *and* revision of a Git repository.
+* `https://github.com/NixOS/patchelf/archive/master.tar.gz`: A tarball
+ flake.
+
+## Flake reference attributes
+
+The following generic flake reference attributes are supported:
+
+* `dir`: The subdirectory of the flake in which `flake.nix` is
+ located. This parameter enables having multiple flakes in a
+ repository or tarball. The default is the root directory of the
+ flake.
+
+* `narHash`: The hash of the NAR serialisation (in SRI format) of the
+ contents of the flake. This is useful for flake types such as
+ tarballs that lack a unique content identifier such as a Git commit
+ hash.
+
+In addition, the following attributes are common to several flake
+reference types:
+
+* `rev`: A Git or Mercurial commit hash.
+
+* `ref`: A Git or Mercurial branch or tag name.
+
+Finally, some attribute are typically not specified by the user, but
+can occur in *locked* flake references and are available to Nix code:
+
+* `revCount`: The number of ancestors of the commit `rev`.
+
+* `lastModified`: The timestamp (in seconds since the Unix epoch) of
+ the last modification of this version of the flake. For
+ Git/Mercurial flakes, this is the commit time of commit *rev*, while
+ for tarball flakes, it's the most recent timestamp of any file
+ inside the tarball.
+
+## Types
+
+Currently the `type` attribute can be one of the following:
+
+* `path`: arbitrary local directories, or local Git trees. The
+ required attribute `path` specifies the path of the flake. The URL
+ form is
+
+ ```
+ [path:]<path>(\?<params)?
+ ```
+
+ where *path* is an absolute path.
+
+ *path* must be a directory in the file system containing a file
+ named `flake.nix`.
+
+ If the directory or any of its parents is a Git repository, then
+ this is essentially equivalent to `git+file://<path>` (see below),
+ except that the `dir` parameter is derived automatically. For
+ example, if `/foo/bar` is a Git repository, then the flake reference
+ `/foo/bar/flake` is equivalent to `/foo/bar?dir=flake`.
+
+ If the directory is not inside a Git repository, then the flake
+ contents is the entire contents of *path*.
+
+ *path* generally must be an absolute path. However, on the command
+ line, it can be a relative path (e.g. `.` or `./foo`) which is
+ interpreted as relative to the current directory. In this case, it
+ must start with `.` to avoid ambiguity with registry lookups
+ (e.g. `nixpkgs` is a registry lookup; `./nixpkgs` is a relative
+ path).
+
+* `git`: Git repositories. The location of the repository is specified
+ by the attribute `url`.
+
+ They have the URL form
+
+ ```
+ git(+http|+https|+ssh|+git|+file|):(//<server>)?<path>(\?<params>)?
+ ```
+
+ The `ref` attribute defaults to `master`.
+
+ The `rev` attribute must denote a commit that exists in the branch
+ or tag specified by the `ref` attribute, since Nix doesn't do a full
+ clone of the remote repository by default (and the Git protocol
+ doesn't allow fetching a `rev` without a known `ref`). The default
+ is the commit currently pointed to by `ref`.
+
+ For example, the following are valid Git flake references:
+
+ * `git+https://example.org/my/repo`
+ * `git+https://example.org/my/repo?dir=flake1`
+ * `git+ssh://git@github.com/NixOS/nix?ref=v1.2.3`
+ * `git://github.com/edolstra/dwarffs?ref=unstable&rev=e486d8d40e626a20e06d792db8cc5ac5aba9a5b4`
+ * `git+file:///home/my-user/some-repo/some-repo`
+
+* `mercurial`: Mercurial repositories. The URL form is similar to the
+ `git` type, except that the URL schema must be one of `hg+http`,
+ `hg+https`, `hg+ssh` or `hg+file`.
+
+* `tarball`: Tarballs. The location of the tarball is specified by the
+ attribute `url`.
+
+ In URL form, the schema must be `http://`, `https://` or `file://`
+ URLs and the extension must be `.zip`, `.tar`, `.tar.gz`, `.tar.xz`
+ or `.tar.bz2`.
+
+* `github`: A more efficient way to fetch repositories from
+ GitHub. The following attributes are required:
+
+ * `owner`: The owner of the repository.
+
+ * `repo`: The name of the repository.
+
+ These are downloaded as tarball archives, rather than
+ through Git. This is often much faster and uses less disk space
+ since it doesn't require fetching the entire history of the
+ repository. On the other hand, it doesn't allow incremental fetching
+ (but full downloads are often faster than incremental fetches!).
+
+ The URL syntax for `github` flakes is:
+
+ ```
+ github:<owner>/<repo>(/<rev-or-ref>)?(\?<params>)?
+ ```
+
+ `<rev-or-ref>` specifies the name of a branch or tag (`ref`), or a
+ commit hash (`rev`). Note that unlike Git, GitHub allows fetching by
+ commit hash without specifying a branch or tag.
+
+ Some examples:
+
+ * `github:edolstra/dwarffs`
+ * `github:edolstra/dwarffs/unstable`
+ * `github:edolstra/dwarffs/d3f2baba8f425779026c6ec04021b2e927f61e31`
+
+* `indirect`: Indirections through the flake registry. These have the
+ form
+
+ ```
+ [flake:]<flake-id>(/<rev-or-ref>(/rev)?)?
+ ```
+
+ These perform a lookup of `<flake-id>` in the flake registry. or
+ example, `nixpkgs` and `nixpkgs/release-20.09` are indirect flake
+ references. The specified `rev` and/or `ref` are merged with the
+ entry in the registry; see [nix registry](./nix3-registry.md) for
+ details.
+
+# Flake format
+
+As an example, here is a simple `flake.nix` that depends on the
+Nixpkgs flake and provides a single package (i.e. an installable
+derivation):
+
+```nix
+{
+ description = "A flake for building Hello World";
+
+ inputs.nixpkgs.url = github:NixOS/nixpkgs/nixos-20.03;
+
+ outputs = { self, nixpkgs }: {
+
+ defaultPackage.x86_64-linux =
+ # Notice the reference to nixpkgs here.
+ with import nixpkgs { system = "x86_64-linux"; };
+ stdenv.mkDerivation {
+ name = "hello";
+ src = self;
+ buildPhase = "gcc -o hello ./hello.c";
+ installPhase = "mkdir -p $out/bin; install -t $out/bin hello";
+ };
+
+ };
+}
+```
+
+The following attributes are supported in `flake.nix`:
+
+* `description`: A short, one-line description of the flake.
+
+* `inputs`: An attrset specifying the dependencies of the flake
+ (described below).
+
+* `outputs`: A function that, given an attribute set containing the
+ outputs of each of the input flakes keyed by their identifier,
+ yields the Nix values provided by this flake. Thus, in the example
+ above, `inputs.nixpkgs` contains the result of the call to the
+ `outputs` function of the `nixpkgs` flake.
+
+ In addition to the outputs of each input, each input in `inputs`
+ also contains some metadata about the inputs. These are:
+
+ * `outPath`: The path in the Nix store of the flake's source tree.
+
+ * `rev`: The commit hash of the flake's repository, if applicable.
+
+ * `revCount`: The number of ancestors of the revision `rev`. This is
+ not available for `github` repositories, since they're fetched as
+ tarballs rather than as Git repositories.
+
+ * `lastModifiedDate`: The commit time of the revision `rev`, in the
+ format `%Y%m%d%H%M%S` (e.g. `20181231100934`). Unlike `revCount`,
+ this is available for both Git and GitHub repositories, so it's
+ useful for generating (hopefully) monotonically increasing version
+ strings.
+
+ * `lastModified`: The commit time of the revision `rev` as an integer
+ denoting the number of seconds since 1970.
+
+ * `narHash`: The SHA-256 (in SRI format) of the NAR serialization of
+ the flake's source tree.
+
+ The value returned by the `outputs` function must be an attribute
+ set. The attributes can have arbitrary values; however, various
+ `nix` subcommands require specific attributes to have a specific
+ value (e.g. `packages.x86_64-linux` must be an attribute set of
+ derivations built for the `x86_64-linux` platform).
+
+## Flake inputs
+
+The attribute `inputs` specifies the dependencies of a flake, as an
+attrset mapping input names to flake references. For example, the
+following specifies a dependency on the `nixpkgs` and `import-cargo`
+repositories:
+
+```nix
+# A GitHub repository.
+inputs.import-cargo = {
+ type = "github";
+ owner = "edolstra";
+ repo = "import-cargo";
+};
+
+# An indirection through the flake registry.
+inputs.nixpkgs = {
+ type = "indirect";
+ id = "nixpkgs";
+};
+```
+
+Alternatively, you can use the URL-like syntax:
+
+```nix
+inputs.import-cargo.url = github:edolstra/import-cargo;
+inputs.nixpkgs.url = "nixpkgs";
+```
+
+Each input is fetched, evaluated and passed to the `outputs` function
+as a set of attributes with the same name as the corresponding
+input. The special input named `self` refers to the outputs and source
+tree of *this* flake. Thus, a typical `outputs` function looks like
+this:
+
+```nix
+outputs = { self, nixpkgs, import-cargo }: {
+ ... outputs ...
+};
+```
+
+It is also possible to omit an input entirely and *only* list it as
+expected function argument to `outputs`. Thus,
+
+```nix
+outputs = { self, nixpkgs }: ...;
+```
+
+without an `inputs.nixpkgs` attribute is equivalent to
+
+```nix
+inputs.nixpkgs = {
+ type = "indirect";
+ id = "nixpkgs";
+};
+```
+
+Repositories that don't contain a `flake.nix` can also be used as
+inputs, by setting the input's `flake` attribute to `false`:
+
+```nix
+inputs.grcov = {
+ type = "github";
+ owner = "mozilla";
+ repo = "grcov";
+ flake = false;
+};
+
+outputs = { self, nixpkgs, grcov }: {
+ packages.x86_64-linux.grcov = stdenv.mkDerivation {
+ src = grcov;
+ ...
+ };
+};
+```
+
+Transitive inputs can be overridden from a `flake.nix` file. For
+example, the following overrides the `nixpkgs` input of the `nixops`
+input:
+
+```nix
+inputs.nixops.inputs.nixpkgs = {
+ type = "github";
+ owner = "my-org";
+ repo = "nixpkgs";
+};
+```
+
+It is also possible to "inherit" an input from another input. This is
+useful to minimize flake dependencies. For example, the following sets
+the `nixpkgs` input of the top-level flake to be equal to the
+`nixpkgs` input of the `dwarffs` input of the top-level flake:
+
+```nix
+inputs.nixops.follows = "dwarffs/nixpkgs";
+```
+
+The value of the `follows` attribute is a `/`-separated sequence of
+input names denoting the path of inputs to be followed from the root
+flake.
+
+Overrides and `follows` can be combined, e.g.
+
+```nix
+inputs.nixops.inputs.nixpkgs.follows = "dwarffs/nixpkgs";
+```
+
+sets the `nixpkgs` input of `nixops` to be the same as the `nixpkgs`
+input of `dwarffs`. It is worth noting, however, that it is generally
+not useful to eliminate transitive `nixpkgs` flake inputs in this
+way. Most flakes provide their functionality through Nixpkgs overlays
+or NixOS modules, which are composed into the top-level flake's
+`nixpkgs` input; so their own `nixpkgs` input is usually irrelevant.
+
+# Lock files
+
+Inputs specified in `flake.nix` are typically "unlocked" in the sense
+that they don't specify an exact revision. To ensure reproducibility,
+Nix will automatically generate and use a *lock file* called
+`flake.lock` in the flake's directory. The lock file contains a graph
+structure isomorphic to the graph of dependencies of the root
+flake. Each node in the graph (except the root node) maps the
+(usually) unlocked input specifications in `flake.nix` to locked input
+specifications. Each node also contains some metadata, such as the
+dependencies (outgoing edges) of the node.
+
+For example, if `flake.nix` has the inputs in the example above, then
+the resulting lock file might be:
+
+```json
+{
+ "version": 7,
+ "root": "n1",
+ "nodes": {
+ "n1": {
+ "inputs": {
+ "nixpkgs": "n2",
+ "import-cargo": "n3",
+ "grcov": "n4"
+ }
+ },
+ "n2": {
+ "inputs": {},
+ "locked": {
+ "owner": "edolstra",
+ "repo": "nixpkgs",
+ "rev": "7f8d4b088e2df7fdb6b513bc2d6941f1d422a013",
+ "type": "github",
+ "lastModified": 1580555482,
+ "narHash": "sha256-OnpEWzNxF/AU4KlqBXM2s5PWvfI5/BS6xQrPvkF5tO8="
+ },
+ "original": {
+ "id": "nixpkgs",
+ "type": "indirect"
+ }
+ },
+ "n3": {
+ "inputs": {},
+ "locked": {
+ "owner": "edolstra",
+ "repo": "import-cargo",
+ "rev": "8abf7b3a8cbe1c8a885391f826357a74d382a422",
+ "type": "github",
+ "lastModified": 1567183309,
+ "narHash": "sha256-wIXWOpX9rRjK5NDsL6WzuuBJl2R0kUCnlpZUrASykSc="
+ },
+ "original": {
+ "owner": "edolstra",
+ "repo": "import-cargo",
+ "type": "github"
+ }
+ },
+ "n4": {
+ "inputs": {},
+ "locked": {
+ "owner": "mozilla",
+ "repo": "grcov",
+ "rev": "989a84bb29e95e392589c4e73c29189fd69a1d4e",
+ "type": "github",
+ "lastModified": 1580729070,
+ "narHash": "sha256-235uMxYlHxJ5y92EXZWAYEsEb6mm+b069GAd+BOIOxI="
+ },
+ "original": {
+ "owner": "mozilla",
+ "repo": "grcov",
+ "type": "github"
+ },
+ "flake": false
+ }
+ }
+}
+```
+
+This graph has 4 nodes: the root flake, and its 3 dependencies. The
+nodes have arbitrary labels (e.g. `n1`). The label of the root node of
+the graph is specified by the `root` attribute. Nodes contain the
+following fields:
+
+* `inputs`: The dependencies of this node, as a mapping from input
+ names (e.g. `nixpkgs`) to node labels (e.g. `n2`).
+
+* `original`: The original input specification from `flake.lock`, as a
+ set of `builtins.fetchTree` arguments.
+
+* `locked`: The locked input specification, as a set of
+ `builtins.fetchTree` arguments. Thus, in the example above, when we
+ build this flake, the input `nixpkgs` is mapped to revision
+ `7f8d4b088e2df7fdb6b513bc2d6941f1d422a013` of the `edolstra/nixpkgs`
+ repository on GitHub.
+
+ It also includes the attribute `narHash`, specifying the expected
+ contents of the tree in the Nix store (as computed by `nix
+ hash-path`), and may include input-type-specific attributes such as
+ the `lastModified` or `revCount`. The main reason for these
+ attributes is to allow flake inputs to be substituted from a binary
+ cache: `narHash` allows the store path to be computed, while the
+ other attributes are necessary because they provide information not
+ stored in the store path.
+
+* `flake`: A Boolean denoting whether this is a flake or non-flake
+ dependency. Corresponds to the `flake` attribute in the `inputs`
+ attribute in `flake.nix`.
+
+The `original` and `locked` attributes are omitted for the root
+node. This is because we cannot record the commit hash or content hash
+of the root flake, since modifying `flake.lock` will invalidate these.
+
+The graph representation of lock files allows circular dependencies
+between flakes. For example, here are two flakes that reference each
+other:
+
+```nix
+{
+ inputs.b = ... location of flake B ...;
+ # Tell the 'b' flake not to fetch 'a' again, to ensure its 'a' is
+ # *this* 'a'.
+ inputs.b.inputs.a.follows = "";
+ outputs = { self, b }: {
+ foo = 123 + b.bar;
+ xyzzy = 1000;
+ };
+}
+```
+
+and
+
+```nix
+{
+ inputs.a = ... location of flake A ...;
+ inputs.a.inputs.b.follows = "";
+ outputs = { self, a }: {
+ bar = 456 + a.xyzzy;
+ };
+}
+```
+
+Lock files transitively lock direct as well as indirect
+dependencies. That is, if a lock file exists and is up to date, Nix
+will not look at the lock files of dependencies. However, lock file
+generation itself *does* use the lock files of dependencies by
+default.
+
+)""
diff --git a/src/nix/hash.cc b/src/nix/hash.cc
index 1d23bb0e2..4535e4ab0 100644
--- a/src/nix/hash.cc
+++ b/src/nix/hash.cc
@@ -8,7 +8,7 @@
using namespace nix;
-struct CmdHash : Command
+struct CmdHashBase : Command
{
FileIngestionMethod mode;
Base base = SRI;
@@ -17,20 +17,43 @@ struct CmdHash : Command
std::vector<std::string> paths;
std::optional<std::string> modulus;
- CmdHash(FileIngestionMethod mode) : mode(mode)
+ CmdHashBase(FileIngestionMethod mode) : mode(mode)
{
- mkFlag(0, "sri", "print hash in SRI format", &base, SRI);
- mkFlag(0, "base64", "print hash in base-64", &base, Base64);
- mkFlag(0, "base32", "print hash in base-32 (Nix-specific)", &base, Base32);
- mkFlag(0, "base16", "print hash in base-16", &base, Base16);
+ addFlag({
+ .longName = "sri",
+ .description = "Print the hash in SRI format.",
+ .handler = {&base, SRI},
+ });
+
+ addFlag({
+ .longName = "base64",
+ .description = "Print the hash in base-64 format.",
+ .handler = {&base, Base64},
+ });
+
+ addFlag({
+ .longName = "base32",
+ .description = "Print the hash in base-32 (Nix-specific) format.",
+ .handler = {&base, Base32},
+ });
+
+ addFlag({
+ .longName = "base16",
+ .description = "Print the hash in base-16 format.",
+ .handler = {&base, Base16},
+ });
+
addFlag(Flag::mkHashTypeFlag("type", &ht));
+
#if 0
- mkFlag()
- .longName("modulo")
- .description("compute hash modulo specified string")
- .labels({"modulus"})
- .dest(&modulus);
- #endif
+ addFlag({
+ .longName = "modulo",
+ .description = "Compute the hash modulo the specified string.",
+ .labels = {"modulus"},
+ .handler = {&modulus},
+ });
+ #endif\
+
expectArgs({
.label = "paths",
.handler = {&paths},
@@ -40,19 +63,16 @@ struct CmdHash : Command
std::string description() override
{
- const char* d;
switch (mode) {
case FileIngestionMethod::Flat:
- d = "print cryptographic hash of a regular file";
- break;
+ return "print cryptographic hash of a regular file";
case FileIngestionMethod::Recursive:
- d = "print cryptographic hash of the NAR serialisation of a path";
+ return "print cryptographic hash of the NAR serialisation of a path";
+ default:
+ assert(false);
};
- return d;
}
- Category category() override { return catUtility; }
-
void run() override
{
for (auto path : paths) {
@@ -74,14 +94,11 @@ struct CmdHash : Command
Hash h = hashSink->finish().first;
if (truncate && h.hashSize > 20) h = compressHash(h, 20);
- logger->stdout(h.to_string(base, base == SRI));
+ logger->cout(h.to_string(base, base == SRI));
}
}
};
-static RegisterCommand rCmdHashFile("hash-file", [](){ return make_ref<CmdHash>(FileIngestionMethod::Flat); });
-static RegisterCommand rCmdHashPath("hash-path", [](){ return make_ref<CmdHash>(FileIngestionMethod::Recursive); });
-
struct CmdToBase : Command
{
Base base;
@@ -103,19 +120,43 @@ struct CmdToBase : Command
"SRI");
}
+ void run() override
+ {
+ for (auto s : args)
+ logger->cout(Hash::parseAny(s, ht).to_string(base, base == SRI));
+ }
+};
+
+struct CmdHash : NixMultiCommand
+{
+ CmdHash()
+ : MultiCommand({
+ {"file", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Flat);; }},
+ {"path", []() { return make_ref<CmdHashBase>(FileIngestionMethod::Recursive); }},
+ {"to-base16", []() { return make_ref<CmdToBase>(Base16); }},
+ {"to-base32", []() { return make_ref<CmdToBase>(Base32); }},
+ {"to-base64", []() { return make_ref<CmdToBase>(Base64); }},
+ {"to-sri", []() { return make_ref<CmdToBase>(SRI); }},
+ })
+ { }
+
+ std::string description() override
+ {
+ return "compute and convert cryptographic hashes";
+ }
+
Category category() override { return catUtility; }
void run() override
{
- for (auto s : args)
- logger->stdout(Hash::parseAny(s, ht).to_string(base, base == SRI));
+ if (!command)
+ throw UsageError("'nix hash' requires a sub-command.");
+ command->second->prepare();
+ command->second->run();
}
};
-static RegisterCommand rCmdToBase16("to-base16", [](){ return make_ref<CmdToBase>(Base16); });
-static RegisterCommand rCmdToBase32("to-base32", [](){ return make_ref<CmdToBase>(Base32); });
-static RegisterCommand rCmdToBase64("to-base64", [](){ return make_ref<CmdToBase>(Base64); });
-static RegisterCommand rCmdToSRI("to-sri", [](){ return make_ref<CmdToBase>(SRI); });
+static auto rCmdHash = registerCommand<CmdHash>("hash");
/* Legacy nix-hash command. */
static int compatNixHash(int argc, char * * argv)
@@ -149,7 +190,7 @@ static int compatNixHash(int argc, char * * argv)
});
if (op == opHash) {
- CmdHash cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive);
+ CmdHashBase cmd(flat ? FileIngestionMethod::Flat : FileIngestionMethod::Recursive);
cmd.ht = ht;
cmd.base = base32 ? Base32 : Base16;
cmd.truncate = truncate;
diff --git a/src/nix/help.md b/src/nix/help.md
new file mode 100644
index 000000000..734f35028
--- /dev/null
+++ b/src/nix/help.md
@@ -0,0 +1,17 @@
+R""(
+
+# Examples
+
+* Show help about `nix` in general:
+
+ ```console
+ # nix help
+ ```
+
+* Show help about a particular subcommand:
+
+ ```console
+ # nix help flake info
+ ```
+
+)""
diff --git a/src/nix/key-convert-secret-to-public.md b/src/nix/key-convert-secret-to-public.md
new file mode 100644
index 000000000..3adc18502
--- /dev/null
+++ b/src/nix/key-convert-secret-to-public.md
@@ -0,0 +1,19 @@
+R""(
+
+# Examples
+
+* Convert a secret key to a public key:
+
+ ```console
+ # echo cache.example.org-0:E7lAO+MsPwTFfPXsdPtW8GKui/5ho4KQHVcAGnX+Tti1V4dUxoVoqLyWJ4YESuZJwQ67GVIksDt47og+tPVUZw== \
+ | nix key convert-secret-to-public
+ cache.example.org-0:tVeHVMaFaKi8lieGBErmScEOuxlSJLA7eO6IPrT1VGc=
+ ```
+
+# Description
+
+This command reads a Ed25519 secret key from standard input, and
+writes the corresponding public key to standard output. For more
+details, see [nix key generate-secret](./nix3-key-generate-secret.md).
+
+)""
diff --git a/src/nix/key-generate-secret.md b/src/nix/key-generate-secret.md
new file mode 100644
index 000000000..4938f637c
--- /dev/null
+++ b/src/nix/key-generate-secret.md
@@ -0,0 +1,48 @@
+R""(
+
+# Examples
+
+* Generate a new secret key:
+
+ ```console
+ # nix key generate-secret --key-name cache.example.org-1 > ./secret-key
+ ```
+
+ We can then use this key to sign the closure of the Hello package:
+
+ ```console
+ # nix build nixpkgs#hello
+ # nix store sign --key-file ./secret-key --recursive ./result
+ ```
+
+ Finally, we can verify the store paths using the corresponding
+ public key:
+
+ ```
+ # nix store verify --trusted-public-keys $(nix key convert-secret-to-public < ./secret-key) ./result
+ ```
+
+# Description
+
+This command generates a new Ed25519 secret key for signing store
+paths and prints it on standard output. Use `nix key
+convert-secret-to-public` to get the corresponding public key for
+verifying signed store paths.
+
+The mandatory argument `--key-name` specifies a key name (such as
+`cache.example.org-1). It is used to look up keys on the client when
+it verifies signatures. It can be anything, but it’s suggested to use
+the host name of your cache (e.g. `cache.example.org`) with a suffix
+denoting the number of the key (to be incremented every time you need
+to revoke a key).
+
+# Format
+
+Both secret and public keys are represented as the key name followed
+by a base-64 encoding of the Ed25519 key data, e.g.
+
+```
+cache.example.org-0:E7lAO+MsPwTFfPXsdPtW8GKui/5ho4KQHVcAGnX+Tti1V4dUxoVoqLyWJ4YESuZJwQ67GVIksDt47og+tPVUZw==
+```
+
+)""
diff --git a/src/nix/local.mk b/src/nix/local.mk
index f37b73384..83b6dd08b 100644
--- a/src/nix/local.mk
+++ b/src/nix/local.mk
@@ -12,12 +12,11 @@ nix_SOURCES := \
$(wildcard src/nix-daemon/*.cc) \
$(wildcard src/nix-env/*.cc) \
$(wildcard src/nix-instantiate/*.cc) \
- $(wildcard src/nix-prefetch-url/*.cc) \
$(wildcard src/nix-store/*.cc) \
-nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain
+nix_CXXFLAGS += -I src/libutil -I src/libstore -I src/libfetchers -I src/libexpr -I src/libmain -I src/libcmd
-nix_LIBS = libexpr libmain libfetchers libstore libutil
+nix_LIBS = libexpr libmain libfetchers libstore libutil libcmd
nix_LDFLAGS = -pthread $(SODIUM_LIBS) $(EDITLINE_LIBS) $(BOOST_LDFLAGS) -llowdown
diff --git a/src/nix/log.cc b/src/nix/log.cc
index 33a3053f5..638bb5073 100644
--- a/src/nix/log.cc
+++ b/src/nix/log.cc
@@ -13,22 +13,11 @@ struct CmdLog : InstallableCommand
return "show the build log of the specified packages or paths, if available";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To get the build log of GNU Hello:",
- "nix log nixpkgs#hello"
- },
- Example{
- "To get the build log of a specific path:",
- "nix log /nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1"
- },
- Example{
- "To get a build log from a specific binary cache:",
- "nix log --store https://cache.nixos.org nixpkgs#hello"
- },
- };
+ return
+ #include "log.md"
+ ;
}
Category category() override { return catSecondary; }
@@ -41,18 +30,18 @@ struct CmdLog : InstallableCommand
subs.push_front(store);
- auto b = installable->toBuildable();
+ auto b = installable->toDerivedPathWithHints();
RunPager pager;
for (auto & sub : subs) {
auto log = std::visit(overloaded {
- [&](BuildableOpaque bo) {
+ [&](DerivedPathWithHints::Opaque bo) {
return sub->getBuildLog(bo.path);
},
- [&](BuildableFromDrv bfd) {
+ [&](DerivedPathWithHints::Built bfd) {
return sub->getBuildLog(bfd.drvPath);
},
- }, b);
+ }, b.raw());
if (!log) continue;
stopProgressBar();
printInfo("got build log for '%s' from '%s'", installable->what(), sub->getUri());
diff --git a/src/nix/log.md b/src/nix/log.md
new file mode 100644
index 000000000..1c76226a3
--- /dev/null
+++ b/src/nix/log.md
@@ -0,0 +1,40 @@
+R""(
+
+# Examples
+
+* Get the build log of GNU Hello:
+
+ ```console
+ # nix log nixpkgs#hello
+ ```
+
+* Get the build log of a specific store path:
+
+ ```console
+ # nix log /nix/store/lmngj4wcm9rkv3w4dfhzhcyij3195hiq-thunderbird-52.2.1
+ ```
+
+* Get a build log from a specific binary cache:
+
+ ```console
+ # nix log --store https://cache.nixos.org nixpkgs#hello
+ ```
+
+# Description
+
+This command prints the log of a previous build of the derivation
+*installable* on standard output.
+
+Nix looks for build logs in two places:
+
+* In the directory `/nix/var/log/nix/drvs`, which contains logs for
+ locally built derivations.
+
+* In the binary caches listed in the `substituters` setting. Logs
+ should be named `<cache>/log/<base-name-of-store-path>`, where
+ `store-path` is a derivation,
+ e.g. `https://cache.nixos.org/log/dvmig8jgrdapvbyxb1rprckdmdqx08kv-hello-2.10.drv`.
+ For non-derivation store paths, Nix will first try to determine the
+ deriver by fetching the `.narinfo` file for this store path.
+
+)""
diff --git a/src/nix/ls.cc b/src/nix/ls.cc
index baca54431..c1dc9a95b 100644
--- a/src/nix/ls.cc
+++ b/src/nix/ls.cc
@@ -17,9 +17,26 @@ struct MixLs : virtual Args, MixJSON
MixLs()
{
- mkFlag('R', "recursive", "list subdirectories recursively", &recursive);
- mkFlag('l', "long", "show more file information", &verbose);
- mkFlag('d', "directory", "show directories rather than their contents", &showDirectory);
+ addFlag({
+ .longName = "recursive",
+ .shortName = 'R',
+ .description = "List subdirectories recursively.",
+ .handler = {&recursive, true},
+ });
+
+ addFlag({
+ .longName = "long",
+ .shortName = 'l',
+ .description = "Show detailed file information.",
+ .handler = {&verbose, true},
+ });
+
+ addFlag({
+ .longName = "directory",
+ .shortName = 'd',
+ .description = "Show directories rather than their contents.",
+ .handler = {&showDirectory, true},
+ });
}
void listText(ref<FSAccessor> accessor)
@@ -37,11 +54,11 @@ struct MixLs : virtual Args, MixJSON
auto line = fmt("%s %20d %s", tp, st.fileSize, relPath);
if (st.type == FSAccessor::Type::tSymlink)
line += " -> " + accessor->readLink(curPath);
- logger->stdout(line);
+ logger->cout(line);
if (recursive && st.type == FSAccessor::Type::tDirectory)
doPath(st, curPath, relPath, false);
} else {
- logger->stdout(relPath);
+ logger->cout(relPath);
if (recursive) {
auto st = accessor->stat(curPath);
if (st.type == FSAccessor::Type::tDirectory)
@@ -75,6 +92,8 @@ struct MixLs : virtual Args, MixJSON
if (json) {
JSONPlaceholder jsonRoot(std::cout);
+ if (showDirectory)
+ throw UsageError("'--directory' is useless with '--json'");
listNar(jsonRoot, accessor, path, recursive);
} else
listText(accessor);
@@ -92,22 +111,17 @@ struct CmdLsStore : StoreCommand, MixLs
});
}
- Examples examples() override
- {
- return {
- Example{
- "To list the contents of a store path in a binary cache:",
- "nix ls-store --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10"
- },
- };
- }
-
std::string description() override
{
return "show information about a path in the Nix store";
}
- Category category() override { return catUtility; }
+ std::string doc() override
+ {
+ return
+ #include "store-ls.md"
+ ;
+ }
void run(ref<Store> store) override
{
@@ -129,14 +143,11 @@ struct CmdLsNar : Command, MixLs
expectArg("path", &path);
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To list a specific file in a NAR:",
- "nix ls-nar -l hello.nar /bin/hello"
- },
- };
+ return
+ #include "nar-ls.md"
+ ;
}
std::string description() override
@@ -144,13 +155,11 @@ struct CmdLsNar : Command, MixLs
return "show information about a path inside a NAR file";
}
- Category category() override { return catUtility; }
-
void run() override
{
list(makeNarAccessor(make_ref<std::string>(readFile(narPath))));
}
};
-static auto rCmdLsStore = registerCommand<CmdLsStore>("ls-store");
-static auto rCmdLsNar = registerCommand<CmdLsNar>("ls-nar");
+static auto rCmdLsStore = registerCommand2<CmdLsStore>({"store", "ls"});
+static auto rCmdLsNar = registerCommand2<CmdLsNar>({"nar", "ls"});
diff --git a/src/nix/main.cc b/src/nix/main.cc
index 5056ceb78..008482be3 100644
--- a/src/nix/main.cc
+++ b/src/nix/main.cc
@@ -52,14 +52,18 @@ static bool haveInternet()
}
std::string programPath;
+char * * savedArgv;
+
+struct HelpRequested { };
struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
{
bool printBuildLogs = false;
bool useNet = true;
bool refresh = false;
+ bool showVersion = false;
- NixArgs() : MultiCommand(*RegisterCommand::commands), MixCommonArgs("nix")
+ NixArgs() : MultiCommand(RegisterCommand::getCommandsFor({})), MixCommonArgs("nix")
{
categories.clear();
categories[Command::catDefault] = "Main commands";
@@ -69,88 +73,137 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs
addFlag({
.longName = "help",
- .description = "show usage information",
- .handler = {[&]() { if (!completions) showHelpAndExit(); }},
- });
-
- addFlag({
- .longName = "help-config",
- .description = "show configuration options",
- .handler = {[&]() {
- std::cout << "The following configuration options are available:\n\n";
- Table2 tbl;
- std::map<std::string, Config::SettingInfo> settings;
- globalConfig.getSettings(settings);
- for (const auto & s : settings)
- tbl.emplace_back(s.first, s.second.description);
- printTable(std::cout, tbl);
- throw Exit();
- }},
+ .description = "Show usage information.",
+ .handler = {[&]() { throw HelpRequested(); }},
});
addFlag({
.longName = "print-build-logs",
.shortName = 'L',
- .description = "print full build logs on stderr",
+ .description = "Print full build logs on standard error.",
+ .category = loggingCategory,
.handler = {[&]() {setLogFormat(LogFormat::barWithLogs); }},
});
addFlag({
.longName = "version",
- .description = "show version information",
- .handler = {[&]() { if (!completions) printVersion(programName); }},
+ .description = "Show version information.",
+ .handler = {[&]() { showVersion = true; }},
});
addFlag({
- .longName = "no-net",
- .description = "disable substituters and consider all previously downloaded files up-to-date",
+ .longName = "offline",
+ .aliases = {"no-net"}, // FIXME: remove
+ .description = "Disable substituters and consider all previously downloaded files up-to-date.",
.handler = {[&]() { useNet = false; }},
});
addFlag({
.longName = "refresh",
- .description = "consider all previously downloaded files out-of-date",
+ .description = "Consider all previously downloaded files out-of-date.",
.handler = {[&]() { refresh = true; }},
});
-
- deprecatedAliases.insert({"dev-shell", "develop"});
}
- void printFlags(std::ostream & out) override
+ std::map<std::string, std::vector<std::string>> aliases = {
+ {"add-to-store", {"store", "add-path"}},
+ {"cat-nar", {"nar", "cat"}},
+ {"cat-store", {"store", "cat"}},
+ {"copy-sigs", {"store", "copy-sigs"}},
+ {"dev-shell", {"develop"}},
+ {"diff-closures", {"store", "diff-closures"}},
+ {"dump-path", {"store", "dump-path"}},
+ {"hash-file", {"hash", "file"}},
+ {"hash-path", {"hash", "path"}},
+ {"ls-nar", {"nar", "ls"}},
+ {"ls-store", {"store", "ls"}},
+ {"make-content-addressable", {"store", "make-content-addressable"}},
+ {"optimise-store", {"store", "optimise"}},
+ {"ping-store", {"store", "ping"}},
+ {"sign-paths", {"store", "sign"}},
+ {"to-base16", {"hash", "to-base16"}},
+ {"to-base32", {"hash", "to-base32"}},
+ {"to-base64", {"hash", "to-base64"}},
+ {"verify", {"store", "verify"}},
+ };
+
+ bool aliasUsed = false;
+
+ Strings::iterator rewriteArgs(Strings & args, Strings::iterator pos) override
{
- Args::printFlags(out);
- std::cout <<
- "\n"
- "In addition, most configuration settings can be overriden using '--" ANSI_ITALIC "name value" ANSI_NORMAL "'.\n"
- "Boolean settings can be overriden using '--" ANSI_ITALIC "name" ANSI_NORMAL "' or '--no-" ANSI_ITALIC "name" ANSI_NORMAL "'. See 'nix\n"
- "--help-config' for a list of configuration settings.\n";
+ if (aliasUsed || command || pos == args.end()) return pos;
+ auto arg = *pos;
+ auto i = aliases.find(arg);
+ if (i == aliases.end()) return pos;
+ warn("'%s' is a deprecated alias for '%s'",
+ arg, concatStringsSep(" ", i->second));
+ pos = args.erase(pos);
+ for (auto j = i->second.rbegin(); j != i->second.rend(); ++j)
+ pos = args.insert(pos, *j);
+ aliasUsed = true;
+ return pos;
}
- void printHelp(const string & programName, std::ostream & out) override
+ std::string description() override
{
- MultiCommand::printHelp(programName, out);
+ return "a tool for reproducible and declarative configuration management";
+ }
-#if 0
- out << "\nFor full documentation, run 'man " << programName << "' or 'man " << programName << "-" ANSI_ITALIC "COMMAND" ANSI_NORMAL "'.\n";
-#endif
+ std::string doc() override
+ {
+ return
+ #include "nix.md"
+ ;
+ }
- std::cout << "\nNote: this program is " ANSI_RED "EXPERIMENTAL" ANSI_NORMAL " and subject to change.\n";
+ // Plugins may add new subcommands.
+ void pluginsInited() override
+ {
+ commands = RegisterCommand::getCommandsFor({});
}
+};
+
+static void showHelp(std::vector<std::string> subcommand)
+{
+ showManPage(subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand)));
+}
+
+struct CmdHelp : Command
+{
+ std::vector<std::string> subcommand;
- void showHelpAndExit()
+ CmdHelp()
{
- printHelp(programName, std::cout);
- throw Exit();
+ expectArgs({
+ .label = "subcommand",
+ .handler = {&subcommand},
+ });
}
std::string description() override
{
- return "a tool for reproducible and declarative configuration management";
+ return "show help about `nix` or a particular subcommand";
+ }
+
+ std::string doc() override
+ {
+ return
+ #include "help.md"
+ ;
+ }
+
+ void run() override
+ {
+ showHelp(subcommand);
}
};
+static auto rCmdHelp = registerCommand<CmdHelp>("help");
+
void mainWrapped(int argc, char * * argv)
{
+ savedArgv = argv;
+
/* The chroot helper needs to be run before any threads have been
started. */
if (argc > 0 && argv[0] == chrootHelperName) {
@@ -169,7 +222,7 @@ void mainWrapped(int argc, char * * argv)
if (legacy) return legacy(argc, argv);
}
- verbosity = lvlWarn;
+ verbosity = lvlNotice;
settings.verboseBuild = false;
evalSettings.pureEval = true;
@@ -191,7 +244,7 @@ void mainWrapped(int argc, char * * argv)
auto builtins = state.baseEnv.values[0]->attrs;
for (auto & builtin : *builtins) {
auto b = nlohmann::json::object();
- if (builtin.value->type != tPrimOp) continue;
+ if (!builtin.value->isPrimOp()) continue;
auto primOp = builtin.value->primOp;
if (!primOp->doc) continue;
b["arity"] = primOp->arity;
@@ -214,15 +267,31 @@ void mainWrapped(int argc, char * * argv)
try {
args.parseCmdline(argvToStrings(argc, argv));
+ } catch (HelpRequested &) {
+ std::vector<std::string> subcommand;
+ MultiCommand * command = &args;
+ while (command) {
+ if (command && command->command) {
+ subcommand.push_back(command->command->first);
+ command = dynamic_cast<MultiCommand *>(&*command->command->second);
+ } else
+ break;
+ }
+ showHelp(subcommand);
+ return;
} catch (UsageError &) {
if (!completions) throw;
}
if (completions) return;
- initPlugins();
+ if (args.showVersion) {
+ printVersion(programName);
+ return;
+ }
- if (!args.command) args.showHelpAndExit();
+ if (!args.command)
+ throw UsageError("no subcommand specified");
if (args.command->first != "repl"
&& args.command->first != "doctor"
@@ -236,18 +305,21 @@ void mainWrapped(int argc, char * * argv)
if (!args.useNet) {
// FIXME: should check for command line overrides only.
- if (!settings.useSubstitutes.overriden)
+ if (!settings.useSubstitutes.overridden)
settings.useSubstitutes = false;
- if (!settings.tarballTtl.overriden)
+ if (!settings.tarballTtl.overridden)
settings.tarballTtl = std::numeric_limits<unsigned int>::max();
- if (!fileTransferSettings.tries.overriden)
+ if (!fileTransferSettings.tries.overridden)
fileTransferSettings.tries = 0;
- if (!fileTransferSettings.connectTimeout.overriden)
+ if (!fileTransferSettings.connectTimeout.overridden)
fileTransferSettings.connectTimeout = 1;
}
- if (args.refresh)
+ if (args.refresh) {
settings.tarballTtl = 0;
+ settings.ttlNegativeNarInfoCache = 0;
+ settings.ttlPositiveNarInfoCache = 0;
+ }
args.command->second->prepare();
args.command->second->run();
@@ -257,6 +329,10 @@ void mainWrapped(int argc, char * * argv)
int main(int argc, char * * argv)
{
+ // Increase the default stack size for the evaluator and for
+ // libstdc++'s std::regex.
+ nix::setStackSize(64 * 1024 * 1024);
+
return nix::handleExceptions(argv[0], [&]() {
nix::mainWrapped(argc, argv);
});
diff --git a/src/nix/make-content-addressable.cc b/src/nix/make-content-addressable.cc
index df3ec5194..f5bdc7e65 100644
--- a/src/nix/make-content-addressable.cc
+++ b/src/nix/make-content-addressable.cc
@@ -15,25 +15,16 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
std::string description() override
{
- return "rewrite a path or closure to content-addressable form";
+ return "rewrite a path or closure to content-addressed form";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To create a content-addressable representation of GNU Hello (but not its dependencies):",
- "nix make-content-addressable nixpkgs#hello"
- },
- Example{
- "To compute a content-addressable representation of the current NixOS system closure:",
- "nix make-content-addressable -r /run/current-system"
- },
- };
+ return
+ #include "make-content-addressable.md"
+ ;
}
- Category category() override { return catUtility; }
-
void run(ref<Store> store, StorePaths storePaths) override
{
auto paths = store->topoSortPaths(StorePathSet(storePaths.begin(), storePaths.end()));
@@ -73,7 +64,7 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
*sink.s = rewriteStrings(*sink.s, rewrites);
HashModuloSink hashModuloSink(htSHA256, oldHashPart);
- hashModuloSink((unsigned char *) sink.s->data(), sink.s->size());
+ hashModuloSink(*sink.s);
auto narHash = hashModuloSink.finish().first;
@@ -90,11 +81,11 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
};
if (!json)
- printInfo("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
+ notice("rewrote '%s' to '%s'", pathS, store->printStorePath(info.path));
auto source = sinkToSource([&](Sink & nextSink) {
RewritingSink rsink2(oldHashPart, std::string(info.path.hashPart()), nextSink);
- rsink2((unsigned char *) sink.s->data(), sink.s->size());
+ rsink2(*sink.s);
rsink2.flush();
});
@@ -108,4 +99,4 @@ struct CmdMakeContentAddressable : StorePathsCommand, MixJSON
}
};
-static auto rCmdMakeContentAddressable = registerCommand<CmdMakeContentAddressable>("make-content-addressable");
+static auto rCmdMakeContentAddressable = registerCommand2<CmdMakeContentAddressable>({"store", "make-content-addressable"});
diff --git a/src/nix/make-content-addressable.md b/src/nix/make-content-addressable.md
new file mode 100644
index 000000000..3dd847edc
--- /dev/null
+++ b/src/nix/make-content-addressable.md
@@ -0,0 +1,59 @@
+R""(
+
+# Examples
+
+* Create a content-addressed representation of the closure of GNU Hello:
+
+ ```console
+ # nix store make-content-addressable -r nixpkgs#hello
+ …
+ rewrote '/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10' to '/nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10'
+ ```
+
+ Since the resulting paths are content-addressed, they are always
+ trusted and don't need signatures to copied to another store:
+
+ ```console
+ # nix copy --to /tmp/nix --trusted-public-keys '' /nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10
+ ```
+
+ By contrast, the original closure is input-addressed, so it does
+ need signatures to be trusted:
+
+ ```console
+ # nix copy --to /tmp/nix --trusted-public-keys '' nixpkgs#hello
+ cannot add path '/nix/store/zy9wbxwcygrwnh8n2w9qbbcr6zk87m26-libunistring-0.9.10' because it lacks a valid signature
+ ```
+
+* Create a content-addressed representation of the current NixOS
+ system closure:
+
+ ```console
+ # nix store make-content-addressable -r /run/current-system
+ ```
+
+# Description
+
+This command converts the closure of the store paths specified by
+*installables* to content-addressed form. Nix store paths are usually
+*input-addressed*, meaning that the hash part of the store path is
+computed from the contents of the derivation (i.e., the build-time
+dependency graph). Input-addressed paths need to be signed by a
+trusted key if you want to import them into a store, because we need
+to trust that the contents of the path were actually built by the
+derivation.
+
+By contrast, in a *content-addressed* path, the hash part is computed
+from the contents of the path. This allows the contents of the path to
+be verified without any additional information such as
+signatures. This means that a command like
+
+```console
+# nix store build /nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10 \
+ --substituters https://my-cache.example.org
+```
+
+will succeed even if the binary cache `https://my-cache.example.org`
+doesn't present any signatures.
+
+)""
diff --git a/src/nix/nar-cat.md b/src/nix/nar-cat.md
new file mode 100644
index 000000000..55c481a28
--- /dev/null
+++ b/src/nix/nar-cat.md
@@ -0,0 +1,19 @@
+R""(
+
+# Examples
+
+* List a file in a NAR and pipe it through `gunzip`:
+
+ ```console
+ # nix nar cat ./hello.nar /share/man/man1/hello.1.gz | gunzip
+ .\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.46.4.
+ .TH HELLO "1" "November 2014" "hello 2.10" "User Commands"
+ …
+ ```
+
+# Description
+
+This command prints on standard output the contents of the regular
+file *path* inside the NAR file *nar*.
+
+)""
diff --git a/src/nix/nar-dump-path.md b/src/nix/nar-dump-path.md
new file mode 100644
index 000000000..26191ad25
--- /dev/null
+++ b/src/nix/nar-dump-path.md
@@ -0,0 +1,17 @@
+R""(
+
+# Examples
+
+* To serialise directory `foo` as a NAR:
+
+ ```console
+ # nix nar dump-path ./foo > foo.nar
+ ```
+
+# Description
+
+This command generates a NAR file containing the serialisation of
+*path*, which must contain only regular files, directories and
+symbolic links. The NAR is written to standard output.
+
+)""
diff --git a/src/nix/nar-ls.md b/src/nix/nar-ls.md
new file mode 100644
index 000000000..d373f9715
--- /dev/null
+++ b/src/nix/nar-ls.md
@@ -0,0 +1,24 @@
+R""(
+
+# Examples
+
+* To list a specific file in a NAR:
+
+ ```console
+ # nix nar ls -l ./hello.nar /bin/hello
+ -r-xr-xr-x 38184 hello
+ ```
+
+* To recursively list the contents of a directory inside a NAR, in JSON
+ format:
+
+ ```console
+ # nix nar ls --json -R ./hello.nar /bin
+ {"type":"directory","entries":{"hello":{"type":"regular","size":38184,"executable":true,"narOffset":400}}}
+ ```
+
+# Description
+
+This command shows information about a *path* inside NAR file *nar*.
+
+)""
diff --git a/src/nix/nar.cc b/src/nix/nar.cc
new file mode 100644
index 000000000..dbb043d9b
--- /dev/null
+++ b/src/nix/nar.cc
@@ -0,0 +1,33 @@
+#include "command.hh"
+
+using namespace nix;
+
+struct CmdNar : NixMultiCommand
+{
+ CmdNar() : MultiCommand(RegisterCommand::getCommandsFor({"nar"}))
+ { }
+
+ std::string description() override
+ {
+ return "create or inspect NAR files";
+ }
+
+ std::string doc() override
+ {
+ return
+ #include "nar.md"
+ ;
+ }
+
+ Category category() override { return catUtility; }
+
+ void run() override
+ {
+ if (!command)
+ throw UsageError("'nix nar' requires a sub-command.");
+ command->second->prepare();
+ command->second->run();
+ }
+};
+
+static auto rCmdNar = registerCommand<CmdNar>("nar");
diff --git a/src/nix/nar.md b/src/nix/nar.md
new file mode 100644
index 000000000..a83b5c764
--- /dev/null
+++ b/src/nix/nar.md
@@ -0,0 +1,13 @@
+R""(
+
+# Description
+
+`nix nar` provides several subcommands for creating and inspecting
+*Nix Archives* (NARs).
+
+# File format
+
+For the definition of the NAR file format, see Figure 5.2 in
+https://edolstra.github.io/pubs/phd-thesis.pdf.
+
+)""
diff --git a/src/nix/nix.md b/src/nix/nix.md
new file mode 100644
index 000000000..d10de7c01
--- /dev/null
+++ b/src/nix/nix.md
@@ -0,0 +1,119 @@
+R""(
+
+# Examples
+
+* Create a new flake:
+
+ ```console
+ # nix flake new hello
+ # cd hello
+ ```
+
+* Build the flake in the current directory:
+
+ ```console
+ # nix build
+ # ./result/bin/hello
+ Hello, world!
+ ```
+
+* Run the flake in the current directory:
+
+ ```console
+ # nix run
+ Hello, world!
+ ```
+
+* Start a development shell for hacking on this flake:
+
+ ```console
+ # nix develop
+ # unpackPhase
+ # cd hello-*
+ # configurePhase
+ # buildPhase
+ # ./hello
+ Hello, world!
+ # installPhase
+ # ../outputs/out/bin/hello
+ Hello, world!
+ ```
+
+# Description
+
+Nix is a tool for building software, configurations and other
+artifacts in a reproducible and declarative way. For more information,
+see the [Nix homepage](https://nixos.org/) or the [Nix
+manual](https://nixos.org/manual/nix/stable/).
+
+# Installables
+
+Many `nix` subcommands operate on one or more *installables*. These are
+command line arguments that represent something that can be built in
+the Nix store. Here are the recognised types of installables:
+
+* **Flake output attributes**: `nixpkgs#hello`
+
+ These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a
+ flake reference and *attrpath* is an optional attribute path. For
+ more information on flakes, see [the `nix flake` manual
+ page](./nix3-flake.md). Flake references are most commonly a flake
+ identifier in the flake registry (e.g. `nixpkgs`) or a path
+ (e.g. `/path/to/my-flake` or `.`).
+
+ If *attrpath* is omitted, Nix tries some default values; for most
+ subcommands, the default is `defaultPackage.`*system*
+ (e.g. `defaultPackage.x86_64-linux`), but some subcommands have
+ other defaults. If *attrpath* *is* specified, *attrpath* is
+ interpreted as relative to one or more prefixes; for most
+ subcommands, these are `packages.`*system*,
+ `legacyPackages.*system*` and the empty prefix. Thus, on
+ `x86_64-linux` `nix build nixpkgs#hello` will try to build the
+ attributes `packages.x86_64-linux.hello`,
+ `legacyPackages.x86_64-linux.hello` and `hello`.
+
+* **Store paths**: `/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10`
+
+ These are paths inside the Nix store, or symlinks that resolve to a
+ path in the Nix store.
+
+* **Store derivations**: `/nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv`
+
+ Store derivations are store paths with extension `.drv` and are a
+ low-level representation of a build-time dependency graph used
+ internally by Nix. By default, if you pass a store derivation to a
+ `nix` subcommand, it will operate on the *output paths* of the
+ derivation. For example, `nix path-info` prints information about
+ the output paths:
+
+ ```console
+ # nix path-info --json /nix/store/p7gp6lxdg32h4ka1q398wd9r2zkbbz2v-hello-2.10.drv
+ [{"path":"/nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10",…}]
+ ```
+
+ If you want to operate on the store derivation itself, pass the
+ `--derivation` flag.
+
+* **Nix attributes**: `--file /path/to/nixpkgs hello`
+
+ When the `-f` / `--file` *path* option is given, installables are
+ interpreted as attribute paths referencing a value returned by
+ evaluating the Nix file *path*.
+
+* **Nix expressions**: `--expr '(import <nixpkgs> {}).hello.overrideDerivation (prev: { name = "my-hello"; })'`.
+
+ When the `--expr` option is given, all installables are interpreted
+ as Nix expressions. You may need to specify `--impure` if the
+ expression references impure inputs (such as `<nixpkgs>`).
+
+For most commands, if no installable is specified, the default is `.`,
+i.e. Nix will operate on the default flake output attribute of the
+flake in the current directory.
+
+# Nix stores
+
+Most `nix` subcommands operate on a *Nix store*.
+
+TODO: list store types, options
+
+)""
diff --git a/src/nix/optimise-store.cc b/src/nix/optimise-store.cc
index 51a7a9756..985006e5a 100644
--- a/src/nix/optimise-store.cc
+++ b/src/nix/optimise-store.cc
@@ -13,22 +13,17 @@ struct CmdOptimiseStore : StoreCommand
return "replace identical files in the store by hard links";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To optimise the Nix store:",
- "nix optimise-store"
- },
- };
+ return
+ #include "optimise-store.md"
+ ;
}
- Category category() override { return catUtility; }
-
void run(ref<Store> store) override
{
store->optimiseStore();
}
};
-static auto rCmdOptimiseStore = registerCommand<CmdOptimiseStore>("optimise-store");
+static auto rCmdOptimiseStore = registerCommand2<CmdOptimiseStore>({"store", "optimise"});
diff --git a/src/nix/optimise-store.md b/src/nix/optimise-store.md
new file mode 100644
index 000000000..f6fb66f97
--- /dev/null
+++ b/src/nix/optimise-store.md
@@ -0,0 +1,23 @@
+R""(
+
+# Examples
+
+* Optimise the Nix store:
+
+ ```console
+ nix store optimise
+ ```
+
+# Description
+
+This command deduplicates the Nix store: it scans the store for
+regular files with identical contents, and replaces them with hard
+links to a single instance.
+
+Note that you can also set `auto-optimise-store` to `true` in
+`nix.conf` to perform this optimisation incrementally whenever a new
+path is added to the Nix store. To make this efficient, Nix maintains
+a content-addressed index of all the files in the Nix store in the
+directory `/nix/store/.links/`.
+
+)""
diff --git a/src/nix/path-info.cc b/src/nix/path-info.cc
index 63cf885f9..518cd5568 100644
--- a/src/nix/path-info.cc
+++ b/src/nix/path-info.cc
@@ -18,10 +18,32 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
CmdPathInfo()
{
- mkFlag('s', "size", "print size of the NAR dump of each path", &showSize);
- mkFlag('S', "closure-size", "print sum size of the NAR dumps of the closure of each path", &showClosureSize);
- mkFlag('h', "human-readable", "with -s and -S, print sizes like 1K 234M 5.67G etc.", &humanReadable);
- mkFlag(0, "sigs", "show signatures", &showSigs);
+ addFlag({
+ .longName = "size",
+ .shortName = 's',
+ .description = "Print the size of the NAR serialisation of each path.",
+ .handler = {&showSize, true},
+ });
+
+ addFlag({
+ .longName = "closure-size",
+ .shortName = 'S',
+ .description = "Print the sum of the sizes of the NAR serialisations of the closure of each path.",
+ .handler = {&showClosureSize, true},
+ });
+
+ addFlag({
+ .longName = "human-readable",
+ .shortName = 'h',
+ .description = "With `-s` and `-S`, print sizes in a human-friendly format such as `5.67G`.",
+ .handler = {&humanReadable, true},
+ });
+
+ addFlag({
+ .longName = "sigs",
+ .description = "Show signatures.",
+ .handler = {&showSigs, true},
+ });
}
std::string description() override
@@ -29,38 +51,15 @@ struct CmdPathInfo : StorePathsCommand, MixJSON
return "query information about store paths";
}
- Category category() override { return catSecondary; }
-
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To show the closure sizes of every path in the current NixOS system closure, sorted by size:",
- "nix path-info -rS /run/current-system | sort -nk2"
- },
- Example{
- "To show a package's closure size and all its dependencies with human readable sizes:",
- "nix path-info -rsSh nixpkgs#rust"
- },
- Example{
- "To check the existence of a path in a binary cache:",
- "nix path-info -r /nix/store/7qvk5c91...-geeqie-1.1 --store https://cache.nixos.org/"
- },
- Example{
- "To print the 10 most recently added paths (using --json and the jq(1) command):",
- "nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path'"
- },
- Example{
- "To show the size of the entire Nix store:",
- "nix path-info --json --all | jq 'map(.narSize) | add'"
- },
- Example{
- "To show every path whose closure is bigger than 1 GB, sorted by closure size:",
- "nix path-info --json --all -S | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])'"
- },
- };
+ return
+ #include "path-info.md"
+ ;
}
+ Category category() override { return catSecondary; }
+
void printSize(uint64_t value)
{
if (!humanReadable) {
diff --git a/src/nix/path-info.md b/src/nix/path-info.md
new file mode 100644
index 000000000..76a83e39d
--- /dev/null
+++ b/src/nix/path-info.md
@@ -0,0 +1,94 @@
+R""(
+
+# Examples
+
+* Print the store path produced by `nixpkgs#hello`:
+
+ ```console
+ # nix path-info nixpkgs#hello
+ /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10
+ ```
+
+* Show the closure sizes of every path in the current NixOS system
+ closure, sorted by size:
+
+ ```console
+ # nix path-info -rS /run/current-system | sort -nk2
+ /nix/store/hl5xwp9kdrd1zkm0idm3kkby9q66z404-empty 96
+ /nix/store/27324qvqhnxj3rncazmxc4mwy79kz8ha-nameservers 112
+ …
+ /nix/store/539jkw9a8dyry7clcv60gk6na816j7y8-etc 5783255504
+ /nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65 5887562256
+ ```
+
+* Show a package's closure size and all its dependencies with human
+ readable sizes:
+
+ ```console
+ # nix path-info -rsSh nixpkgs#rustc
+ /nix/store/01rrgsg5zk3cds0xgdsq40zpk6g51dz9-ncurses-6.2-dev 386.7K 69.1M
+ /nix/store/0q783wnvixpqz6dxjp16nw296avgczam-libpfm-4.11.0 5.9M 37.4M
+ …
+ ```
+
+* Check the existence of a path in a binary cache:
+
+ ```console
+ # nix path-info -r /nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1 --store https://cache.nixos.org/
+ path '/nix/store/blzxgyvrk32ki6xga10phr4sby2xf25q-geeqie-1.5.1' is not valid
+
+ ```
+
+* Print the 10 most recently added paths (using --json and the jq(1)
+ command):
+
+ ```console
+ # nix path-info --json --all | jq -r 'sort_by(.registrationTime)[-11:-1][].path'
+ ```
+
+* Show the size of the entire Nix store:
+
+ ```console
+ # nix path-info --json --all | jq 'map(.narSize) | add'
+ 49812020936
+ ```
+
+* Show every path whose closure is bigger than 1 GB, sorted by closure
+ size:
+
+ ```console
+ # nix path-info --json --all -S \
+ | jq 'map(select(.closureSize > 1e9)) | sort_by(.closureSize) | map([.path, .closureSize])'
+ [
+ …,
+ [
+ "/nix/store/zqamz3cz4dbzfihki2mk7a63mbkxz9xq-nixos-system-machine-20.09.20201112.3090c65",
+ 5887562256
+ ]
+ ]
+ ```
+
+* Print the path of the store derivation produced by `nixpkgs#hello`:
+
+ ```console
+ # nix path-info --derivation nixpkgs#hello
+ /nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv
+ ```
+
+# Description
+
+This command shows information about the store paths produced by
+*installables*, or about all paths in the store if you pass `--all`.
+
+By default, this command only prints the store paths. You can get
+additional information by passing flags such as `--closure-size`,
+--size`, `--sigs` or `--json`.
+
+> **Warning**
+>
+> Note that `nix path-info` does not build or substitute the
+> *installables* you specify. Thus, if the corresponding store paths
+> don't already exist, this command will fail. You can use `nix build`
+> to ensure that they exist.
+
+)""
diff --git a/src/nix/ping-store.cc b/src/nix/ping-store.cc
index 8db78d591..62b645b06 100644
--- a/src/nix/ping-store.cc
+++ b/src/nix/ping-store.cc
@@ -8,25 +8,20 @@ struct CmdPingStore : StoreCommand
{
std::string description() override
{
- return "test whether a store can be opened";
+ return "test whether a store can be accessed";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To test whether connecting to a remote Nix store via SSH works:",
- "nix ping-store --store ssh://mac1"
- },
- };
+ return
+ #include "ping-store.md"
+ ;
}
- Category category() override { return catUtility; }
-
void run(ref<Store> store) override
{
store->connect();
}
};
-static auto rCmdPingStore = registerCommand<CmdPingStore>("ping-store");
+static auto rCmdPingStore = registerCommand2<CmdPingStore>({"store", "ping"});
diff --git a/src/nix/ping-store.md b/src/nix/ping-store.md
new file mode 100644
index 000000000..8c846791b
--- /dev/null
+++ b/src/nix/ping-store.md
@@ -0,0 +1,33 @@
+R""(
+
+# Examples
+
+* Test whether connecting to a remote Nix store via SSH works:
+
+ ```console
+ # nix store ping --store ssh://mac1
+ ```
+
+* Test whether a URL is a valid binary cache:
+
+ ```console
+ # nix store ping --store https://cache.nixos.org
+ ```
+
+* Test whether the Nix daemon is up and running:
+
+ ```console
+ # nix store ping --store daemon
+ ```
+
+# Description
+
+This command tests whether a particular Nix store (specified by the
+argument `--store` *url*) can be accessed. What this means is
+dependent on the type of the store. For instance, for an SSH store it
+means that Nix can connect to the specified machine.
+
+If the command succeeds, Nix returns a exit code of 0 and does not
+print any output.
+
+)""
diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc
new file mode 100644
index 000000000..b7da3ea5a
--- /dev/null
+++ b/src/nix/prefetch.cc
@@ -0,0 +1,317 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "store-api.hh"
+#include "filetransfer.hh"
+#include "finally.hh"
+#include "progress-bar.hh"
+#include "tarfile.hh"
+#include "attr-path.hh"
+#include "eval-inline.hh"
+#include "legacy.hh"
+
+#include <nlohmann/json.hpp>
+
+using namespace nix;
+
+/* If ‘url’ starts with ‘mirror://’, then resolve it using the list of
+ mirrors defined in Nixpkgs. */
+string resolveMirrorUrl(EvalState & state, string url)
+{
+ if (url.substr(0, 9) != "mirror://") return url;
+
+ std::string s(url, 9);
+ auto p = s.find('/');
+ if (p == std::string::npos) throw Error("invalid mirror URL '%s'", url);
+ std::string mirrorName(s, 0, p);
+
+ Value vMirrors;
+ // FIXME: use nixpkgs flake
+ state.eval(state.parseExprFromString("import <nixpkgs/pkgs/build-support/fetchurl/mirrors.nix>", "."), vMirrors);
+ state.forceAttrs(vMirrors);
+
+ auto mirrorList = vMirrors.attrs->find(state.symbols.create(mirrorName));
+ if (mirrorList == vMirrors.attrs->end())
+ throw Error("unknown mirror name '%s'", mirrorName);
+ state.forceList(*mirrorList->value);
+
+ if (mirrorList->value->listSize() < 1)
+ throw Error("mirror URL '%s' did not expand to anything", url);
+
+ auto mirror = state.forceString(*mirrorList->value->listElems()[0]);
+ return mirror + (hasSuffix(mirror, "/") ? "" : "/") + string(s, p + 1);
+}
+
+std::tuple<StorePath, Hash> prefetchFile(
+ ref<Store> store,
+ std::string_view url,
+ std::optional<std::string> name,
+ HashType hashType,
+ std::optional<Hash> expectedHash,
+ bool unpack,
+ bool executable)
+{
+ auto ingestionMethod = unpack || executable ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
+
+ /* Figure out a name in the Nix store. */
+ if (!name) {
+ name = baseNameOf(url);
+ if (name->empty())
+ throw Error("cannot figure out file name for '%s'", url);
+ }
+
+ std::optional<StorePath> storePath;
+ std::optional<Hash> hash;
+
+ /* If an expected hash is given, the file may already exist in
+ the store. */
+ if (expectedHash) {
+ hashType = expectedHash->type;
+ storePath = store->makeFixedOutputPath(ingestionMethod, *expectedHash, *name);
+ if (store->isValidPath(*storePath))
+ hash = expectedHash;
+ else
+ storePath.reset();
+ }
+
+ if (!storePath) {
+
+ AutoDelete tmpDir(createTempDir(), true);
+ Path tmpFile = (Path) tmpDir + "/tmp";
+
+ /* Download the file. */
+ {
+ auto mode = 0600;
+ if (executable)
+ mode = 0700;
+
+ AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_CREAT | O_EXCL, mode);
+ if (!fd) throw SysError("creating temporary file '%s'", tmpFile);
+
+ FdSink sink(fd.get());
+
+ FileTransferRequest req(url);
+ req.decompress = false;
+ getFileTransfer()->download(std::move(req), sink);
+ }
+
+ /* Optionally unpack the file. */
+ if (unpack) {
+ Activity act(*logger, lvlChatty, actUnknown,
+ fmt("unpacking '%s'", url));
+ Path unpacked = (Path) tmpDir + "/unpacked";
+ createDirs(unpacked);
+ unpackTarfile(tmpFile, unpacked);
+
+ /* If the archive unpacks to a single file/directory, then use
+ that as the top-level. */
+ auto entries = readDirectory(unpacked);
+ if (entries.size() == 1)
+ tmpFile = unpacked + "/" + entries[0].name;
+ else
+ tmpFile = unpacked;
+ }
+
+ Activity act(*logger, lvlChatty, actUnknown,
+ fmt("adding '%s' to the store", url));
+
+ auto info = store->addToStoreSlow(*name, tmpFile, ingestionMethod, hashType, expectedHash);
+ storePath = info.path;
+ assert(info.ca);
+ hash = getContentAddressHash(*info.ca);
+ }
+
+ return {storePath.value(), hash.value()};
+}
+
+static int main_nix_prefetch_url(int argc, char * * argv)
+{
+ {
+ HashType ht = htSHA256;
+ std::vector<string> args;
+ bool printPath = getEnv("PRINT_PATH") == "1";
+ bool fromExpr = false;
+ string attrPath;
+ bool unpack = false;
+ bool executable = false;
+ std::optional<std::string> name;
+
+ struct MyArgs : LegacyArgs, MixEvalArgs
+ {
+ using LegacyArgs::LegacyArgs;
+ };
+
+ MyArgs myArgs(std::string(baseNameOf(argv[0])), [&](Strings::iterator & arg, const Strings::iterator & end) {
+ if (*arg == "--help")
+ showManPage("nix-prefetch-url");
+ else if (*arg == "--version")
+ printVersion("nix-prefetch-url");
+ else if (*arg == "--type") {
+ string s = getArg(*arg, arg, end);
+ ht = parseHashType(s);
+ }
+ else if (*arg == "--print-path")
+ printPath = true;
+ else if (*arg == "--attr" || *arg == "-A") {
+ fromExpr = true;
+ attrPath = getArg(*arg, arg, end);
+ }
+ else if (*arg == "--unpack")
+ unpack = true;
+ else if (*arg == "--executable")
+ executable = true;
+ else if (*arg == "--name")
+ name = getArg(*arg, arg, end);
+ else if (*arg != "" && arg->at(0) == '-')
+ return false;
+ else
+ args.push_back(*arg);
+ return true;
+ });
+
+ myArgs.parseCmdline(argvToStrings(argc, argv));
+
+ if (args.size() > 2)
+ throw UsageError("too many arguments");
+
+ Finally f([]() { stopProgressBar(); });
+
+ if (isatty(STDERR_FILENO))
+ startProgressBar();
+
+ auto store = openStore();
+ auto state = std::make_unique<EvalState>(myArgs.searchPath, store);
+
+ Bindings & autoArgs = *myArgs.getAutoArgs(*state);
+
+ /* If -A is given, get the URL from the specified Nix
+ expression. */
+ string url;
+ if (!fromExpr) {
+ if (args.empty())
+ throw UsageError("you must specify a URL");
+ url = args[0];
+ } else {
+ Path path = resolveExprPath(lookupFileArg(*state, args.empty() ? "." : args[0]));
+ Value vRoot;
+ state->evalFile(path, vRoot);
+ Value & v(*findAlongAttrPath(*state, attrPath, autoArgs, vRoot).first);
+ state->forceAttrs(v);
+
+ /* Extract the URL. */
+ auto attr = v.attrs->find(state->symbols.create("urls"));
+ if (attr == v.attrs->end())
+ throw Error("attribute set does not contain a 'urls' attribute");
+ state->forceList(*attr->value);
+ if (attr->value->listSize() < 1)
+ throw Error("'urls' list is empty");
+ url = state->forceString(*attr->value->listElems()[0]);
+
+ /* Extract the hash mode. */
+ attr = v.attrs->find(state->symbols.create("outputHashMode"));
+ if (attr == v.attrs->end())
+ printInfo("warning: this does not look like a fetchurl call");
+ else
+ unpack = state->forceString(*attr->value) == "recursive";
+
+ /* Extract the name. */
+ if (!name) {
+ attr = v.attrs->find(state->symbols.create("name"));
+ if (attr != v.attrs->end())
+ name = state->forceString(*attr->value);
+ }
+ }
+
+ std::optional<Hash> expectedHash;
+ if (args.size() == 2)
+ expectedHash = Hash::parseAny(args[1], ht);
+
+ auto [storePath, hash] = prefetchFile(
+ store, resolveMirrorUrl(*state, url), name, ht, expectedHash, unpack, executable);
+
+ stopProgressBar();
+
+ if (!printPath)
+ printInfo("path is '%s'", store->printStorePath(storePath));
+
+ std::cout << printHash16or32(hash) << std::endl;
+ if (printPath)
+ std::cout << store->printStorePath(storePath) << std::endl;
+
+ return 0;
+ }
+}
+
+static RegisterLegacyCommand r_nix_prefetch_url("nix-prefetch-url", main_nix_prefetch_url);
+
+struct CmdStorePrefetchFile : StoreCommand, MixJSON
+{
+ std::string url;
+ bool executable = false;
+ std::optional<std::string> name;
+ HashType hashType = htSHA256;
+ std::optional<Hash> expectedHash;
+
+ CmdStorePrefetchFile()
+ {
+ addFlag({
+ .longName = "name",
+ .description = "Override the name component of the resulting store path. It defaults to the base name of *url*.",
+ .labels = {"name"},
+ .handler = {&name}
+ });
+
+ addFlag({
+ .longName = "expected-hash",
+ .description = "The expected hash of the file.",
+ .labels = {"hash"},
+ .handler = {[&](std::string s) {
+ expectedHash = Hash::parseAny(s, hashType);
+ }}
+ });
+
+ addFlag(Flag::mkHashTypeFlag("hash-type", &hashType));
+
+ addFlag({
+ .longName = "executable",
+ .description =
+ "Make the resulting file executable. Note that this causes the "
+ "resulting hash to be a NAR hash rather than a flat file hash.",
+ .handler = {&executable, true},
+ });
+
+ expectArg("url", &url);
+ }
+
+ Category category() override { return catUtility; }
+
+ std::string description() override
+ {
+ return "download a file into the Nix store";
+ }
+
+ std::string doc() override
+ {
+ return
+ #include "store-prefetch-file.md"
+ ;
+ }
+ void run(ref<Store> store) override
+ {
+ auto [storePath, hash] = prefetchFile(store, url, name, hashType, expectedHash, false, executable);
+
+ if (json) {
+ auto res = nlohmann::json::object();
+ res["storePath"] = store->printStorePath(storePath);
+ res["hash"] = hash.to_string(SRI, true);
+ logger->cout(res.dump());
+ } else {
+ notice("Downloaded '%s' to '%s' (hash '%s').",
+ url,
+ store->printStorePath(storePath),
+ hash.to_string(SRI, true));
+ }
+ }
+};
+
+static auto rCmdStorePrefetchFile = registerCommand2<CmdStorePrefetchFile>({"store", "prefetch-file"});
diff --git a/src/nix/print-dev-env.md b/src/nix/print-dev-env.md
new file mode 100644
index 000000000..b80252acf
--- /dev/null
+++ b/src/nix/print-dev-env.md
@@ -0,0 +1,19 @@
+R""(
+
+# Examples
+
+* Apply the build environment of GNU hello to the current shell:
+
+ ```console
+ # . <(nix print-dev-env nixpkgs#hello)
+ ```
+
+# Description
+
+This command prints a shell script that can be sourced by `b`ash and
+that sets the environment variables and shell functions defined by the
+build process of *installable*. This allows you to get a similar build
+environment in your current shell rather than in a subshell (as with
+`nix develop`).
+
+)""
diff --git a/src/nix/profile-diff-closures.md b/src/nix/profile-diff-closures.md
new file mode 100644
index 000000000..295d1252b
--- /dev/null
+++ b/src/nix/profile-diff-closures.md
@@ -0,0 +1,28 @@
+R""(
+
+# Examples
+
+* Show what changed between each version of the NixOS system
+ profile:
+
+ ```console
+ # nix profile diff-closures --profile /nix/var/nix/profiles/system
+ Version 13 -> 14:
+ acpi-call: 2020-04-07-5.8.13 → 2020-04-07-5.8.14
+ aws-sdk-cpp: -6723.1 KiB
+ …
+
+ Version 14 -> 15:
+ acpi-call: 2020-04-07-5.8.14 → 2020-04-07-5.8.16
+ attica: -996.2 KiB
+ breeze-icons: -78713.5 KiB
+ brotli: 1.0.7 → 1.0.9, +44.2 KiB
+ ```
+
+# Description
+
+This command shows the difference between the closures of subsequent
+versions of a profile. See [`nix store
+diff-closures`](nix3-store-diff-closures.md) for details.
+
+)""
diff --git a/src/nix/profile-history.md b/src/nix/profile-history.md
new file mode 100644
index 000000000..d0fe40c82
--- /dev/null
+++ b/src/nix/profile-history.md
@@ -0,0 +1,26 @@
+R""(
+
+# Examples
+
+* Show the changes between each version of your default profile:
+
+ ```console
+ # nix profile history
+ Version 508 -> 509:
+ flake:nixpkgs#legacyPackages.x86_64-linux.awscli: ∅ -> 1.17.13
+
+ Version 509 -> 510:
+ flake:nixpkgs#legacyPackages.x86_64-linux.awscli: 1.17.13 -> 1.18.211
+ ```
+
+# Description
+
+This command shows what packages were added, removed or upgraded
+between subsequent versions of a profile. It only shows top-level
+packages, not dependencies; for that, use [`nix profile
+diff-closures`](./nix3-profile-diff-closures.md).
+
+The addition of a package to a profile is denoted by the string `∅ ->`
+*version*, whereas the removal is denoted by *version* `-> ∅`.
+
+)""
diff --git a/src/nix/profile-install.md b/src/nix/profile-install.md
new file mode 100644
index 000000000..e3009491e
--- /dev/null
+++ b/src/nix/profile-install.md
@@ -0,0 +1,27 @@
+R""(
+
+# Examples
+
+* Install a package from Nixpkgs:
+
+ ```console
+ # nix profile install nixpkgs#hello
+ ```
+
+* Install a package from a specific branch of Nixpkgs:
+
+ ```console
+ # nix profile install nixpkgs/release-20.09#hello
+ ```
+
+* Install a package from a specific revision of Nixpkgs:
+
+ ```console
+ # nix profile install nixpkgs/d73407e8e6002646acfdef0e39ace088bacc83da#hello
+ ```
+
+# Description
+
+This command adds *installables* to a Nix profile.
+
+)""
diff --git a/src/nix/profile-list.md b/src/nix/profile-list.md
new file mode 100644
index 000000000..5c29c0b02
--- /dev/null
+++ b/src/nix/profile-list.md
@@ -0,0 +1,31 @@
+R""(
+
+# Examples
+
+* Show what packages are installed in the default profile:
+
+ ```console
+ # nix profile list
+ 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.spotify /nix/store/akpdsid105phbbvknjsdh7hl4v3fhjkr-spotify-1.1.46.916.g416cacf1
+ 1 flake:nixpkgs#legacyPackages.x86_64-linux.zoom-us github:NixOS/nixpkgs/c23db78bbd474c4d0c5c3c551877523b4a50db06#legacyPackages.x86_64-linux.zoom-us /nix/store/89pmjmbih5qpi7accgacd17ybpgp4xfm-zoom-us-5.4.53350.1027
+ 2 flake:blender-bin#defaultPackage.x86_64-linux github:edolstra/nix-warez/d09d7eea893dcb162e89bc67f6dc1ced14abfc27?dir=blender#defaultPackage.x86_64-linux /nix/store/zfgralhqjnam662kqsgq6isjw8lhrflz-blender-bin-2.91.0
+ ```
+
+# Description
+
+This command shows what packages are currently installed in a
+profile. The output consists of one line per package, with the
+following fields:
+
+* An integer that can be used to unambiguously identify the package in
+ invocations of `nix profile remove` and `nix profile upgrade`.
+
+* The original ("mutable") flake reference and output attribute path
+ used at installation time.
+
+* The immutable flake reference to which the mutable flake reference
+ was resolved.
+
+* The store path(s) of the package.
+
+)""
diff --git a/src/nix/profile-remove.md b/src/nix/profile-remove.md
new file mode 100644
index 000000000..dcf825da9
--- /dev/null
+++ b/src/nix/profile-remove.md
@@ -0,0 +1,32 @@
+R""(
+
+# Examples
+
+* Remove a package by position:
+
+ ```console
+ # nix profile remove 3
+ ```
+
+* Remove a package by attribute path:
+
+ ```console
+ # nix profile remove packages.x86_64-linux.hello
+ ```
+
+* Remove all packages:
+ ```console
+ # nix profile remove '.*'
+ ```
+
+* Remove a package by store path:
+
+ ```console
+ # nix profile remove /nix/store/rr3y0c6zyk7kjjl8y19s4lsrhn4aiq1z-hello-2.10
+ ```
+
+# Description
+
+This command removes a package from a profile.
+
+)""
diff --git a/src/nix/profile-upgrade.md b/src/nix/profile-upgrade.md
new file mode 100644
index 000000000..2bd5d256d
--- /dev/null
+++ b/src/nix/profile-upgrade.md
@@ -0,0 +1,41 @@
+R""(
+
+# Examples
+
+* Upgrade all packages that were installed using a mutable flake
+ reference:
+
+ ```console
+ # nix profile upgrade '.*'
+ ```
+
+* Upgrade a specific package:
+
+ ```console
+ # nix profile upgrade packages.x86_64-linux.hello
+ ```
+
+* Upgrade a specific profile element by number:
+
+ ```console
+ # nix profile info
+ 0 flake:nixpkgs#legacyPackages.x86_64-linux.spotify …
+
+ # nix profile upgrade 0
+ ```
+
+# Description
+
+This command upgrades a previously installed package in a Nix profile,
+by fetching and evaluating the latest version of the flake from which
+the package was installed.
+
+> **Warning**
+>
+> This only works if you used a *mutable* flake reference at
+> installation time, e.g. `nixpkgs#hello`. It does not work if you
+> used an *immutable* flake reference
+> (e.g. `github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a#hello`),
+> since in that case the "latest version" is always the same.
+
+)""
diff --git a/src/nix/profile.cc b/src/nix/profile.cc
index 01aef2f9b..667904cd2 100644
--- a/src/nix/profile.cc
+++ b/src/nix/profile.cc
@@ -8,6 +8,7 @@
#include "flake/flakeref.hh"
#include "../nix-env/user-env.hh"
#include "profiles.hh"
+#include "names.hh"
#include <nlohmann/json.hpp>
#include <regex>
@@ -21,6 +22,13 @@ struct ProfileElementSource
FlakeRef resolvedRef;
std::string attrPath;
// FIXME: output names
+
+ bool operator < (const ProfileElementSource & other) const
+ {
+ return
+ std::pair(originalRef.to_string(), attrPath) <
+ std::pair(other.originalRef.to_string(), other.attrPath);
+ }
};
struct ProfileElement
@@ -29,6 +37,29 @@ struct ProfileElement
std::optional<ProfileElementSource> source;
bool active = true;
// FIXME: priority
+
+ std::string describe() const
+ {
+ if (source)
+ return fmt("%s#%s", source->originalRef, source->attrPath);
+ StringSet names;
+ for (auto & path : storePaths)
+ names.insert(DrvName(path.name()).name);
+ return concatStringsSep(", ", names);
+ }
+
+ std::string versions() const
+ {
+ StringSet versions;
+ for (auto & path : storePaths)
+ versions.insert(DrvName(path.name()).version);
+ return showVersions(versions);
+ }
+
+ bool operator < (const ProfileElement & other) const
+ {
+ return std::tuple(describe(), storePaths) < std::tuple(other.describe(), other.storePaths);
+ }
};
struct ProfileManifest
@@ -142,6 +173,46 @@ struct ProfileManifest
return std::move(info.path);
}
+
+ static void printDiff(const ProfileManifest & prev, const ProfileManifest & cur, std::string_view indent)
+ {
+ auto prevElems = prev.elements;
+ std::sort(prevElems.begin(), prevElems.end());
+
+ auto curElems = cur.elements;
+ std::sort(curElems.begin(), curElems.end());
+
+ auto i = prevElems.begin();
+ auto j = curElems.begin();
+
+ bool changes = false;
+
+ while (i != prevElems.end() || j != curElems.end()) {
+ if (j != curElems.end() && (i == prevElems.end() || i->describe() > j->describe())) {
+ std::cout << fmt("%s%s: ∅ -> %s\n", indent, j->describe(), j->versions());
+ changes = true;
+ ++j;
+ }
+ else if (i != prevElems.end() && (j == curElems.end() || i->describe() < j->describe())) {
+ std::cout << fmt("%s%s: %s -> ∅\n", indent, i->describe(), i->versions());
+ changes = true;
+ ++i;
+ }
+ else {
+ auto v1 = i->versions();
+ auto v2 = j->versions();
+ if (v1 != v2) {
+ std::cout << fmt("%s%s: %s -> %s\n", indent, i->describe(), v1, v2);
+ changes = true;
+ }
+ ++i;
+ ++j;
+ }
+ }
+
+ if (!changes)
+ std::cout << fmt("%sNo changes.\n", indent);
+ }
};
struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
@@ -151,29 +222,18 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
return "install a package into a profile";
}
- Examples examples() override
- {
- return {
- Example{
- "To install a package from Nixpkgs:",
- "nix profile install nixpkgs#hello"
- },
- Example{
- "To install a package from a specific branch of Nixpkgs:",
- "nix profile install nixpkgs/release-19.09#hello"
- },
- Example{
- "To install a package from a specific revision of Nixpkgs:",
- "nix profile install nixpkgs/1028bb33859f8dfad7f98e1c8d185f3d1aaa7340#hello"
- },
- };
+ std::string doc() override
+ {
+ return
+ #include "profile-install.md"
+ ;
}
void run(ref<Store> store) override
{
ProfileManifest manifest(*getEvalState(), *profile);
- std::vector<StorePathWithOutputs> pathsToBuild;
+ std::vector<DerivedPath> pathsToBuild;
for (auto & installable : installables) {
if (auto installable2 = std::dynamic_pointer_cast<InstallableFlake>(installable)) {
@@ -189,11 +249,34 @@ struct CmdProfileInstall : InstallablesCommand, MixDefaultProfile
attrPath,
};
- pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME
+ pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, StringSet{drv.outputName}});
manifest.elements.emplace_back(std::move(element));
- } else
- throw UnimplementedError("'nix profile install' does not support argument '%s'", installable->what());
+ } else {
+ auto buildables = build(store, Realise::Outputs, {installable}, bmNormal);
+
+ for (auto & buildable : buildables) {
+ ProfileElement element;
+
+ std::visit(overloaded {
+ [&](DerivedPathWithHints::Opaque bo) {
+ pathsToBuild.push_back(bo);
+ element.storePaths.insert(bo.path);
+ },
+ [&](DerivedPathWithHints::Built bfd) {
+ // TODO: Why are we querying if we know the output
+ // names already? Is it just to figure out what the
+ // default one is?
+ for (auto & output : store->queryDerivationOutputMap(bfd.drvPath)) {
+ pathsToBuild.push_back(DerivedPath::Built{bfd.drvPath, {output.first}});
+ element.storePaths.insert(output.second);
+ }
+ },
+ }, buildable.raw());
+
+ manifest.elements.emplace_back(std::move(element));
+ }
+ }
}
store->buildPaths(pathsToBuild);
@@ -220,9 +303,8 @@ public:
std::vector<Matcher> res;
for (auto & s : _matchers) {
- size_t n;
- if (string2Int(s, n))
- res.push_back(n);
+ if (auto n = string2Int<size_t>(s))
+ res.push_back(*n);
else if (store->isStorePath(s))
res.push_back(s);
else
@@ -257,26 +339,11 @@ struct CmdProfileRemove : virtual EvalCommand, MixDefaultProfile, MixProfileElem
return "remove packages from a profile";
}
- Examples examples() override
- {
- return {
- Example{
- "To remove a package by attribute path:",
- "nix profile remove packages.x86_64-linux.hello"
- },
- Example{
- "To remove all packages:",
- "nix profile remove '.*'"
- },
- Example{
- "To remove a package by store path:",
- "nix profile remove /nix/store/rr3y0c6zyk7kjjl8y19s4lsrhn4aiq1z-hello-2.10"
- },
- Example{
- "To remove a package by position:",
- "nix profile remove 3"
- },
- };
+ std::string doc() override
+ {
+ return
+ #include "profile-remove.md"
+ ;
}
void run(ref<Store> store) override
@@ -310,18 +377,11 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
return "upgrade packages using their most recent flake";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To upgrade all packages that were installed using a mutable flake reference:",
- "nix profile upgrade '.*'"
- },
- Example{
- "To upgrade a specific package:",
- "nix profile upgrade packages.x86_64-linux.hello"
- },
- };
+ return
+ #include "profile-upgrade.md"
+ ;
}
void run(ref<Store> store) override
@@ -331,7 +391,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
auto matchers = getMatchers(store);
// FIXME: code duplication
- std::vector<StorePathWithOutputs> pathsToBuild;
+ std::vector<DerivedPath> pathsToBuild;
for (size_t i = 0; i < manifest.elements.size(); ++i) {
auto & element(manifest.elements[i]);
@@ -342,7 +402,13 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
Activity act(*logger, lvlChatty, actUnknown,
fmt("checking '%s' for updates", element.source->attrPath));
- InstallableFlake installable(getEvalState(), FlakeRef(element.source->originalRef), {element.source->attrPath}, {}, lockFlags);
+ InstallableFlake installable(
+ this,
+ getEvalState(),
+ FlakeRef(element.source->originalRef),
+ {element.source->attrPath},
+ {},
+ lockFlags);
auto [attrPath, resolvedRef, drv] = installable.toDerivation();
@@ -360,7 +426,7 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
attrPath,
};
- pathsToBuild.push_back({drv.drvPath, StringSet{"out"}}); // FIXME
+ pathsToBuild.push_back(DerivedPath::Built{drv.drvPath, {"out"}}); // FIXME
}
}
@@ -370,21 +436,18 @@ struct CmdProfileUpgrade : virtual SourceExprCommand, MixDefaultProfile, MixProf
}
};
-struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile
+struct CmdProfileList : virtual EvalCommand, virtual StoreCommand, MixDefaultProfile
{
std::string description() override
{
return "list installed packages";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To show what packages are installed in the default profile:",
- "nix profile info"
- },
- };
+ return
+ #include "profile-list.md"
+ ;
}
void run(ref<Store> store) override
@@ -393,7 +456,7 @@ struct CmdProfileInfo : virtual EvalCommand, virtual StoreCommand, MixDefaultPro
for (size_t i = 0; i < manifest.elements.size(); ++i) {
auto & element(manifest.elements[i]);
- logger->stdout("%d %s %s %s", i,
+ logger->cout("%d %s %s %s", i,
element.source ? element.source->originalRef.to_string() + "#" + element.source->attrPath : "-",
element.source ? element.source->resolvedRef.to_string() + "#" + element.source->attrPath : "-",
concatStringsSep(" ", store->printStorePathSet(element.storePaths)));
@@ -405,17 +468,14 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile
{
std::string description() override
{
- return "show the closure difference between each generation of a profile";
+ return "show the closure difference between each version of a profile";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To show what changed between each generation of the NixOS system profile:",
- "nix profile diff-closure --profile /nix/var/nix/profiles/system"
- },
- };
+ return
+ #include "profile-diff-closures.md"
+ ;
}
void run(ref<Store> store) override
@@ -429,7 +489,7 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile
if (prevGen) {
if (!first) std::cout << "\n";
first = false;
- std::cout << fmt("Generation %d -> %d:\n", prevGen->number, gen.number);
+ std::cout << fmt("Version %d -> %d:\n", prevGen->number, gen.number);
printClosureDiff(store,
store->followLinksToStorePath(prevGen->path),
store->followLinksToStorePath(gen.path),
@@ -441,6 +501,48 @@ struct CmdProfileDiffClosures : virtual StoreCommand, MixDefaultProfile
}
};
+struct CmdProfileHistory : virtual StoreCommand, EvalCommand, MixDefaultProfile
+{
+ std::string description() override
+ {
+ return "show all versions of a profile";
+ }
+
+ std::string doc() override
+ {
+ return
+ #include "profile-history.md"
+ ;
+ }
+
+ void run(ref<Store> store) override
+ {
+ auto [gens, curGen] = findGenerations(*profile);
+
+ std::optional<std::pair<Generation, ProfileManifest>> prevGen;
+ bool first = true;
+
+ for (auto & gen : gens) {
+ ProfileManifest manifest(*getEvalState(), gen.path);
+
+ if (!first) std::cout << "\n";
+ first = false;
+
+ if (prevGen)
+ std::cout << fmt("Version %d -> %d:\n", prevGen->first.number, gen.number);
+ else
+ std::cout << fmt("Version %d:\n", gen.number);
+
+ ProfileManifest::printDiff(
+ prevGen ? prevGen->second : ProfileManifest(),
+ manifest,
+ " ");
+
+ prevGen = {gen, std::move(manifest)};
+ }
+ }
+};
+
struct CmdProfile : NixMultiCommand
{
CmdProfile()
@@ -448,8 +550,9 @@ struct CmdProfile : NixMultiCommand
{"install", []() { return make_ref<CmdProfileInstall>(); }},
{"remove", []() { return make_ref<CmdProfileRemove>(); }},
{"upgrade", []() { return make_ref<CmdProfileUpgrade>(); }},
- {"info", []() { return make_ref<CmdProfileInfo>(); }},
+ {"list", []() { return make_ref<CmdProfileList>(); }},
{"diff-closures", []() { return make_ref<CmdProfileDiffClosures>(); }},
+ {"history", []() { return make_ref<CmdProfileHistory>(); }},
})
{ }
@@ -458,6 +561,13 @@ struct CmdProfile : NixMultiCommand
return "manage Nix profiles";
}
+ std::string doc() override
+ {
+ return
+ #include "profile.md"
+ ;
+ }
+
void run() override
{
if (!command)
diff --git a/src/nix/profile.md b/src/nix/profile.md
new file mode 100644
index 000000000..d3ddcd3d1
--- /dev/null
+++ b/src/nix/profile.md
@@ -0,0 +1,107 @@
+R""(
+
+# Description
+
+`nix profile` allows you to create and manage *Nix profiles*. A Nix
+profile is a set of packages that can be installed and upgraded
+independently from each other. Nix profiles are versioned, allowing
+them to be rolled back easily.
+
+# Default profile
+
+The default profile used by `nix profile` is `$HOME/.nix-profile`,
+which, if it does not exist, is created as a symlink to
+`/nix/var/nix/profiles/per-user/default` if Nix is invoked by the
+`root` user, or `/nix/var/nix/profiles/per-user/`*username* otherwise.
+
+You can specify another profile location using `--profile` *path*.
+
+# Filesystem layout
+
+Profiles are versioned as follows. When using profile *path*, *path*
+is a symlink to *path*`-`*N*, where *N* is the current *version* of
+the profile. In turn, *path*`-`*N* is a symlink to a path in the Nix
+store. For example:
+
+```console
+$ ls -l /nix/var/nix/profiles/per-user/alice/profile*
+lrwxrwxrwx 1 alice users 14 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile -> profile-7-link
+lrwxrwxrwx 1 alice users 51 Oct 28 16:18 /nix/var/nix/profiles/per-user/alice/profile-5-link -> /nix/store/q69xad13ghpf7ir87h0b2gd28lafjj1j-profile
+lrwxrwxrwx 1 alice users 51 Oct 29 13:20 /nix/var/nix/profiles/per-user/alice/profile-6-link -> /nix/store/6bvhpysd7vwz7k3b0pndn7ifi5xr32dg-profile
+lrwxrwxrwx 1 alice users 51 Nov 25 14:35 /nix/var/nix/profiles/per-user/alice/profile-7-link -> /nix/store/mp0x6xnsg0b8qhswy6riqvimai4gm677-profile
+```
+
+Each of these symlinks is a root for the Nix garbage collector.
+
+The contents of the store path corresponding to each version of the
+profile is a tree of symlinks to the files of the installed packages,
+e.g.
+
+```console
+$ ll -R /nix/var/nix/profiles/per-user/eelco/profile-7-link/
+/nix/var/nix/profiles/per-user/eelco/profile-7-link/:
+total 20
+dr-xr-xr-x 2 root root 4096 Jan 1 1970 bin
+-r--r--r-- 2 root root 1402 Jan 1 1970 manifest.json
+dr-xr-xr-x 4 root root 4096 Jan 1 1970 share
+
+/nix/var/nix/profiles/per-user/eelco/profile-7-link/bin:
+total 20
+lrwxrwxrwx 5 root root 79 Jan 1 1970 chromium -> /nix/store/ijm5k0zqisvkdwjkc77mb9qzb35xfi4m-chromium-86.0.4240.111/bin/chromium
+lrwxrwxrwx 7 root root 87 Jan 1 1970 spotify -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/bin/spotify
+lrwxrwxrwx 3 root root 79 Jan 1 1970 zoom-us -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/bin/zoom-us
+
+/nix/var/nix/profiles/per-user/eelco/profile-7-link/share/applications:
+total 12
+lrwxrwxrwx 4 root root 120 Jan 1 1970 chromium-browser.desktop -> /nix/store/4cf803y4vzfm3gyk3vzhzb2327v0kl8a-chromium-unwrapped-86.0.4240.111/share/applications/chromium-browser.desktop
+lrwxrwxrwx 7 root root 110 Jan 1 1970 spotify.desktop -> /nix/store/w9182874m1bl56smps3m5zjj36jhp3rn-spotify-1.1.26.501.gbe11e53b-15/share/applications/spotify.desktop
+lrwxrwxrwx 3 root root 107 Jan 1 1970 us.zoom.Zoom.desktop -> /nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927/share/applications/us.zoom.Zoom.desktop
+
+…
+```
+
+The file `manifest.json` records the provenance of the packages that
+are installed in this version of the profile. It looks like this:
+
+```json
+{
+ "version": 1,
+ "elements": [
+ {
+ "active": true,
+ "attrPath": "legacyPackages.x86_64-linux.zoom-us",
+ "originalUri": "flake:nixpkgs",
+ "storePaths": [
+ "/nix/store/wbhg2ga8f3h87s9h5k0slxk0m81m4cxl-zoom-us-5.3.469451.0927"
+ ],
+ "uri": "github:NixOS/nixpkgs/13d0c311e3ae923a00f734b43fd1d35b47d8943a"
+ },
+ …
+ ]
+}
+```
+
+Each object in the array `elements` denotes an installed package and
+has the following fields:
+
+* `originalUri`: The [flake reference](./nix3-flake.md) specified by
+ the user at the time of installation (e.g. `nixpkgs`). This is also
+ the flake reference that will be used by `nix profile upgrade`.
+
+* `uri`: The immutable flake reference to which `originalUri`
+ resolved.
+
+* `attrPath`: The flake output attribute that provided this
+ package. Note that this is not necessarily the attribute that the
+ user specified, but the one resulting from applying the default
+ attribute paths and prefixes; for instance, `hello` might resolve to
+ `packages.x86_64-linux.hello` and the empty string to
+ `defaultPackage.x86_64-linux`.
+
+* `storePath`: The paths in the Nix store containing the package.
+
+* `active`: Whether the profile contains symlinks to the files of this
+ package. If set to false, the package is kept in the Nix store, but
+ is not "visible" in the profile's symlink tree.
+
+)""
diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc
new file mode 100644
index 000000000..9ee9ccb91
--- /dev/null
+++ b/src/nix/realisation.cc
@@ -0,0 +1,78 @@
+#include "command.hh"
+#include "common-args.hh"
+
+#include <nlohmann/json.hpp>
+
+using namespace nix;
+
+struct CmdRealisation : virtual NixMultiCommand
+{
+ CmdRealisation() : MultiCommand(RegisterCommand::getCommandsFor({"realisation"}))
+ { }
+
+ std::string description() override
+ {
+ return "manipulate a Nix realisation";
+ }
+
+ Category category() override { return catUtility; }
+
+ void run() override
+ {
+ if (!command)
+ throw UsageError("'nix realisation' requires a sub-command.");
+ command->second->prepare();
+ command->second->run();
+ }
+};
+
+static auto rCmdRealisation = registerCommand<CmdRealisation>("realisation");
+
+struct CmdRealisationInfo : RealisedPathsCommand, MixJSON
+{
+ std::string description() override
+ {
+ return "query information about one or several realisations";
+ }
+
+ std::string doc() override
+ {
+ return
+ #include "realisation/info.md"
+ ;
+ }
+
+ Category category() override { return catSecondary; }
+
+ void run(ref<Store> store, std::vector<RealisedPath> paths) override
+ {
+ settings.requireExperimentalFeature("ca-derivations");
+ if (json) {
+ nlohmann::json res = nlohmann::json::array();
+ for (auto & path : paths) {
+ nlohmann::json currentPath;
+ if (auto realisation = std::get_if<Realisation>(&path.raw))
+ currentPath = realisation->toJSON();
+ else
+ currentPath["opaquePath"] = store->printStorePath(path.path());
+
+ res.push_back(currentPath);
+ }
+ std::cout << res.dump();
+ }
+ else {
+ for (auto & path : paths) {
+ if (auto realisation = std::get_if<Realisation>(&path.raw)) {
+ std::cout <<
+ realisation->id.to_string() << " " <<
+ store->printStorePath(realisation->outPath);
+ } else
+ std::cout << store->printStorePath(path.path());
+
+ std::cout << std::endl;
+ }
+ }
+ }
+};
+
+static auto rCmdRealisationInfo = registerCommand2<CmdRealisationInfo>({"realisation", "info"});
diff --git a/src/nix/realisation/info.md b/src/nix/realisation/info.md
new file mode 100644
index 000000000..852240f44
--- /dev/null
+++ b/src/nix/realisation/info.md
@@ -0,0 +1,15 @@
+R"MdBoundary(
+# Description
+
+Display some informations about the given realisation
+
+# Examples
+
+Show some information about the realisation of the `hello` package:
+
+```console
+$ nix realisation info nixpkgs#hello --json
+[{"id":"sha256:3d382378a00588e064ee30be96dd0fa7e7df7cf3fbcace85a0e7b7dada1eef25!out","outPath":"fd3m7xawvrqcg98kgz5hc2vk3x9q0lh7-hello"}]
+```
+
+)MdBoundary"
diff --git a/src/nix/registry-add.md b/src/nix/registry-add.md
new file mode 100644
index 000000000..80a31996a
--- /dev/null
+++ b/src/nix/registry-add.md
@@ -0,0 +1,33 @@
+R""(
+
+# Examples
+
+* Set the `nixpkgs` flake identifier to a specific branch of Nixpkgs:
+
+ ```console
+ # nix registry add nixpkgs github:NixOS/nixpkgs/nixos-20.03
+ ```
+
+* Pin `nixpkgs` to a specific revision:
+
+ ```console
+ # nix registry add nixpkgs github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a
+ ```
+
+* Add an entry that redirects a specific branch of `nixpkgs` to
+ another fork:
+
+ ```console
+ # nix registry add nixpkgs/nixos-20.03 ~/Dev/nixpkgs
+ ```
+
+# Description
+
+This command adds an entry to the user registry that maps flake
+reference *from-url* to flake reference *to-url*. If an entry for
+*from-url* already exists, it is overwritten.
+
+Entries can be removed using [`nix registry
+remove`](./nix3-registry-remove.md).
+
+)""
diff --git a/src/nix/registry-list.md b/src/nix/registry-list.md
new file mode 100644
index 000000000..30b6e29d8
--- /dev/null
+++ b/src/nix/registry-list.md
@@ -0,0 +1,29 @@
+R""(
+
+# Examples
+
+* Show the contents of all registries:
+
+ ```console
+ # nix registry list
+ user flake:dwarffs github:edolstra/dwarffs/d181d714fd36eb06f4992a1997cd5601e26db8f5
+ system flake:nixpkgs path:/nix/store/fxl9mrm5xvzam0lxi9ygdmksskx4qq8s-source?lastModified=1605220118&narHash=sha256-Und10ixH1WuW0XHYMxxuHRohKYb45R%2fT8CwZuLd2D2Q=&rev=3090c65041104931adda7625d37fa874b2b5c124
+ global flake:blender-bin github:edolstra/nix-warez?dir=blender
+ global flake:dwarffs github:edolstra/dwarffs
+ …
+ ```
+
+# Description
+
+This command displays the contents of all registries on standard
+output. Each line represents one registry entry in the format *type*
+*from* *to*, where *type* denotes the registry containing the entry:
+
+* `flags`: entries specified on the command line using `--override-flake`.
+* `user`: the user registry.
+* `system`: the system registry.
+* `global`: the global registry.
+
+See the [`nix registry` manual page](./nix3-registry.md) for more details.
+
+)""
diff --git a/src/nix/registry-pin.md b/src/nix/registry-pin.md
new file mode 100644
index 000000000..6e97e003e
--- /dev/null
+++ b/src/nix/registry-pin.md
@@ -0,0 +1,38 @@
+R""(
+
+# Examples
+
+* Pin `nixpkgs` to its most recent Git revision:
+
+ ```console
+ # nix registry pin nixpkgs
+ ```
+
+ Afterwards the user registry will have an entry like this:
+
+ ```console
+ nix registry list | grep '^user '
+ user flake:nixpkgs github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a
+ ```
+
+ and `nix flake info` will say:
+
+ ```console
+ # nix flake info nixpkgs
+ Resolved URL: github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a
+ Locked URL: github:NixOS/nixpkgs/925b70cd964ceaedee26fde9b19cc4c4f081196a
+ …
+ ```
+
+# Description
+
+This command adds an entry to the user registry that maps flake
+reference *url* to the corresponding *locked* flake reference, that
+is, a flake reference that specifies an exact revision or content
+hash. This ensures that until this registry entry is removed, all uses
+of *url* will resolve to exactly the same flake.
+
+Entries can be removed using [`nix registry
+remove`](./nix3-registry-remove.md).
+
+)""
diff --git a/src/nix/registry-remove.md b/src/nix/registry-remove.md
new file mode 100644
index 000000000..4c0eb4947
--- /dev/null
+++ b/src/nix/registry-remove.md
@@ -0,0 +1,16 @@
+R""(
+
+# Examples
+
+* Remove the entry `nixpkgs` from the user registry:
+
+ ```console
+ # nix registry remove nixpkgs
+ ```
+
+# Description
+
+This command removes from the user registry any entry for flake
+reference *url*.
+
+)""
diff --git a/src/nix/registry.cc b/src/nix/registry.cc
index 8e8983ad0..f9719600f 100644
--- a/src/nix/registry.cc
+++ b/src/nix/registry.cc
@@ -17,6 +17,13 @@ struct CmdRegistryList : StoreCommand
return "list available Nix flakes";
}
+ std::string doc() override
+ {
+ return
+ #include "registry-list.md"
+ ;
+ }
+
void run(nix::ref<nix::Store> store) override
{
using namespace fetchers;
@@ -26,7 +33,7 @@ struct CmdRegistryList : StoreCommand
for (auto & registry : registries) {
for (auto & entry : registry->entries) {
// FIXME: format nicely
- logger->stdout("%s %s %s",
+ logger->cout("%s %s %s",
registry->type == Registry::Flag ? "flags " :
registry->type == Registry::User ? "user " :
registry->type == Registry::System ? "system" :
@@ -47,6 +54,13 @@ struct CmdRegistryAdd : MixEvalArgs, Command
return "add/replace flake in user flake registry";
}
+ std::string doc() override
+ {
+ return
+ #include "registry-add.md"
+ ;
+ }
+
CmdRegistryAdd()
{
expectArg("from-url", &fromUrl);
@@ -75,6 +89,13 @@ struct CmdRegistryRemove : virtual Args, MixEvalArgs, Command
return "remove flake from user flake registry";
}
+ std::string doc() override
+ {
+ return
+ #include "registry-remove.md"
+ ;
+ }
+
CmdRegistryRemove()
{
expectArg("url", &url);
@@ -97,6 +118,13 @@ struct CmdRegistryPin : virtual Args, EvalCommand
return "pin a flake to its current version in user flake registry";
}
+ std::string doc() override
+ {
+ return
+ #include "registry-pin.md"
+ ;
+ }
+
CmdRegistryPin()
{
expectArg("url", &url);
@@ -132,6 +160,13 @@ struct CmdRegistry : virtual NixMultiCommand
return "manage the flake registry";
}
+ std::string doc() override
+ {
+ return
+ #include "registry.md"
+ ;
+ }
+
Category category() override { return catSecondary; }
void run() override
diff --git a/src/nix/registry.md b/src/nix/registry.md
new file mode 100644
index 000000000..557e5795b
--- /dev/null
+++ b/src/nix/registry.md
@@ -0,0 +1,98 @@
+R""(
+
+# Description
+
+`nix flake` provides subcommands for managing *flake
+registries*. Flake registries are a convenience feature that allows
+you to refer to flakes using symbolic identifiers such as `nixpkgs`,
+rather than full URLs such as `git://github.com/NixOS/nixpkgs`. You
+can use these identifiers on the command line (e.g. when you do `nix
+run nixpkgs#hello`) or in flake input specifications in `flake.nix`
+files. The latter are automatically resolved to full URLs and recorded
+in the flake's `flake.lock` file.
+
+In addition, the flake registry allows you to redirect arbitrary flake
+references (e.g. `github:NixOS/patchelf`) to another location, such as
+a local fork.
+
+There are multiple registries. These are, in order from lowest to
+highest precedence:
+
+* The global registry, which is a file downloaded from the URL
+ specified by the setting `flake-registry`. It is cached locally and
+ updated automatically when it's older than `tarball-ttl`
+ seconds. The default global registry is kept in [a GitHub
+ repository](https://github.com/NixOS/flake-registry).
+
+* The system registry, which is shared by all users. The default
+ location is `/etc/nix/registry.json`. On NixOS, the system registry
+ can be specified using the NixOS option `nix.registry`.
+
+* The user registry `~/.config/nix/registry.json`. This registry can
+ be modified by commands such as `nix flake pin`.
+
+* Overrides specified on the command line using the option
+ `--override-flake`.
+
+# Registry format
+
+A registry is a JSON file with the following format:
+
+```json
+{
+ "version": 2,
+ [
+ {
+ "from": {
+ "type": "indirect",
+ "id": "nixpkgs"
+ },
+ "to": {
+ "type": "github",
+ "owner": "NixOS",
+ "repo": "nixpkgs"
+ }
+ },
+ ...
+ ]
+}
+```
+
+That is, it contains a list of objects with attributes `from` and
+`to`, both of which contain a flake reference in attribute
+representation. (For example, `{"type": "indirect", "id": "nixpkgs"}`
+is the attribute representation of `nixpkgs`, while `{"type":
+"github", "owner": "NixOS", "repo": "nixpkgs"}` is the attribute
+representation of `github:NixOS/nixpkgs`.)
+
+Given some flake reference *R*, a registry entry is used if its
+`from` flake reference *matches* *R*. *R* is then replaced by the
+*unification* of the `to` flake reference with *R*.
+
+# Matching
+
+The `from` flake reference in a registry entry *matches* some flake
+reference *R* if the attributes in `from` are the same as the
+attributes in `R`. For example:
+
+* `nixpkgs` matches with `nixpkgs`.
+
+* `nixpkgs` matches with `nixpkgs/nixos-20.09`.
+
+* `nixpkgs/nixos-20.09` does not match with `nixpkgs`.
+
+* `nixpkgs` does not match with `git://github.com/NixOS/patchelf`.
+
+# Unification
+
+The `to` flake reference in a registry entry is *unified* with some flake
+reference *R* by taking `to` and applying the `rev` and `ref`
+attributes from *R*, if specified. For example:
+
+* `github:NixOS/nixpkgs` unified with `nixpkgs` produces `github:NixOS/nixpkgs`.
+
+* `github:NixOS/nixpkgs` unified with `nixpkgs/nixos-20.09` produces `github:NixOS/nixpkgs/nixos-20.09`.
+
+* `github:NixOS/nixpkgs/master` unified with `nixpkgs/nixos-20.09` produces `github:NixOS/nixpkgs/nixos-20.09`.
+
+)""
diff --git a/src/nix/repl.cc b/src/nix/repl.cc
index 9ff386b1d..eed79c332 100644
--- a/src/nix/repl.cc
+++ b/src/nix/repl.cc
@@ -212,7 +212,7 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
try {
if (!removeWhitespace(input).empty() && !processLine(input)) return;
} catch (ParseError & e) {
- if (e.msg().find("unexpected $end") != std::string::npos) {
+ if (e.msg().find("unexpected end of file") != std::string::npos) {
// For parse errors on incomplete input, we continue waiting for the next line of
// input without clearing the input so far.
continue;
@@ -220,9 +220,9 @@ void NixRepl::mainLoop(const std::vector<std::string> & files)
printMsg(lvlError, e.msg());
}
} catch (Error & e) {
- printMsg(lvlError, e.msg());
+ printMsg(lvlError, e.msg());
} catch (Interrupted & e) {
- printMsg(lvlError, e.msg());
+ printMsg(lvlError, e.msg());
}
// We handled the current input fully, so we should clear it
@@ -343,24 +343,6 @@ StringSet NixRepl::completePrefix(string prefix)
}
-static int runProgram(const string & program, const Strings & args)
-{
- Strings args2(args);
- args2.push_front(program);
-
- Pid pid;
- pid = fork();
- if (pid == -1) throw SysError("forking");
- if (pid == 0) {
- restoreAffinity();
- execvp(program.c_str(), stringsToCharPtrs(args2).data());
- _exit(1);
- }
-
- return pid.wait();
-}
-
-
bool isVarName(const string & s)
{
if (s.size() == 0) return false;
@@ -405,6 +387,7 @@ bool NixRepl::processLine(string line)
}
if (command == ":?" || command == ":help") {
+ // FIXME: convert to Markdown, include in the 'nix repl' manpage.
std::cout
<< "The following commands are available:\n"
<< "\n"
@@ -446,11 +429,11 @@ bool NixRepl::processLine(string line)
Pos pos;
- if (v.type == tPath || v.type == tString) {
+ if (v.type() == nPath || v.type() == nString) {
PathSet context;
auto filename = state->coerceToString(noPos, v, context);
pos.file = state->symbols.create(filename);
- } else if (v.type == tLambda) {
+ } else if (v.isLambda()) {
pos = v.lambda.fun->pos;
} else {
// assume it's a derivation
@@ -461,7 +444,7 @@ bool NixRepl::processLine(string line)
auto args = editorFor(pos);
auto editor = args.front();
args.pop_front();
- runProgram(editor, args);
+ runProgram(editor, true, args);
// Reload right after exiting the editor
state->resetFileCache();
@@ -480,7 +463,7 @@ bool NixRepl::processLine(string line)
state->callFunction(f, v, result, Pos());
StorePath drvPath = getDerivationPath(result);
- runProgram(settings.nixBinDir + "/nix-shell", Strings{state->store->printStorePath(drvPath)});
+ runProgram(settings.nixBinDir + "/nix-shell", true, {state->store->printStorePath(drvPath)});
}
else if (command == ":b" || command == ":i" || command == ":s") {
@@ -493,16 +476,18 @@ bool NixRepl::processLine(string line)
/* We could do the build in this process using buildPaths(),
but doing it in a child makes it easier to recover from
problems / SIGINT. */
- if (runProgram(settings.nixBinDir + "/nix", Strings{"build", "--no-link", drvPathRaw}) == 0) {
+ try {
+ runProgram(settings.nixBinDir + "/nix", true, {"build", "--no-link", drvPathRaw});
auto drv = state->store->readDerivation(drvPath);
std::cout << std::endl << "this derivation produced the following outputs:" << std::endl;
for (auto & i : drv.outputsAndOptPaths(*state->store))
std::cout << fmt(" %s -> %s\n", i.first, state->store->printStorePath(*i.second.second));
+ } catch (ExecError &) {
}
} else if (command == ":i") {
- runProgram(settings.nixBinDir + "/nix-env", Strings{"-i", drvPathRaw});
+ runProgram(settings.nixBinDir + "/nix-env", true, {"-i", drvPathRaw});
} else {
- runProgram(settings.nixBinDir + "/nix-shell", Strings{drvPathRaw});
+ runProgram(settings.nixBinDir + "/nix-shell", true, {drvPathRaw});
}
}
@@ -551,9 +536,7 @@ bool NixRepl::processLine(string line)
{
Expr * e = parseString(string(line, p + 1));
Value & v(*state->allocValue());
- v.type = tThunk;
- v.thunk.env = env;
- v.thunk.expr = e;
+ v.mkThunk(env, e);
addVarToScope(state->symbols.create(name), v);
} else {
Value v;
@@ -669,31 +652,31 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
state->forceValue(v);
- switch (v.type) {
+ switch (v.type()) {
- case tInt:
+ case nInt:
str << ANSI_CYAN << v.integer << ANSI_NORMAL;
break;
- case tBool:
+ case nBool:
str << ANSI_CYAN << (v.boolean ? "true" : "false") << ANSI_NORMAL;
break;
- case tString:
+ case nString:
str << ANSI_YELLOW;
printStringValue(str, v.string.s);
str << ANSI_NORMAL;
break;
- case tPath:
+ case nPath:
str << ANSI_GREEN << v.path << ANSI_NORMAL; // !!! escaping?
break;
- case tNull:
+ case nNull:
str << ANSI_CYAN "null" ANSI_NORMAL;
break;
- case tAttrs: {
+ case nAttrs: {
seen.insert(&v);
bool isDrv = state->isDerivation(v);
@@ -738,9 +721,7 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
break;
}
- case tList1:
- case tList2:
- case tListN:
+ case nList:
seen.insert(&v);
str << "[ ";
@@ -761,22 +742,21 @@ std::ostream & NixRepl::printValue(std::ostream & str, Value & v, unsigned int m
str << "]";
break;
- case tLambda: {
- std::ostringstream s;
- s << v.lambda.fun->pos;
- str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL;
- break;
- }
-
- case tPrimOp:
- str << ANSI_MAGENTA "«primop»" ANSI_NORMAL;
- break;
-
- case tPrimOpApp:
- str << ANSI_BLUE "«primop-app»" ANSI_NORMAL;
+ case nFunction:
+ if (v.isLambda()) {
+ std::ostringstream s;
+ s << v.lambda.fun->pos;
+ str << ANSI_BLUE "«lambda @ " << filterANSIEscapes(s.str()) << "»" ANSI_NORMAL;
+ } else if (v.isPrimOp()) {
+ str << ANSI_MAGENTA "«primop»" ANSI_NORMAL;
+ } else if (v.isPrimOpApp()) {
+ str << ANSI_BLUE "«primop-app»" ANSI_NORMAL;
+ } else {
+ abort();
+ }
break;
- case tFloat:
+ case nFloat:
str << v.fpoint;
break;
@@ -806,14 +786,11 @@ struct CmdRepl : StoreCommand, MixEvalArgs
return "start an interactive environment for evaluating Nix expressions";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "Display all special commands within the REPL:",
- "nix repl\nnix-repl> :?"
- }
- };
+ return
+ #include "repl.md"
+ ;
}
void run(ref<Store> store) override
diff --git a/src/nix/repl.md b/src/nix/repl.md
new file mode 100644
index 000000000..bba60f871
--- /dev/null
+++ b/src/nix/repl.md
@@ -0,0 +1,57 @@
+R""(
+
+# Examples
+
+* Display all special commands within the REPL:
+
+ ```console
+ # nix repl
+ nix-repl> :?
+ ```
+
+* Evaluate some simple Nix expressions:
+
+ ```console
+ # nix repl
+
+ nix-repl> 1 + 2
+ 3
+
+ nix-repl> map (x: x * 2) [1 2 3]
+ [ 2 4 6 ]
+ ```
+
+* Interact with Nixpkgs in the REPL:
+
+ ```console
+ # nix repl '<nixpkgs>'
+
+ Loading '<nixpkgs>'...
+ Added 12428 variables.
+
+ nix-repl> emacs.name
+ "emacs-27.1"
+
+ nix-repl> emacs.drvPath
+ "/nix/store/lp0sjrhgg03y2n0l10n70rg0k7hhyz0l-emacs-27.1.drv"
+
+ nix-repl> drv = runCommand "hello" { buildInputs = [ hello ]; } "hello > $out"
+
+ nix-repl> :b x
+ this derivation produced the following outputs:
+ out -> /nix/store/0njwbgwmkwls0w5dv9mpc1pq5fj39q0l-hello
+
+ nix-repl> builtins.readFile drv
+ "Hello, world!\n"
+ ```
+
+# Description
+
+This command provides an interactive environment for evaluating Nix
+expressions. (REPL stands for 'read–eval–print loop'.)
+
+On startup, it loads the Nix expressions named *files* and adds them
+into the lexical scope. You can load addition files using the `:l
+<filename>` command, or reload all files using `:r`.
+
+)""
diff --git a/src/nix/run.cc b/src/nix/run.cc
index 790784382..b5d8ab38a 100644
--- a/src/nix/run.cc
+++ b/src/nix/run.cc
@@ -22,15 +22,16 @@ std::string chrootHelperName = "__run_in_chroot";
struct RunCommon : virtual Command
{
+
+ using Command::run;
+
void runProgram(ref<Store> store,
const std::string & program,
const Strings & args)
{
stopProgressBar();
- restoreSignals();
-
- restoreAffinity();
+ restoreProcessContext();
/* If this is a diverted store (i.e. its "logical" location
(typically /nix/store) differs from its "physical" location
@@ -59,6 +60,9 @@ struct RunCommon : virtual Command
struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment
{
+
+ using InstallablesCommand::run;
+
std::vector<std::string> command = { getEnv("SHELL").value_or("bash") };
CmdShell()
@@ -66,7 +70,7 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment
addFlag({
.longName = "command",
.shortName = 'c',
- .description = "command and arguments to be executed; defaults to '$SHELL'",
+ .description = "Command and arguments to be executed, defaulting to `$SHELL`",
.labels = {"command", "args"},
.handler = {[&](std::vector<std::string> ss) {
if (ss.empty()) throw UsageError("--command requires at least one argument");
@@ -80,26 +84,11 @@ struct CmdShell : InstallablesCommand, RunCommon, MixEnvironment
return "run a shell in which the specified packages are available";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To start a shell providing GNU Hello from NixOS 20.03:",
- "nix shell nixpkgs/nixos-20.03#hello"
- },
- Example{
- "To start a shell providing youtube-dl from your 'nixpkgs' channel:",
- "nix shell nixpkgs#youtube-dl"
- },
- Example{
- "To run GNU Hello:",
- "nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!'"
- },
- Example{
- "To run GNU Hello in a chroot store:",
- "nix shell --store ~/my-nix nixpkgs#hello -c hello"
- },
- };
+ return
+ #include "shell.md"
+ ;
}
void run(ref<Store> store) override
@@ -144,6 +133,8 @@ static auto rCmdShell = registerCommand<CmdShell>("shell");
struct CmdRun : InstallableCommand, RunCommon
{
+ using InstallableCommand::run;
+
std::vector<std::string> args;
CmdRun()
@@ -160,22 +151,11 @@ struct CmdRun : InstallableCommand, RunCommon
return "run a Nix application";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To run Blender:",
- "nix run blender-bin"
- },
- Example{
- "To run vim from nixpkgs:",
- "nix run nixpkgs#vim"
- },
- Example{
- "To run vim from nixpkgs with arguments:",
- "nix run nixpkgs#vim -- --help"
- },
- };
+ return
+ #include "run.md"
+ ;
}
Strings getDefaultFlakeAttrPaths() override
@@ -200,7 +180,7 @@ struct CmdRun : InstallableCommand, RunCommon
auto app = installable->toApp(*state);
- state->store->buildPaths(app.context);
+ state->store->buildPaths(toDerivedPaths(app.context));
Strings allArgs{app.program};
for (auto & i : args) allArgs.push_back(i);
@@ -250,14 +230,16 @@ void chrootHelper(int argc, char * * argv)
for (auto entry : readDirectory("/")) {
auto src = "/" + entry.name;
- auto st = lstat(src);
- if (!S_ISDIR(st.st_mode)) continue;
Path dst = tmpDir + "/" + entry.name;
if (pathExists(dst)) continue;
- if (mkdir(dst.c_str(), 0700) == -1)
- throw SysError("creating directory '%s'", dst);
- if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1)
- throw SysError("mounting '%s' on '%s'", src, dst);
+ auto st = lstat(src);
+ if (S_ISDIR(st.st_mode)) {
+ if (mkdir(dst.c_str(), 0700) == -1)
+ throw SysError("creating directory '%s'", dst);
+ if (mount(src.c_str(), dst.c_str(), "", MS_BIND | MS_REC, 0) == -1)
+ throw SysError("mounting '%s' on '%s'", src, dst);
+ } else if (S_ISLNK(st.st_mode))
+ createSymlink(readLink(src), dst);
}
char * cwd = getcwd(0, 0);
diff --git a/src/nix/run.md b/src/nix/run.md
new file mode 100644
index 000000000..a76750376
--- /dev/null
+++ b/src/nix/run.md
@@ -0,0 +1,88 @@
+R""(
+
+# Examples
+
+* Run the default app from the `blender-bin` flake:
+
+ ```console
+ # nix run blender-bin
+ ```
+
+* Run a non-default app from the `blender-bin` flake:
+
+ ```console
+ # nix run blender-bin#blender_2_83
+ ```
+
+ Tip: you can find apps provided by this flake by running `nix flake
+ show blender-bin`.
+
+* Run `vim` from the `nixpkgs` flake:
+
+ ```console
+ # nix run nixpkgs#vim
+ ```
+
+ Note that `vim` (as of the time of writing of this page) is not an
+ app but a package. Thus, Nix runs the eponymous file from the `vim`
+ package.
+
+* Run `vim` with arguments:
+
+ ```console
+ # nix run nixpkgs#vim -- --help
+ ```
+
+# Description
+
+`nix run` builds and runs *installable*, which must evaluate to an
+*app* or a regular Nix derivation.
+
+If *installable* evaluates to an *app* (see below), it executes the
+program specified by the app definition.
+
+If *installable* evaluates to a derivation, it will try to execute the
+program `<out>/bin/<name>`, where *out* is the primary output store
+path of the derivation and *name* is the `meta.mainProgram` attribute
+of the derivation if it exists, and otherwise the name part of the
+value of the `name` attribute of the derivation (e.g. if `name` is set
+to `hello-1.10`, it will run `$out/bin/hello`).
+
+# Flake output attributes
+
+If no flake output attribute is given, `nix run` tries the following
+flake output attributes:
+
+* `defaultApp.<system>`
+
+* `defaultPackage.<system>`
+
+If an attribute *name* is given, `nix run` tries the following flake
+output attributes:
+
+* `apps.<system>.<name>`
+
+* `packages.<system>.<name>`
+
+* `legacyPackages.<system>.<name>`
+
+# Apps
+
+An app is specified by a flake output attribute named
+`apps.<system>.<name>` or `defaultApp.<system>`. It looks like this:
+
+```nix
+apps.x86_64-linux.blender_2_79 = {
+ type = "app";
+ program = "${self.packages.x86_64-linux.blender_2_79}/bin/blender";
+};
+```
+
+The only supported attributes are:
+
+* `type` (required): Must be set to `app`.
+
+* `program` (required): The full path of the executable to run. It
+ must reside in the Nix store.
+
+)""
diff --git a/src/nix/search.cc b/src/nix/search.cc
index d4326dc84..c52a48d4e 100644
--- a/src/nix/search.cc
+++ b/src/nix/search.cc
@@ -41,29 +41,14 @@ struct CmdSearch : InstallableCommand, MixJSON
std::string description() override
{
- return "query available packages";
+ return "search for packages";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To show all packages in the flake in the current directory:",
- "nix search"
- },
- Example{
- "To show packages in the 'nixpkgs' flake containing 'blender' in its name or description:",
- "nix search nixpkgs blender"
- },
- Example{
- "To search for Firefox or Chromium:",
- "nix search nixpkgs 'firefox|chromium'"
- },
- Example{
- "To search for packages containing 'git' and either 'frontend' or 'gui':",
- "nix search nixpkgs git 'frontend|gui'"
- }
- };
+ return
+ #include "search.md"
+ ;
}
Strings getDefaultFlakeAttrPaths() override
@@ -96,9 +81,9 @@ struct CmdSearch : InstallableCommand, MixJSON
uint64_t results = 0;
- std::function<void(eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath)> visit;
+ std::function<void(eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse)> visit;
- visit = [&](eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath)
+ visit = [&](eval_cache::AttrCursor & cursor, const std::vector<Symbol> & attrPath, bool initialRecurse)
{
Activity act(*logger, lvlInfo, actUnknown,
fmt("evaluating '%s'", concatStringsSep(".", attrPath)));
@@ -109,7 +94,7 @@ struct CmdSearch : InstallableCommand, MixJSON
auto cursor2 = cursor.getAttr(attr);
auto attrPath2(attrPath);
attrPath2.push_back(attr);
- visit(*cursor2, attrPath2);
+ visit(*cursor2, attrPath2, false);
}
};
@@ -147,13 +132,13 @@ struct CmdSearch : InstallableCommand, MixJSON
jsonElem.attr("description", description);
} else {
auto name2 = hilite(name.name, nameMatch, "\e[0;2m");
- if (results > 1) logger->stdout("");
- logger->stdout(
+ if (results > 1) logger->cout("");
+ logger->cout(
"* %s%s",
wrap("\e[0;1m", hilite(attrPath2, attrPathMatch, "\e[0;1m")),
name.version != "" ? " (" + name.version + ")" : "");
if (description != "")
- logger->stdout(
+ logger->cout(
" %s", hilite(description, descriptionMatch, ANSI_NORMAL));
}
}
@@ -165,6 +150,9 @@ struct CmdSearch : InstallableCommand, MixJSON
|| (attrPath[0] == "packages" && attrPath.size() <= 2))
recurse();
+ else if (initialRecurse)
+ recurse();
+
else if (attrPath[0] == "legacyPackages" && attrPath.size() > 2) {
auto attr = cursor.maybeGetAttr(state->sRecurseForDerivations);
if (attr && attr->getBool())
@@ -178,7 +166,7 @@ struct CmdSearch : InstallableCommand, MixJSON
};
for (auto & [cursor, prefix] : installable->getCursors(*state))
- visit(*cursor, parseAttrPath(*state, prefix));
+ visit(*cursor, parseAttrPath(*state, prefix), true);
if (!json && !results)
throw Error("no results for the given search term(s)!");
diff --git a/src/nix/search.md b/src/nix/search.md
new file mode 100644
index 000000000..d182788a6
--- /dev/null
+++ b/src/nix/search.md
@@ -0,0 +1,72 @@
+R""(
+
+# Examples
+
+* Show all packages in the `nixpkgs` flake:
+
+ ```console
+ # nix search nixpkgs
+ * legacyPackages.x86_64-linux.AMB-plugins (0.8.1)
+ A set of ambisonics ladspa plugins
+
+ * legacyPackages.x86_64-linux.ArchiSteamFarm (4.3.1.0)
+ Application with primary purpose of idling Steam cards from multiple accounts simultaneously
+ …
+ ```
+
+* Show packages in the `nixpkgs` flake containing `blender` in its
+ name or description:
+
+ ```console
+ # nix search nixpkgs blender
+ * legacyPackages.x86_64-linux.blender (2.91.0)
+ 3D Creation/Animation/Publishing System
+ ```
+
+* Search for packages underneath the attribute `gnome3` in Nixpkgs:
+
+ ```console
+ # nix search nixpkgs#gnome3 vala
+ * legacyPackages.x86_64-linux.gnome3.vala (0.48.9)
+ Compiler for GObject type system
+ ```
+
+* Show all packages in the flake in the current directory:
+
+ ```console
+ # nix search
+ ```
+
+* Search for Firefox or Chromium:
+
+ ```console
+ # nix search nixpkgs 'firefox|chromium'
+ ```
+
+* Search for packages containing `git'`and either `frontend` or `gui`:
+
+ ```console
+ # nix search nixpkgs git 'frontend|gui'
+ ```
+
+# Description
+
+`nix search` searches *installable* (which must be evaluatable, e.g. a
+flake) for packages whose name or description matches all of the
+regular expressions *regex*. For each matching package, It prints the
+full attribute name (from the root of the installable), the version
+and the `meta.description` field, highlighting the substrings that
+were matched by the regular expressions. If no regular expressions are
+specified, all packages are shown.
+
+# Flake output attributes
+
+If no flake output attribute is given, `nix search` searches for
+packages:
+
+* Directly underneath `packages.<system>`.
+
+* Underneath `legacyPackages.<system>`, recursing into attribute sets
+ that contain an attribute `recurseForDerivations = true`.
+
+)""
diff --git a/src/nix/shell.md b/src/nix/shell.md
new file mode 100644
index 000000000..2a379e03f
--- /dev/null
+++ b/src/nix/shell.md
@@ -0,0 +1,48 @@
+R""(
+
+# Examples
+
+* Start a shell providing `youtube-dl` from the `nixpkgs` flake:
+
+ ```console
+ # nix shell nixpkgs#youtube-dl
+ # youtube-dl --version
+ 2020.11.01.1
+ ```
+
+* Start a shell providing GNU Hello from NixOS 20.03:
+
+ ```console
+ # nix shell nixpkgs/nixos-20.03#hello
+ ```
+
+* Run GNU Hello:
+
+ ```console
+ # nix shell nixpkgs#hello -c hello --greeting 'Hi everybody!'
+ Hi everybody!
+ ```
+
+* Run GNU Hello in a chroot store:
+
+ ```console
+ # nix shell --store ~/my-nix nixpkgs#hello -c hello
+ ```
+
+* Start a shell providing GNU Hello in a chroot store:
+
+ ```console
+ # nix shell --store ~/my-nix nixpkgs#hello nixpkgs#bashInteractive -c bash
+ ```
+
+ Note that it's necessary to specify `bash` explicitly because your
+ default shell (e.g. `/bin/bash`) generally will not exist in the
+ chroot.
+
+# Description
+
+`nix shell` runs a command in an environment in which the `$PATH`
+variable provides the specified *installables*. If not command is
+specified, it starts the default shell of your user account.
+
+)""
diff --git a/src/nix/show-config.cc b/src/nix/show-config.cc
index 1ef54a33a..91721219b 100644
--- a/src/nix/show-config.cc
+++ b/src/nix/show-config.cc
@@ -20,12 +20,12 @@ struct CmdShowConfig : Command, MixJSON
{
if (json) {
// FIXME: use appropriate JSON types (bool, ints, etc).
- logger->stdout("%s", globalConfig.toJSON().dump());
+ logger->cout("%s", globalConfig.toJSON().dump());
} else {
std::map<std::string, Config::SettingInfo> settings;
globalConfig.getSettings(settings);
for (auto & s : settings)
- logger->stdout("%s = %s", s.first, s.second.value);
+ logger->cout("%s = %s", s.first, s.second.value);
}
}
};
diff --git a/src/nix/show-derivation.cc b/src/nix/show-derivation.cc
index 2542537d3..2588a011d 100644
--- a/src/nix/show-derivation.cc
+++ b/src/nix/show-derivation.cc
@@ -1,4 +1,5 @@
// FIXME: integrate this with nix path-info?
+// FIXME: rename to 'nix store show-derivation' or 'nix debug show-derivation'?
#include "command.hh"
#include "common-args.hh"
@@ -18,7 +19,7 @@ struct CmdShowDerivation : InstallablesCommand
addFlag({
.longName = "recursive",
.shortName = 'r',
- .description = "include the dependencies of the specified derivations",
+ .description = "Include the dependencies of the specified derivations.",
.handler = {&recursive, true}
});
}
@@ -28,18 +29,11 @@ struct CmdShowDerivation : InstallablesCommand
return "show the contents of a store derivation";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To show the store derivation that results from evaluating the Hello package:",
- "nix show-derivation nixpkgs#hello"
- },
- Example{
- "To show the full derivation graph (if available) that produced your NixOS system:",
- "nix show-derivation -r /run/current-system"
- },
- };
+ return
+ #include "show-derivation.md"
+ ;
}
Category category() override { return catUtility; }
@@ -82,6 +76,7 @@ struct CmdShowDerivation : InstallablesCommand
[&](DerivationOutputCAFloating dof) {
outputObj.attr("hashAlgo", makeFileIngestionPrefix(dof.method) + printHashType(dof.hashType));
},
+ [&](DerivationOutputDeferred) {},
}, output.output);
}
}
@@ -101,7 +96,7 @@ struct CmdShowDerivation : InstallablesCommand
}
}
- drvObj.attr("platform", drv.platform);
+ drvObj.attr("system", drv.platform);
drvObj.attr("builder", drv.builder);
{
diff --git a/src/nix/show-derivation.md b/src/nix/show-derivation.md
new file mode 100644
index 000000000..aa863899c
--- /dev/null
+++ b/src/nix/show-derivation.md
@@ -0,0 +1,103 @@
+R""(
+
+# Examples
+
+* Show the store derivation that results from evaluating the Hello
+ package:
+
+ ```console
+ # nix show-derivation nixpkgs#hello
+ {
+ "/nix/store/s6rn4jz1sin56rf4qj5b5v8jxjm32hlk-hello-2.10.drv": {
+ …
+ }
+ }
+ ```
+
+* Show the full derivation graph (if available) that produced your
+ NixOS system:
+
+ ```console
+ # nix show-derivation -r /run/current-system
+ ```
+
+* Print all files fetched using `fetchurl` by Firefox's dependency
+ graph:
+
+ ```console
+ # nix show-derivation -r nixpkgs#firefox \
+ | jq -r '.[] | select(.outputs.out.hash and .env.urls) | .env.urls' \
+ | uniq | sort
+ ```
+
+ Note that `.outputs.out.hash` selects *fixed-output derivations*
+ (derivations that produce output with a specified content hash),
+ while `.env.urls` selects derivations with a `urls` attribute.
+
+# Description
+
+This command prints on standard output a JSON representation of the
+store derivations to which *installables* evaluate. Store derivations
+are used internally by Nix. They are store paths with extension `.drv`
+that represent the build-time dependency graph to which a Nix
+expression evaluates.
+
+By default, this command only shows top-level derivations, but with
+`--recursive`, it also shows their dependencies.
+
+The JSON output is a JSON object whose keys are the store paths of the
+derivations, and whose values are a JSON object with the following
+fields:
+
+* `outputs`: Information about the output paths of the
+ derivation. This is a JSON object with one member per output, where
+ the key is the output name and the value is a JSON object with these
+ fields:
+
+ * `path`: The output path.
+ * `hashAlgo`: For fixed-output derivations, the hashing algorithm
+ (e.g. `sha256`), optionally prefixed by `r:` if `hash` denotes a
+ NAR hash rather than a flat file hash.
+ * `hash`: For fixed-output derivations, the expected content hash in
+ base-16.
+
+ Example:
+
+ ```json
+ "outputs": {
+ "out": {
+ "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source",
+ "hashAlgo": "r:sha256",
+ "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62"
+ }
+ }
+ ```
+
+* `inputSrcs`: A list of store paths on which this derivation depends.
+
+* `inputDrvs`: A JSON object specifying the derivations on which this
+ derivation depends, and what outputs of those derivations. For
+ example,
+
+ ```json
+ "inputDrvs": {
+ "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"],
+ "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"]
+ }
+ ```
+
+ specifies that this derivation depends on the `dev` output of
+ `curl`, and the `out` output of `unzip`.
+
+* `system`: The system type on which this derivation is to be built
+ (e.g. `x86_64-linux`).
+
+* `builder`: The absolute path of the program to be executed to run
+ the build. Typically this is the `bash` shell
+ (e.g. `/nix/store/r3j288vpmczbl500w6zz89gyfa4nr0b1-bash-4.4-p23/bin/bash`).
+
+* `args`: The command-line arguments passed to the `builder`.
+
+* `env`: The environment passed to the `builder`.
+
+)""
diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc
index 44916c77f..c64b472b6 100644
--- a/src/nix/sigs.cc
+++ b/src/nix/sigs.cc
@@ -16,7 +16,7 @@ struct CmdCopySigs : StorePathsCommand
addFlag({
.longName = "substituter",
.shortName = 's',
- .description = "use signatures from specified store",
+ .description = "Copy signatures from the specified store.",
.labels = {"store-uri"},
.handler = {[&](std::string s) { substituterUris.push_back(s); }},
});
@@ -24,11 +24,9 @@ struct CmdCopySigs : StorePathsCommand
std::string description() override
{
- return "copy path signatures from substituters (like binary caches)";
+ return "copy store path signatures from substituters";
}
- Category category() override { return catUtility; }
-
void run(ref<Store> store, StorePaths storePaths) override
{
if (substituterUris.empty())
@@ -92,18 +90,18 @@ struct CmdCopySigs : StorePathsCommand
}
};
-static auto rCmdCopySigs = registerCommand<CmdCopySigs>("copy-sigs");
+static auto rCmdCopySigs = registerCommand2<CmdCopySigs>({"store", "copy-sigs"});
-struct CmdSignPaths : StorePathsCommand
+struct CmdSign : StorePathsCommand
{
Path secretKeyFile;
- CmdSignPaths()
+ CmdSign()
{
addFlag({
.longName = "key-file",
.shortName = 'k',
- .description = "file containing the secret signing key",
+ .description = "File containing the secret signing key.",
.labels = {"file"},
.handler = {&secretKeyFile},
.completer = completePath
@@ -112,11 +110,9 @@ struct CmdSignPaths : StorePathsCommand
std::string description() override
{
- return "sign the specified paths";
+ return "sign store paths";
}
- Category category() override { return catUtility; }
-
void run(ref<Store> store, StorePaths storePaths) override
{
if (secretKeyFile.empty())
@@ -144,4 +140,89 @@ struct CmdSignPaths : StorePathsCommand
}
};
-static auto rCmdSignPaths = registerCommand<CmdSignPaths>("sign-paths");
+static auto rCmdSign = registerCommand2<CmdSign>({"store", "sign"});
+
+struct CmdKeyGenerateSecret : Command
+{
+ std::optional<std::string> keyName;
+
+ CmdKeyGenerateSecret()
+ {
+ addFlag({
+ .longName = "key-name",
+ .description = "Identifier of the key (e.g. `cache.example.org-1`).",
+ .labels = {"name"},
+ .handler = {&keyName},
+ });
+ }
+
+ std::string description() override
+ {
+ return "generate a secret key for signing store paths";
+ }
+
+ std::string doc() override
+ {
+ return
+ #include "key-generate-secret.md"
+ ;
+ }
+
+ void run() override
+ {
+ if (!keyName)
+ throw UsageError("required argument '--key-name' is missing");
+
+ std::cout << SecretKey::generate(*keyName).to_string();
+ }
+};
+
+struct CmdKeyConvertSecretToPublic : Command
+{
+ std::string description() override
+ {
+ return "generate a public key for verifying store paths from a secret key read from standard input";
+ }
+
+ std::string doc() override
+ {
+ return
+ #include "key-convert-secret-to-public.md"
+ ;
+ }
+
+ void run() override
+ {
+ SecretKey secretKey(drainFD(STDIN_FILENO));
+ std::cout << secretKey.toPublicKey().to_string();
+ }
+};
+
+struct CmdKey : NixMultiCommand
+{
+ CmdKey()
+ : MultiCommand({
+ {"generate-secret", []() { return make_ref<CmdKeyGenerateSecret>(); }},
+ {"convert-secret-to-public", []() { return make_ref<CmdKeyConvertSecretToPublic>(); }},
+ })
+ {
+ }
+
+ std::string description() override
+ {
+ return "generate and convert Nix signing keys";
+ }
+
+ Category category() override { return catUtility; }
+
+ void run() override
+ {
+ if (!command)
+ throw UsageError("'nix flake' requires a sub-command.");
+ settings.requireExperimentalFeature("flakes");
+ command->second->prepare();
+ command->second->run();
+ }
+};
+
+static auto rCmdKey = registerCommand<CmdKey>("key");
diff --git a/src/nix/store-cat.md b/src/nix/store-cat.md
new file mode 100644
index 000000000..da2073473
--- /dev/null
+++ b/src/nix/store-cat.md
@@ -0,0 +1,19 @@
+R""(
+
+# Examples
+
+* Show the contents of a file in a binary cache:
+
+ ```console
+ # nix store cat --store https://cache.nixos.org/ \
+ /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello | hexdump -C | head -n1
+ 00000000 7f 45 4c 46 02 01 01 00 00 00 00 00 00 00 00 00 |.ELF............|
+ ```
+
+# Description
+
+This command prints on standard output the contents of the regular
+file *path* in a Nix store. *path* can be a top-level store path or
+any file inside a store path.
+
+)""
diff --git a/src/nix/store-delete.cc b/src/nix/store-delete.cc
new file mode 100644
index 000000000..10245978e
--- /dev/null
+++ b/src/nix/store-delete.cc
@@ -0,0 +1,44 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "store-api.hh"
+
+using namespace nix;
+
+struct CmdStoreDelete : StorePathsCommand
+{
+ GCOptions options { .action = GCOptions::gcDeleteSpecific };
+
+ CmdStoreDelete()
+ {
+ addFlag({
+ .longName = "ignore-liveness",
+ .description = "Do not check whether the paths are reachable from a root.",
+ .handler = {&options.ignoreLiveness, true}
+ });
+ }
+
+ std::string description() override
+ {
+ return "delete paths from the Nix store";
+ }
+
+ std::string doc() override
+ {
+ return
+ #include "store-delete.md"
+ ;
+ }
+
+ void run(ref<Store> store, std::vector<StorePath> storePaths) override
+ {
+ for (auto & path : storePaths)
+ options.pathsToDelete.insert(path);
+
+ GCResults results;
+ PrintFreed freed(true, results);
+ store->collectGarbage(options, results);
+ }
+};
+
+static auto rCmdStoreDelete = registerCommand2<CmdStoreDelete>({"store", "delete"});
diff --git a/src/nix/store-delete.md b/src/nix/store-delete.md
new file mode 100644
index 000000000..db535f87c
--- /dev/null
+++ b/src/nix/store-delete.md
@@ -0,0 +1,24 @@
+R""(
+
+# Examples
+
+* Delete a specific store path:
+
+ ```console
+ # nix store delete /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10
+ ```
+
+# Description
+
+This command deletes the store paths specified by *installables*. ,
+but only if it is safe to do so; that is, when the path is not
+reachable from a root of the garbage collector. This means that you
+can only delete paths that would also be deleted by `nix store
+gc`. Thus, `nix store delete` is a more targeted version of `nix store
+gc`.
+
+With the option `--ignore-liveness`, reachability from the roots is
+ignored. However, the path still won't be deleted if there are other
+paths in the store that refer to it (i.e., depend on it).
+
+)""
diff --git a/src/nix/store-dump-path.md b/src/nix/store-dump-path.md
new file mode 100644
index 000000000..4ef563526
--- /dev/null
+++ b/src/nix/store-dump-path.md
@@ -0,0 +1,23 @@
+R""(
+
+# Examples
+
+* To get a NAR containing the GNU Hello package:
+
+ ```console
+ # nix store dump-path nixpkgs#hello > hello.nar
+ ```
+
+* To get a NAR from the binary cache https://cache.nixos.org/:
+
+ ```console
+ # nix store dump-path --store https://cache.nixos.org/ \
+ /nix/store/7crrmih8c52r8fbnqb933dxrsp44md93-glibc-2.25 > glibc.nar
+ ```
+
+# Description
+
+This command generates a NAR file containing the serialisation of the
+store path *installable*. The NAR is written to standard output.
+
+)""
diff --git a/src/nix/store-gc.cc b/src/nix/store-gc.cc
new file mode 100644
index 000000000..a2d74066e
--- /dev/null
+++ b/src/nix/store-gc.cc
@@ -0,0 +1,43 @@
+#include "command.hh"
+#include "common-args.hh"
+#include "shared.hh"
+#include "store-api.hh"
+
+using namespace nix;
+
+struct CmdStoreGC : StoreCommand, MixDryRun
+{
+ GCOptions options;
+
+ CmdStoreGC()
+ {
+ addFlag({
+ .longName = "max",
+ .description = "Stop after freeing *n* bytes of disk space.",
+ .labels = {"n"},
+ .handler = {&options.maxFreed}
+ });
+ }
+
+ std::string description() override
+ {
+ return "perform garbage collection on a Nix store";
+ }
+
+ std::string doc() override
+ {
+ return
+ #include "store-gc.md"
+ ;
+ }
+
+ void run(ref<Store> store) override
+ {
+ options.action = dryRun ? GCOptions::gcReturnDead : GCOptions::gcDeleteDead;
+ GCResults results;
+ PrintFreed freed(options.action == GCOptions::gcDeleteDead, results);
+ store->collectGarbage(options, results);
+ }
+};
+
+static auto rCmdStoreGC = registerCommand2<CmdStoreGC>({"store", "gc"});
diff --git a/src/nix/store-gc.md b/src/nix/store-gc.md
new file mode 100644
index 000000000..956b3c872
--- /dev/null
+++ b/src/nix/store-gc.md
@@ -0,0 +1,21 @@
+R""(
+
+# Examples
+
+* Delete unreachable paths in the Nix store:
+
+ ```console
+ # nix store gc
+ ```
+
+* Delete up to 1 gigabyte of garbage:
+
+ ```console
+ # nix store gc --max 1G
+ ```
+
+# Description
+
+This command deletes unreachable paths in the Nix store.
+
+)""
diff --git a/src/nix/store-ls.md b/src/nix/store-ls.md
new file mode 100644
index 000000000..836efce42
--- /dev/null
+++ b/src/nix/store-ls.md
@@ -0,0 +1,27 @@
+R""(
+
+# Examples
+
+* To list the contents of a store path in a binary cache:
+
+ ```console
+ # nix store ls --store https://cache.nixos.org/ -lR /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10
+ dr-xr-xr-x 0 ./bin
+ -r-xr-xr-x 38184 ./bin/hello
+ dr-xr-xr-x 0 ./share
+ …
+ ```
+
+* To show information about a specific file in a binary cache:
+
+ ```console
+ # nix store ls --store https://cache.nixos.org/ -l /nix/store/0i2jd68mp5g6h2sa5k9c85rb80sn8hi9-hello-2.10/bin/hello
+ -r-xr-xr-x 38184 hello
+ ```
+
+# Description
+
+This command shows information about *path* in a Nix store. *path* can
+be a top-level store path or any file inside a store path.
+
+)""
diff --git a/src/nix/store-prefetch-file.md b/src/nix/store-prefetch-file.md
new file mode 100644
index 000000000..f9fdcbc57
--- /dev/null
+++ b/src/nix/store-prefetch-file.md
@@ -0,0 +1,32 @@
+R""(
+
+# Examples
+
+* Download a file to the Nix store:
+
+ ```console
+ # nix store prefetch-file https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz
+ Downloaded 'https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz' to
+ '/nix/store/vbdbi42hgnc4h7pyqzp6h2yf77kw93aw-source' (hash
+ 'sha256-qKheVd5D0BervxMDbt+1hnTKE2aRWC8XCAwc0SeHt6s=').
+ ```
+
+* Download a file and get the SHA-512 hash:
+
+ ```console
+ # nix store prefetch-file --json --hash-type sha512 \
+ https://releases.nixos.org/nix/nix-2.3.10/nix-2.3.10.tar.xz \
+ | jq -r .hash
+ sha512-6XJxfym0TNH9knxeH4ZOvns6wElFy3uahunl2hJgovACCMEMXSy42s69zWVyGJALXTI+86tpDJGlIcAySEKBbA==
+ ```
+
+# Description
+
+This command downloads the file *url* to the Nix store. It prints out
+the resulting store path and the cryptographic hash of the contents of
+the file.
+
+The name component of the store path defaults to the last component of
+*url*, but this can be overridden using `--name`.
+
+)""
diff --git a/src/nix/store-repair.cc b/src/nix/store-repair.cc
new file mode 100644
index 000000000..1c7a4392e
--- /dev/null
+++ b/src/nix/store-repair.cc
@@ -0,0 +1,27 @@
+#include "command.hh"
+#include "store-api.hh"
+
+using namespace nix;
+
+struct CmdStoreRepair : StorePathsCommand
+{
+ std::string description() override
+ {
+ return "repair store paths";
+ }
+
+ std::string doc() override
+ {
+ return
+ #include "store-repair.md"
+ ;
+ }
+
+ void run(ref<Store> store, std::vector<StorePath> storePaths) override
+ {
+ for (auto & path : storePaths)
+ store->repairPath(path);
+ }
+};
+
+static auto rStoreRepair = registerCommand2<CmdStoreRepair>({"store", "repair"});
diff --git a/src/nix/store-repair.md b/src/nix/store-repair.md
new file mode 100644
index 000000000..92d2205a9
--- /dev/null
+++ b/src/nix/store-repair.md
@@ -0,0 +1,32 @@
+R""(
+
+# Examples
+
+* Repair a store path, after determining that it is corrupt:
+
+ ```console
+ # nix store verify /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10
+ path '/nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10' was
+ modified! expected hash
+ 'sha256:1hd5vnh6xjk388gdk841vflicy8qv7qzj2hb7xlyh8lpb43j921l', got
+ 'sha256:1a25lf78x5wi6pfkrxalf0n13kdaca0bqmjqnp7wfjza2qz5ssgl'
+
+ # nix store repair /nix/store/yb5q57zxv6hgqql42d5r8b5k5mcq6kay-hello-2.10
+ ```
+
+# Description
+
+This command attempts to "repair" the store paths specified by
+*installables* by redownloading them using the available
+substituters. If no substitutes are available, then repair is not
+possible.
+
+> **Warning**
+>
+> During repair, there is a very small time window during which the old
+> path (if it exists) is moved out of the way and replaced with the new
+> path. If repair is interrupted in between, then the system may be left
+> in a broken state (e.g., if the path contains a critical system
+> component like the GNU C Library).
+
+)""
diff --git a/src/nix/store.cc b/src/nix/store.cc
new file mode 100644
index 000000000..44e53c7c7
--- /dev/null
+++ b/src/nix/store.cc
@@ -0,0 +1,26 @@
+#include "command.hh"
+
+using namespace nix;
+
+struct CmdStore : virtual NixMultiCommand
+{
+ CmdStore() : MultiCommand(RegisterCommand::getCommandsFor({"store"}))
+ { }
+
+ std::string description() override
+ {
+ return "manipulate a Nix store";
+ }
+
+ Category category() override { return catUtility; }
+
+ void run() override
+ {
+ if (!command)
+ throw UsageError("'nix store' requires a sub-command.");
+ command->second->prepare();
+ command->second->run();
+ }
+};
+
+static auto rCmdStore = registerCommand<CmdStore>("store");
diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc
index 66ecc5b34..9cd567896 100644
--- a/src/nix/upgrade-nix.cc
+++ b/src/nix/upgrade-nix.cc
@@ -19,14 +19,14 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
addFlag({
.longName = "profile",
.shortName = 'p',
- .description = "the Nix profile to upgrade",
+ .description = "The path to the Nix profile to upgrade.",
.labels = {"profile-dir"},
.handler = {&profileDir}
});
addFlag({
.longName = "nix-store-paths-url",
- .description = "URL of the file that contains the store paths of the latest Nix release",
+ .description = "The URL of the file that contains the store paths of the latest Nix release.",
.labels = {"url"},
.handler = {&storePathsUrl}
});
@@ -37,18 +37,11 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
return "upgrade Nix to the latest stable version";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To upgrade Nix to the latest stable version:",
- "nix upgrade-nix"
- },
- Example{
- "To upgrade Nix in a specific profile:",
- "nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile"
- },
- };
+ return
+ #include "upgrade-nix.md"
+ ;
}
Category category() override { return catNixInstallation; }
@@ -68,10 +61,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand
if (dryRun) {
stopProgressBar();
- logWarning({
- .name = "Version update",
- .hint = hintfmt("would upgrade to version %s", version)
- });
+ warn("would upgrade to version %s", version);
return;
}
diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md
new file mode 100644
index 000000000..4d27daad9
--- /dev/null
+++ b/src/nix/upgrade-nix.md
@@ -0,0 +1,28 @@
+R""(
+
+# Examples
+
+* Upgrade Nix to the latest stable version:
+
+ ```console
+ # nix upgrade-nix
+ ```
+
+* Upgrade Nix in a specific profile:
+
+ ```console
+ # nix upgrade-nix -p /nix/var/nix/profiles/per-user/alice/profile
+ ```
+
+# Description
+
+This command upgrades Nix to the latest version. By default, it
+locates the directory containing the `nix` binary in the `$PATH`
+environment variable. If that directory is a Nix profile, it will
+upgrade the `nix` package in that profile to the latest stable binary
+release.
+
+You cannot use this command to upgrade Nix in the system profile of a
+NixOS system (that is, if `nix` is found in `/run/current-system`).
+
+)""
diff --git a/src/nix/verify.cc b/src/nix/verify.cc
index ec7333d03..1721c7f16 100644
--- a/src/nix/verify.cc
+++ b/src/nix/verify.cc
@@ -18,16 +18,33 @@ struct CmdVerify : StorePathsCommand
CmdVerify()
{
- mkFlag(0, "no-contents", "do not verify the contents of each store path", &noContents);
- mkFlag(0, "no-trust", "do not verify whether each store path is trusted", &noTrust);
+ addFlag({
+ .longName = "no-contents",
+ .description = "Do not verify the contents of each store path.",
+ .handler = {&noContents, true},
+ });
+
+ addFlag({
+ .longName = "no-trust",
+ .description = "Do not verify whether each store path is trusted.",
+ .handler = {&noTrust, true},
+ });
+
addFlag({
.longName = "substituter",
.shortName = 's',
- .description = "use signatures from specified store",
+ .description = "Use signatures from the specified store.",
.labels = {"store-uri"},
.handler = {[&](std::string s) { substituterUris.push_back(s); }}
});
- mkIntFlag('n', "sigs-needed", "require that each path has at least N valid signatures", &sigsNeeded);
+
+ addFlag({
+ .longName = "sigs-needed",
+ .shortName = 'n',
+ .description = "Require that each path has at least *n* valid signatures.",
+ .labels = {"n"},
+ .handler = {&sigsNeeded}
+ });
}
std::string description() override
@@ -35,22 +52,13 @@ struct CmdVerify : StorePathsCommand
return "verify the integrity of store paths";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To verify the entire Nix store:",
- "nix verify --all"
- },
- Example{
- "To check whether each path in the closure of Firefox has at least 2 signatures:",
- "nix verify -r -n2 --no-contents $(type -p firefox)"
- },
- };
+ return
+ #include "verify.md"
+ ;
}
- Category category() override { return catSecondary; }
-
void run(ref<Store> store, StorePaths storePaths) override
{
std::vector<ref<Store>> substituters;
@@ -102,14 +110,10 @@ struct CmdVerify : StorePathsCommand
if (hash.first != info->narHash) {
corrupted++;
act2.result(resCorruptedPath, store->printStorePath(info->path));
- logError({
- .name = "Hash error - path modified",
- .hint = hintfmt(
- "path '%s' was modified! expected hash '%s', got '%s'",
- store->printStorePath(info->path),
- info->narHash.to_string(Base32, true),
- hash.first.to_string(Base32, true))
- });
+ printError("path '%s' was modified! expected hash '%s', got '%s'",
+ store->printStorePath(info->path),
+ info->narHash.to_string(Base32, true),
+ hash.first.to_string(Base32, true));
}
}
@@ -157,12 +161,7 @@ struct CmdVerify : StorePathsCommand
if (!good) {
untrusted++;
act2.result(resUntrustedPath, store->printStorePath(info->path));
- logError({
- .name = "Untrusted path",
- .hint = hintfmt("path '%s' is untrusted",
- store->printStorePath(info->path))
- });
-
+ printError("path '%s' is untrusted", store->printStorePath(info->path));
}
}
@@ -189,4 +188,4 @@ struct CmdVerify : StorePathsCommand
}
};
-static auto rCmdVerify = registerCommand<CmdVerify>("verify");
+static auto rCmdVerify = registerCommand2<CmdVerify>({"store", "verify"});
diff --git a/src/nix/verify.md b/src/nix/verify.md
new file mode 100644
index 000000000..1c43792e7
--- /dev/null
+++ b/src/nix/verify.md
@@ -0,0 +1,49 @@
+R""(
+
+# Examples
+
+* Verify the entire Nix store:
+
+ ```console
+ # nix store verify --all
+ ```
+
+* Check whether each path in the closure of Firefox has at least 2
+ signatures:
+
+ ```console
+ # nix store verify -r -n2 --no-contents $(type -p firefox)
+ ```
+
+* Verify a store path in the binary cache `https://cache.nixos.org/`:
+
+ ```console
+ # nix store verify --store https://cache.nixos.org/ \
+ /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10
+ ```
+
+# Description
+
+This command verifies the integrity of the store paths *installables*,
+or, if `--all` is given, the entire Nix store. For each path, it
+checks that
+
+* its contents match the NAR hash recorded in the Nix database; and
+
+* it is *trusted*, that is, it is signed by at least one trusted
+ signing key, is content-addressed, or is built locally ("ultimately
+ trusted").
+
+# Exit status
+
+The exit status of this command is the sum of the following values:
+
+* **1** if any path is corrupted (i.e. its contents don't match the
+ recorded NAR hash).
+
+* **2** if any path is untrusted.
+
+* **4** if any path couldn't be verified for any other reason (such as
+ an I/O error).
+
+)""
diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc
index 63bf087e6..7a4ca5172 100644
--- a/src/nix/why-depends.cc
+++ b/src/nix/why-depends.cc
@@ -40,7 +40,7 @@ struct CmdWhyDepends : SourceExprCommand
addFlag({
.longName = "all",
.shortName = 'a',
- .description = "show all edges in the dependency graph leading from 'package' to 'dependency', rather than just a shortest path",
+ .description = "Show all edges in the dependency graph leading from *package* to *dependency*, rather than just a shortest path.",
.handler = {&all, true},
});
}
@@ -50,22 +50,11 @@ struct CmdWhyDepends : SourceExprCommand
return "show why a package has another package in its closure";
}
- Examples examples() override
+ std::string doc() override
{
- return {
- Example{
- "To show one path through the dependency graph leading from Hello to Glibc:",
- "nix why-depends nixpkgs#hello nixpkgs#glibc"
- },
- Example{
- "To show all files and paths in the dependency graph leading from Thunderbird to libX11:",
- "nix why-depends --all nixpkgs#thunderbird nixpkgs#xorg.libX11"
- },
- Example{
- "To show why Glibc depends on itself:",
- "nix why-depends nixpkgs#glibc nixpkgs#glibc"
- },
- };
+ return
+ #include "why-depends.md"
+ ;
}
Category category() override { return catSecondary; }
@@ -156,7 +145,7 @@ struct CmdWhyDepends : SourceExprCommand
auto pathS = store->printStorePath(node.path);
assert(node.dist != inf);
- logger->stdout("%s%s%s%s" ANSI_NORMAL,
+ logger->cout("%s%s%s%s" ANSI_NORMAL,
firstPad,
node.visited ? "\e[38;5;244m" : "",
firstPad != "" ? "→ " : "",
diff --git a/src/nix/why-depends.md b/src/nix/why-depends.md
new file mode 100644
index 000000000..dc13619e1
--- /dev/null
+++ b/src/nix/why-depends.md
@@ -0,0 +1,80 @@
+R""(
+
+# Examples
+
+* Show one path through the dependency graph leading from Hello to
+ Glibc:
+
+ ```console
+ # nix why-depends nixpkgs#hello nixpkgs#glibc
+ /nix/store/v5sv61sszx301i0x6xysaqzla09nksnd-hello-2.10
+ └───bin/hello: …...................../nix/store/9l06v7fc38c1x3r2iydl15ksgz0ysb82-glibc-2.32/lib/ld-linux-x86-64.…
+ → /nix/store/9l06v7fc38c1x3r2iydl15ksgz0ysb82-glibc-2.32
+ ```
+
+* Show all files and paths in the dependency graph leading from
+ Thunderbird to libX11:
+
+ ```console
+ # nix why-depends --all nixpkgs#thunderbird nixpkgs#xorg.libX11
+ /nix/store/qfc8729nzpdln1h0hvi1ziclsl3m84sr-thunderbird-78.5.1
+ ├───lib/thunderbird/libxul.so: …6wrw-libxcb-1.14/lib:/nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0/lib:/nix/store/ssf…
+ │ → /nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0
+ ├───lib/thunderbird/libxul.so: …pxyc-libXt-1.2.0/lib:/nix/store/1qj29ipxl2fyi2b13l39hdircq17gnk0-libXdamage-1.1.5/lib:/nix/store…
+ │ → /nix/store/1qj29ipxl2fyi2b13l39hdircq17gnk0-libXdamage-1.1.5
+ │ ├───lib/libXdamage.so.1.1.0: …-libXfixes-5.0.3/lib:/nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0/lib:/nix/store/9l0…
+ │ │ → /nix/store/adzfjjh8w25vdr0xdx9x16ah4f5rqrw5-libX11-1.7.0
+ …
+ ```
+
+* Show why Glibc depends on itself:
+
+ ```console
+ # nix why-depends nixpkgs#glibc nixpkgs#glibc
+ /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31
+ └───lib/ld-2.31.so: …che Do not use /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31/etc/ld.so.cache. --…
+ → /nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31
+ ```
+
+* Show why Geeqie has a build-time dependency on `systemd`:
+
+ ```console
+ # nix why-depends --derivation nixpkgs#geeqie nixpkgs#systemd
+ /nix/store/drrpq2fqlrbj98bmazrnww7hm1in3wgj-geeqie-1.4.drv
+ └───/: …atch.drv",["out"]),("/nix/store/qzh8dyq3lfbk3i1acbp7x9wh3il2imiv-gtk+3-3.24.21.drv",["dev"]),("/…
+ → /nix/store/qzh8dyq3lfbk3i1acbp7x9wh3il2imiv-gtk+3-3.24.21.drv
+ └───/: …16.0.drv",["dev"]),("/nix/store/8kp79fyslf3z4m3dpvlh6w46iaadz5c2-cups-2.3.3.drv",["dev"]),("/nix…
+ → /nix/store/8kp79fyslf3z4m3dpvlh6w46iaadz5c2-cups-2.3.3.drv
+ └───/: ….3.1.drv",["out"]),("/nix/store/yd3ihapyi5wbz1kjacq9dbkaq5v5hqjg-systemd-246.4.drv",["dev"]),("/…
+ → /nix/store/yd3ihapyi5wbz1kjacq9dbkaq5v5hqjg-systemd-246.4.drv
+ ```
+
+# Description
+
+Nix automatically determines potential runtime dependencies between
+store paths by scanning for the *hash parts* of store paths. For
+instance, if there exists a store path
+`/nix/store/9df65igwjmf2wbw0gbrrgair6piqjgmi-glibc-2.31`, and a file
+inside another store path contains the string `9df65igw…`, then the
+latter store path *refers* to the former, and thus might need it at
+runtime. Nix always maintains the existence of the transitive closure
+of a store path under the references relationship; it is therefore not
+possible to install a store path without having all of its references
+present.
+
+Sometimes Nix packages end up with unexpected runtime dependencies;
+for instance, a reference to a compiler might accidentally end up in a
+binary, causing the former to be in the latter's closure. This kind of
+*closure size bloat* is undesirable.
+
+`nix why-depends` allows you to diagnose the cause of such issues. It
+shows why the store path *package* depends on the store path
+*dependency*, by showing a shortest sequence in the references graph
+from the former to the latter. Also, for each node along this path, it
+shows a file fragment containing a reference to the next store path in
+the sequence.
+
+To show why derivation *package* has a build-time rather than runtime
+dependency on derivation *dependency*, use `--derivation`.
+
+)""
diff --git a/src/resolve-system-dependencies/resolve-system-dependencies.cc b/src/resolve-system-dependencies/resolve-system-dependencies.cc
index d30227e4e..27cf53a45 100644
--- a/src/resolve-system-dependencies/resolve-system-dependencies.cc
+++ b/src/resolve-system-dependencies/resolve-system-dependencies.cc
@@ -39,18 +39,12 @@ std::set<std::string> runResolver(const Path & filename)
throw SysError("statting '%s'", filename);
if (!S_ISREG(st.st_mode)) {
- logError({
- .name = "Regular MACH file",
- .hint = hintfmt("file '%s' is not a regular file", filename)
- });
+ printError("file '%s' is not a regular MACH binary", filename);
return {};
}
if (st.st_size < sizeof(mach_header_64)) {
- logError({
- .name = "File too short",
- .hint = hintfmt("file '%s' is too short for a MACH binary", filename)
- });
+ printError("file '%s' is too short for a MACH binary", filename);
return {};
}
@@ -72,19 +66,13 @@ std::set<std::string> runResolver(const Path & filename)
}
}
if (mach64_offset == 0) {
- logError({
- .name = "No mach64 blobs",
- .hint = hintfmt("Could not find any mach64 blobs in file '%1%', continuing...", filename)
- });
+ printError("could not find any mach64 blobs in file '%1%', continuing...", filename);
return {};
}
} else if (magic == MH_MAGIC_64 || magic == MH_CIGAM_64) {
mach64_offset = 0;
} else {
- logError({
- .name = "Magic number",
- .hint = hintfmt("Object file has unknown magic number '%1%', skipping it...", magic)
- });
+ printError("Object file has unknown magic number '%1%', skipping it...", magic);
return {};
}