aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/build-remote/build-remote.cc2
-rw-r--r--src/libexpr/eval.cc9
-rw-r--r--src/libmain/common-args.cc4
-rw-r--r--src/libstore/binary-cache-store.cc2
-rw-r--r--src/libstore/binary-cache-store.hh2
-rw-r--r--src/libstore/build.cc36
-rw-r--r--src/libstore/local-store.cc3
-rw-r--r--src/libstore/local-store.hh3
-rw-r--r--src/libstore/pathlocks.cc6
-rw-r--r--src/libstore/pathlocks.hh6
-rw-r--r--src/libstore/store-api.cc6
-rw-r--r--src/libutil/compression.cc32
-rw-r--r--src/libutil/compression.hh4
-rw-r--r--src/nix/build.cc2
-rw-r--r--src/nix/command.hh2
-rw-r--r--src/nix/installables.cc4
-rw-r--r--tests/nix-copy-closure.nix6
-rw-r--r--tests/remote-builds.nix10
-rw-r--r--tests/user-envs.sh3
19 files changed, 90 insertions, 52 deletions
diff --git a/src/build-remote/build-remote.cc b/src/build-remote/build-remote.cc
index 50eb6b29e..dbf8fe1b8 100644
--- a/src/build-remote/build-remote.cc
+++ b/src/build-remote/build-remote.cc
@@ -243,7 +243,7 @@ connected:
if (!missing.empty()) {
Activity act(*logger, lvlTalkative, actUnknown, fmt("copying outputs from '%s'", storeUri));
- setenv("NIX_HELD_LOCKS", concatStringsSep(" ", missing).c_str(), 1); /* FIXME: ugly */
+ store->locksHeld.insert(missing.begin(), missing.end()); /* FIXME: ugly */
copyPaths(ref<Store>(sshStore), store, missing, NoRepair, NoCheckSigs, substitute);
}
diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc
index 0b0a0f7b1..11195af77 100644
--- a/src/libexpr/eval.cc
+++ b/src/libexpr/eval.cc
@@ -1700,10 +1700,13 @@ void EvalState::printStats()
printMsg(v, format(" time elapsed: %1%") % cpuTime);
printMsg(v, format(" size of a value: %1%") % sizeof(Value));
printMsg(v, format(" size of an attr: %1%") % sizeof(Attr));
- printMsg(v, format(" environments allocated: %1% (%2% bytes)") % nrEnvs % bEnvs);
- printMsg(v, format(" list elements: %1% (%2% bytes)") % nrListElems % bLists);
+ printMsg(v, format(" environments allocated count: %1%") % nrEnvs);
+ printMsg(v, format(" environments allocated bytes: %1%") % bEnvs);
+ printMsg(v, format(" list elements count: %1%") % nrListElems);
+ printMsg(v, format(" list elements bytes: %1%") % bLists);
printMsg(v, format(" list concatenations: %1%") % nrListConcats);
- printMsg(v, format(" values allocated: %1% (%2% bytes)") % nrValues % bValues);
+ printMsg(v, format(" values allocated count: %1%") % nrValues);
+ printMsg(v, format(" values allocated bytes: %1%") % bValues);
printMsg(v, format(" sets allocated: %1% (%2% bytes)") % nrAttrsets % bAttrsets);
printMsg(v, format(" right-biased unions: %1%") % nrOpUpdates);
printMsg(v, format(" values copied in right-biased unions: %1%") % nrOpUpdateValuesCopied);
diff --git a/src/libmain/common-args.cc b/src/libmain/common-args.cc
index d3aac6aba..bcc05c2cd 100644
--- a/src/libmain/common-args.cc
+++ b/src/libmain/common-args.cc
@@ -37,6 +37,10 @@ MixCommonArgs::MixCommonArgs(const string & programName)
std::string cat = "config";
settings.convertToArgs(*this, cat);
+
+ // Backward compatibility hack: nix-env already had a --system flag.
+ if (programName == "nix-env") longFlags.erase("system");
+
hiddenCategories.insert(cat);
}
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index ab971dd8b..d1b278b8e 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -149,7 +149,7 @@ void BinaryCacheStore::addToStore(const ValidPathInfo & info, const ref<std::str
/* Compress the NAR. */
narInfo->compression = compression;
auto now1 = std::chrono::steady_clock::now();
- auto narCompressed = compress(compression, *nar);
+ auto narCompressed = compress(compression, *nar, parallelCompression);
auto now2 = std::chrono::steady_clock::now();
narInfo->fileHash = hashString(htSHA256, *narCompressed);
narInfo->fileSize = narCompressed->size();
diff --git a/src/libstore/binary-cache-store.hh b/src/libstore/binary-cache-store.hh
index 8492ff600..e20b96844 100644
--- a/src/libstore/binary-cache-store.hh
+++ b/src/libstore/binary-cache-store.hh
@@ -19,6 +19,8 @@ public:
const Setting<bool> writeNARListing{this, false, "write-nar-listing", "whether to write a JSON file listing the files in each NAR"};
const Setting<Path> secretKeyFile{this, "", "secret-key", "path to secret key used to sign the binary cache"};
const Setting<Path> localNarCache{this, "", "local-nar-cache", "path to a local cache of NARs"};
+ const Setting<bool> parallelCompression{this, false, "parallel-compression",
+ "enable multi-threading compression, available for xz only currently"};
private:
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 9f669f7e4..cc69ff1c7 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -1335,19 +1335,6 @@ void DerivationGoal::tryToBuild()
{
trace("trying to build");
- /* Check for the possibility that some other goal in this process
- has locked the output since we checked in haveDerivation().
- (It can't happen between here and the lockPaths() call below
- because we're not allowing multi-threading.) If so, put this
- goal to sleep until another goal finishes, then try again. */
- for (auto & i : drv->outputs)
- if (pathIsLockedByMe(worker.store.toRealPath(i.second.path))) {
- debug(format("putting derivation '%1%' to sleep because '%2%' is locked by another goal")
- % drvPath % i.second.path);
- worker.waitForAnyGoal(shared_from_this());
- return;
- }
-
/* Obtain locks on all output paths. The locks are automatically
released when we exit this function or Nix crashes. If we
can't acquire the lock, then continue; hopefully some other
@@ -3688,8 +3675,8 @@ void SubstitutionGoal::tryNext()
&& !sub->isTrusted
&& !info->checkSignatures(worker.store, worker.store.publicKeys))
{
- printInfo(format("warning: substituter '%s' does not have a valid signature for path '%s'")
- % sub->getUri() % storePath);
+ printError("warning: substituter '%s' does not have a valid signature for path '%s'",
+ sub->getUri(), storePath);
tryNext();
return;
}
@@ -3739,6 +3726,17 @@ void SubstitutionGoal::tryToRun()
return;
}
+ /* If the store path is already locked (probably by a
+ DerivationGoal), then put this goal to sleep. Note: we don't
+ acquire a lock here since that breaks addToStore(), so below we
+ handle an AlreadyLocked exception from addToStore(). The check
+ here is just an optimisation to prevent having to redo a
+ download due to a locked path. */
+ if (pathIsLockedByMe(worker.store.toRealPath(storePath))) {
+ worker.waitForAWhile(shared_from_this());
+ return;
+ }
+
maintainRunningSubstitutions = std::make_unique<MaintainCount<uint64_t>>(worker.runningSubstitutions);
worker.updateProgress();
@@ -3778,8 +3776,14 @@ void SubstitutionGoal::finished()
try {
promise.get_future().get();
+ } catch (AlreadyLocked & e) {
+ /* Probably a DerivationGoal is already building this store
+ path. Sleep for a while and try again. */
+ state = &SubstitutionGoal::init;
+ worker.waitForAWhile(shared_from_this());
+ return;
} catch (Error & e) {
- printInfo(e.msg());
+ printError(e.msg());
/* Try the next substitute. */
state = &SubstitutionGoal::tryNext;
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 7afecc1cf..4afe51ea9 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -992,8 +992,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, const ref<std::string> &
/* Lock the output path. But don't lock if we're being called
from a build hook (whose parent process already acquired a
lock on this path). */
- Strings locksHeld = tokenizeString<Strings>(getEnv("NIX_HELD_LOCKS"));
- if (find(locksHeld.begin(), locksHeld.end(), info.path) == locksHeld.end())
+ if (!locksHeld.count(info.path))
outputLock.lockPaths({realPath});
if (repair || !isValidPath(info.path)) {
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 30bef3a79..bbd50e1c1 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -104,6 +104,9 @@ private:
public:
+ // Hack for build-remote.cc.
+ PathSet locksHeld = tokenizeString<PathSet>(getEnv("NIX_HELD_LOCKS"));
+
/* Initialise the local store, upgrading the schema if
necessary. */
LocalStore(const Params & params);
diff --git a/src/libstore/pathlocks.cc b/src/libstore/pathlocks.cc
index 587f29598..08d1efdbe 100644
--- a/src/libstore/pathlocks.cc
+++ b/src/libstore/pathlocks.cc
@@ -113,8 +113,10 @@ bool PathLocks::lockPaths(const PathSet & _paths,
{
auto lockedPaths(lockedPaths_.lock());
- if (lockedPaths->count(lockPath))
- throw Error("deadlock: trying to re-acquire self-held lock '%s'", lockPath);
+ if (lockedPaths->count(lockPath)) {
+ if (!wait) return false;
+ throw AlreadyLocked("deadlock: trying to re-acquire self-held lock '%s'", lockPath);
+ }
lockedPaths->insert(lockPath);
}
diff --git a/src/libstore/pathlocks.hh b/src/libstore/pathlocks.hh
index 2a7de6114..db51f950a 100644
--- a/src/libstore/pathlocks.hh
+++ b/src/libstore/pathlocks.hh
@@ -2,10 +2,8 @@
#include "util.hh"
-
namespace nix {
-
/* Open (possibly create) a lock file and return the file descriptor.
-1 is returned if create is false and the lock could not be opened
because it doesn't exist. Any other error throws an exception. */
@@ -18,6 +16,7 @@ enum LockType { ltRead, ltWrite, ltNone };
bool lockFile(int fd, LockType lockType, bool wait);
+MakeError(AlreadyLocked, Error);
class PathLocks
{
@@ -38,9 +37,6 @@ public:
void setDeletion(bool deletePaths);
};
-
-// FIXME: not thread-safe!
bool pathIsLockedByMe(const Path & path);
-
}
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index 7abb300a9..4d43ef082 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -896,7 +896,11 @@ std::list<ref<Store>> getDefaultSubstituters()
auto addStore = [&](const std::string & uri) {
if (done.count(uri)) return;
done.insert(uri);
- stores.push_back(openStore(uri));
+ try {
+ stores.push_back(openStore(uri));
+ } catch (Error & e) {
+ printError("warning: %s", e.what());
+ }
};
for (auto uri : settings.substituters.get())
diff --git a/src/libutil/compression.cc b/src/libutil/compression.cc
index 5e2631ba3..ed15761b3 100644
--- a/src/libutil/compression.cc
+++ b/src/libutil/compression.cc
@@ -151,10 +151,10 @@ static ref<std::string> decompressBrotli(const std::string & in)
#endif // HAVE_BROTLI
}
-ref<std::string> compress(const std::string & method, const std::string & in)
+ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel)
{
StringSink ssink;
- auto sink = makeCompressionSink(method, ssink);
+ auto sink = makeCompressionSink(method, ssink, parallel);
(*sink)(in);
sink->finish();
return ssink.s;
@@ -189,10 +189,28 @@ struct XzSink : CompressionSink
lzma_stream strm = LZMA_STREAM_INIT;
bool finished = false;
- XzSink(Sink & nextSink) : nextSink(nextSink)
+ XzSink(Sink & nextSink, const bool parallel) : nextSink(nextSink)
{
- lzma_ret ret = lzma_easy_encoder(
- &strm, 6, LZMA_CHECK_CRC64);
+ lzma_ret ret;
+ if (parallel) {
+ lzma_mt mt_options = {};
+ mt_options.flags = 0;
+ mt_options.timeout = 300; // Using the same setting as the xz cmd line
+ mt_options.preset = LZMA_PRESET_DEFAULT;
+ mt_options.filters = NULL;
+ mt_options.check = LZMA_CHECK_CRC64;
+ mt_options.threads = lzma_cputhreads();
+ mt_options.block_size = 0;
+ if (mt_options.threads == 0)
+ mt_options.threads = 1;
+ // FIXME: maybe use lzma_stream_encoder_mt_memusage() to control the
+ // number of threads.
+ ret = lzma_stream_encoder_mt(
+ &strm, &mt_options);
+ } else
+ ret = lzma_easy_encoder(
+ &strm, 6, LZMA_CHECK_CRC64);
+
if (ret != LZMA_OK)
throw CompressionError("unable to initialise lzma encoder");
// FIXME: apply the x86 BCJ filter?
@@ -449,12 +467,12 @@ struct BrotliSink : CompressionSink
};
#endif // HAVE_BROTLI
-ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink)
+ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel)
{
if (method == "none")
return make_ref<NoneSink>(nextSink);
else if (method == "xz")
- return make_ref<XzSink>(nextSink);
+ return make_ref<XzSink>(nextSink, parallel);
else if (method == "bzip2")
return make_ref<BzipSink>(nextSink);
else if (method == "br")
diff --git a/src/libutil/compression.hh b/src/libutil/compression.hh
index e3e6f5a99..a0d7530d7 100644
--- a/src/libutil/compression.hh
+++ b/src/libutil/compression.hh
@@ -8,7 +8,7 @@
namespace nix {
-ref<std::string> compress(const std::string & method, const std::string & in);
+ref<std::string> compress(const std::string & method, const std::string & in, const bool parallel = false);
ref<std::string> decompress(const std::string & method, const std::string & in);
@@ -17,7 +17,7 @@ struct CompressionSink : BufferedSink
virtual void finish() = 0;
};
-ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink);
+ref<CompressionSink> makeCompressionSink(const std::string & method, Sink & nextSink, const bool parallel = false);
MakeError(UnknownCompressionMethod, Error);
diff --git a/src/nix/build.cc b/src/nix/build.cc
index f7c99f12d..b4f21b32d 100644
--- a/src/nix/build.cc
+++ b/src/nix/build.cc
@@ -50,7 +50,7 @@ struct CmdBuild : MixDryRun, InstallablesCommand
void run(ref<Store> store) override
{
- auto buildables = toBuildables(store, dryRun ? DryRun : Build, installables);
+ auto buildables = build(store, dryRun ? DryRun : Build, installables);
for (size_t i = 0; i < buildables.size(); ++i) {
auto & b(buildables[i]);
diff --git a/src/nix/command.hh b/src/nix/command.hh
index a7863c49f..97a6fee7f 100644
--- a/src/nix/command.hh
+++ b/src/nix/command.hh
@@ -198,7 +198,7 @@ std::shared_ptr<Installable> parseInstallable(
SourceExprCommand & cmd, ref<Store> store, const std::string & installable,
bool useDefaultInstallables);
-Buildables toBuildables(ref<Store> store, RealiseMode mode,
+Buildables build(ref<Store> store, RealiseMode mode,
std::vector<std::shared_ptr<Installable>> installables);
PathSet toStorePaths(ref<Store> store, RealiseMode mode,
diff --git a/src/nix/installables.cc b/src/nix/installables.cc
index c3b06c22e..a3fdd8a28 100644
--- a/src/nix/installables.cc
+++ b/src/nix/installables.cc
@@ -253,7 +253,7 @@ std::shared_ptr<Installable> parseInstallable(
return installables.front();
}
-Buildables toBuildables(ref<Store> store, RealiseMode mode,
+Buildables build(ref<Store> store, RealiseMode mode,
std::vector<std::shared_ptr<Installable>> installables)
{
if (mode != Build)
@@ -291,7 +291,7 @@ PathSet toStorePaths(ref<Store> store, RealiseMode mode,
{
PathSet outPaths;
- for (auto & b : toBuildables(store, mode, installables))
+ for (auto & b : build(store, mode, installables))
for (auto & output : b.outputs)
outPaths.insert(output.second);
diff --git a/tests/nix-copy-closure.nix b/tests/nix-copy-closure.nix
index be0a4a683..0dc147fb3 100644
--- a/tests/nix-copy-closure.nix
+++ b/tests/nix-copy-closure.nix
@@ -29,10 +29,10 @@ makeTest (let pkgA = pkgs.cowsay; pkgB = pkgs.wget; pkgC = pkgs.hello; in {
startAll;
# Create an SSH key on the client.
- my $key = `${pkgs.openssh}/bin/ssh-keygen -t dsa -f key -N ""`;
+ my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`;
$client->succeed("mkdir -m 700 /root/.ssh");
- $client->copyFileFromHost("key", "/root/.ssh/id_dsa");
- $client->succeed("chmod 600 /root/.ssh/id_dsa");
+ $client->copyFileFromHost("key", "/root/.ssh/id_ed25519");
+ $client->succeed("chmod 600 /root/.ssh/id_ed25519");
# Install the SSH key on the server.
$server->succeed("mkdir -m 700 /root/.ssh");
diff --git a/tests/remote-builds.nix b/tests/remote-builds.nix
index 75704ace2..d7a4b2198 100644
--- a/tests/remote-builds.nix
+++ b/tests/remote-builds.nix
@@ -46,13 +46,13 @@ in
nix.buildMachines =
[ { hostName = "slave1";
sshUser = "root";
- sshKey = "/root/.ssh/id_dsa";
+ sshKey = "/root/.ssh/id_ed25519";
system = "i686-linux";
maxJobs = 1;
}
{ hostName = "slave2";
sshUser = "root";
- sshKey = "/root/.ssh/id_dsa";
+ sshKey = "/root/.ssh/id_ed25519";
system = "i686-linux";
maxJobs = 1;
}
@@ -70,10 +70,10 @@ in
startAll;
# Create an SSH key on the client.
- my $key = `${pkgs.openssh}/bin/ssh-keygen -t dsa -f key -N ""`;
+ my $key = `${pkgs.openssh}/bin/ssh-keygen -t ed25519 -f key -N ""`;
$client->succeed("mkdir -p -m 700 /root/.ssh");
- $client->copyFileFromHost("key", "/root/.ssh/id_dsa");
- $client->succeed("chmod 600 /root/.ssh/id_dsa");
+ $client->copyFileFromHost("key", "/root/.ssh/id_ed25519");
+ $client->succeed("chmod 600 /root/.ssh/id_ed25519");
# Install the SSH key on the slaves.
$client->waitForUnit("network.target");
diff --git a/tests/user-envs.sh b/tests/user-envs.sh
index c4192fdc5..ba6392311 100644
--- a/tests/user-envs.sh
+++ b/tests/user-envs.sh
@@ -24,6 +24,9 @@ rm -f $HOME/.nix-defexpr
ln -s $(pwd)/user-envs.nix $HOME/.nix-defexpr
nix-env -qa '*' --description | grep -q silly
+# Query the system.
+nix-env -qa '*' --system | grep -q $system
+
# Install "foo-1.0".
nix-env -i foo-1.0
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 19:27:55 +0000