aboutsummaryrefslogtreecommitdiff
path: root/src/libstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore')
-rw-r--r--src/libstore/build.cc176
-rw-r--r--src/libstore/builtins/fetchurl.cc10
-rw-r--r--src/libstore/derivations.cc2
-rw-r--r--src/libstore/filetransfer.cc (renamed from src/libstore/download.cc)283
-rw-r--r--src/libstore/filetransfer.hh (renamed from src/libstore/download.hh)78
-rw-r--r--src/libstore/gc.cc11
-rw-r--r--src/libstore/globals.cc58
-rw-r--r--src/libstore/globals.hh17
-rw-r--r--src/libstore/http-binary-cache-store.cc35
-rw-r--r--src/libstore/s3-binary-cache-store.cc6
-rw-r--r--src/libstore/s3.hh4
-rw-r--r--src/libstore/store-api.cc25
-rw-r--r--src/libstore/store-api.hh1
13 files changed, 309 insertions, 397 deletions
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index be52e637e..272c4f1ae 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -7,7 +7,7 @@
#include "affinity.hh"
#include "builtins.hh"
#include "builtins/buildenv.hh"
-#include "download.hh"
+#include "filetransfer.hh"
#include "finally.hh"
#include "compression.hh"
#include "json.hh"
@@ -33,7 +33,6 @@
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/utsname.h>
-#include <sys/select.h>
#include <sys/resource.h>
#include <sys/socket.h>
#include <sys/un.h>
@@ -43,6 +42,7 @@
#include <errno.h>
#include <cstring>
#include <termios.h>
+#include <poll.h>
#include <pwd.h>
#include <grp.h>
@@ -361,7 +361,7 @@ public:
{
actDerivations.progress(doneBuilds, expectedBuilds + doneBuilds, runningBuilds, failedBuilds);
actSubstitutions.progress(doneSubstitutions, expectedSubstitutions + doneSubstitutions, runningSubstitutions, failedSubstitutions);
- act.setExpected(actDownload, expectedDownloadSize + doneDownloadSize);
+ act.setExpected(actFileTransfer, expectedDownloadSize + doneDownloadSize);
act.setExpected(actCopyPath, expectedNarSize + doneNarSize);
}
};
@@ -507,9 +507,10 @@ private:
Path fnUserLock;
AutoCloseFD fdUserLock;
+ bool isEnabled = false;
string user;
- uid_t uid;
- gid_t gid;
+ uid_t uid = 0;
+ gid_t gid = 0;
std::vector<gid_t> supplementaryGIDs;
public:
@@ -522,7 +523,9 @@ public:
uid_t getGID() { assert(gid); return gid; }
std::vector<gid_t> getSupplementaryGIDs() { return supplementaryGIDs; }
- bool enabled() { return uid != 0; }
+ bool findFreeUser();
+
+ bool enabled() { return isEnabled; }
};
@@ -530,6 +533,11 @@ public:
UserLock::UserLock()
{
assert(settings.buildUsersGroup != "");
+ createDirs(settings.nixStateDir + "/userpool");
+}
+
+bool UserLock::findFreeUser() {
+ if (enabled()) return true;
/* Get the members of the build-users-group. */
struct group * gr = getgrnam(settings.buildUsersGroup.get().c_str());
@@ -559,7 +567,6 @@ UserLock::UserLock()
throw Error(format("the user '%1%' in the group '%2%' does not exist")
% i % settings.buildUsersGroup);
- createDirs(settings.nixStateDir + "/userpool");
fnUserLock = (format("%1%/userpool/%2%") % settings.nixStateDir % pw->pw_uid).str();
@@ -590,16 +597,13 @@ UserLock::UserLock()
supplementaryGIDs.resize(ngroups);
#endif
- return;
+ isEnabled = true;
+ return true;
}
}
-
- throw Error(format("all build users are currently in use; "
- "consider creating additional users and adding them to the '%1%' group")
- % settings.buildUsersGroup);
+ return false;
}
-
void UserLock::kill()
{
killUser(uid);
@@ -928,6 +932,7 @@ private:
void closureRepaired();
void inputsRealised();
void tryToBuild();
+ void tryLocalBuild();
void buildDone();
/* Is the build hook willing to perform the build? */
@@ -999,6 +1004,8 @@ private:
Goal::amDone(result);
}
+ void started();
+
void done(BuildResult::Status status, const string & msg = "");
StorePathSet exportReferences(const StorePathSet & storePaths);
@@ -1386,6 +1393,19 @@ void DerivationGoal::inputsRealised()
result = BuildResult();
}
+void DerivationGoal::started() {
+ auto msg = fmt(
+ buildMode == bmRepair ? "repairing outputs of '%s'" :
+ buildMode == bmCheck ? "checking outputs of '%s'" :
+ nrRounds > 1 ? "building '%s' (round %d/%d)" :
+ "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
+ fmt("building '%s'", worker.store.printStorePath(drvPath));
+ if (hook) msg += fmt(" on '%s'", machineName);
+ act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
+ Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
+ mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
+ worker.updateProgress();
+}
void DerivationGoal::tryToBuild()
{
@@ -1437,20 +1457,6 @@ void DerivationGoal::tryToBuild()
supported for local builds. */
bool buildLocally = buildMode != bmNormal || parsedDrv->willBuildLocally();
- auto started = [&]() {
- auto msg = fmt(
- buildMode == bmRepair ? "repairing outputs of '%s'" :
- buildMode == bmCheck ? "checking outputs of '%s'" :
- nrRounds > 1 ? "building '%s' (round %d/%d)" :
- "building '%s'", worker.store.printStorePath(drvPath), curRound, nrRounds);
- fmt("building '%s'", worker.store.printStorePath(drvPath));
- if (hook) msg += fmt(" on '%s'", machineName);
- act = std::make_unique<Activity>(*logger, lvlInfo, actBuild, msg,
- Logger::Fields{worker.store.printStorePath(drvPath), hook ? machineName : "", curRound, nrRounds});
- mcRunningBuilds = std::make_unique<MaintainCount<uint64_t>>(worker.runningBuilds);
- worker.updateProgress();
- };
-
/* Is the build hook willing to accept this job? */
if (!buildLocally) {
switch (tryBuildHook()) {
@@ -1483,6 +1489,34 @@ void DerivationGoal::tryToBuild()
return;
}
+ state = &DerivationGoal::tryLocalBuild;
+ worker.wakeUp(shared_from_this());
+}
+
+void DerivationGoal::tryLocalBuild() {
+
+ /* If `build-users-group' is not empty, then we have to build as
+ one of the members of that group. */
+ if (settings.buildUsersGroup != "" && getuid() == 0) {
+#if defined(__linux__) || defined(__APPLE__)
+ if (!buildUser) buildUser = std::make_unique<UserLock>();
+
+ if (buildUser->findFreeUser()) {
+ /* Make sure that no other processes are executing under this
+ uid. */
+ buildUser->kill();
+ } else {
+ debug("waiting for build users");
+ worker.waitForAWhile(shared_from_this());
+ return;
+ }
+#else
+ /* Don't know how to block the creation of setuid/setgid
+ binaries on this platform. */
+ throw Error("build users are not supported on this platform for security reasons");
+#endif
+ }
+
try {
/* Okay, we have to build. */
@@ -1680,6 +1714,7 @@ void DerivationGoal::buildDone()
}
if (buildMode == bmCheck) {
+ deleteTmpDir(true);
done(BuildResult::Built);
return;
}
@@ -1942,22 +1977,6 @@ void DerivationGoal::startBuilder()
#endif
}
- /* If `build-users-group' is not empty, then we have to build as
- one of the members of that group. */
- if (settings.buildUsersGroup != "" && getuid() == 0) {
-#if defined(__linux__) || defined(__APPLE__)
- buildUser = std::make_unique<UserLock>();
-
- /* Make sure that no other processes are executing under this
- uid. */
- buildUser->kill();
-#else
- /* Don't know how to block the creation of setuid/setgid
- binaries on this platform. */
- throw Error("build users are not supported on this platform for security reasons");
-#endif
- }
-
/* Create a temporary directory where the build will take
place. */
tmpDir = createTempDir("", "nix-build-" + std::string(drvPath.name()), false, false, 0700);
@@ -2161,7 +2180,7 @@ void DerivationGoal::startBuilder()
if (needsHashRewrite()) {
if (pathExists(homeDir))
- throw Error(format("directory '%1%' exists; please remove it") % homeDir);
+ throw Error(format("home directory '%1%' exists; please remove it to assure purity of builds without sandboxing") % homeDir);
/* We're not doing a chroot build, but we have some valid
output paths. Since we can't just overwrite or delete
@@ -2249,10 +2268,13 @@ void DerivationGoal::startBuilder()
if (chown(slaveName.c_str(), buildUser->getUID(), 0))
throw SysError("changing owner of pseudoterminal slave");
- } else {
+ }
+#if __APPLE__
+ else {
if (grantpt(builderOut.readSide.get()))
throw SysError("granting access to pseudoterminal slave");
}
+#endif
#if 0
// Mount the pt in the sandbox so that the "tty" command works.
@@ -2465,7 +2487,7 @@ void DerivationGoal::initTmpDir() {
auto hash = hashString(htSHA256, i.first);
string fn = ".attr-" + hash.to_string(Base32, false);
Path p = tmpDir + "/" + fn;
- writeFile(p, i.second);
+ writeFile(p, rewriteStrings(i.second, inputRewrites));
chownToBuilder(p);
env[i.first + "Path"] = tmpDirInSandbox + "/" + fn;
}
@@ -3151,7 +3173,7 @@ void DerivationGoal::runChild()
// Only use nss functions to resolve hosts and
// services. Don’t use it for anything else that may
// be configured for this system. This limits the
- // potential impurities introduced in fixed outputs.
+ // potential impurities introduced in fixed-outputs.
writeFile(chrootRootDir + "/etc/nsswitch.conf", "hosts: files dns\nservices: files\n");
ss.push_back("/etc/services");
@@ -3536,6 +3558,29 @@ StorePathSet parseReferenceSpecifiers(Store & store, const BasicDerivation & drv
}
+static void moveCheckToStore(const Path & src, const Path & dst)
+{
+ /* For the rename of directory to succeed, we must be running as root or
+ the directory must be made temporarily writable (to update the
+ directory's parent link ".."). */
+ struct stat st;
+ if (lstat(src.c_str(), &st) == -1) {
+ throw SysError(format("getting attributes of path '%1%'") % src);
+ }
+
+ bool changePerm = (geteuid() && S_ISDIR(st.st_mode) && !(st.st_mode & S_IWUSR));
+
+ if (changePerm)
+ chmod_(src, st.st_mode | S_IWUSR);
+
+ if (rename(src.c_str(), dst.c_str()))
+ throw SysError(format("renaming '%1%' to '%2%'") % src % dst);
+
+ if (changePerm)
+ chmod_(dst, st.st_mode);
+}
+
+
void DerivationGoal::registerOutputs()
{
/* When using a build hook, the build hook can register the output
@@ -3654,7 +3699,8 @@ void DerivationGoal::registerOutputs()
/* The output path should be a regular file without execute permission. */
if (!S_ISREG(st.st_mode) || (st.st_mode & S_IXUSR) != 0)
throw BuildError(
- format("output path '%1%' should be a non-executable regular file") % path);
+ format("output path '%1%' should be a non-executable regular file "
+ "since recursive hashing is not enabled (outputHashMode=flat)") % path);
}
/* Check the hash. In hash mode, move the path produced by
@@ -3716,8 +3762,7 @@ void DerivationGoal::registerOutputs()
if (settings.runDiffHook || settings.keepFailed) {
Path dst = worker.store.toRealPath(path + checkSuffix);
deletePath(dst);
- if (rename(actualPath.c_str(), dst.c_str()))
- throw SysError(format("renaming '%1%' to '%2%'") % actualPath % dst);
+ moveCheckToStore(actualPath, dst);
handleDiffHook(
buildUser ? buildUser->getUID() : getuid(),
@@ -3725,10 +3770,10 @@ void DerivationGoal::registerOutputs()
path, dst, worker.store.printStorePath(drvPath), tmpDir);
throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs from '%s'",
- worker.store.printStorePath(drvPath), path, dst);
+ worker.store.printStorePath(drvPath), worker.store.toRealPath(path), dst);
} else
throw NotDeterministic("derivation '%s' may not be deterministic: output '%s' differs",
- worker.store.printStorePath(drvPath), path);
+ worker.store.printStorePath(drvPath), worker.store.toRealPath(path));
}
/* Since we verified the build, it's now ultimately trusted. */
@@ -4770,8 +4815,7 @@ void Worker::waitForInput()
terminated. */
bool useTimeout = false;
- struct timeval timeout;
- timeout.tv_usec = 0;
+ long timeout = 0;
auto before = steady_time_point::clock::now();
/* If we're monitoring for silence on stdout/stderr, or if there
@@ -4789,7 +4833,7 @@ void Worker::waitForInput()
nearest = std::min(nearest, i.timeStarted + std::chrono::seconds(settings.buildTimeout));
}
if (nearest != steady_time_point::max()) {
- timeout.tv_sec = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
+ timeout = std::max(1L, (long) std::chrono::duration_cast<std::chrono::seconds>(nearest - before).count());
useTimeout = true;
}
@@ -4798,32 +4842,30 @@ void Worker::waitForInput()
if (!waitingForAWhile.empty()) {
useTimeout = true;
if (lastWokenUp == steady_time_point::min())
- printError("waiting for locks or build slots...");
+ printError("waiting for locks, build slots or build users...");
if (lastWokenUp == steady_time_point::min() || lastWokenUp > before) lastWokenUp = before;
- timeout.tv_sec = std::max(1L,
+ timeout = std::max(1L,
(long) std::chrono::duration_cast<std::chrono::seconds>(
lastWokenUp + std::chrono::seconds(settings.pollInterval) - before).count());
} else lastWokenUp = steady_time_point::min();
if (useTimeout)
- vomit("sleeping %d seconds", timeout.tv_sec);
+ vomit("sleeping %d seconds", timeout);
/* Use select() to wait for the input side of any logger pipe to
become `available'. Note that `available' (i.e., non-blocking)
includes EOF. */
- fd_set fds;
- FD_ZERO(&fds);
- int fdMax = 0;
+ std::vector<struct pollfd> pollStatus;
+ std::map <int, int> fdToPollStatus;
for (auto & i : children) {
for (auto & j : i.fds) {
- if (j >= FD_SETSIZE)
- throw Error("reached FD_SETSIZE limit");
- FD_SET(j, &fds);
- if (j >= fdMax) fdMax = j + 1;
+ pollStatus.push_back((struct pollfd) { .fd = j, .events = POLLIN });
+ fdToPollStatus[j] = pollStatus.size() - 1;
}
}
- if (select(fdMax, &fds, 0, 0, useTimeout ? &timeout : 0) == -1) {
+ if (poll(pollStatus.data(), pollStatus.size(),
+ useTimeout ? timeout * 1000 : -1) == -1) {
if (errno == EINTR) return;
throw SysError("waiting for input");
}
@@ -4844,7 +4886,7 @@ void Worker::waitForInput()
set<int> fds2(j->fds);
std::vector<unsigned char> buffer(4096);
for (auto & k : fds2) {
- if (FD_ISSET(k, &fds)) {
+ if (pollStatus.at(fdToPollStatus.at(k)).revents) {
ssize_t rd = read(k, buffer.data(), buffer.size());
// FIXME: is there a cleaner way to handle pt close
// than EIO? Is this even standard?
diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc
index f6ae5d2e6..486babf14 100644
--- a/src/libstore/builtins/fetchurl.cc
+++ b/src/libstore/builtins/fetchurl.cc
@@ -1,5 +1,5 @@
#include "builtins.hh"
-#include "download.hh"
+#include "filetransfer.hh"
#include "store-api.hh"
#include "archive.hh"
#include "compression.hh"
@@ -26,9 +26,9 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
auto mainUrl = getAttr("url");
bool unpack = get(drv.env, "unpack").value_or("") == "1";
- /* Note: have to use a fresh downloader here because we're in
+ /* Note: have to use a fresh fileTransfer here because we're in
a forked process. */
- auto downloader = makeDownloader();
+ auto fileTransfer = makeFileTransfer();
auto fetch = [&](const std::string & url) {
@@ -36,13 +36,13 @@ void builtinFetchurl(const BasicDerivation & drv, const std::string & netrcData)
/* No need to do TLS verification, because we check the hash of
the result anyway. */
- DownloadRequest request(url);
+ FileTransferRequest request(url);
request.verifyTLS = false;
request.decompress = false;
auto decompressor = makeDecompressionSink(
unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink);
- downloader->download(std::move(request), *decompressor);
+ fileTransfer->download(std::move(request), *decompressor);
decompressor->finish();
});
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 5934c1912..c68e7b16b 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -378,7 +378,7 @@ Hash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOutput
if (h == drvHashes.end()) {
assert(store.isValidPath(i.first));
h = drvHashes.insert_or_assign(i.first.clone(), hashDerivationModulo(store,
- readDerivation(store, store.toRealPath(store.printStorePath(i.first))), false)).first;
+ readDerivation(store, store.toRealPath(i.first)), false)).first;
}
inputs2.insert_or_assign(h->second.to_string(Base16, false), i.second);
}
diff --git a/src/libstore/download.cc b/src/libstore/filetransfer.cc
index 8a9b65899..e9684b3d4 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/filetransfer.cc
@@ -1,14 +1,10 @@
-#include "download.hh"
+#include "filetransfer.hh"
#include "util.hh"
#include "globals.hh"
-#include "hash.hh"
#include "store-api.hh"
-#include "archive.hh"
#include "s3.hh"
#include "compression.hh"
-#include "pathlocks.hh"
#include "finally.hh"
-#include "tarfile.hh"
#ifdef ENABLE_S3
#include <aws/core/client/ClientConfiguration.h>
@@ -31,13 +27,9 @@ using namespace std::string_literals;
namespace nix {
-DownloadSettings downloadSettings;
+FileTransferSettings fileTransferSettings;
-static GlobalConfig::Register r1(&downloadSettings);
-
-CachedDownloadRequest::CachedDownloadRequest(const std::string & uri)
- : uri(uri), ttl(settings.tarballTtl)
-{ }
+static GlobalConfig::Register r1(&fileTransferSettings);
std::string resolveUri(const std::string & uri)
{
@@ -47,21 +39,21 @@ std::string resolveUri(const std::string & uri)
return uri;
}
-struct CurlDownloader : public Downloader
+struct curlFileTransfer : public FileTransfer
{
CURLM * curlm = 0;
std::random_device rd;
std::mt19937 mt19937;
- struct DownloadItem : public std::enable_shared_from_this<DownloadItem>
+ struct TransferItem : public std::enable_shared_from_this<TransferItem>
{
- CurlDownloader & downloader;
- DownloadRequest request;
- DownloadResult result;
+ curlFileTransfer & fileTransfer;
+ FileTransferRequest request;
+ FileTransferResult result;
Activity act;
bool done = false; // whether either the success or failure function has been called
- Callback<DownloadResult> callback;
+ Callback<FileTransferResult> callback;
CURL * req = 0;
bool active = false; // whether the handle has been added to the multi object
std::string status;
@@ -80,19 +72,26 @@ struct CurlDownloader : public Downloader
curl_off_t writtenToSink = 0;
- DownloadItem(CurlDownloader & downloader,
- const DownloadRequest & request,
- Callback<DownloadResult> && callback)
- : downloader(downloader)
+ TransferItem(curlFileTransfer & fileTransfer,
+ const FileTransferRequest & request,
+ Callback<FileTransferResult> && callback)
+ : fileTransfer(fileTransfer)
, request(request)
- , act(*logger, lvlTalkative, actDownload,
+ , act(*logger, lvlTalkative, actFileTransfer,
fmt(request.data ? "uploading '%s'" : "downloading '%s'", request.uri),
{request.uri}, request.parentAct)
, callback(std::move(callback))
, finalSink([this](const unsigned char * data, size_t len) {
if (this->request.dataCallback) {
- writtenToSink += len;
- this->request.dataCallback((char *) data, len);
+ long httpStatus = 0;
+ curl_easy_getinfo(req, CURLINFO_RESPONSE_CODE, &httpStatus);
+
+ /* Only write data to the sink if this is a
+ successful response. */
+ if (httpStatus == 0 || httpStatus == 200 || httpStatus == 201 || httpStatus == 206) {
+ writtenToSink += len;
+ this->request.dataCallback((char *) data, len);
+ }
} else
this->result.data->append((char *) data, len);
})
@@ -103,17 +102,17 @@ struct CurlDownloader : public Downloader
requestHeaders = curl_slist_append(requestHeaders, ("Content-Type: " + request.mimeType).c_str());
}
- ~DownloadItem()
+ ~TransferItem()
{
if (req) {
if (active)
- curl_multi_remove_handle(downloader.curlm, req);
+ curl_multi_remove_handle(fileTransfer.curlm, req);
curl_easy_cleanup(req);
}
if (requestHeaders) curl_slist_free_all(requestHeaders);
try {
if (!done)
- fail(DownloadError(Interrupted, format("download of '%s' was interrupted") % request.uri));
+ fail(FileTransferError(Interrupted, format("download of '%s' was interrupted") % request.uri));
} catch (...) {
ignoreException();
}
@@ -157,7 +156,7 @@ struct CurlDownloader : public Downloader
static size_t writeCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
{
- return ((DownloadItem *) userp)->writeCallback(contents, size, nmemb);
+ return ((TransferItem *) userp)->writeCallback(contents, size, nmemb);
}
size_t headerCallback(void * contents, size_t size, size_t nmemb)
@@ -199,7 +198,7 @@ struct CurlDownloader : public Downloader
static size_t headerCallbackWrapper(void * contents, size_t size, size_t nmemb, void * userp)
{
- return ((DownloadItem *) userp)->headerCallback(contents, size, nmemb);
+ return ((TransferItem *) userp)->headerCallback(contents, size, nmemb);
}
int progressCallback(double dltotal, double dlnow)
@@ -214,7 +213,7 @@ struct CurlDownloader : public Downloader
static int progressCallbackWrapper(void * userp, double dltotal, double dlnow, double ultotal, double ulnow)
{
- return ((DownloadItem *) userp)->progressCallback(dltotal, dlnow);
+ return ((TransferItem *) userp)->progressCallback(dltotal, dlnow);
}
static int debugCallback(CURL * handle, curl_infotype type, char * data, size_t size, void * userptr)
@@ -238,7 +237,7 @@ struct CurlDownloader : public Downloader
static size_t readCallbackWrapper(char *buffer, size_t size, size_t nitems, void * userp)
{
- return ((DownloadItem *) userp)->readCallback(buffer, size, nitems);
+ return ((TransferItem *) userp)->readCallback(buffer, size, nitems);
}
void init()
@@ -249,7 +248,7 @@ struct CurlDownloader : public Downloader
if (verbosity >= lvlVomit) {
curl_easy_setopt(req, CURLOPT_VERBOSE, 1);
- curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, DownloadItem::debugCallback);
+ curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback);
}
curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str());
@@ -258,19 +257,19 @@ struct CurlDownloader : public Downloader
curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1);
curl_easy_setopt(req, CURLOPT_USERAGENT,
("curl/" LIBCURL_VERSION " Nix/" + nixVersion +
- (downloadSettings.userAgentSuffix != "" ? " " + downloadSettings.userAgentSuffix.get() : "")).c_str());
+ (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : "")).c_str());
#if LIBCURL_VERSION_NUM >= 0x072b00
curl_easy_setopt(req, CURLOPT_PIPEWAIT, 1);
#endif
#if LIBCURL_VERSION_NUM >= 0x072f00
- if (downloadSettings.enableHttp2)
+ if (fileTransferSettings.enableHttp2)
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);
else
curl_easy_setopt(req, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
#endif
- curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, DownloadItem::writeCallbackWrapper);
+ curl_easy_setopt(req, CURLOPT_WRITEFUNCTION, TransferItem::writeCallbackWrapper);
curl_easy_setopt(req, CURLOPT_WRITEDATA, this);
- curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, DownloadItem::headerCallbackWrapper);
+ curl_easy_setopt(req, CURLOPT_HEADERFUNCTION, TransferItem::headerCallbackWrapper);
curl_easy_setopt(req, CURLOPT_HEADERDATA, this);
curl_easy_setopt(req, CURLOPT_PROGRESSFUNCTION, progressCallbackWrapper);
@@ -298,10 +297,10 @@ struct CurlDownloader : public Downloader
curl_easy_setopt(req, CURLOPT_SSL_VERIFYHOST, 0);
}
- curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, downloadSettings.connectTimeout.get());
+ curl_easy_setopt(req, CURLOPT_CONNECTTIMEOUT, fileTransferSettings.connectTimeout.get());
curl_easy_setopt(req, CURLOPT_LOW_SPEED_LIMIT, 1L);
- curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, downloadSettings.stalledDownloadTimeout.get());
+ curl_easy_setopt(req, CURLOPT_LOW_SPEED_TIME, fileTransferSettings.stalledDownloadTimeout.get());
/* If no file exist in the specified path, curl continues to work
anyway as if netrc support was disabled. */
@@ -390,6 +389,7 @@ struct CurlDownloader : public Downloader
case CURLE_SSL_CACERT_BADFILE:
case CURLE_TOO_MANY_REDIRECTS:
case CURLE_WRITE_ERROR:
+ case CURLE_UNSUPPORTED_PROTOCOL:
err = Misc;
break;
default: // Shut up warnings
@@ -401,14 +401,14 @@ struct CurlDownloader : public Downloader
auto exc =
code == CURLE_ABORTED_BY_CALLBACK && _isInterrupted
- ? DownloadError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
+ ? FileTransferError(Interrupted, fmt("%s of '%s' was interrupted", request.verb(), request.uri))
: httpStatus != 0
- ? DownloadError(err,
+ ? FileTransferError(err,
fmt("unable to %s '%s': HTTP error %d",
request.verb(), request.uri, httpStatus)
+ (code == CURLE_OK ? "" : fmt(" (curl error: %s)", curl_easy_strerror(code)))
)
- : DownloadError(err,
+ : FileTransferError(err,
fmt("unable to %s '%s': %s (%d)",
request.verb(), request.uri, curl_easy_strerror(code), code));
@@ -422,13 +422,13 @@ struct CurlDownloader : public Downloader
|| writtenToSink == 0
|| (acceptRanges && encoding.empty())))
{
- int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(downloader.mt19937));
+ int ms = request.baseRetryTimeMs * std::pow(2.0f, attempt - 1 + std::uniform_real_distribution<>(0.0, 0.5)(fileTransfer.mt19937));
if (writtenToSink)
warn("%s; retrying from offset %d in %d ms", exc.what(), writtenToSink, ms);
else
warn("%s; retrying in %d ms", exc.what(), ms);
embargo = std::chrono::steady_clock::now() + std::chrono::milliseconds(ms);
- downloader.enqueueItem(shared_from_this());
+ fileTransfer.enqueueItem(shared_from_this());
}
else
fail(exc);
@@ -439,12 +439,12 @@ struct CurlDownloader : public Downloader
struct State
{
struct EmbargoComparator {
- bool operator() (const std::shared_ptr<DownloadItem> & i1, const std::shared_ptr<DownloadItem> & i2) {
+ bool operator() (const std::shared_ptr<TransferItem> & i1, const std::shared_ptr<TransferItem> & i2) {
return i1->embargo > i2->embargo;
}
};
bool quit = false;
- std::priority_queue<std::shared_ptr<DownloadItem>, std::vector<std::shared_ptr<DownloadItem>>, EmbargoComparator> incoming;
+ std::priority_queue<std::shared_ptr<TransferItem>, std::vector<std::shared_ptr<TransferItem>>, EmbargoComparator> incoming;
};
Sync<State> state_;
@@ -456,7 +456,7 @@ struct CurlDownloader : public Downloader
std::thread workerThread;
- CurlDownloader()
+ curlFileTransfer()
: mt19937(rd())
{
static std::once_flag globalInit;
@@ -469,7 +469,7 @@ struct CurlDownloader : public Downloader
#endif
#if LIBCURL_VERSION_NUM >= 0x071e00 // Max connections requires >= 7.30.0
curl_multi_setopt(curlm, CURLMOPT_MAX_TOTAL_CONNECTIONS,
- downloadSettings.httpConnections.get());
+ fileTransferSettings.httpConnections.get());
#endif
wakeupPipe.create();
@@ -478,7 +478,7 @@ struct CurlDownloader : public Downloader
workerThread = std::thread([&]() { workerThreadEntry(); });
}
- ~CurlDownloader()
+ ~curlFileTransfer()
{
stopWorkerThread();
@@ -504,7 +504,7 @@ struct CurlDownloader : public Downloader
stopWorkerThread();
});
- std::map<CURL *, std::shared_ptr<DownloadItem>> items;
+ std::map<CURL *, std::shared_ptr<TransferItem>> items;
bool quit = false;
@@ -561,7 +561,7 @@ struct CurlDownloader : public Downloader
throw SysError("reading curl wakeup socket");
}
- std::vector<std::shared_ptr<DownloadItem>> incoming;
+ std::vector<std::shared_ptr<TransferItem>> incoming;
auto now = std::chrono::steady_clock::now();
{
@@ -609,7 +609,7 @@ struct CurlDownloader : public Downloader
}
}
- void enqueueItem(std::shared_ptr<DownloadItem> item)
+ void enqueueItem(std::shared_ptr<TransferItem> item)
{
if (item->request.data
&& !hasPrefix(item->request.uri, "http://")
@@ -641,8 +641,8 @@ struct CurlDownloader : public Downloader
}
#endif
- void enqueueDownload(const DownloadRequest & request,
- Callback<DownloadResult> callback) override
+ void enqueueFileTransfer(const FileTransferRequest & request,
+ Callback<FileTransferResult> callback) override
{
/* Ugly hack to support s3:// URIs. */
if (hasPrefix(request.uri, "s3://")) {
@@ -660,9 +660,9 @@ struct CurlDownloader : public Downloader
// FIXME: implement ETag
auto s3Res = s3Helper.getObject(bucketName, key);
- DownloadResult res;
+ FileTransferResult res;
if (!s3Res.data)
- throw DownloadError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
+ throw FileTransferError(NotFound, fmt("S3 object '%s' does not exist", request.uri));
res.data = s3Res.data;
callback(std::move(res));
#else
@@ -672,26 +672,26 @@ struct CurlDownloader : public Downloader
return;
}
- enqueueItem(std::make_shared<DownloadItem>(*this, request, std::move(callback)));
+ enqueueItem(std::make_shared<TransferItem>(*this, request, std::move(callback)));
}
};
-ref<Downloader> getDownloader()
+ref<FileTransfer> getFileTransfer()
{
- static ref<Downloader> downloader = makeDownloader();
- return downloader;
+ static ref<FileTransfer> fileTransfer = makeFileTransfer();
+ return fileTransfer;
}
-ref<Downloader> makeDownloader()
+ref<FileTransfer> makeFileTransfer()
{
- return make_ref<CurlDownloader>();
+ return make_ref<curlFileTransfer>();
}
-std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest & request)
+std::future<FileTransferResult> FileTransfer::enqueueFileTransfer(const FileTransferRequest & request)
{
- auto promise = std::make_shared<std::promise<DownloadResult>>();
- enqueueDownload(request,
- {[promise](std::future<DownloadResult> fut) {
+ auto promise = std::make_shared<std::promise<FileTransferResult>>();
+ enqueueFileTransfer(request,
+ {[promise](std::future<FileTransferResult> fut) {
try {
promise->set_value(fut.get());
} catch (...) {
@@ -701,15 +701,21 @@ std::future<DownloadResult> Downloader::enqueueDownload(const DownloadRequest &
return promise->get_future();
}
-DownloadResult Downloader::download(const DownloadRequest & request)
+FileTransferResult FileTransfer::download(const FileTransferRequest & request)
+{
+ return enqueueFileTransfer(request).get();
+}
+
+FileTransferResult FileTransfer::upload(const FileTransferRequest & request)
{
- return enqueueDownload(request).get();
+ /* Note: this method is the same as download, but helps in readability */
+ return enqueueFileTransfer(request).get();
}
-void Downloader::download(DownloadRequest && request, Sink & sink)
+void FileTransfer::download(FileTransferRequest && request, Sink & sink)
{
/* Note: we can't call 'sink' via request.dataCallback, because
- that would cause the sink to execute on the downloader
+ that would cause the sink to execute on the fileTransfer
thread. If 'sink' is a coroutine, this will fail. Also, if the
sink is expensive (e.g. one that does decompression and writing
to the Nix store), it would stall the download thread too much.
@@ -755,8 +761,8 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
state->avail.notify_one();
};
- enqueueDownload(request,
- {[_state](std::future<DownloadResult> fut) {
+ enqueueFileTransfer(request,
+ {[_state](std::future<FileTransferResult> fut) {
auto state(_state->lock());
state->quit = true;
try {
@@ -801,141 +807,6 @@ void Downloader::download(DownloadRequest && request, Sink & sink)
}
}
-CachedDownloadResult Downloader::downloadCached(
- ref<Store> store, const CachedDownloadRequest & request)
-{
- auto url = resolveUri(request.uri);
-
- auto name = request.name;
- if (name == "") {
- auto p = url.rfind('/');
- if (p != string::npos) name = string(url, p + 1);
- }
-
- std::optional<StorePath> expectedStorePath;
- if (request.expectedHash) {
- auto method = request.unpack ? FileIngestionMethod::Recursive : FileIngestionMethod::Flat;
- expectedStorePath = store->makeFixedOutputPath(method, request.expectedHash, name);
- if (store->isValidPath(*expectedStorePath)) {
- CachedDownloadResult result;
- result.storePath = store->printStorePath(*expectedStorePath);
- result.path = store->toRealPath(result.storePath);
- return result;
- }
- }
-
- Path cacheDir = getCacheDir() + "/nix/tarballs";
- createDirs(cacheDir);
-
- string urlHash = hashString(htSHA256, name + std::string("\0"s) + url).to_string(Base32, false);
-
- Path dataFile = cacheDir + "/" + urlHash + ".info";
- Path fileLink = cacheDir + "/" + urlHash + "-file";
-
- PathLocks lock({fileLink}, fmt("waiting for lock on '%1%'...", fileLink));
-
- std::optional<StorePath> storePath;
-
- string expectedETag;
-
- bool skip = false;
-
- CachedDownloadResult result;
-
- if (pathExists(fileLink) && pathExists(dataFile)) {
- storePath = store->parseStorePath(readLink(fileLink));
- // FIXME
- store->addTempRoot(*storePath);
- if (store->isValidPath(*storePath)) {
- auto ss = tokenizeString<vector<string>>(readFile(dataFile), "\n");
- if (ss.size() >= 3 && ss[0] == url) {
- time_t lastChecked;
- if (string2Int(ss[2], lastChecked) && (uint64_t) lastChecked + request.ttl >= (uint64_t) time(0)) {
- skip = true;
- result.effectiveUri = request.uri;
- result.etag = ss[1];
- } else if (!ss[1].empty()) {
- debug(format("verifying previous ETag '%1%'") % ss[1]);
- expectedETag = ss[1];
- }
- }
- } else
- storePath.reset();
- }
-
- if (!skip) {
-
- try {
- DownloadRequest request2(url);
- request2.expectedETag = expectedETag;
- auto res = download(request2);
- result.effectiveUri = res.effectiveUri;
- result.etag = res.etag;
-
- if (!res.cached) {
- StringSink sink;
- dumpString(*res.data, sink);
- Hash hash = hashString(request.expectedHash ? request.expectedHash.type : htSHA256, *res.data);
- ValidPathInfo info(store->makeFixedOutputPath(FileIngestionMethod::Flat, hash, name));
- info.narHash = hashString(htSHA256, *sink.s);
- info.narSize = sink.s->size();
- info.ca = makeFixedOutputCA(FileIngestionMethod::Flat, hash);
- store->addToStore(info, sink.s, NoRepair, NoCheckSigs);
- storePath = info.path.clone();
- }
-
- assert(storePath);
- replaceSymlink(store->printStorePath(*storePath), fileLink);
-
- writeFile(dataFile, url + "\n" + res.etag + "\n" + std::to_string(time(0)) + "\n");
- } catch (DownloadError & e) {
- if (!storePath) throw;
- warn("warning: %s; using cached result", e.msg());
- result.etag = expectedETag;
- }
- }
-
- if (request.unpack) {
- Path unpackedLink = cacheDir + "/" + ((std::string) storePath->to_string()) + "-unpacked";
- PathLocks lock2({unpackedLink}, fmt("waiting for lock on '%1%'...", unpackedLink));
- std::optional<StorePath> unpackedStorePath;
- if (pathExists(unpackedLink)) {
- unpackedStorePath = store->parseStorePath(readLink(unpackedLink));
- // FIXME
- store->addTempRoot(*unpackedStorePath);
- if (!store->isValidPath(*unpackedStorePath))
- unpackedStorePath.reset();
- }
- if (!unpackedStorePath) {
- printInfo("unpacking '%s'...", url);
- Path tmpDir = createTempDir();
- AutoDelete autoDelete(tmpDir, true);
- unpackTarfile(store->toRealPath(store->printStorePath(*storePath)), tmpDir);
- auto members = readDirectory(tmpDir);
- if (members.size() != 1)
- throw nix::Error("tarball '%s' contains an unexpected number of top-level files", url);
- auto topDir = tmpDir + "/" + members.begin()->name;
- unpackedStorePath = store->addToStore(name, topDir, FileIngestionMethod::Recursive, htSHA256, defaultPathFilter, NoRepair);
- }
- replaceSymlink(store->printStorePath(*unpackedStorePath), unpackedLink);
- storePath = std::move(*unpackedStorePath);
- }
-
- if (expectedStorePath && *storePath != *expectedStorePath) {
- unsigned int statusCode = 102;
- Hash gotHash = request.unpack
- ? hashPath(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath))).first
- : hashFile(request.expectedHash.type, store->toRealPath(store->printStorePath(*storePath)));
- throw nix::Error(statusCode, "hash mismatch in file downloaded from '%s':\n wanted: %s\n got: %s",
- url, request.expectedHash.to_string(), gotHash.to_string());
- }
-
- result.storePath = store->printStorePath(*storePath);
- result.path = store->toRealPath(result.storePath);
- return result;
-}
-
-
bool isUri(const string & s)
{
if (s.compare(0, 8, "channel:") == 0) return true;
diff --git a/src/libstore/download.hh b/src/libstore/filetransfer.hh
index 5a131c704..2347f363d 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/filetransfer.hh
@@ -9,7 +9,7 @@
namespace nix {
-struct DownloadSettings : Config
+struct FileTransferSettings : Config
{
Setting<bool> enableHttp2{this, true, "http2",
"Whether to enable HTTP/2 support."};
@@ -31,15 +31,15 @@ struct DownloadSettings : Config
"How often Nix will attempt to download a file before giving up."};
};
-extern DownloadSettings downloadSettings;
+extern FileTransferSettings fileTransferSettings;
-struct DownloadRequest
+struct FileTransferRequest
{
std::string uri;
std::string expectedETag;
bool verifyTLS = true;
bool head = false;
- size_t tries = downloadSettings.tries;
+ size_t tries = fileTransferSettings.tries;
unsigned int baseRetryTimeMs = 250;
ActivityId parentAct;
bool decompress = true;
@@ -47,7 +47,7 @@ struct DownloadRequest
std::string mimeType;
std::function<void(char *, size_t)> dataCallback;
- DownloadRequest(const std::string & uri)
+ FileTransferRequest(const std::string & uri)
: uri(uri), parentAct(getCurActivity()) { }
std::string verb()
@@ -56,7 +56,7 @@ struct DownloadRequest
}
};
-struct DownloadResult
+struct FileTransferResult
{
bool cached = false;
std::string etag;
@@ -65,74 +65,52 @@ struct DownloadResult
uint64_t bodySize = 0;
};
-struct CachedDownloadRequest
-{
- std::string uri;
- bool unpack = false;
- std::string name;
- Hash expectedHash;
- unsigned int ttl;
-
- CachedDownloadRequest(const std::string & uri);
- CachedDownloadRequest() = delete;
-};
-
-struct CachedDownloadResult
-{
- // Note: 'storePath' may be different from 'path' when using a
- // chroot store.
- Path storePath;
- Path path;
- std::optional<std::string> etag;
- std::string effectiveUri;
-};
-
class Store;
-struct Downloader
+struct FileTransfer
{
- virtual ~Downloader() { }
+ virtual ~FileTransfer() { }
- /* Enqueue a download request, returning a future to the result of
- the download. The future may throw a DownloadError
+ /* Enqueue a data transfer request, returning a future to the result of
+ the download. The future may throw a FileTransferError
exception. */
- virtual void enqueueDownload(const DownloadRequest & request,
- Callback<DownloadResult> callback) = 0;
+ virtual void enqueueFileTransfer(const FileTransferRequest & request,
+ Callback<FileTransferResult> callback) = 0;
- std::future<DownloadResult> enqueueDownload(const DownloadRequest & request);
+ std::future<FileTransferResult> enqueueFileTransfer(const FileTransferRequest & request);
/* Synchronously download a file. */
- DownloadResult download(const DownloadRequest & request);
+ FileTransferResult download(const FileTransferRequest & request);
+
+ /* Synchronously upload a file. */
+ FileTransferResult upload(const FileTransferRequest & request);
/* Download a file, writing its data to a sink. The sink will be
invoked on the thread of the caller. */
- void download(DownloadRequest && request, Sink & sink);
-
- /* Check if the specified file is already in ~/.cache/nix/tarballs
- and is more recent than ‘tarball-ttl’ seconds. Otherwise,
- use the recorded ETag to verify if the server has a more
- recent version, and if so, download it to the Nix store. */
- CachedDownloadResult downloadCached(ref<Store> store, const CachedDownloadRequest & request);
+ void download(FileTransferRequest && request, Sink & sink);
enum Error { NotFound, Forbidden, Misc, Transient, Interrupted };
};
-/* Return a shared Downloader object. Using this object is preferred
+/* Return a shared FileTransfer object. Using this object is preferred
because it enables connection reuse and HTTP/2 multiplexing. */
-ref<Downloader> getDownloader();
+ref<FileTransfer> getFileTransfer();
-/* Return a new Downloader object. */
-ref<Downloader> makeDownloader();
+/* Return a new FileTransfer object. */
+ref<FileTransfer> makeFileTransfer();
-class DownloadError : public Error
+class FileTransferError : public Error
{
public:
- Downloader::Error error;
- DownloadError(Downloader::Error error, const FormatOrString & fs)
+ FileTransfer::Error error;
+ FileTransferError(FileTransfer::Error error, const FormatOrString & fs)
: Error(fs), error(error)
{ }
};
bool isUri(const string & s);
+/* Resolve deprecated 'channel:<foo>' URLs. */
+std::string resolveUri(const std::string & uri);
+
}
diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc
index 0c3d89611..95a4bc934 100644
--- a/src/libstore/gc.cc
+++ b/src/libstore/gc.cc
@@ -202,6 +202,11 @@ void LocalStore::findTempRoots(FDs & fds, Roots & tempRoots, bool censor)
/* Read the `temproots' directory for per-process temporary root
files. */
for (auto & i : readDirectory(tempRootsDir)) {
+ if (i.name[0] == '.') {
+ // Ignore hidden files. Some package managers (notably portage) create
+ // those to keep the directory alive.
+ continue;
+ }
Path path = tempRootsDir + "/" + i.name;
pid_t pid = std::stoi(i.name);
@@ -414,7 +419,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
try {
auto mapFile = fmt("/proc/%s/maps", ent->d_name);
- auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile, true), "\n");
+ auto mapLines = tokenizeString<std::vector<string>>(readFile(mapFile), "\n");
for (const auto & line : mapLines) {
auto match = std::smatch{};
if (std::regex_match(line, match, mapRegex))
@@ -422,7 +427,7 @@ void LocalStore::findRuntimeRoots(Roots & roots, bool censor)
}
auto envFile = fmt("/proc/%s/environ", ent->d_name);
- auto envString = readFile(envFile, true);
+ auto envString = readFile(envFile);
auto env_end = std::sregex_iterator{};
for (auto i = std::sregex_iterator{envString.begin(), envString.end(), storePathRegex}; i != env_end; ++i)
unchecked[i->str()].emplace(envFile);
@@ -884,7 +889,7 @@ void LocalStore::autoGC(bool sync)
if (statvfs(realStoreDir.c_str(), &st))
throw SysError("getting filesystem info about '%s'", realStoreDir);
- return (uint64_t) st.f_bavail * st.f_bsize;
+ return (uint64_t) st.f_bavail * st.f_frsize;
};
std::shared_future<void> future;
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index 7e97f3c22..bee94cbd8 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -31,6 +31,7 @@ Settings::Settings()
, nixLogDir(canonPath(getEnv("NIX_LOG_DIR").value_or(NIX_LOG_DIR)))
, nixStateDir(canonPath(getEnv("NIX_STATE_DIR").value_or(NIX_STATE_DIR)))
, nixConfDir(canonPath(getEnv("NIX_CONF_DIR").value_or(NIX_CONF_DIR)))
+ , nixUserConfFiles(getUserConfigFiles())
, nixLibexecDir(canonPath(getEnv("NIX_LIBEXEC_DIR").value_or(NIX_LIBEXEC_DIR)))
, nixBinDir(canonPath(getEnv("NIX_BIN_DIR").value_or(NIX_BIN_DIR)))
, nixManDir(canonPath(NIX_MAN_DIR))
@@ -77,11 +78,27 @@ void loadConfFile()
~/.nix/nix.conf or the command line. */
globalConfig.resetOverriden();
+ auto files = settings.nixUserConfFiles;
+ for (auto file = files.rbegin(); file != files.rend(); file++) {
+ globalConfig.applyConfigFile(*file);
+ }
+}
+
+std::vector<Path> getUserConfigFiles()
+{
+ // Use the paths specified in NIX_USER_CONF_FILES if it has been defined
+ auto nixConfFiles = getEnv("NIX_USER_CONF_FILES");
+ if (nixConfFiles.has_value()) {
+ return tokenizeString<std::vector<string>>(nixConfFiles.value(), ":");
+ }
+
+ // Use the paths specified by the XDG spec
+ std::vector<Path> files;
auto dirs = getConfigDirs();
- // Iterate over them in reverse so that the ones appearing first in the path take priority
- for (auto dir = dirs.rbegin(); dir != dirs.rend(); dir++) {
- globalConfig.applyConfigFile(*dir + "/nix/nix.conf");
+ for (auto & dir : dirs) {
+ files.insert(files.end(), dir + "/nix/nix.conf");
}
+ return files;
}
unsigned int Settings::getDefaultCores()
@@ -113,7 +130,7 @@ bool Settings::isExperimentalFeatureEnabled(const std::string & name)
void Settings::requireExperimentalFeature(const std::string & name)
{
if (!isExperimentalFeatureEnabled(name))
- throw Error("experimental Nix feature '%s' is disabled", name);
+ throw Error("experimental Nix feature '%1%' is disabled; use '--experimental-features %1%' to override", name);
}
bool Settings::isWSL1()
@@ -150,21 +167,24 @@ template<> void BaseSetting<SandboxMode>::toJSON(JSONPlaceholder & out)
template<> void BaseSetting<SandboxMode>::convertToArg(Args & args, const std::string & category)
{
- args.mkFlag()
- .longName(name)
- .description("Enable sandboxing.")
- .handler([=](std::vector<std::string> ss) { override(smEnabled); })
- .category(category);
- args.mkFlag()
- .longName("no-" + name)
- .description("Disable sandboxing.")
- .handler([=](std::vector<std::string> ss) { override(smDisabled); })
- .category(category);
- args.mkFlag()
- .longName("relaxed-" + name)
- .description("Enable sandboxing, but allow builds to disable it.")
- .handler([=](std::vector<std::string> ss) { override(smRelaxed); })
- .category(category);
+ args.addFlag({
+ .longName = name,
+ .description = "Enable sandboxing.",
+ .category = category,
+ .handler = {[=]() { override(smEnabled); }}
+ });
+ args.addFlag({
+ .longName = "no-" + name,
+ .description = "Disable sandboxing.",
+ .category = category,
+ .handler = {[=]() { override(smDisabled); }}
+ });
+ args.addFlag({
+ .longName = "relaxed-" + name,
+ .description = "Enable sandboxing, but allow builds to disable it.",
+ .category = category,
+ .handler = {[=]() { override(smRelaxed); }}
+ });
}
void MaxBuildJobsSetting::set(const std::string & str)
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 3aa3653f3..da95fd3ae 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -53,9 +53,12 @@ public:
/* The directory where state is stored. */
Path nixStateDir;
- /* The directory where configuration files are stored. */
+ /* The directory where system configuration files are stored. */
Path nixConfDir;
+ /* A list of user configuration files to load. */
+ std::vector<Path> nixUserConfFiles;
+
/* The directory where internal helper programs are stored. */
Path nixLibexecDir;
@@ -351,12 +354,21 @@ public:
Setting<Paths> pluginFiles{this, {}, "plugin-files",
"Plugins to dynamically load at nix initialization time."};
+ Setting<std::string> githubAccessToken{this, "", "github-access-token",
+ "GitHub access token to get access to GitHub data through the GitHub API for github:<..> flakes."};
+
Setting<Strings> experimentalFeatures{this, {}, "experimental-features",
"Experimental Nix features to enable."};
bool isExperimentalFeatureEnabled(const std::string & name);
void requireExperimentalFeature(const std::string & name);
+
+ Setting<bool> allowDirty{this, true, "allow-dirty",
+ "Whether to allow dirty Git/Mercurial trees."};
+
+ Setting<bool> warnDirty{this, true, "warn-dirty",
+ "Whether to warn about dirty Git/Mercurial trees."};
};
@@ -369,6 +381,9 @@ void initPlugins();
void loadConfFile();
+// Used by the Settings constructor
+std::vector<Path> getUserConfigFiles();
+
extern const string nixVersion;
}
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index 011794c62..451a64785 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -1,5 +1,5 @@
#include "binary-cache-store.hh"
-#include "download.hh"
+#include "filetransfer.hh"
#include "globals.hh"
#include "nar-info-disk-cache.hh"
@@ -85,14 +85,14 @@ protected:
checkEnabled();
try {
- DownloadRequest request(cacheUri + "/" + path);
+ FileTransferRequest request(cacheUri + "/" + path);
request.head = true;
- getDownloader()->download(request);
+ getFileTransfer()->download(request);
return true;
- } catch (DownloadError & e) {
+ } catch (FileTransferError & e) {
/* S3 buckets return 403 if a file doesn't exist and the
bucket is unlistable, so treat 403 as 404. */
- if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
+ if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
return false;
maybeDisable();
throw;
@@ -103,19 +103,19 @@ protected:
const std::string & data,
const std::string & mimeType) override
{
- auto req = DownloadRequest(cacheUri + "/" + path);
+ auto req = FileTransferRequest(cacheUri + "/" + path);
req.data = std::make_shared<string>(data); // FIXME: inefficient
req.mimeType = mimeType;
try {
- getDownloader()->download(req);
- } catch (DownloadError & e) {
+ getFileTransfer()->upload(req);
+ } catch (FileTransferError & e) {
throw UploadToHTTP("while uploading to HTTP binary cache at '%s': %s", cacheUri, e.msg());
}
}
- DownloadRequest makeRequest(const std::string & path)
+ FileTransferRequest makeRequest(const std::string & path)
{
- DownloadRequest request(cacheUri + "/" + path);
+ FileTransferRequest request(cacheUri + "/" + path);
return request;
}
@@ -124,9 +124,9 @@ protected:
checkEnabled();
auto request(makeRequest(path));
try {
- getDownloader()->download(std::move(request), sink);
- } catch (DownloadError & e) {
- if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
+ getFileTransfer()->download(std::move(request), sink);
+ } catch (FileTransferError & e) {
+ if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
throw NoSuchBinaryCacheFile("file '%s' does not exist in binary cache '%s'", path, getUri());
maybeDisable();
throw;
@@ -142,12 +142,12 @@ protected:
auto callbackPtr = std::make_shared<decltype(callback)>(std::move(callback));
- getDownloader()->enqueueDownload(request,
- {[callbackPtr, this](std::future<DownloadResult> result) {
+ getFileTransfer()->enqueueFileTransfer(request,
+ {[callbackPtr, this](std::future<FileTransferResult> result) {
try {
(*callbackPtr)(result.get().data);
- } catch (DownloadError & e) {
- if (e.error == Downloader::NotFound || e.error == Downloader::Forbidden)
+ } catch (FileTransferError & e) {
+ if (e.error == FileTransfer::NotFound || e.error == FileTransfer::Forbidden)
return (*callbackPtr)(std::shared_ptr<std::string>());
maybeDisable();
callbackPtr->rethrow();
@@ -174,4 +174,3 @@ static RegisterStoreImplementation regStore([](
});
}
-
diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc
index f2e4b63e0..b24e7b7d6 100644
--- a/src/libstore/s3-binary-cache-store.cc
+++ b/src/libstore/s3-binary-cache-store.cc
@@ -6,7 +6,7 @@
#include "nar-info-disk-cache.hh"
#include "globals.hh"
#include "compression.hh"
-#include "download.hh"
+#include "filetransfer.hh"
#include "istringstream_nocopy.hh"
#include <aws/core/Aws.h>
@@ -132,7 +132,7 @@ ref<Aws::Client::ClientConfiguration> S3Helper::makeConfig(const string & region
return res;
}
-S3Helper::DownloadResult S3Helper::getObject(
+S3Helper::FileTransferResult S3Helper::getObject(
const std::string & bucketName, const std::string & key)
{
debug("fetching 's3://%s/%s'...", bucketName, key);
@@ -146,7 +146,7 @@ S3Helper::DownloadResult S3Helper::getObject(
return Aws::New<std::stringstream>("STRINGSTREAM");
});
- DownloadResult res;
+ FileTransferResult res;
auto now1 = std::chrono::steady_clock::now();
diff --git a/src/libstore/s3.hh b/src/libstore/s3.hh
index ef5f23d0f..2042bffcf 100644
--- a/src/libstore/s3.hh
+++ b/src/libstore/s3.hh
@@ -18,13 +18,13 @@ struct S3Helper
ref<Aws::Client::ClientConfiguration> makeConfig(const std::string & region, const std::string & scheme, const std::string & endpoint);
- struct DownloadResult
+ struct FileTransferResult
{
std::shared_ptr<std::string> data;
unsigned int durationMs;
};
- DownloadResult getObject(
+ FileTransferResult getObject(
const std::string & bucketName, const std::string & key);
};
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index ca1409588..86cb20a26 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -6,6 +6,7 @@
#include "thread-pool.hh"
#include "json.hh"
#include "derivations.hh"
+#include "url.hh"
#include <future>
@@ -40,7 +41,7 @@ Path Store::followLinksToStore(std::string_view _path) const
path = absPath(target, dirOf(path));
}
if (!isInStore(path))
- throw Error(format("path '%1%' is not in the Nix store") % path);
+ throw NotInStore("path '%1%' is not in the Nix store", path);
return path;
}
@@ -873,27 +874,7 @@ std::pair<std::string, Store::Params> splitUriAndParams(const std::string & uri_
Store::Params params;
auto q = uri.find('?');
if (q != std::string::npos) {
- for (auto s : tokenizeString<Strings>(uri.substr(q + 1), "&")) {
- auto e = s.find('=');
- if (e != std::string::npos) {
- auto value = s.substr(e + 1);
- std::string decoded;
- for (size_t i = 0; i < value.size(); ) {
- if (value[i] == '%') {
- if (i + 2 >= value.size())
- throw Error("invalid URI parameter '%s'", value);
- try {
- decoded += std::stoul(std::string(value, i + 1, 2), 0, 16);
- i += 3;
- } catch (...) {
- throw Error("invalid URI parameter '%s'", value);
- }
- } else
- decoded += value[i++];
- }
- params[s.substr(0, e)] = decoded;
- }
- }
+ params = decodeQuery(uri.substr(q + 1));
uri = uri_.substr(0, q);
}
return {uri, params};
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 32c24c500..3def209a6 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -28,6 +28,7 @@ MakeError(InvalidPath, Error);
MakeError(Unsupported, Error);
MakeError(SubstituteGone, Error);
MakeError(SubstituterDisabled, Error);
+MakeError(NotInStore, Error);
struct BasicDerivation;
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 15:26:39 +0000