aboutsummaryrefslogtreecommitdiff
path: root/src/libstore
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore')
-rw-r--r--src/libstore/binary-cache-store.cc2
-rw-r--r--src/libstore/build.cc50
-rw-r--r--src/libstore/derivations.cc7
-rw-r--r--src/libstore/derivations.hh2
-rw-r--r--src/libstore/download.cc54
-rw-r--r--src/libstore/download.hh14
-rw-r--r--src/libstore/globals.cc3
-rw-r--r--src/libstore/globals.hh3
-rw-r--r--src/libstore/http-binary-cache-store.cc3
-rw-r--r--src/libstore/local-fs-store.cc8
-rw-r--r--src/libstore/local-store.cc56
-rw-r--r--src/libstore/local-store.hh4
-rw-r--r--src/libstore/nar-accessor.cc2
-rw-r--r--src/libstore/nar-info-disk-cache.cc13
-rw-r--r--src/libstore/nar-info.cc7
-rw-r--r--src/libstore/remote-store.cc35
-rw-r--r--src/libstore/schema.sql3
-rw-r--r--src/libstore/sqlite.cc13
-rw-r--r--src/libstore/sqlite.hh10
-rw-r--r--src/libstore/store-api.cc60
-rw-r--r--src/libstore/store-api.hh75
21 files changed, 281 insertions, 143 deletions
diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc
index 801ecd368..e71ea6a57 100644
--- a/src/libstore/binary-cache-store.cc
+++ b/src/libstore/binary-cache-store.cc
@@ -254,7 +254,7 @@ struct BinaryCacheStoreAccessor : public FSAccessor
std::string restPath = std::string(path, storePath.size());
if (!store->isValidPath(storePath))
- throw Error(format("path ‘%1%’ is not a valid store path") % storePath);
+ throw InvalidPath(format("path ‘%1%’ is not a valid store path") % storePath);
auto i = nars.find(storePath);
if (i != nars.end()) return {i->second, restPath};
diff --git a/src/libstore/build.cc b/src/libstore/build.cc
index 10ae574f9..e0eb702a4 100644
--- a/src/libstore/build.cc
+++ b/src/libstore/build.cc
@@ -193,6 +193,7 @@ bool CompareGoalPtrs::operator() (const GoalPtr & a, const GoalPtr & b) {
struct Child
{
WeakGoalPtr goal;
+ Goal * goal2; // ugly hackery
set<int> fds;
bool respectTimeouts;
bool inBuildSlot;
@@ -284,7 +285,7 @@ public:
false if there is no sense in waking up goals that are sleeping
because they can't run yet (e.g., there is no free build slot,
or the hook would still say `postpone'). */
- void childTerminated(GoalPtr goal, bool wakeSleepers = true);
+ void childTerminated(Goal * goal, bool wakeSleepers = true);
/* Put `goal' to sleep until a build slot becomes available (which
might be right away). */
@@ -652,18 +653,15 @@ HookInstance::~HookInstance()
//////////////////////////////////////////////////////////////////////
-typedef map<string, string> HashRewrites;
+typedef map<std::string, std::string> StringRewrites;
-string rewriteHashes(string s, const HashRewrites & rewrites)
+std::string rewriteStrings(std::string s, const StringRewrites & rewrites)
{
for (auto & i : rewrites) {
- assert(i.first.size() == i.second.size());
size_t j = 0;
- while ((j = s.find(i.first, j)) != string::npos) {
- debug(format("rewriting @ %1%") % j);
- s.replace(j, i.second.size(), i.second);
- }
+ while ((j = s.find(i.first, j)) != string::npos)
+ s.replace(j, i.first.size(), i.second);
}
return s;
}
@@ -782,7 +780,7 @@ private:
#endif
/* Hash rewriting. */
- HashRewrites rewritesToTmp, rewritesFromTmp;
+ StringRewrites inputRewrites, outputRewrites;
typedef map<Path, Path> RedirectedOutputs;
RedirectedOutputs redirectedOutputs;
@@ -938,7 +936,7 @@ DerivationGoal::~DerivationGoal()
void DerivationGoal::killChild()
{
if (pid != -1) {
- worker.childTerminated(shared_from_this());
+ worker.childTerminated(this);
if (buildUser.enabled()) {
/* If we're using a build user, then there is a tricky
@@ -1412,7 +1410,7 @@ void DerivationGoal::buildDone()
debug(format("builder process for ‘%1%’ finished") % drvPath);
/* So the child is gone now. */
- worker.childTerminated(shared_from_this());
+ worker.childTerminated(this);
/* Close the read side of the logger pipe. */
if (hook) {
@@ -1774,6 +1772,10 @@ void DerivationGoal::startBuilder()
for (auto & i : varNames) env[i] = getEnv(i);
}
+ /* Substitute output placeholders with the actual output paths. */
+ for (auto & output : drv->outputs)
+ inputRewrites[hashPlaceholder(output.first)] = output.second.path;
+
/* The `exportReferencesGraph' feature allows the references graph
to be passed to a builder. This attribute should be a list of
pairs [name1 path1 name2 path2 ...]. The references graph of
@@ -2418,7 +2420,7 @@ void DerivationGoal::runChild()
/* Fill in the environment. */
Strings envStrs;
for (auto & i : env)
- envStrs.push_back(rewriteHashes(i.first + "=" + i.second, rewritesToTmp));
+ envStrs.push_back(rewriteStrings(i.first + "=" + i.second, inputRewrites));
/* If we are running in `build-users' mode, then switch to the
user we allocated above. Make sure that we drop all root
@@ -2560,7 +2562,7 @@ void DerivationGoal::runChild()
}
for (auto & i : drv->args)
- args.push_back(rewriteHashes(i, rewritesToTmp));
+ args.push_back(rewriteStrings(i, inputRewrites));
restoreSIGPIPE();
@@ -2682,7 +2684,7 @@ void DerivationGoal::registerOutputs()
/* Apply hash rewriting if necessary. */
bool rewritten = false;
- if (!rewritesFromTmp.empty()) {
+ if (!outputRewrites.empty()) {
printMsg(lvlError, format("warning: rewriting hashes in ‘%1%’; cross fingers") % path);
/* Canonicalise first. This ensures that the path we're
@@ -2694,7 +2696,7 @@ void DerivationGoal::registerOutputs()
StringSink sink;
dumpPath(actualPath, sink);
deletePath(actualPath);
- sink.s = make_ref<std::string>(rewriteHashes(*sink.s, rewritesFromTmp));
+ sink.s = make_ref<std::string>(rewriteStrings(*sink.s, outputRewrites));
StringSource source(*sink.s);
restorePath(actualPath, source);
@@ -2910,7 +2912,7 @@ Path DerivationGoal::openLogFile()
string baseName = baseNameOf(drvPath);
/* Create a log file. */
- Path dir = (format("%1%/%2%/%3%/") % settings.nixLogDir % drvsLogDir % string(baseName, 0, 2)).str();
+ Path dir = (format("%1%/%2%/%3%/") % worker.store.logDir % drvsLogDir % string(baseName, 0, 2)).str();
createDirs(dir);
Path logFileName = (format("%1%/%2%%3%")
@@ -3033,8 +3035,8 @@ Path DerivationGoal::addHashRewrite(const Path & path)
Path p = worker.store.storeDir + "/" + h2 + string(path, worker.store.storeDir.size() + 33);
deletePath(p);
assert(path.size() == p.size());
- rewritesToTmp[h1] = h2;
- rewritesFromTmp[h2] = h1;
+ inputRewrites[h1] = h2;
+ outputRewrites[h2] = h1;
redirectedOutputs[path] = p;
return p;
}
@@ -3140,8 +3142,9 @@ SubstitutionGoal::~SubstitutionGoal()
{
try {
if (thr.joinable()) {
+ // FIXME: signal worker thread to quit.
thr.join();
- //worker.childTerminated(shared_from_this()); // FIXME
+ worker.childTerminated(this);
}
} catch (...) {
ignoreException();
@@ -3213,7 +3216,7 @@ void SubstitutionGoal::tryNext()
/* Bail out early if this substituter lacks a valid
signature. LocalStore::addToStore() also checks for this, but
only after we've downloaded the path. */
- if (worker.store.requireSigs && !info->checkSignatures(worker.store.publicKeys)) {
+ if (worker.store.requireSigs && !info->checkSignatures(worker.store, worker.store.publicKeys)) {
printMsg(lvlInfo, format("warning: substituter ‘%s’ does not have a valid signature for path ‘%s’")
% sub->getUri() % storePath);
tryNext();
@@ -3296,7 +3299,7 @@ void SubstitutionGoal::finished()
trace("substitute finished");
thr.join();
- worker.childTerminated(shared_from_this());
+ worker.childTerminated(this);
try {
promise.get_future().get();
@@ -3449,6 +3452,7 @@ void Worker::childStarted(GoalPtr goal, const set<int> & fds,
{
Child child;
child.goal = goal;
+ child.goal2 = goal.get();
child.fds = fds;
child.timeStarted = child.lastOutput = time(0);
child.inBuildSlot = inBuildSlot;
@@ -3458,10 +3462,10 @@ void Worker::childStarted(GoalPtr goal, const set<int> & fds,
}
-void Worker::childTerminated(GoalPtr goal, bool wakeSleepers)
+void Worker::childTerminated(Goal * goal, bool wakeSleepers)
{
auto i = std::find_if(children.begin(), children.end(),
- [&](const Child & child) { return child.goal.lock() == goal; });
+ [&](const Child & child) { return child.goal2 == goal; });
assert(i != children.end());
if (i->inBuildSlot) {
diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc
index 7dcf71d46..f051f10bd 100644
--- a/src/libstore/derivations.cc
+++ b/src/libstore/derivations.cc
@@ -390,4 +390,11 @@ Sink & operator << (Sink & out, const BasicDerivation & drv)
}
+std::string hashPlaceholder(const std::string & outputName)
+{
+ // FIXME: memoize?
+ return "/" + printHash32(hashString(htSHA256, "nix-output:" + outputName));
+}
+
+
}
diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh
index 974de78c5..9717a81e4 100644
--- a/src/libstore/derivations.hh
+++ b/src/libstore/derivations.hh
@@ -117,4 +117,6 @@ struct Sink;
Source & readDerivation(Source & in, Store & store, BasicDerivation & drv);
Sink & operator << (Sink & out, const BasicDerivation & drv);
+std::string hashPlaceholder(const std::string & outputName);
+
}
diff --git a/src/libstore/download.cc b/src/libstore/download.cc
index 9cc433228..5cb2b497a 100644
--- a/src/libstore/download.cc
+++ b/src/libstore/download.cc
@@ -8,6 +8,7 @@
#include <curl/curl.h>
#include <iostream>
+#include <thread>
namespace nix {
@@ -194,9 +195,17 @@ struct CurlDownloader : public Downloader
if (res != CURLE_OK) {
Error err =
httpStatus == 404 ? NotFound :
- httpStatus == 403 ? Forbidden : Misc;
- throw DownloadError(err, format("unable to download ‘%1%’: %2% (%3%)")
- % url % curl_easy_strerror(res) % res);
+ httpStatus == 403 ? Forbidden :
+ (httpStatus == 408 || httpStatus == 500 || httpStatus == 503
+ || httpStatus == 504 || httpStatus == 522 || httpStatus == 524
+ || res == CURLE_COULDNT_RESOLVE_HOST) ? Transient :
+ Misc;
+ if (res == CURLE_HTTP_RETURNED_ERROR && httpStatus != -1)
+ throw DownloadError(err, format("unable to download ‘%s’: HTTP error %d")
+ % url % httpStatus);
+ else
+ throw DownloadError(err, format("unable to download ‘%s’: %s (%d)")
+ % url % curl_easy_strerror(res) % res);
}
char *effectiveUrlCStr;
@@ -211,15 +220,27 @@ struct CurlDownloader : public Downloader
DownloadResult download(string url, const DownloadOptions & options) override
{
- DownloadResult res;
- if (fetch(resolveUri(url), options)) {
- res.cached = false;
- res.data = data;
- } else
- res.cached = true;
- res.effectiveUrl = effectiveUrl;
- res.etag = etag;
- return res;
+ size_t attempt = 0;
+
+ while (true) {
+ try {
+ DownloadResult res;
+ if (fetch(resolveUri(url), options)) {
+ res.cached = false;
+ res.data = data;
+ } else
+ res.cached = true;
+ res.effectiveUrl = effectiveUrl;
+ res.etag = etag;
+ return res;
+ } catch (DownloadError & e) {
+ attempt++;
+ if (e.error != Transient || attempt >= options.tries) throw;
+ auto ms = options.baseRetryTimeMs * (1 << (attempt - 1));
+ printMsg(lvlError, format("warning: %s; retrying in %d ms") % e.what() % ms);
+ std::this_thread::sleep_for(std::chrono::milliseconds(ms));
+ }
+ }
}
};
@@ -228,7 +249,7 @@ ref<Downloader> makeDownloader()
return make_ref<CurlDownloader>();
}
-Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, const Hash & expectedHash)
+Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpack, string name, const Hash & expectedHash)
{
string ignored;
return downloadCached(store, url_, unpack, ignored, expectedHash);
@@ -238,9 +259,10 @@ Path Downloader::downloadCached(ref<Store> store, const string & url_, bool unpa
{
auto url = resolveUri(url_);
- string name;
- auto p = url.rfind('/');
- if (p != string::npos) name = string(url, p + 1);
+ if (name == "") {
+ auto p = url.rfind('/');
+ if (p != string::npos) name = string(url, p + 1);
+ }
Path expectedStorePath;
if (expectedHash) {
diff --git a/src/libstore/download.hh b/src/libstore/download.hh
index d17e14400..cb7de6ef1 100644
--- a/src/libstore/download.hh
+++ b/src/libstore/download.hh
@@ -9,10 +9,12 @@ namespace nix {
struct DownloadOptions
{
- string expectedETag;
- bool verifyTLS{true};
- enum { yes, no, automatic } showProgress{yes};
- bool head{false};
+ std::string expectedETag;
+ bool verifyTLS = true;
+ enum { yes, no, automatic } showProgress = yes;
+ bool head = false;
+ size_t tries = 1;
+ unsigned int baseRetryTimeMs = 100;
};
struct DownloadResult
@@ -29,7 +31,7 @@ struct Downloader
{
virtual DownloadResult download(string url, const DownloadOptions & options) = 0;
- Path downloadCached(ref<Store> store, const string & url, bool unpack,
+ Path downloadCached(ref<Store> store, const string & url, bool unpack, string name = "",
const Hash & expectedHash = Hash());
/* Need to overload because can't have an rvalue default value for non-const reference */
@@ -37,7 +39,7 @@ struct Downloader
Path downloadCached(ref<Store> store, const string & url, bool unpack,
string & effectiveUrl, const Hash & expectedHash = Hash());
- enum Error { NotFound, Forbidden, Misc };
+ enum Error { NotFound, Forbidden, Misc, Transient };
};
ref<Downloader> makeDownloader();
diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc
index c12178e40..ecf81e8eb 100644
--- a/src/libstore/globals.cc
+++ b/src/libstore/globals.cc
@@ -69,7 +69,6 @@ void Settings::processEnvironment()
nixDataDir = canonPath(getEnv("NIX_DATA_DIR", NIX_DATA_DIR));
nixLogDir = canonPath(getEnv("NIX_LOG_DIR", NIX_LOG_DIR));
nixStateDir = canonPath(getEnv("NIX_STATE_DIR", NIX_STATE_DIR));
- nixDBPath = getEnv("NIX_DB_DIR", nixStateDir + "/db");
nixConfDir = canonPath(getEnv("NIX_CONF_DIR", NIX_CONF_DIR));
nixLibexecDir = canonPath(getEnv("NIX_LIBEXEC_DIR", NIX_LIBEXEC_DIR));
nixBinDir = canonPath(getEnv("NIX_BIN_DIR", NIX_BIN_DIR));
@@ -77,7 +76,7 @@ void Settings::processEnvironment()
// should be set with the other config options, but depends on nixLibexecDir
#ifdef __APPLE__
- preBuildHook = nixLibexecDir + "/nix/resolve-system-dependencies.pl";
+ preBuildHook = nixLibexecDir + "/nix/resolve-system-dependencies";
#endif
}
diff --git a/src/libstore/globals.hh b/src/libstore/globals.hh
index 65f763ace..3194193bc 100644
--- a/src/libstore/globals.hh
+++ b/src/libstore/globals.hh
@@ -51,9 +51,6 @@ struct Settings {
/* The directory where state is stored. */
Path nixStateDir;
- /* The directory where we keep the SQLite database. */
- Path nixDBPath;
-
/* The directory where configuration files are stored. */
Path nixConfDir;
diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc
index da80b636c..bdcd2fd39 100644
--- a/src/libstore/http-binary-cache-store.cc
+++ b/src/libstore/http-binary-cache-store.cc
@@ -58,6 +58,7 @@ protected:
DownloadOptions options;
options.showProgress = DownloadOptions::no;
options.head = true;
+ options.tries = 5;
downloader->download(cacheUri + "/" + path, options);
return true;
} catch (DownloadError & e) {
@@ -79,6 +80,8 @@ protected:
auto downloader(downloaders.get());
DownloadOptions options;
options.showProgress = DownloadOptions::no;
+ options.tries = 5;
+ options.baseRetryTimeMs = 1000;
try {
return downloader->download(cacheUri + "/" + path, options).data;
} catch (DownloadError & e) {
diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc
index b1b9dc29e..4571a2211 100644
--- a/src/libstore/local-fs-store.cc
+++ b/src/libstore/local-fs-store.cc
@@ -7,7 +7,9 @@ namespace nix {
LocalFSStore::LocalFSStore(const Params & params)
: Store(params)
- , stateDir(get(params, "state", settings.nixStateDir))
+ , rootDir(get(params, "root"))
+ , stateDir(canonPath(get(params, "state", rootDir != "" ? rootDir + "/nix/var/nix" : settings.nixStateDir)))
+ , logDir(canonPath(get(params, "log", rootDir != "" ? rootDir + "/nix/var/log/nix" : settings.nixLogDir)))
{
}
@@ -21,7 +23,7 @@ struct LocalStoreAccessor : public FSAccessor
{
Path storePath = store->toStorePath(path);
if (!store->isValidPath(storePath))
- throw Error(format("path ‘%1%’ is not a valid store path") % storePath);
+ throw InvalidPath(format("path ‘%1%’ is not a valid store path") % storePath);
return store->getRealStoreDir() + std::string(path, store->storeDir.size());
}
@@ -79,7 +81,7 @@ void LocalFSStore::narFromPath(const Path & path, Sink & sink)
{
if (!isValidPath(path))
throw Error(format("path ‘%s’ is not valid") % path);
- dumpPath(path, sink);
+ dumpPath(getRealStoreDir() + std::string(path, storeDir.size()), sink);
}
}
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index 96ce6a0d8..10056f2f1 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -38,8 +38,8 @@ namespace nix {
LocalStore::LocalStore(const Params & params)
: LocalFSStore(params)
- , realStoreDir(get(params, "real", storeDir))
- , dbDir(get(params, "state", "") != "" ? get(params, "state", "") + "/db" : settings.nixDBPath)
+ , realStoreDir(get(params, "real", rootDir != "" ? rootDir + "/nix/store" : storeDir))
+ , dbDir(stateDir + "/db")
, linksDir(realStoreDir + "/.links")
, reservedPath(dbDir + "/reserved")
, schemaPath(dbDir + "/schema")
@@ -181,17 +181,20 @@ LocalStore::LocalStore(const Params & params)
if (curSchema < 8) {
SQLiteTxn txn(state->db);
- if (sqlite3_exec(state->db, "alter table ValidPaths add column ultimate integer", 0, 0, 0) != SQLITE_OK)
- throwSQLiteError(state->db, "upgrading database schema");
- if (sqlite3_exec(state->db, "alter table ValidPaths add column sigs text", 0, 0, 0) != SQLITE_OK)
- throwSQLiteError(state->db, "upgrading database schema");
+ state->db.exec("alter table ValidPaths add column ultimate integer");
+ state->db.exec("alter table ValidPaths add column sigs text");
txn.commit();
}
if (curSchema < 9) {
SQLiteTxn txn(state->db);
- if (sqlite3_exec(state->db, "drop table FailedPaths", 0, 0, 0) != SQLITE_OK)
- throwSQLiteError(state->db, "upgrading database schema");
+ state->db.exec("drop table FailedPaths");
+ txn.commit();
+ }
+
+ if (curSchema < 10) {
+ SQLiteTxn txn(state->db);
+ state->db.exec("alter table ValidPaths add column ca text");
txn.commit();
}
@@ -204,13 +207,13 @@ LocalStore::LocalStore(const Params & params)
/* Prepare SQL statements. */
state->stmtRegisterValidPath.create(state->db,
- "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs) values (?, ?, ?, ?, ?, ?, ?);");
+ "insert into ValidPaths (path, hash, registrationTime, deriver, narSize, ultimate, sigs, ca) values (?, ?, ?, ?, ?, ?, ?, ?);");
state->stmtUpdatePathInfo.create(state->db,
- "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ? where path = ?;");
+ "update ValidPaths set narSize = ?, hash = ?, ultimate = ?, sigs = ?, ca = ? where path = ?;");
state->stmtAddReference.create(state->db,
"insert or replace into Refs (referrer, reference) values (?, ?);");
state->stmtQueryPathInfo.create(state->db,
- "select id, hash, registrationTime, deriver, narSize, ultimate, sigs from ValidPaths where path = ?;");
+ "select id, hash, registrationTime, deriver, narSize, ultimate, sigs, ca from ValidPaths where path = ?;");
state->stmtQueryReferences.create(state->db,
"select path from Refs join ValidPaths on reference = id where referrer = ?;");
state->stmtQueryReferrers.create(state->db,
@@ -279,8 +282,7 @@ void LocalStore::openDB(State & state, bool create)
if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK)
throwSQLiteError(db, "setting timeout");
- if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK)
- throwSQLiteError(db, "enabling foreign keys");
+ db.exec("pragma foreign_keys = 1");
/* !!! check whether sqlite has been built with foreign key
support */
@@ -290,8 +292,7 @@ void LocalStore::openDB(State & state, bool create)
all. This can cause database corruption if the system
crashes. */
string syncMode = settings.fsyncMetadata ? "normal" : "off";
- if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
- throwSQLiteError(db, "setting synchronous mode");
+ db.exec("pragma synchronous = " + syncMode);
/* Set the SQLite journal mode. WAL mode is fastest, so it's the
default. */
@@ -319,8 +320,7 @@ void LocalStore::openDB(State & state, bool create)
const char * schema =
#include "schema.sql.hh"
;
- if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK)
- throwSQLiteError(db, "initialising database schema");
+ db.exec(schema);
}
}
@@ -527,6 +527,7 @@ uint64_t LocalStore::addValidPath(State & state,
(info.narSize, info.narSize != 0)
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
+ (info.ca, !info.ca.empty())
.exec();
uint64_t id = sqlite3_last_insert_rowid(state.db);
@@ -609,6 +610,9 @@ std::shared_ptr<ValidPathInfo> LocalStore::queryPathInfoUncached(const Path & pa
s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 6);
if (s) info->sigs = tokenizeString<StringSet>(s, " ");
+ s = (const char *) sqlite3_column_text(state->stmtQueryPathInfo, 7);
+ if (s) info->ca = s;
+
/* Get the references. */
auto useQueryReferences(state->stmtQueryReferences.use()(info->id));
@@ -628,6 +632,7 @@ void LocalStore::updatePathInfo(State & state, const ValidPathInfo & info)
("sha256:" + printHash(info.narHash))
(info.ultimate ? 1 : 0, info.ultimate)
(concatStringsSep(" ", info.sigs), !info.sigs.empty())
+ (info.ca, !info.ca.empty())
(info.path)
.exec();
}
@@ -755,7 +760,7 @@ Path LocalStore::queryPathFromHashPart(const string & hashPart)
Path prefix = storeDir + "/" + hashPart;
- return retrySQLite<Path>([&]() {
+ return retrySQLite<Path>([&]() -> std::string {
auto state(_state.lock());
auto useQueryPathFromHashPart(state->stmtQueryPathFromHashPart.use()(prefix));
@@ -898,7 +903,7 @@ void LocalStore::addToStore(const ValidPathInfo & info, const std::string & nar,
throw Error(format("hash mismatch importing path ‘%s’; expected hash ‘%s’, got ‘%s’") %
info.path % info.narHash.to_string() % h.to_string());
- if (requireSigs && !dontCheckSigs && !info.checkSignatures(publicKeys))
+ if (requireSigs && !dontCheckSigs && !info.checkSignatures(*this, publicKeys))
throw Error(format("cannot import path ‘%s’ because it lacks a valid signature") % info.path);
addTempRoot(info.path);
@@ -983,6 +988,7 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name,
info.narHash = hash.first;
info.narSize = hash.second;
info.ultimate = true;
+ info.ca = "fixed:" + (recursive ? (std::string) "r:" : "") + h.to_string();
registerValidPath(info);
}
@@ -1014,7 +1020,8 @@ Path LocalStore::addToStore(const string & name, const Path & _srcPath,
Path LocalStore::addTextToStore(const string & name, const string & s,
const PathSet & references, bool repair)
{
- Path dstPath = computeStorePathForText(name, s, references);
+ auto hash = hashString(htSHA256, s);
+ auto dstPath = makeTextPath(name, hash, references);
addTempRoot(dstPath);
@@ -1034,16 +1041,17 @@ Path LocalStore::addTextToStore(const string & name, const string & s,
StringSink sink;
dumpString(s, sink);
- auto hash = hashString(htSHA256, *sink.s);
+ auto narHash = hashString(htSHA256, *sink.s);
optimisePath(realPath);
ValidPathInfo info;
info.path = dstPath;
- info.narHash = hash;
+ info.narHash = narHash;
info.narSize = sink.s->size();
info.references = references;
info.ultimate = true;
+ info.ca = "text:" + hash.to_string();
registerValidPath(info);
}
@@ -1282,9 +1290,7 @@ void LocalStore::upgradeStore7()
void LocalStore::vacuumDB()
{
auto state(_state.lock());
-
- if (sqlite3_exec(state->db, "vacuum;", 0, 0, 0) != SQLITE_OK)
- throwSQLiteError(state->db, "vacuuming SQLite database");
+ state->db.exec("vacuum");
}
diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh
index 7bfc4ad34..5b5960cf2 100644
--- a/src/libstore/local-store.hh
+++ b/src/libstore/local-store.hh
@@ -17,8 +17,8 @@ namespace nix {
/* Nix store and database schema version. Version 1 (or 0) was Nix <=
0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10.
Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is
- Nix 1.0. Version 7 is Nix 1.3. Version 9 is 1.12. */
-const int nixSchemaVersion = 9;
+ Nix 1.0. Version 7 is Nix 1.3. Version 10 is 1.12. */
+const int nixSchemaVersion = 10;
extern string drvsLogDir;
diff --git a/src/libstore/nar-accessor.cc b/src/libstore/nar-accessor.cc
index 8896862be..ded19c05d 100644
--- a/src/libstore/nar-accessor.cc
+++ b/src/libstore/nar-accessor.cc
@@ -27,7 +27,7 @@ struct NarIndexer : ParseSink, StringSource
Path currentPath;
std::string currentStart;
- bool isExec;
+ bool isExec = false;
NarIndexer(const std::string & nar) : StringSource(nar)
{
diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc
index 172a918ff..d28ff42c7 100644
--- a/src/libstore/nar-info-disk-cache.cc
+++ b/src/libstore/nar-info-disk-cache.cc
@@ -78,21 +78,16 @@ public:
Path dbPath = getCacheDir() + "/nix/binary-cache-v5.sqlite";
createDirs(dirOf(dbPath));
- if (sqlite3_open_v2(dbPath.c_str(), &state->db.db,
- SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK)
- throw Error(format("cannot open store cache ‘%s’") % dbPath);
+ state->db = SQLite(dbPath);
if (sqlite3_busy_timeout(state->db, 60 * 60 * 1000) != SQLITE_OK)
throwSQLiteError(state->db, "setting timeout");
// We can always reproduce the cache.
- if (sqlite3_exec(state->db, "pragma synchronous = off", 0, 0, 0) != SQLITE_OK)
- throwSQLiteError(state->db, "making database asynchronous");
- if (sqlite3_exec(state->db, "pragma main.journal_mode = truncate", 0, 0, 0) != SQLITE_OK)
- throwSQLiteError(state->db, "setting journal mode");
+ state->db.exec("pragma synchronous = off");
+ state->db.exec("pragma main.journal_mode = truncate");
- if (sqlite3_exec(state->db, schema, 0, 0, 0) != SQLITE_OK)
- throwSQLiteError(state->db, "initialising database schema");
+ state->db.exec(schema);
state->insertCache.create(state->db,
"insert or replace into BinaryCaches(url, timestamp, storeDir, wantMassQuery, priority) values (?, ?, ?, ?, ?)");
diff --git a/src/libstore/nar-info.cc b/src/libstore/nar-info.cc
index b0a8d77c2..201cac671 100644
--- a/src/libstore/nar-info.cc
+++ b/src/libstore/nar-info.cc
@@ -67,6 +67,10 @@ NarInfo::NarInfo(const Store & store, const std::string & s, const std::string &
system = value;
else if (name == "Sig")
sigs.insert(value);
+ else if (name == "CA") {
+ if (!ca.empty()) corrupt();
+ ca = value;
+ }
pos = eol + 1;
}
@@ -101,6 +105,9 @@ std::string NarInfo::to_string() const
for (auto sig : sigs)
res += "Sig: " + sig + "\n";
+ if (!ca.empty())
+ res += "CA: " + ca + "\n";
+
return res;
}
diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc
index ab05c3844..94075f3b9 100644
--- a/src/libstore/remote-store.cc
+++ b/src/libstore/remote-store.cc
@@ -94,6 +94,8 @@ ref<RemoteStore::Connection> RemoteStore::openConnection()
conn->daemonVersion = readInt(conn->from);
if (GET_PROTOCOL_MAJOR(conn->daemonVersion) != GET_PROTOCOL_MAJOR(PROTOCOL_VERSION))
throw Error("Nix daemon protocol version not supported");
+ if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 10)
+ throw Error("the Nix daemon version is too old");
conn->to << PROTOCOL_VERSION;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 14) {
@@ -127,17 +129,13 @@ void RemoteStore::setOptions(ref<Connection> conn)
<< settings.tryFallback
<< verbosity
<< settings.maxBuildJobs
- << settings.maxSilentTime;
- if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 2)
- conn->to << settings.useBuildHook;
- if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 4)
- conn->to << (settings.verboseBuild ? lvlError : lvlVomit)
- << 0 // obsolete log type
- << 0 /* obsolete print build trace */;
- if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 6)
- conn->to << settings.buildCores;
- if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 10)
- conn->to << settings.useSubstitutes;
+ << settings.maxSilentTime
+ << settings.useBuildHook
+ << (settings.verboseBuild ? lvlError : lvlVomit)
+ << 0 // obsolete log type
+ << 0 /* obsolete print build trace */
+ << settings.buildCores
+ << settings.useSubstitutes;
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 12) {
Settings::SettingsMap overrides = settings.getOverrides();
@@ -213,8 +211,6 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
auto conn(connections->get());
- if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 3) return;
-
if (GET_PROTOCOL_MINOR(conn->daemonVersion) < 12) {
for (auto & i : paths) {
@@ -227,7 +223,7 @@ void RemoteStore::querySubstitutablePathInfos(const PathSet & paths,
if (info.deriver != "") assertStorePath(info.deriver);
info.references = readStorePaths<PathSet>(*this, conn->from);
info.downloadSize = readLongLong(conn->from);
- info.narSize = GET_PROTOCOL_MINOR(conn->daemonVersion) >= 7 ? readLongLong(conn->from) : 0;
+ info.narSize = readLongLong(conn->from);
infos[i] = info;
}
@@ -277,6 +273,7 @@ std::shared_ptr<ValidPathInfo> RemoteStore::queryPathInfoUncached(const Path & p
if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 16) {
info->ultimate = readInt(conn->from) != 0;
info->sigs = readStrings<StringSet>(conn->from);
+ info->ca = readString(conn->from);
}
return info;
}
@@ -481,11 +478,11 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results)
{
auto conn(connections->get());
- conn->to << wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness
- << options.maxFreed << 0;
- if (GET_PROTOCOL_MINOR(conn->daemonVersion) >= 5)
+ conn->to
+ << wopCollectGarbage << options.action << options.pathsToDelete << options.ignoreLiveness
+ << options.maxFreed
/* removed options */
- conn->to << 0 << 0;
+ << 0 << 0 << 0;
conn->processStderr();
@@ -562,7 +559,7 @@ void RemoteStore::Connection::processStderr(Sink * sink, Source * source)
}
if (msg == STDERR_ERROR) {
string error = readString(from);
- unsigned int status = GET_PROTOCOL_MINOR(daemonVersion) >= 8 ? readInt(from) : 1;
+ unsigned int status = readInt(from);
throw Error(format("%1%") % error, status);
}
else if (msg != STDERR_LAST)
diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql
index 91878af15..09c71a2b8 100644
--- a/src/libstore/schema.sql
+++ b/src/libstore/schema.sql
@@ -6,7 +6,8 @@ create table if not exists ValidPaths (
deriver text,
narSize integer,
ultimate integer, -- null implies "false"
- sigs text -- space-separated
+ sigs text, -- space-separated
+ ca text -- if not null, an assertion that the path is content-addressed; see ValidPathInfo
);
create table if not exists Refs (
diff --git a/src/libstore/sqlite.cc b/src/libstore/sqlite.cc
index 816f9984d..ea0b843f5 100644
--- a/src/libstore/sqlite.cc
+++ b/src/libstore/sqlite.cc
@@ -35,6 +35,13 @@ namespace nix {
throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db));
}
+SQLite::SQLite(const Path & path)
+{
+ if (sqlite3_open_v2(path.c_str(), &db,
+ SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK)
+ throw Error(format("cannot open SQLite database ‘%s’") % path);
+}
+
SQLite::~SQLite()
{
try {
@@ -45,6 +52,12 @@ SQLite::~SQLite()
}
}
+void SQLite::exec(const std::string & stmt)
+{
+ if (sqlite3_exec(db, stmt.c_str(), 0, 0, 0) != SQLITE_OK)
+ throwSQLiteError(db, format("executing SQLite statement ‘%s’") % stmt);
+}
+
void SQLiteStmt::create(sqlite3 * db, const string & s)
{
checkInterrupt();
diff --git a/src/libstore/sqlite.hh b/src/libstore/sqlite.hh
index d6b4a8d91..7c1ed5382 100644
--- a/src/libstore/sqlite.hh
+++ b/src/libstore/sqlite.hh
@@ -13,10 +13,16 @@ namespace nix {
/* RAII wrapper to close a SQLite database automatically. */
struct SQLite
{
- sqlite3 * db;
- SQLite() { db = 0; }
+ sqlite3 * db = 0;
+ SQLite() { }
+ SQLite(const Path & path);
+ SQLite(const SQLite & from) = delete;
+ SQLite& operator = (const SQLite & from) = delete;
+ SQLite& operator = (SQLite && from) { db = from.db; from.db = 0; return *this; }
~SQLite();
operator sqlite3 * () { return db; }
+
+ void exec(const std::string & stmt);
};
/* RAII wrapper to create and destroy SQLite prepared statements. */
diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc
index af002dcc8..5dd56f905 100644
--- a/src/libstore/store-api.cc
+++ b/src/libstore/store-api.cc
@@ -202,6 +202,22 @@ Path Store::makeFixedOutputPath(bool recursive,
}
+Path Store::makeTextPath(const string & name, const Hash & hash,
+ const PathSet & references) const
+{
+ assert(hash.type == htSHA256);
+ /* Stuff the references (if any) into the type. This is a bit
+ hacky, but we can't put them in `s' since that would be
+ ambiguous. */
+ string type = "text";
+ for (auto & i : references) {
+ type += ":";
+ type += i;
+ }
+ return makeStorePath(type, hash, name);
+}
+
+
std::pair<Path, Hash> Store::computeStorePathForPath(const Path & srcPath,
bool recursive, HashType hashAlgo, PathFilter & filter) const
{
@@ -215,16 +231,7 @@ std::pair<Path, Hash> Store::computeStorePathForPath(const Path & srcPath,
Path Store::computeStorePathForText(const string & name, const string & s,
const PathSet & references) const
{
- Hash hash = hashString(htSHA256, s);
- /* Stuff the references (if any) into the type. This is a bit
- hacky, but we can't put them in `s' since that would be
- ambiguous. */
- string type = "text";
- for (auto & i : references) {
- type += ":";
- type += i;
- }
- return makeStorePath(type, hash, name);
+ return makeTextPath(name, hashString(htSHA256, s), references);
}
@@ -432,9 +439,38 @@ void ValidPathInfo::sign(const SecretKey & secretKey)
}
-unsigned int ValidPathInfo::checkSignatures(const PublicKeys & publicKeys) const
+bool ValidPathInfo::isContentAddressed(const Store & store) const
+{
+ auto warn = [&]() {
+ printMsg(lvlError, format("warning: path ‘%s’ claims to be content-addressed but isn't") % path);
+ };
+
+ if (hasPrefix(ca, "text:")) {
+ auto hash = parseHash(std::string(ca, 5));
+ if (store.makeTextPath(storePathToName(path), hash, references) == path)
+ return true;
+ else
+ warn();
+ }
+
+ else if (hasPrefix(ca, "fixed:")) {
+ bool recursive = ca.compare(6, 2, "r:") == 0;
+ auto hash = parseHash(std::string(ca, recursive ? 8 : 6));
+ if (store.makeFixedOutputPath(recursive, hash, storePathToName(path)) == path)
+ return true;
+ else
+ warn();
+ }
+
+ return false;
+}
+
+
+size_t ValidPathInfo::checkSignatures(const Store & store, const PublicKeys & publicKeys) const
{
- unsigned int good = 0;
+ if (isContentAddressed(store)) return maxSigs;
+
+ size_t good = 0;
for (auto & sig : sigs)
if (checkSignature(publicKeys, sig))
good++;
diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh
index 0b80312d6..41fc58fc4 100644
--- a/src/libstore/store-api.hh
+++ b/src/libstore/store-api.hh
@@ -16,6 +16,13 @@
namespace nix {
+struct BasicDerivation;
+struct Derivation;
+class FSAccessor;
+class NarInfoDiskCache;
+class Store;
+
+
/* Size of the hash part of store paths, in base-32 characters. */
const size_t storePathHashLen = 32; // i.e. 160 bits
@@ -109,6 +116,34 @@ struct ValidPathInfo
StringSet sigs; // note: not necessarily verified
+ /* If non-empty, an assertion that the path is content-addressed,
+ i.e., that the store path is computed from a cryptographic hash
+ of the contents of the path, plus some other bits of data like
+ the "name" part of the path. Such a path doesn't need
+ signatures, since we don't have to trust anybody's claim that
+ the path is the output of a particular derivation. (In the
+ extensional store model, we have to trust that the *contents*
+ of an output path of a derivation were actually produced by
+ that derivation. In the intensional model, we have to trust
+ that a particular output path was produced by a derivation; the
+ path name then implies the contents.)
+
+ Ideally, the content-addressability assertion would just be a
+ Boolean, and the store path would be computed from
+ ‘storePathToName(path)’, ‘narHash’ and ‘references’. However,
+ 1) we've accumulated several types of content-addressed paths
+ over the years; and 2) fixed-output derivations support
+ multiple hash algorithms and serialisation methods (flat file
+ vs NAR). Thus, ‘ca’ has one of the following forms:
+
+ * ‘text:sha256:<sha256 hash of file contents>’: For paths
+ computed by makeTextPath() / addTextToStore().
+
+ * ‘fixed:<r?>:<ht>:<h>’: For paths computed by
+ makeFixedOutputPath() / addToStore().
+ */
+ std::string ca;
+
bool operator == (const ValidPathInfo & i) const
{
return
@@ -117,19 +152,25 @@ struct ValidPathInfo
&& references == i.references;
}
- /* Return a fingerprint of the store path to be used in binary
- cache signatures. It contains the store path, the base-32
- SHA-256 hash of the NAR serialisation of the path, the size of
- the NAR, and the sorted references. The size field is strictly
- speaking superfluous, but might prevent endless/excessive data
- attacks. */
+ /* Return a fingerprint of the store path to be used in binary
+ cache signatures. It contains the store path, the base-32
+ SHA-256 hash of the NAR serialisation of the path, the size of
+ the NAR, and the sorted references. The size field is strictly
+ speaking superfluous, but might prevent endless/excessive data
+ attacks. */
std::string fingerprint() const;
void sign(const SecretKey & secretKey);
+ /* Return true iff the path is verifiably content-addressed. */
+ bool isContentAddressed(const Store & store) const;
+
+ static const size_t maxSigs = std::numeric_limits<size_t>::max();
+
/* Return the number of signatures on this .narinfo that were
- produced by one of the specified keys. */
- unsigned int checkSignatures(const PublicKeys & publicKeys) const;
+ produced by one of the specified keys, or maxSigs if the path
+ is content-addressed. */
+ size_t checkSignatures(const Store & store, const PublicKeys & publicKeys) const;
/* Verify a single signature. */
bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const;
@@ -169,12 +210,6 @@ struct BuildResult
};
-struct BasicDerivation;
-struct Derivation;
-class FSAccessor;
-class NarInfoDiskCache;
-
-
class Store : public std::enable_shared_from_this<Store>
{
public:
@@ -234,10 +269,12 @@ public:
Path makeFixedOutputPath(bool recursive,
const Hash & hash, const string & name) const;
- /* This is the preparatory part of addToStore() and
- addToStoreFixed(); it computes the store path to which srcPath
- is to be copied. Returns the store path and the cryptographic
- hash of the contents of srcPath. */
+ Path makeTextPath(const string & name, const Hash & hash,
+ const PathSet & references) const;
+
+ /* This is the preparatory part of addToStore(); it computes the
+ store path to which srcPath is to be copied. Returns the store
+ path and the cryptographic hash of the contents of srcPath. */
std::pair<Path, Hash> computeStorePathForPath(const Path & srcPath,
bool recursive = true, HashType hashAlgo = htSHA256,
PathFilter & filter = defaultPathFilter) const;
@@ -491,7 +528,9 @@ protected:
class LocalFSStore : public Store
{
public:
+ const Path rootDir;
const Path stateDir;
+ const Path logDir;
LocalFSStore(const Params & params);
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 04:17:30 +0000