aboutsummaryrefslogtreecommitdiff
path: root/src/libstore/local-store.cc
diff options
context:
space:
mode:
Diffstat (limited to 'src/libstore/local-store.cc')
-rw-r--r--src/libstore/local-store.cc925
1 files changed, 507 insertions, 418 deletions
diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc
index a83ba55e2..8a690b913 100644
--- a/src/libstore/local-store.cc
+++ b/src/libstore/local-store.cc
@@ -3,9 +3,9 @@
#include "globals.hh"
#include "archive.hh"
#include "pathlocks.hh"
-#include "aterm.hh"
#include "derivations-ast.hh"
#include "worker-protocol.hh"
+#include "derivations.hh"
#include <iostream>
#include <algorithm>
@@ -18,10 +18,137 @@
#include <errno.h>
#include <stdio.h>
+#include <sqlite3.h>
+
namespace nix {
+class SQLiteError : public Error
+{
+public:
+ SQLiteError(sqlite3 * db, const format & f)
+ : Error(format("%1%: %2%") % f.str() % sqlite3_errmsg(db))
+ {
+ }
+};
+
+
+SQLite::~SQLite()
+{
+ try {
+ if (db && sqlite3_close(db) != SQLITE_OK)
+ throw SQLiteError(db, "closing database");
+ } catch (...) {
+ ignoreException();
+ }
+}
+
+
+void SQLiteStmt::create(sqlite3 * db, const string & s)
+{
+ checkInterrupt();
+ assert(!stmt);
+ if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK)
+ throw SQLiteError(db, "creating statement");
+ this->db = db;
+}
+
+
+void SQLiteStmt::reset()
+{
+ assert(stmt);
+ if (sqlite3_reset(stmt) != SQLITE_OK)
+ throw SQLiteError(db, "resetting statement");
+ curArg = 1;
+}
+
+
+SQLiteStmt::~SQLiteStmt()
+{
+ try {
+ if (stmt && sqlite3_finalize(stmt) != SQLITE_OK)
+ throw SQLiteError(db, "finalizing statement");
+ } catch (...) {
+ ignoreException();
+ }
+}
+
+
+void SQLiteStmt::bind(const string & value)
+{
+ if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK)
+ throw SQLiteError(db, "binding argument");
+}
+
+
+void SQLiteStmt::bind(int value)
+{
+ if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK)
+ throw SQLiteError(db, "binding argument");
+}
+
+
+void SQLiteStmt::bind()
+{
+ if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK)
+ throw SQLiteError(db, "binding argument");
+}
+
+
+/* Helper class to ensure that prepared statements are reset when
+ leaving the scope that uses them. Unfinished prepared statements
+ prevent transactions from being aborted, and can cause locks to be
+ kept when they should be released. */
+struct SQLiteStmtUse
+{
+ SQLiteStmt & stmt;
+ SQLiteStmtUse(SQLiteStmt & stmt) : stmt(stmt)
+ {
+ stmt.reset();
+ }
+ ~SQLiteStmtUse()
+ {
+ try {
+ stmt.reset();
+ } catch (...) {
+ ignoreException();
+ }
+ }
+};
+
+
+struct SQLiteTxn
+{
+ bool active;
+ sqlite3 * db;
+
+ SQLiteTxn(sqlite3 * db) : active(false) {
+ this->db = db;
+ if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK)
+ throw SQLiteError(db, "starting transaction");
+ active = true;
+ }
+
+ void commit()
+ {
+ if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK)
+ throw SQLiteError(db, "committing transaction");
+ active = false;
+ }
+
+ ~SQLiteTxn()
+ {
+ try {
+ if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK)
+ throw SQLiteError(db, "aborting transaction");
+ } catch (...) {
+ ignoreException();
+ }
+ }
+};
+
+
void checkStoreNotSymlink()
{
if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return;
@@ -46,13 +173,13 @@ LocalStore::LocalStore()
schemaPath = nixDBPath + "/schema";
- if (readOnlyMode) return;
+ if (readOnlyMode) {
+ openDB(false);
+ return;
+ }
/* Create missing state directories if they don't already exist. */
createDirs(nixStore);
- createDirs(nixDBPath + "/info");
- createDirs(nixDBPath + "/referrer");
- createDirs(nixDBPath + "/failed");
Path profilesDir = nixStateDir + "/profiles";
createDirs(nixStateDir + "/profiles");
createDirs(nixStateDir + "/temproots");
@@ -65,12 +192,15 @@ LocalStore::LocalStore()
checkStoreNotSymlink();
+ /* Acquire the big fat lock in shared mode to make sure that no
+ schema upgrade is in progress. */
try {
Path globalLockPath = nixDBPath + "/big-lock";
globalLock = openLockFile(globalLockPath.c_str(), true);
} catch (SysError & e) {
if (e.errNo != EACCES) throw;
readOnlyMode = true;
+ openDB(false);
return;
}
@@ -78,33 +208,55 @@ LocalStore::LocalStore()
printMsg(lvlError, "waiting for the big Nix store lock...");
lockFile(globalLock, ltRead, true);
}
-
+
+ /* Check the current database schema and if necessary do an
+ upgrade. */
int curSchema = getSchema();
if (curSchema > nixSchemaVersion)
throw Error(format("current Nix store schema is version %1%, but I only support %2%")
% curSchema % nixSchemaVersion);
- if (curSchema == 0) { /* new store */
- curSchema = nixSchemaVersion;
+
+ else if (curSchema == 0) { /* new store */
+ curSchema = nixSchemaVersion;
+ openDB(true);
writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str());
}
- if (curSchema == 1) throw Error("your Nix store is no longer supported");
- if (curSchema < nixSchemaVersion) upgradeStore12();
+
+ else if (curSchema < nixSchemaVersion) {
+ if (curSchema < 5)
+ throw Error(
+ "Your Nix store has a database in Berkeley DB format,\n"
+ "which is no longer supported. To convert to the new format,\n"
+ "please upgrade Nix to version 0.12 first.");
+
+ if (!lockFile(globalLock, ltWrite, false)) {
+ printMsg(lvlError, "waiting for exclusive access to the Nix store...");
+ lockFile(globalLock, ltWrite, true);
+ }
+
+ /* Get the schema version again, because another process may
+ have performed the upgrade already. */
+ curSchema = getSchema();
+
+ if (curSchema < 6) upgradeStore6();
- doFsync = queryBoolSetting("fsync-metadata", false);
+ writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str());
+
+ lockFile(globalLock, ltRead, true);
+ }
+
+ else openDB(false);
}
LocalStore::~LocalStore()
{
try {
- flushDelayedUpdates();
-
foreach (RunningSubstituters::iterator, i, runningSubstituters) {
i->second.to.close();
i->second.from.close();
i->second.pid.wait(true);
}
-
} catch (...) {
ignoreException();
}
@@ -123,6 +275,67 @@ int LocalStore::getSchema()
}
+void LocalStore::openDB(bool create)
+{
+ /* Open the Nix database. */
+ if (sqlite3_open_v2((nixDBPath + "/db.sqlite").c_str(), &db.db,
+ SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK)
+ throw Error("cannot open SQLite database");
+
+ if (sqlite3_busy_timeout(db, 60000) != SQLITE_OK)
+ throw SQLiteError(db, "setting timeout");
+
+ if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK)
+ throw SQLiteError(db, "enabling foreign keys");
+
+ /* !!! check whether sqlite has been built with foreign key
+ support */
+
+ /* Whether SQLite should fsync(). "Normal" synchronous mode
+ should be safe enough. If the user asks for it, don't sync at
+ all. This can cause database corruption if the system
+ crashes. */
+ string syncMode = queryBoolSetting("fsync-metadata", true) ? "normal" : "off";
+ if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK)
+ throw SQLiteError(db, "setting synchronous mode");
+
+ /* Use `truncate' journal mode, which should be a bit faster. */
+ if (sqlite3_exec(db, "pragma main.journal_mode = truncate;", 0, 0, 0) != SQLITE_OK)
+ throw SQLiteError(db, "setting journal mode");
+
+ /* Initialise the database schema, if necessary. */
+ if (create) {
+#include "schema.sql.hh"
+ if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK)
+ throw SQLiteError(db, "initialising database schema");
+ }
+
+ /* Prepare SQL statements. */
+ stmtRegisterValidPath.create(db,
+ "insert or replace into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);");
+ stmtAddReference.create(db,
+ "insert or replace into Refs (referrer, reference) values (?, ?);");
+ stmtQueryPathInfo.create(db,
+ "select id, hash, registrationTime, deriver from ValidPaths where path = ?;");
+ stmtQueryReferences.create(db,
+ "select path from Refs join ValidPaths on reference = id where referrer = ?;");
+ stmtQueryReferrers.create(db,
+ "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);");
+ stmtInvalidatePath.create(db,
+ "delete from ValidPaths where path = ?;");
+ stmtRegisterFailedPath.create(db,
+ "insert into FailedPaths (path, time) values (?, ?);");
+ stmtHasPathFailed.create(db,
+ "select time from FailedPaths where path = ?;");
+ stmtAddDerivationOutput.create(db,
+ "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);");
+ stmtQueryValidDerivers.create(db,
+ "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;");
+ stmtQueryDerivationOutputs.create(db,
+ "select id, path from DerivationOutputs where drv = ?;");
+}
+
+
void canonicalisePathMetaData(const Path & path, bool recurse)
{
checkInterrupt();
@@ -197,98 +410,6 @@ void canonicalisePathMetaData(const Path & path)
}
-static Path infoFileFor(const Path & path)
-{
- string baseName = baseNameOf(path);
- return (format("%1%/info/%2%") % nixDBPath % baseName).str();
-}
-
-
-static Path referrersFileFor(const Path & path)
-{
- string baseName = baseNameOf(path);
- return (format("%1%/referrer/%2%") % nixDBPath % baseName).str();
-}
-
-
-static Path failedFileFor(const Path & path)
-{
- string baseName = baseNameOf(path);
- return (format("%1%/failed/%2%") % nixDBPath % baseName).str();
-}
-
-
-static Path tmpFileForAtomicUpdate(const Path & path)
-{
- return (format("%1%/.%2%.%3%") % dirOf(path) % getpid() % baseNameOf(path)).str();
-}
-
-
-void LocalStore::appendReferrer(const Path & from, const Path & to, bool lock)
-{
- Path referrersFile = referrersFileFor(from);
-
- PathLocks referrersLock;
- if (lock) {
- referrersLock.lockPaths(singleton<PathSet, Path>(referrersFile));
- referrersLock.setDeletion(true);
- }
-
- AutoCloseFD fd = open(referrersFile.c_str(), O_WRONLY | O_APPEND | O_CREAT, 0666);
- if (fd == -1) throw SysError(format("opening file `%1%'") % referrersFile);
-
- string s = " " + to;
- writeFull(fd, (const unsigned char *) s.c_str(), s.size());
-
- if (doFsync) fsync(fd);
-}
-
-
-/* Atomically update the referrers file. If `purge' is true, the set
- of referrers is set to `referrers'. Otherwise, the current set of
- referrers is purged of invalid paths. */
-void LocalStore::rewriteReferrers(const Path & path, bool purge, PathSet referrers)
-{
- Path referrersFile = referrersFileFor(path);
-
- PathLocks referrersLock(singleton<PathSet, Path>(referrersFile));
- referrersLock.setDeletion(true);
-
- if (purge)
- /* queryReferrers() purges invalid paths, so that's all we
- need. */
- queryReferrers(path, referrers);
-
- Path tmpFile = tmpFileForAtomicUpdate(referrersFile);
-
- AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666);
- if (fd == -1) throw SysError(format("opening file `%1%'") % referrersFile);
-
- string s;
- foreach (PathSet::const_iterator, i, referrers) {
- s += " "; s += *i;
- }
-
- writeFull(fd, (const unsigned char *) s.c_str(), s.size());
-
- if (doFsync) fsync(fd);
-
- fd.close(); /* for Windows; can't rename open file */
-
- if (rename(tmpFile.c_str(), referrersFile.c_str()) == -1)
- throw SysError(format("cannot rename `%1%' to `%2%'") % tmpFile % referrersFile);
-}
-
-
-void LocalStore::flushDelayedUpdates()
-{
- foreach (PathSet::iterator, i, delayedUpdates) {
- rewriteReferrers(*i, true, PathSet());
- }
- delayedUpdates.clear();
-}
-
-
void LocalStore::registerValidPath(const Path & path,
const Hash & hash, const PathSet & references,
const Path & deriver)
@@ -302,76 +423,88 @@ void LocalStore::registerValidPath(const Path & path,
}
-void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidity)
+unsigned long long LocalStore::addValidPath(const ValidPathInfo & info)
{
- Path infoFile = infoFileFor(info.path);
-
- ValidPathInfo oldInfo;
- if (pathExists(infoFile)) oldInfo = queryPathInfo(info.path);
-
- /* Note that it's possible for infoFile to already exist. */
-
- /* Acquire a lock on each referrer file. This prevents those
- paths from being invalidated. (It would be a violation of the
- store invariants if we registered info.path as valid while some
- of its references are invalid.) NB: there can be no deadlock
- here since we're acquiring the locks in sorted order. */
- PathSet lockNames;
- foreach (PathSet::const_iterator, i, info.references)
- if (*i != info.path) lockNames.insert(referrersFileFor(*i));
- PathLocks referrerLocks(lockNames);
- referrerLocks.setDeletion(true);
-
- string refs;
- foreach (PathSet::const_iterator, i, info.references) {
- if (!refs.empty()) refs += " ";
- refs += *i;
-
- if (!ignoreValidity && *i != info.path && !isValidPath(*i))
- throw Error(format("cannot register `%1%' as valid, because its reference `%2%' isn't valid")
- % info.path % *i);
-
- /* Update the referrer mapping for *i. This must be done
- before the info file is written to maintain the invariant
- that if `path' is a valid path, then all its references
- have referrer mappings back to `path'. A " " is prefixed
- to separate it from the previous entry. It's not suffixed
- to deal with interrupted partial writes to this file. */
- if (oldInfo.references.find(*i) == oldInfo.references.end())
- appendReferrer(*i, info.path, false);
+ SQLiteStmtUse use(stmtRegisterValidPath);
+ stmtRegisterValidPath.bind(info.path);
+ stmtRegisterValidPath.bind("sha256:" + printHash(info.hash));
+ stmtRegisterValidPath.bind(info.registrationTime);
+ if (info.deriver != "")
+ stmtRegisterValidPath.bind(info.deriver);
+ else
+ stmtRegisterValidPath.bind(); // null
+ if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE)
+ throw SQLiteError(db, format("registering valid path `%1%' in database") % info.path);
+ unsigned long long id = sqlite3_last_insert_rowid(db);
+
+ /* If this is a derivation, then store the derivation outputs in
+ the database. This is useful for the garbage collector: it can
+ efficiently query whether a path is an output of some
+ derivation. */
+ if (isDerivation(info.path)) {
+ ATerm t = ATreadFromNamedFile(info.path.c_str());
+ if (!t) throw Error(format("cannot read derivation `%1%'") % info.path);
+ Derivation drv = parseDerivation(t);
+ foreach (DerivationOutputs::iterator, i, drv.outputs) {
+ SQLiteStmtUse use(stmtAddDerivationOutput);
+ stmtAddDerivationOutput.bind(id);
+ stmtAddDerivationOutput.bind(i->first);
+ stmtAddDerivationOutput.bind(i->second.path);
+ if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE)
+ throw SQLiteError(db, format("adding derivation output for `%1%' in database") % info.path);
+ }
}
+ return id;
+}
+
+
+void LocalStore::addReference(unsigned long long referrer, unsigned long long reference)
+{
+ SQLiteStmtUse use(stmtAddReference);
+ stmtAddReference.bind(referrer);
+ stmtAddReference.bind(reference);
+ if (sqlite3_step(stmtAddReference) != SQLITE_DONE)
+ throw SQLiteError(db, "adding reference to database");
+}
+
+
+void LocalStore::registerValidPath(const ValidPathInfo & info)
+{
assert(info.hash.type == htSHA256);
+ ValidPathInfo info2(info);
+ if (info2.registrationTime == 0) info2.registrationTime = time(0);
+
+ SQLiteTxn txn(db);
+
+ unsigned long long id = addValidPath(info2);
- string s = (format(
- "Hash: sha256:%1%\n"
- "References: %2%\n"
- "Deriver: %3%\n"
- "Registered-At: %4%\n")
- % printHash(info.hash) % refs % info.deriver %
- (oldInfo.registrationTime ? oldInfo.registrationTime : time(0))).str();
-
- /* Atomically rewrite the info file. */
- Path tmpFile = tmpFileForAtomicUpdate(infoFile);
- writeFile(tmpFile, s, doFsync);
- if (rename(tmpFile.c_str(), infoFile.c_str()) == -1)
- throw SysError(format("cannot rename `%1%' to `%2%'") % tmpFile % infoFile);
-
- pathInfoCache[info.path] = info;
+ foreach (PathSet::const_iterator, i, info2.references)
+ addReference(id, queryValidPathId(*i));
+
+ txn.commit();
}
void LocalStore::registerFailedPath(const Path & path)
{
- /* Write an empty file in the .../failed directory to denote the
- failure of the builder for `path'. */
- writeFile(failedFileFor(path), "");
+ if (hasPathFailed(path)) return;
+ SQLiteStmtUse use(stmtRegisterFailedPath);
+ stmtRegisterFailedPath.bind(path);
+ stmtRegisterFailedPath.bind(time(0));
+ if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE)
+ throw SQLiteError(db, format("registering failed path `%1%'") % path);
}
bool LocalStore::hasPathFailed(const Path & path)
{
- return pathExists(failedFileFor(path));
+ SQLiteStmtUse use(stmtHasPathFailed);
+ stmtHasPathFailed.bind(path);
+ int res = sqlite3_step(stmtHasPathFailed);
+ if (res != SQLITE_DONE && res != SQLITE_ROW)
+ throw SQLiteError(db, "querying whether path failed");
+ return res == SQLITE_ROW;
}
@@ -389,91 +522,91 @@ Hash parseHashField(const Path & path, const string & s)
}
-ValidPathInfo LocalStore::queryPathInfo(const Path & path, bool ignoreErrors)
+ValidPathInfo LocalStore::queryPathInfo(const Path & path)
{
- ValidPathInfo res;
- res.path = path;
+ ValidPathInfo info;
+ info.path = path;
assertStorePath(path);
- if (!isValidPath(path))
- throw Error(format("path `%1%' is not valid") % path);
+ /* Get the path info. */
+ SQLiteStmtUse use1(stmtQueryPathInfo);
- std::map<Path, ValidPathInfo>::iterator lookup = pathInfoCache.find(path);
- if (lookup != pathInfoCache.end()) return lookup->second;
+ stmtQueryPathInfo.bind(path);
- /* Read the info file. */
- Path infoFile = infoFileFor(path);
- if (!pathExists(infoFile))
- throw Error(format("path `%1%' is not valid") % path);
- string info = readFile(infoFile);
+ int r = sqlite3_step(stmtQueryPathInfo);
+ if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path);
+ if (r != SQLITE_ROW) throw SQLiteError(db, "querying path in database");
- /* Parse it. */
- Strings lines = tokenizeString(info, "\n");
+ info.id = sqlite3_column_int(stmtQueryPathInfo, 0);
- foreach (Strings::iterator, i, lines) {
- string::size_type p = i->find(':');
- if (p == string::npos) {
- if (!ignoreErrors)
- throw Error(format("corrupt line in `%1%': %2%") % infoFile % *i);
- continue; /* bad line */
- }
- string name(*i, 0, p);
- string value(*i, p + 2);
- if (name == "References") {
- Strings refs = tokenizeString(value, " ");
- res.references = PathSet(refs.begin(), refs.end());
- } else if (name == "Deriver") {
- res.deriver = value;
- } else if (name == "Hash") {
- try {
- res.hash = parseHashField(path, value);
- } catch (Error & e) {
- if (!ignoreErrors) throw;
- printMsg(lvlError, format("cannot parse hash field in `%1%': %2%") % infoFile % e.msg());
- }
- } else if (name == "Registered-At") {
- int n = 0;
- string2Int(value, n);
- res.registrationTime = n;
- }
+ const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1);
+ assert(s);
+ info.hash = parseHashField(path, s);
+
+ info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2);
+
+ s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3);
+ if (s) info.deriver = s;
+
+ /* Get the references. */
+ SQLiteStmtUse use2(stmtQueryReferences);
+
+ stmtQueryReferences.bind(info.id);
+
+ while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) {
+ s = (const char *) sqlite3_column_text(stmtQueryReferences, 0);
+ assert(s);
+ info.references.insert(s);
}
- return pathInfoCache[path] = res;
+ if (r != SQLITE_DONE)
+ throw SQLiteError(db, format("error getting references of `%1%'") % path);
+
+ return info;
}
-bool LocalStore::isValidPath(const Path & path)
+unsigned long long LocalStore::queryValidPathId(const Path & path)
{
- /* Files in the info directory starting with a `.' are temporary
- files. */
- if (baseNameOf(path).at(0) == '.') return false;
-
- /* A path is valid if its info file exists and has a non-zero
- size. (The non-zero size restriction is to be robust to
- certain kinds of filesystem corruption, particularly with
- ext4.) */
- Path infoFile = infoFileFor(path);
+ SQLiteStmtUse use(stmtQueryPathInfo);
+ stmtQueryPathInfo.bind(path);
+ int res = sqlite3_step(stmtQueryPathInfo);
+ if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0);
+ if (res == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path);
+ throw SQLiteError(db, "querying path in database");
+}
- struct stat st;
- if (lstat(infoFile.c_str(), &st)) {
- if (errno == ENOENT) return false;
- throw SysError(format("getting status of `%1%'") % infoFile);
- }
- if (st.st_size == 0) return false;
-
- return true;
+bool LocalStore::isValidPath(const Path & path)
+{
+ SQLiteStmtUse use(stmtQueryPathInfo);
+ stmtQueryPathInfo.bind(path);
+ int res = sqlite3_step(stmtQueryPathInfo);
+ if (res != SQLITE_DONE && res != SQLITE_ROW)
+ throw SQLiteError(db, "querying path in database");
+ return res == SQLITE_ROW;
}
PathSet LocalStore::queryValidPaths()
{
- PathSet paths;
- Strings entries = readDirectory(nixDBPath + "/info");
- foreach (Strings::iterator, i, entries)
- if (i->at(0) != '.') paths.insert(nixStore + "/" + *i);
- return paths;
+ SQLiteStmt stmt;
+ stmt.create(db, "select path from ValidPaths");
+
+ PathSet res;
+
+ int r;
+ while ((r = sqlite3_step(stmt)) == SQLITE_ROW) {
+ const char * s = (const char *) sqlite3_column_text(stmt, 0);
+ assert(s);
+ res.insert(s);
+ }
+
+ if (r != SQLITE_DONE)
+ throw SQLiteError(db, "error getting valid paths");
+
+ return res;
}
@@ -485,45 +618,73 @@ void LocalStore::queryReferences(const Path & path,
}
-bool LocalStore::queryReferrersInternal(const Path & path, PathSet & referrers)
+void LocalStore::queryReferrers(const Path & path, PathSet & referrers)
{
- bool allValid = true;
-
- if (!isValidPath(path))
- throw Error(format("path `%1%' is not valid") % path);
+ assertStorePath(path);
- /* No locking is necessary here: updates are only done by
- appending or by atomically replacing the file. When appending,
- there is a possibility that we see a partial entry, but it will
- just be filtered out below (the partially written path will not
- be valid, so it will be ignored). */
+ SQLiteStmtUse use(stmtQueryReferrers);
- Path referrersFile = referrersFileFor(path);
- if (!pathExists(referrersFile)) return true;
-
- AutoCloseFD fd = open(referrersFile.c_str(), O_RDONLY);
- if (fd == -1) throw SysError(format("opening file `%1%'") % referrersFile);
+ stmtQueryReferrers.bind(path);
- Paths refs = tokenizeString(readFile(fd), " ");
+ int r;
+ while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) {
+ const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0);
+ assert(s);
+ referrers.insert(s);
+ }
+
+ if (r != SQLITE_DONE)
+ throw SQLiteError(db, format("error getting references of `%1%'") % path);
+}
- foreach (Paths::iterator, i, refs)
- /* Referrers can be invalid (see registerValidPath() for the
- invariant), so we only return one if it is valid. */
- if (isStorePath(*i) && isValidPath(*i)) referrers.insert(*i); else allValid = false;
- return allValid;
+Path LocalStore::queryDeriver(const Path & path)
+{
+ return queryPathInfo(path).deriver;
}
-void LocalStore::queryReferrers(const Path & path, PathSet & referrers)
+PathSet LocalStore::queryValidDerivers(const Path & path)
{
- queryReferrersInternal(path, referrers);
+ assertStorePath(path);
+
+ SQLiteStmtUse use(stmtQueryValidDerivers);
+ stmtQueryValidDerivers.bind(path);
+
+ PathSet derivers;
+ int r;
+ while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) {
+ const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1);
+ assert(s);
+ derivers.insert(s);
+ }
+
+ if (r != SQLITE_DONE)
+ throw SQLiteError(db, format("error getting valid derivers of `%1%'") % path);
+
+ return derivers;
}
-Path LocalStore::queryDeriver(const Path & path)
+PathSet LocalStore::queryDerivationOutputs(const Path & path)
{
- return queryPathInfo(path).deriver;
+ SQLiteTxn txn(db);
+
+ SQLiteStmtUse use(stmtQueryDerivationOutputs);
+ stmtQueryDerivationOutputs.bind(queryValidPathId(path));
+
+ PathSet outputs;
+ int r;
+ while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) {
+ const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1);
+ assert(s);
+ outputs.insert(s);
+ }
+
+ if (r != SQLITE_DONE)
+ throw SQLiteError(db, format("error getting outputs of `%1%'") % path);
+
+ return outputs;
}
@@ -637,39 +798,19 @@ Hash LocalStore::queryPathHash(const Path & path)
}
-static void dfsVisit(std::map<Path, ValidPathInfo> & infos,
- const Path & path, PathSet & visited, Paths & sorted)
+void LocalStore::registerValidPaths(const ValidPathInfos & infos)
{
- if (visited.find(path) != visited.end()) return;
- visited.insert(path);
-
- ValidPathInfo & info(infos[path]);
+ SQLiteTxn txn(db);
- foreach (PathSet::iterator, i, info.references)
- if (infos.find(*i) != infos.end())
- dfsVisit(infos, *i, visited, sorted);
-
- sorted.push_back(path);
-}
+ foreach (ValidPathInfos::const_iterator, i, infos) addValidPath(*i);
+ foreach (ValidPathInfos::const_iterator, i, infos) {
+ unsigned long long referrer = queryValidPathId(i->path);
+ foreach (PathSet::iterator, j, i->references)
+ addReference(referrer, queryValidPathId(*j));
+ }
-void LocalStore::registerValidPaths(const ValidPathInfos & infos)
-{
- std::map<Path, ValidPathInfo> infosMap;
-
- /* Sort the paths topologically under the references relation, so
- that if path A is referenced by B, then A is registered before
- B. */
- foreach (ValidPathInfos::const_iterator, i, infos)
- infosMap[i->path] = *i;
-
- PathSet visited;
- Paths sorted;
- foreach (ValidPathInfos::const_iterator, i, infos)
- dfsVisit(infosMap, i->path, visited, sorted);
-
- foreach (Paths::iterator, i, sorted)
- registerValidPath(infosMap[*i]);
+ txn.commit();
}
@@ -679,43 +820,15 @@ void LocalStore::invalidatePath(const Path & path)
{
debug(format("invalidating path `%1%'") % path);
- ValidPathInfo info;
+ SQLiteStmtUse use(stmtInvalidatePath);
- if (pathExists(infoFileFor(path))) {
- info = queryPathInfo(path);
+ stmtInvalidatePath.bind(path);
- /* Remove the info file. */
- Path p = infoFileFor(path);
- if (unlink(p.c_str()) == -1)
- throw SysError(format("unlinking `%1%'") % p);
- }
+ if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE)
+ throw SQLiteError(db, format("invalidating path `%1%' in database") % path);
- /* Remove the referrers file for `path'. */
- Path p = referrersFileFor(path);
- if (pathExists(p) && unlink(p.c_str()) == -1)
- throw SysError(format("unlinking `%1%'") % p);
-
- /* Clear `path' from the info cache. */
- pathInfoCache.erase(path);
- delayedUpdates.erase(path);
-
- /* Cause the referrer files for each path referenced by this one
- to be updated. This has to happen after removing the info file
- to preserve the invariant (see registerValidPath()).
-
- The referrer files are updated lazily in flushDelayedUpdates()
- to prevent quadratic performance in the garbage collector
- (i.e., when N referrers to some path X are deleted, we have to
- rewrite the referrers file for X N times, causing O(N^2) I/O).
-
- What happens if we die before the referrer file can be updated?
- That's not a problem, because stale (invalid) entries in the
- referrer file are ignored by queryReferrers(). Thus a referrer
- file is allowed to have stale entries; removing them is just an
- optimisation. verifyStore() gets rid of them eventually.
- */
- foreach (PathSet::iterator, i, info.references)
- if (*i != path) delayedUpdates.insert(*i);
+ /* Note that the foreign key constraints on the Refs table take
+ care of deleting the references entries for `path'. */
}
@@ -1024,11 +1137,6 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr
assertStorePath(path);
if (isValidPath(path)) {
- /* Acquire a lock on the referrers file to prevent new
- referrers to this path from appearing while we're deleting
- it. */
- PathLocks referrersLock(singleton<PathSet, Path>(referrersFileFor(path)));
- referrersLock.setDeletion(true);
PathSet referrers; queryReferrers(path, referrers);
referrers.erase(path); /* ignore self-references */
if (!referrers.empty())
@@ -1050,68 +1158,26 @@ void LocalStore::verifyStore(bool checkContents)
foreach (PathSet::iterator, i, validPaths2) {
checkInterrupt();
+ /* !!! invalidatePath() will probably fail due to the foreign
+ key constraints on the Ref table. */
if (!isStorePath(*i)) {
printMsg(lvlError, format("path `%1%' is not in the Nix store") % *i);
invalidatePath(*i);
} else if (!pathExists(*i)) {
printMsg(lvlError, format("path `%1%' disappeared") % *i);
invalidatePath(*i);
- } else {
- Path infoFile = infoFileFor(*i);
- struct stat st;
- if (lstat(infoFile.c_str(), &st))
- throw SysError(format("getting status of `%1%'") % infoFile);
- if (st.st_size == 0) {
- printMsg(lvlError, format("removing corrupt info file `%1%'") % infoFile);
- if (unlink(infoFile.c_str()) == -1)
- throw SysError(format("unlinking `%1%'") % infoFile);
- }
- else validPaths.insert(*i);
- }
+ } else validPaths.insert(*i);
}
+ /* Optionally, check the content hashes (slow). */
+ if (checkContents) {
+ printMsg(lvlInfo, "checking hashes");
- /* Check the store path meta-information. */
- printMsg(lvlInfo, "checking path meta-information");
-
- std::map<Path, PathSet> referrersCache;
-
- foreach (PathSet::iterator, i, validPaths) {
- bool update = false;
- ValidPathInfo info = queryPathInfo(*i, true);
-
- /* Check the references: each reference should be valid, and
- it should have a matching referrer. */
- foreach (PathSet::iterator, j, info.references) {
- if (validPaths.find(*j) == validPaths.end()) {
- printMsg(lvlError, format("incomplete closure: `%1%' needs missing `%2%'")
- % *i % *j);
- /* nothing we can do about it... */
- } else {
- if (referrersCache.find(*j) == referrersCache.end())
- queryReferrers(*j, referrersCache[*j]);
- if (referrersCache[*j].find(*i) == referrersCache[*j].end()) {
- printMsg(lvlError, format("adding missing referrer mapping from `%1%' to `%2%'")
- % *j % *i);
- appendReferrer(*j, *i, true);
- }
- }
- }
+ foreach (PathSet::iterator, i, validPaths) {
+ ValidPathInfo info = queryPathInfo(*i);
- /* Check the deriver. (Note that the deriver doesn't have to
- be a valid path.) */
- if (!info.deriver.empty() && !isStorePath(info.deriver)) {
- info.deriver = "";
- update = true;
- }
-
- /* Check the content hash (optionally - slow). */
- if (info.hash.hashSize == 0) {
- printMsg(lvlError, format("re-hashing `%1%'") % *i);
- info.hash = hashPath(htSHA256, *i);
- update = true;
- } else if (checkContents) {
- debug(format("checking contents of `%1%'") % *i);
+ /* Check the content hash (optionally - slow). */
+ printMsg(lvlTalkative, format("checking contents of `%1%'") % *i);
Hash current = hashPath(info.hash.type, *i);
if (current != info.hash) {
printMsg(lvlError, format("path `%1%' was modified! "
@@ -1119,67 +1185,90 @@ void LocalStore::verifyStore(bool checkContents)
% *i % printHash(info.hash) % printHash(current));
}
}
-
- if (update) registerValidPath(info);
}
+}
- referrersCache.clear();
-
- /* Check the referrers. */
- printMsg(lvlInfo, "checking referrers");
+/* Functions for upgrading from the pre-SQLite database. */
- std::map<Path, PathSet> referencesCache;
-
- Strings entries = readDirectory(nixDBPath + "/referrer");
- foreach (Strings::iterator, i, entries) {
- Path from = nixStore + "/" + *i;
-
- if (validPaths.find(from) == validPaths.end()) {
- /* !!! This removes lock files as well. Need to check
- whether that's okay. */
- printMsg(lvlError, format("removing referrers file for invalid `%1%'") % from);
- Path p = referrersFileFor(from);
- if (unlink(p.c_str()) == -1)
- throw SysError(format("unlinking `%1%'") % p);
- continue;
- }
+PathSet LocalStore::queryValidPathsOld()
+{
+ PathSet paths;
+ Strings entries = readDirectory(nixDBPath + "/info");
+ foreach (Strings::iterator, i, entries)
+ if (i->at(0) != '.') paths.insert(nixStore + "/" + *i);
+ return paths;
+}
- PathSet referrers;
- bool allValid = queryReferrersInternal(from, referrers);
- bool update = false;
- if (!allValid) {
- printMsg(lvlError, format("removing some stale referrers for `%1%'") % from);
- update = true;
- }
+ValidPathInfo LocalStore::queryPathInfoOld(const Path & path)
+{
+ ValidPathInfo res;
+ res.path = path;
- /* Each referrer should have a matching reference. */
- PathSet referrersNew;
- foreach (PathSet::iterator, j, referrers) {
- if (referencesCache.find(*j) == referencesCache.end())
- queryReferences(*j, referencesCache[*j]);
- if (referencesCache[*j].find(from) == referencesCache[*j].end()) {
- printMsg(lvlError, format("removing unexpected referrer mapping from `%1%' to `%2%'")
- % from % *j);
- update = true;
- } else referrersNew.insert(*j);
- }
+ /* Read the info file. */
+ string baseName = baseNameOf(path);
+ Path infoFile = (format("%1%/info/%2%") % nixDBPath % baseName).str();
+ if (!pathExists(infoFile))
+ throw Error(format("path `%1%' is not valid") % path);
+ string info = readFile(infoFile);
- if (update) rewriteReferrers(from, false, referrersNew);
+ /* Parse it. */
+ Strings lines = tokenizeString(info, "\n");
+
+ foreach (Strings::iterator, i, lines) {
+ string::size_type p = i->find(':');
+ if (p == string::npos)
+ throw Error(format("corrupt line in `%1%': %2%") % infoFile % *i);
+ string name(*i, 0, p);
+ string value(*i, p + 2);
+ if (name == "References") {
+ Strings refs = tokenizeString(value, " ");
+ res.references = PathSet(refs.begin(), refs.end());
+ } else if (name == "Deriver") {
+ res.deriver = value;
+ } else if (name == "Hash") {
+ res.hash = parseHashField(path, value);
+ } else if (name == "Registered-At") {
+ int n = 0;
+ string2Int(value, n);
+ res.registrationTime = n;
+ }
}
+
+ return res;
}
-/* Upgrade from schema 4 (Nix 0.11) to schema 5 (Nix >= 0.12). The
- old schema uses Berkeley DB, the new one stores store path
- meta-information in files. */
-void LocalStore::upgradeStore12()
+/* Upgrade from schema 5 (Nix 0.12) to schema 6 (Nix >= 0.15). */
+void LocalStore::upgradeStore6()
{
- throw Error(
- "Your Nix store has a database in Berkeley DB format,\n"
- "which is no longer supported. To convert to the new format,\n"
- "please upgrade Nix to version 0.12 first.");
+ printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)...");
+
+ openDB(true);
+
+ PathSet validPaths = queryValidPathsOld();
+
+ SQLiteTxn txn(db);
+
+ foreach (PathSet::iterator, i, validPaths) {
+ addValidPath(queryPathInfoOld(*i));
+ std::cerr << ".";
+ }
+
+ std::cerr << "|";
+
+ foreach (PathSet::iterator, i, validPaths) {
+ ValidPathInfo info = queryPathInfoOld(*i);
+ unsigned long long referrer = queryValidPathId(*i);
+ foreach (PathSet::iterator, j, info.references)
+ addReference(referrer, queryValidPathId(*j));
+ std::cerr << ".";
+ }
+
+ std::cerr << "\n";
+
+ txn.commit();
}
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-23 16:57:57 +0000