From c1a07f94451cfa93aa9ac986188d0e9a536e4b9f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 13:16:59 +0000 Subject: * Convert the Nix database to SQLite. --- src/bin2c/bin2c.c | 4 +- src/libstore/Makefile.am | 5 ++ src/libstore/local-store.cc | 119 +++++++++++++++++++++++++++++++++++++++----- src/libstore/local-store.hh | 8 ++- src/libstore/schema.sql | 27 ++++++++++ 5 files changed, 148 insertions(+), 15 deletions(-) create mode 100644 src/libstore/schema.sql (limited to 'src') diff --git a/src/bin2c/bin2c.c b/src/bin2c/bin2c.c index 18bf81d69..5ed8a5708 100644 --- a/src/bin2c/bin2c.c +++ b/src/bin2c/bin2c.c @@ -14,10 +14,10 @@ int main(int argc, char * * argv) { int c; if (argc != 2) abort(); - print("static unsigned char %s[] = {", argv[1]); + print("static unsigned char %s[] = { ", argv[1]); while ((c = getchar()) != EOF) { print("0x%02x, ", (unsigned char) c); } - print("};\n"); + print("0 };\n"); return 0; } diff --git a/src/libstore/Makefile.am b/src/libstore/Makefile.am index 863871519..0140857b8 100644 --- a/src/libstore/Makefile.am +++ b/src/libstore/Makefile.am @@ -19,5 +19,10 @@ EXTRA_DIST = derivations-ast.def derivations-ast.cc AM_CXXFLAGS = -Wall \ -I$(srcdir)/.. ${aterm_include} -I$(srcdir)/../libutil +local-store.lo: schema.sql.hh + +%.sql.hh: %.sql + ../bin2c/bin2c schema < $< > $@ || (rm $@ && exit 1) + derivations-ast.cc derivations-ast.hh: ../aterm-helper.pl derivations-ast.def $(perl) $(srcdir)/../aterm-helper.pl derivations-ast.hh derivations-ast.cc < $(srcdir)/derivations-ast.def diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7c8db745c..55637dd85 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -22,6 +22,16 @@ namespace nix { +class SQLiteError : public Error +{ +public: + SQLiteError(sqlite3 * db, const format & f) + : Error(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)) + { + } +}; + + void checkStoreNotSymlink() { if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; @@ -42,6 +52,7 @@ void checkStoreNotSymlink() LocalStore::LocalStore() { + db = 0; substitutablePathsLoaded = false; schemaPath = nixDBPath + "/schema"; @@ -50,9 +61,6 @@ LocalStore::LocalStore() /* Create missing state directories if they don't already exist. */ createDirs(nixStore); - createDirs(nixDBPath + "/info"); - createDirs(nixDBPath + "/referrer"); - createDirs(nixDBPath + "/failed"); Path profilesDir = nixStateDir + "/profiles"; createDirs(nixStateDir + "/profiles"); createDirs(nixStateDir + "/temproots"); @@ -88,7 +96,12 @@ LocalStore::LocalStore() writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); } if (curSchema == 1) throw Error("your Nix store is no longer supported"); - if (curSchema < nixSchemaVersion) upgradeStore12(); + if (curSchema < 5) + throw Error( + "Your Nix store has a database in Berkeley DB format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 0.12 first."); + if (curSchema < 6) upgradeStore6(); doFsync = queryBoolSetting("fsync-metadata", false); } @@ -99,6 +112,9 @@ LocalStore::~LocalStore() try { flushDelayedUpdates(); + if (db && sqlite3_close(db) != SQLITE_OK) + throw SQLiteError(db, "closing database"); + foreach (RunningSubstituters::iterator, i, runningSubstituters) { i->second.to.close(); i->second.from.close(); @@ -123,6 +139,22 @@ int LocalStore::getSchema() } +#include "schema.sql.hh" + +void LocalStore::initSchema() +{ + if (sqlite3_open_v2((nixDBPath + "/db.sqlite").c_str(), &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) + throw Error("cannot open SQLite database"); + + if (sqlite3_busy_timeout(db, 60000) != SQLITE_OK) + throw SQLiteError(db, "sett"); + + if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "initialising database schema"); +} + + void canonicalisePathMetaData(const Path & path, bool recurse) { checkInterrupt(); @@ -1171,15 +1203,78 @@ void LocalStore::verifyStore(bool checkContents) } -/* Upgrade from schema 4 (Nix 0.11) to schema 5 (Nix >= 0.12). The - old schema uses Berkeley DB, the new one stores store path - meta-information in files. */ -void LocalStore::upgradeStore12() +/* Upgrade from schema 5 (Nix 0.12) to schema 6 (Nix >= 0.15). */ +void LocalStore::upgradeStore6() { - throw Error( - "Your Nix store has a database in Berkeley DB format,\n" - "which is no longer supported. To convert to the new format,\n" - "please upgrade Nix to version 0.12 first."); + if (!lockFile(globalLock, ltWrite, false)) { + printMsg(lvlError, "waiting for exclusive access to the Nix store..."); + lockFile(globalLock, ltWrite, true); + } + + printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); + + initSchema(); + + PathSet validPaths = queryValidPaths(); + + sqlite3_stmt * registerStmt; + if (sqlite3_prepare_v2(db, "insert into ValidPaths (path, hash, registrationTime) values (?, ?, ?);", + -1, ®isterStmt, 0) != SQLITE_OK) + throw SQLiteError(db, "creating statement"); + + sqlite3_stmt * addRefStmt; + if (sqlite3_prepare_v2(db, "insert into Refs (referrer, reference) values (?, ?);", + -1, &addRefStmt, 0) != SQLITE_OK) + throw SQLiteError(db, "creating statement"); + + if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "running `begin' command"); + + foreach (PathSet::iterator, i, validPaths) { + ValidPathInfo info = queryPathInfo(*i, true); + + if (sqlite3_reset(registerStmt) != SQLITE_OK) + throw SQLiteError(db, "resetting statement"); + if (sqlite3_bind_text(registerStmt, 1, i->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument 1"); + string h = "sha256:" + printHash(info.hash); + if (sqlite3_bind_text(registerStmt, 2, h.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument 2"); + if (sqlite3_bind_int(registerStmt, 3, info.registrationTime) != SQLITE_OK) + throw SQLiteError(db, "binding argument 3"); + if (sqlite3_step(registerStmt) != SQLITE_DONE) + throw SQLiteError(db, "registering valid path in database"); + + foreach (PathSet::iterator, j, info.references) { + if (sqlite3_reset(addRefStmt) != SQLITE_OK) + throw SQLiteError(db, "resetting statement"); + if (sqlite3_bind_text(addRefStmt, 1, i->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument 1"); + if (sqlite3_bind_text(addRefStmt, 2, j->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument 2"); + if (sqlite3_step(addRefStmt) != SQLITE_DONE) + throw SQLiteError(db, "adding reference to database"); + } + + std::cerr << "."; + } + + std::cerr << "\n"; + + if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "running `commit' command"); + + if (sqlite3_finalize(registerStmt) != SQLITE_OK) + throw SQLiteError(db, "finalizing statement"); + + if (sqlite3_finalize(addRefStmt) != SQLITE_OK) + throw SQLiteError(db, "finalizing statement"); + + throw Error("foo"); + + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + + lockFile(globalLock, ltRead, true); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 31f8d9109..f29565680 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -6,6 +6,8 @@ #include "store-api.hh" #include "util.hh" +#include + namespace nix { @@ -161,8 +163,12 @@ private: /* Whether to do an fsync() after writing Nix metadata. */ bool doFsync; + sqlite3 * db; + int getSchema(); + void initSchema(); + void registerValidPath(const ValidPathInfo & info, bool ignoreValidity = false); ValidPathInfo queryPathInfo(const Path & path, bool ignoreErrors = false); @@ -177,7 +183,7 @@ private: void invalidatePath(const Path & path); - void upgradeStore12(); + void upgradeStore6(); struct GCState; diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql new file mode 100644 index 000000000..e24132943 --- /dev/null +++ b/src/libstore/schema.sql @@ -0,0 +1,27 @@ +pragma foreign_keys = on; + +create table if not exists ValidPaths ( + path text primary key not null, + hash text not null, + registrationTime integer not null +); + +create table if not exists Refs ( + referrer text not null, + reference text not null, + primary key (referrer, reference), + foreign key (referrer) references ValidPaths(path) + on delete cascade + deferrable initially deferred, + foreign key (reference) references ValidPaths(path) + on delete restrict + deferrable initially deferred +); + +create table if not exists FailedDerivations ( + path text primary key not null, + time integer not null +); + +create index IndexReferrer on Refs(referrer); +create index IndexReference on Refs(reference); -- cgit v1.2.3 From dbddac0fe91072b402ccb3801c952e3159f0cba4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 13:40:46 +0000 Subject: * Assign an integer id to every row in the ValidPaths table in order to make the Refs table more space-efficient. For instance, this reduces the size of the database on my laptop from 93 MiB to 18 MiB. (It was 72 MiB with the old schema on an ext3 disk with a 1 KiB block size.) --- src/libstore/local-store.cc | 18 ++++++++++++++++-- src/libstore/schema.sql | 21 +++++++++------------ 2 files changed, 25 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 55637dd85..03c5d76b3 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1229,6 +1229,8 @@ void LocalStore::upgradeStore6() if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "running `begin' command"); + + std::map pathToId; foreach (PathSet::iterator, i, validPaths) { ValidPathInfo info = queryPathInfo(*i, true); @@ -1245,12 +1247,24 @@ void LocalStore::upgradeStore6() if (sqlite3_step(registerStmt) != SQLITE_DONE) throw SQLiteError(db, "registering valid path in database"); + pathToId[*i] = sqlite3_last_insert_rowid(db); + + std::cerr << "."; + } + + std::cerr << "|"; + + foreach (PathSet::iterator, i, validPaths) { + ValidPathInfo info = queryPathInfo(*i, true); + foreach (PathSet::iterator, j, info.references) { if (sqlite3_reset(addRefStmt) != SQLITE_OK) throw SQLiteError(db, "resetting statement"); - if (sqlite3_bind_text(addRefStmt, 1, i->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (sqlite3_bind_int(addRefStmt, 1, pathToId[*i]) != SQLITE_OK) throw SQLiteError(db, "binding argument 1"); - if (sqlite3_bind_text(addRefStmt, 2, j->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (pathToId.find(*j) == pathToId.end()) + throw Error(format("path `%1%' referenced by `%2%' is invalid") % *j % *i); + if (sqlite3_bind_int(addRefStmt, 2, pathToId[*j]) != SQLITE_OK) throw SQLiteError(db, "binding argument 2"); if (sqlite3_step(addRefStmt) != SQLITE_DONE) throw SQLiteError(db, "adding reference to database"); diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index e24132943..3c07978fc 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -1,27 +1,24 @@ pragma foreign_keys = on; create table if not exists ValidPaths ( - path text primary key not null, + id integer primary key autoincrement not null, + path text unique not null, hash text not null, registrationTime integer not null ); create table if not exists Refs ( - referrer text not null, - reference text not null, + referrer integer not null, + reference integer not null, primary key (referrer, reference), - foreign key (referrer) references ValidPaths(path) - on delete cascade - deferrable initially deferred, - foreign key (reference) references ValidPaths(path) - on delete restrict - deferrable initially deferred + foreign key (referrer) references ValidPaths(id) on delete cascade, + foreign key (reference) references ValidPaths(id) on delete restrict ); +create index if not exists IndexReferrer on Refs(referrer); +create index if not exists IndexReference on Refs(reference); + create table if not exists FailedDerivations ( path text primary key not null, time integer not null ); - -create index IndexReferrer on Refs(referrer); -create index IndexReference on Refs(reference); -- cgit v1.2.3 From a053d2d8e53f2967c64ab2b204727e4c27f06c0e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 13:48:18 +0000 Subject: * Add the deriver to the ValidPaths table. In principle we could now store all the derivers of a path efficiently. But that opens a big can of worms with respect to garbage collector semantics. --- src/libstore/local-store.cc | 31 +++++++++++++++++++------------ src/libstore/schema.sql | 3 ++- 2 files changed, 21 insertions(+), 13 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 03c5d76b3..35391f3aa 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1217,19 +1217,14 @@ void LocalStore::upgradeStore6() PathSet validPaths = queryValidPaths(); + if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "running `begin' command"); + sqlite3_stmt * registerStmt; - if (sqlite3_prepare_v2(db, "insert into ValidPaths (path, hash, registrationTime) values (?, ?, ?);", + if (sqlite3_prepare_v2(db, "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);", -1, ®isterStmt, 0) != SQLITE_OK) throw SQLiteError(db, "creating statement"); - sqlite3_stmt * addRefStmt; - if (sqlite3_prepare_v2(db, "insert into Refs (referrer, reference) values (?, ?);", - -1, &addRefStmt, 0) != SQLITE_OK) - throw SQLiteError(db, "creating statement"); - - if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "running `begin' command"); - std::map pathToId; foreach (PathSet::iterator, i, validPaths) { @@ -1244,6 +1239,13 @@ void LocalStore::upgradeStore6() throw SQLiteError(db, "binding argument 2"); if (sqlite3_bind_int(registerStmt, 3, info.registrationTime) != SQLITE_OK) throw SQLiteError(db, "binding argument 3"); + if (info.deriver != "") { + if (sqlite3_bind_text(registerStmt, 4, info.deriver.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument 4"); + } else { + if (sqlite3_bind_null(registerStmt, 4) != SQLITE_OK) + throw SQLiteError(db, "binding argument 4"); + } if (sqlite3_step(registerStmt) != SQLITE_DONE) throw SQLiteError(db, "registering valid path in database"); @@ -1252,8 +1254,16 @@ void LocalStore::upgradeStore6() std::cerr << "."; } + if (sqlite3_finalize(registerStmt) != SQLITE_OK) + throw SQLiteError(db, "finalizing statement"); + std::cerr << "|"; + sqlite3_stmt * addRefStmt; + if (sqlite3_prepare_v2(db, "insert into Refs (referrer, reference) values (?, ?);", + -1, &addRefStmt, 0) != SQLITE_OK) + throw SQLiteError(db, "creating statement"); + foreach (PathSet::iterator, i, validPaths) { ValidPathInfo info = queryPathInfo(*i, true); @@ -1278,9 +1288,6 @@ void LocalStore::upgradeStore6() if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "running `commit' command"); - if (sqlite3_finalize(registerStmt) != SQLITE_OK) - throw SQLiteError(db, "finalizing statement"); - if (sqlite3_finalize(addRefStmt) != SQLITE_OK) throw SQLiteError(db, "finalizing statement"); diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index 3c07978fc..dc53f452c 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -4,7 +4,8 @@ create table if not exists ValidPaths ( id integer primary key autoincrement not null, path text unique not null, hash text not null, - registrationTime integer not null + registrationTime integer not null, + deriver text ); create table if not exists Refs ( -- cgit v1.2.3 From e0305bb7a8b24fe1ea8b36cc2e5fe77ab151ae2f Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 14:30:42 +0000 Subject: * Some wrapper objects to ensure that SQLite objects are properly destroyed. --- src/libstore/local-store.cc | 118 +++++++++++++++++++++++++++----------------- src/libstore/local-store.hh | 40 +++++++++++++-- 2 files changed, 109 insertions(+), 49 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 35391f3aa..870644c88 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -18,6 +18,8 @@ #include #include +#include + namespace nix { @@ -32,6 +34,37 @@ public: }; +SQLite::~SQLite() +{ + try { + if (db && sqlite3_close(db) != SQLITE_OK) + throw SQLiteError(db, "closing database"); + } catch (...) { + ignoreException(); + } +} + + +void SQLiteStmt::create(sqlite3 * db, const string & s) +{ + assert(!stmt); + if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) + throw SQLiteError(db, "creating statement"); + this->db = db; +} + + +SQLiteStmt::~SQLiteStmt() +{ + try { + if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) + throw SQLiteError(db, "finalizing statement"); + } catch (...) { + ignoreException(); + } +} + + void checkStoreNotSymlink() { if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; @@ -52,7 +85,6 @@ void checkStoreNotSymlink() LocalStore::LocalStore() { - db = 0; substitutablePathsLoaded = false; schemaPath = nixDBPath + "/schema"; @@ -73,6 +105,8 @@ LocalStore::LocalStore() checkStoreNotSymlink(); + /* Acquire the big fat lock in shared mode to make sure that no + schema upgrade is in progress. */ try { Path globalLockPath = nixDBPath + "/big-lock"; globalLock = openLockFile(globalLockPath.c_str(), true); @@ -86,22 +120,34 @@ LocalStore::LocalStore() printMsg(lvlError, "waiting for the big Nix store lock..."); lockFile(globalLock, ltRead, true); } + + /* Open the Nix database. */ + if (sqlite3_open_v2((nixDBPath + "/db.sqlite").c_str(), &db.db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) + throw Error("cannot open SQLite database"); + + if (sqlite3_busy_timeout(db, 60000) != SQLITE_OK) + throw SQLiteError(db, "setting timeout"); + /* Check the current database schema and if necessary do an + upgrade. */ int curSchema = getSchema(); if (curSchema > nixSchemaVersion) throw Error(format("current Nix store schema is version %1%, but I only support %2%") % curSchema % nixSchemaVersion); if (curSchema == 0) { /* new store */ - curSchema = nixSchemaVersion; + curSchema = nixSchemaVersion; + initSchema(); writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); } - if (curSchema == 1) throw Error("your Nix store is no longer supported"); - if (curSchema < 5) + else if (curSchema == 1) throw Error("your Nix store is no longer supported"); + else if (curSchema < 5) throw Error( "Your Nix store has a database in Berkeley DB format,\n" "which is no longer supported. To convert to the new format,\n" "please upgrade Nix to version 0.12 first."); - if (curSchema < 6) upgradeStore6(); + else if (curSchema < 6) upgradeStore6(); + else prepareStatements(); doFsync = queryBoolSetting("fsync-metadata", false); } @@ -112,9 +158,6 @@ LocalStore::~LocalStore() try { flushDelayedUpdates(); - if (db && sqlite3_close(db) != SQLITE_OK) - throw SQLiteError(db, "closing database"); - foreach (RunningSubstituters::iterator, i, runningSubstituters) { i->second.to.close(); i->second.from.close(); @@ -143,15 +186,19 @@ int LocalStore::getSchema() void LocalStore::initSchema() { - if (sqlite3_open_v2((nixDBPath + "/db.sqlite").c_str(), &db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) - throw Error("cannot open SQLite database"); - - if (sqlite3_busy_timeout(db, 60000) != SQLITE_OK) - throw SQLiteError(db, "sett"); - if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "initialising database schema"); + + prepareStatements(); +} + + +void LocalStore::prepareStatements() +{ + stmtRegisterValidPath.create(db, + "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); + stmtAddReference.create(db, + "insert into Refs (referrer, reference) values (?, ?);"); } @@ -1220,33 +1267,28 @@ void LocalStore::upgradeStore6() if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "running `begin' command"); - sqlite3_stmt * registerStmt; - if (sqlite3_prepare_v2(db, "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);", - -1, ®isterStmt, 0) != SQLITE_OK) - throw SQLiteError(db, "creating statement"); - std::map pathToId; foreach (PathSet::iterator, i, validPaths) { ValidPathInfo info = queryPathInfo(*i, true); - if (sqlite3_reset(registerStmt) != SQLITE_OK) + if (sqlite3_reset(stmtRegisterValidPath) != SQLITE_OK) throw SQLiteError(db, "resetting statement"); - if (sqlite3_bind_text(registerStmt, 1, i->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (sqlite3_bind_text(stmtRegisterValidPath, 1, i->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throw SQLiteError(db, "binding argument 1"); string h = "sha256:" + printHash(info.hash); - if (sqlite3_bind_text(registerStmt, 2, h.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (sqlite3_bind_text(stmtRegisterValidPath, 2, h.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throw SQLiteError(db, "binding argument 2"); - if (sqlite3_bind_int(registerStmt, 3, info.registrationTime) != SQLITE_OK) + if (sqlite3_bind_int(stmtRegisterValidPath, 3, info.registrationTime) != SQLITE_OK) throw SQLiteError(db, "binding argument 3"); if (info.deriver != "") { - if (sqlite3_bind_text(registerStmt, 4, info.deriver.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + if (sqlite3_bind_text(stmtRegisterValidPath, 4, info.deriver.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throw SQLiteError(db, "binding argument 4"); } else { - if (sqlite3_bind_null(registerStmt, 4) != SQLITE_OK) + if (sqlite3_bind_null(stmtRegisterValidPath, 4) != SQLITE_OK) throw SQLiteError(db, "binding argument 4"); } - if (sqlite3_step(registerStmt) != SQLITE_DONE) + if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) throw SQLiteError(db, "registering valid path in database"); pathToId[*i] = sqlite3_last_insert_rowid(db); @@ -1254,29 +1296,21 @@ void LocalStore::upgradeStore6() std::cerr << "."; } - if (sqlite3_finalize(registerStmt) != SQLITE_OK) - throw SQLiteError(db, "finalizing statement"); - std::cerr << "|"; - sqlite3_stmt * addRefStmt; - if (sqlite3_prepare_v2(db, "insert into Refs (referrer, reference) values (?, ?);", - -1, &addRefStmt, 0) != SQLITE_OK) - throw SQLiteError(db, "creating statement"); - foreach (PathSet::iterator, i, validPaths) { ValidPathInfo info = queryPathInfo(*i, true); foreach (PathSet::iterator, j, info.references) { - if (sqlite3_reset(addRefStmt) != SQLITE_OK) + if (sqlite3_reset(stmtAddReference) != SQLITE_OK) throw SQLiteError(db, "resetting statement"); - if (sqlite3_bind_int(addRefStmt, 1, pathToId[*i]) != SQLITE_OK) + if (sqlite3_bind_int(stmtAddReference, 1, pathToId[*i]) != SQLITE_OK) throw SQLiteError(db, "binding argument 1"); if (pathToId.find(*j) == pathToId.end()) throw Error(format("path `%1%' referenced by `%2%' is invalid") % *j % *i); - if (sqlite3_bind_int(addRefStmt, 2, pathToId[*j]) != SQLITE_OK) + if (sqlite3_bind_int(stmtAddReference, 2, pathToId[*j]) != SQLITE_OK) throw SQLiteError(db, "binding argument 2"); - if (sqlite3_step(addRefStmt) != SQLITE_DONE) + if (sqlite3_step(stmtAddReference) != SQLITE_DONE) throw SQLiteError(db, "adding reference to database"); } @@ -1287,12 +1321,6 @@ void LocalStore::upgradeStore6() if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "running `commit' command"); - - if (sqlite3_finalize(addRefStmt) != SQLITE_OK) - throw SQLiteError(db, "finalizing statement"); - - throw Error("foo"); - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); lockFile(globalLock, ltRead, true); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index f29565680..3637a5d07 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -6,7 +6,9 @@ #include "store-api.hh" #include "util.hh" -#include + +class sqlite3; +class sqlite3_stmt; namespace nix { @@ -14,8 +16,9 @@ namespace nix { /* Nix store and database schema version. Version 1 (or 0) was Nix <= 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. - Version 4 is Nix 0.11. Version 5 is Nix 0.12*/ -const int nixSchemaVersion = 5; + Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.14. Version 6 is + Nix 0.15. */ +const int nixSchemaVersion = 6; extern string drvsLogDir; @@ -43,6 +46,28 @@ struct RunningSubstituter }; +/* Wrapper object to close the SQLite database automatically. */ +struct SQLite +{ + sqlite3 * db; + SQLite() { db = 0; } + ~SQLite(); + operator sqlite3 * () { return db; } +}; + + +/* Wrapper object to create and destroy SQLite prepared statements. */ +struct SQLiteStmt +{ + sqlite3 * db; + sqlite3_stmt * stmt; + SQLiteStmt() { stmt = 0; } + void create(sqlite3 * db, const string & s); + ~SQLiteStmt(); + operator sqlite3_stmt * () { return stmt; } +}; + + class LocalStore : public StoreAPI { private: @@ -163,12 +188,19 @@ private: /* Whether to do an fsync() after writing Nix metadata. */ bool doFsync; - sqlite3 * db; + /* The SQLite database object. */ + SQLite db; + + /* Some precompiled SQLite statements. */ + SQLiteStmt stmtRegisterValidPath; + SQLiteStmt stmtAddReference; int getSchema(); void initSchema(); + void prepareStatements(); + void registerValidPath(const ValidPathInfo & info, bool ignoreValidity = false); ValidPathInfo queryPathInfo(const Path & path, bool ignoreErrors = false); -- cgit v1.2.3 From cfb09e0fadf7db240f4f8c35c35c13b192456b89 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 14:40:07 +0000 Subject: * Automatically abort transactions if they go out of scope without committing. --- src/libstore/local-store.cc | 43 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 870644c88..ac6d768a0 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -65,6 +65,37 @@ SQLiteStmt::~SQLiteStmt() } +struct SQLiteTxn +{ + bool active; + sqlite3 * db; + + SQLiteTxn(sqlite3 * db) : active(false) { + this->db = db; + if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "starting transaction"); + active = true; + } + + void commit() + { + if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "committing transaction"); + active = false; + } + + ~SQLiteTxn() + { + try { + if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "aborting transaction"); + } catch (...) { + ignoreException(); + } + } +}; + + void checkStoreNotSymlink() { if (getEnv("NIX_IGNORE_SYMLINK_STORE") == "1") return; @@ -130,7 +161,8 @@ LocalStore::LocalStore() throw SQLiteError(db, "setting timeout"); /* Check the current database schema and if necessary do an - upgrade. */ + upgrade. !!! Race condition: several processes could start + the upgrade at the same time. */ int curSchema = getSchema(); if (curSchema > nixSchemaVersion) throw Error(format("current Nix store schema is version %1%, but I only support %2%") @@ -1264,9 +1296,8 @@ void LocalStore::upgradeStore6() PathSet validPaths = queryValidPaths(); - if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "running `begin' command"); - + SQLiteTxn txn(db); + std::map pathToId; foreach (PathSet::iterator, i, validPaths) { @@ -1319,8 +1350,8 @@ void LocalStore::upgradeStore6() std::cerr << "\n"; - if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "running `commit' command"); + txn.commit(); + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); lockFile(globalLock, ltRead, true); -- cgit v1.2.3 From 885e22b16e3a6ea2a94014d33de31d756fa32eda Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 15:11:08 +0000 Subject: * Implement isValidPath(). --- src/libstore/local-store.cc | 244 +++++++++++++++++--------------------------- src/libstore/local-store.hh | 15 +-- 2 files changed, 100 insertions(+), 159 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ac6d768a0..05133ba55 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -54,6 +54,14 @@ void SQLiteStmt::create(sqlite3 * db, const string & s) } +void SQLiteStmt::reset() +{ + assert(stmt); + if (sqlite3_reset(stmt) != SQLITE_OK) + throw SQLiteError(db, "resetting statement"); +} + + SQLiteStmt::~SQLiteStmt() { try { @@ -188,14 +196,11 @@ LocalStore::LocalStore() LocalStore::~LocalStore() { try { - flushDelayedUpdates(); - foreach (RunningSubstituters::iterator, i, runningSubstituters) { i->second.to.close(); i->second.from.close(); i->second.pid.wait(true); } - } catch (...) { ignoreException(); } @@ -231,6 +236,7 @@ void LocalStore::prepareStatements() "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); stmtAddReference.create(db, "insert into Refs (referrer, reference) values (?, ?);"); + stmtIsValidPath.create(db, "select 1 from ValidPaths where path = ?;"); } @@ -308,98 +314,6 @@ void canonicalisePathMetaData(const Path & path) } -static Path infoFileFor(const Path & path) -{ - string baseName = baseNameOf(path); - return (format("%1%/info/%2%") % nixDBPath % baseName).str(); -} - - -static Path referrersFileFor(const Path & path) -{ - string baseName = baseNameOf(path); - return (format("%1%/referrer/%2%") % nixDBPath % baseName).str(); -} - - -static Path failedFileFor(const Path & path) -{ - string baseName = baseNameOf(path); - return (format("%1%/failed/%2%") % nixDBPath % baseName).str(); -} - - -static Path tmpFileForAtomicUpdate(const Path & path) -{ - return (format("%1%/.%2%.%3%") % dirOf(path) % getpid() % baseNameOf(path)).str(); -} - - -void LocalStore::appendReferrer(const Path & from, const Path & to, bool lock) -{ - Path referrersFile = referrersFileFor(from); - - PathLocks referrersLock; - if (lock) { - referrersLock.lockPaths(singleton(referrersFile)); - referrersLock.setDeletion(true); - } - - AutoCloseFD fd = open(referrersFile.c_str(), O_WRONLY | O_APPEND | O_CREAT, 0666); - if (fd == -1) throw SysError(format("opening file `%1%'") % referrersFile); - - string s = " " + to; - writeFull(fd, (const unsigned char *) s.c_str(), s.size()); - - if (doFsync) fdatasync(fd); -} - - -/* Atomically update the referrers file. If `purge' is true, the set - of referrers is set to `referrers'. Otherwise, the current set of - referrers is purged of invalid paths. */ -void LocalStore::rewriteReferrers(const Path & path, bool purge, PathSet referrers) -{ - Path referrersFile = referrersFileFor(path); - - PathLocks referrersLock(singleton(referrersFile)); - referrersLock.setDeletion(true); - - if (purge) - /* queryReferrers() purges invalid paths, so that's all we - need. */ - queryReferrers(path, referrers); - - Path tmpFile = tmpFileForAtomicUpdate(referrersFile); - - AutoCloseFD fd = open(tmpFile.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666); - if (fd == -1) throw SysError(format("opening file `%1%'") % referrersFile); - - string s; - foreach (PathSet::const_iterator, i, referrers) { - s += " "; s += *i; - } - - writeFull(fd, (const unsigned char *) s.c_str(), s.size()); - - if (doFsync) fdatasync(fd); - - fd.close(); /* for Windows; can't rename open file */ - - if (rename(tmpFile.c_str(), referrersFile.c_str()) == -1) - throw SysError(format("cannot rename `%1%' to `%2%'") % tmpFile % referrersFile); -} - - -void LocalStore::flushDelayedUpdates() -{ - foreach (PathSet::iterator, i, delayedUpdates) { - rewriteReferrers(*i, true, PathSet()); - } - delayedUpdates.clear(); -} - - void LocalStore::registerValidPath(const Path & path, const Hash & hash, const PathSet & references, const Path & deriver) @@ -415,6 +329,7 @@ void LocalStore::registerValidPath(const Path & path, void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidity) { +#if 0 Path infoFile = infoFileFor(info.path); ValidPathInfo oldInfo; @@ -467,22 +382,25 @@ void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidi writeFile(tmpFile, s, doFsync); if (rename(tmpFile.c_str(), infoFile.c_str()) == -1) throw SysError(format("cannot rename `%1%' to `%2%'") % tmpFile % infoFile); - - pathInfoCache[info.path] = info; +#endif } void LocalStore::registerFailedPath(const Path & path) { +#if 0 /* Write an empty file in the .../failed directory to denote the failure of the builder for `path'. */ writeFile(failedFileFor(path), ""); +#endif } bool LocalStore::hasPathFailed(const Path & path) { +#if 0 return pathExists(failedFileFor(path)); +#endif } @@ -502,6 +420,7 @@ Hash parseHashField(const Path & path, const string & s) ValidPathInfo LocalStore::queryPathInfo(const Path & path, bool ignoreErrors) { +#if 0 ValidPathInfo res; res.path = path; @@ -510,9 +429,6 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path, bool ignoreErrors) if (!isValidPath(path)) throw Error(format("path `%1%' is not valid") % path); - std::map::iterator lookup = pathInfoCache.find(path); - if (lookup != pathInfoCache.end()) return lookup->second; - /* Read the info file. */ Path infoFile = infoFileFor(path); if (!pathExists(infoFile)) @@ -550,31 +466,20 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path, bool ignoreErrors) } } - return pathInfoCache[path] = res; + return res; +#endif } bool LocalStore::isValidPath(const Path & path) { - /* Files in the info directory starting with a `.' are temporary - files. */ - if (baseNameOf(path).at(0) == '.') return false; - - /* A path is valid if its info file exists and has a non-zero - size. (The non-zero size restriction is to be robust to - certain kinds of filesystem corruption, particularly with - ext4.) */ - Path infoFile = infoFileFor(path); - - struct stat st; - if (lstat(infoFile.c_str(), &st)) { - if (errno == ENOENT) return false; - throw SysError(format("getting status of `%1%'") % infoFile); - } - - if (st.st_size == 0) return false; - - return true; + stmtIsValidPath.reset(); + if (sqlite3_bind_text(stmtIsValidPath, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + int res = sqlite3_step(stmtIsValidPath); + if (res != SQLITE_DONE && res != SQLITE_ROW) + throw SQLiteError(db, "querying path in database"); + return res == SQLITE_ROW; } @@ -598,6 +503,7 @@ void LocalStore::queryReferences(const Path & path, bool LocalStore::queryReferrersInternal(const Path & path, PathSet & referrers) { +#if 0 bool allValid = true; if (!isValidPath(path)) @@ -623,6 +529,7 @@ bool LocalStore::queryReferrersInternal(const Path & path, PathSet & referrers) if (isStorePath(*i) && isValidPath(*i)) referrers.insert(*i); else allValid = false; return allValid; +#endif } @@ -788,6 +695,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) there are no referrers. */ void LocalStore::invalidatePath(const Path & path) { +#if 0 debug(format("invalidating path `%1%'") % path); ValidPathInfo info; @@ -805,28 +713,7 @@ void LocalStore::invalidatePath(const Path & path) Path p = referrersFileFor(path); if (pathExists(p) && unlink(p.c_str()) == -1) throw SysError(format("unlinking `%1%'") % p); - - /* Clear `path' from the info cache. */ - pathInfoCache.erase(path); - delayedUpdates.erase(path); - - /* Cause the referrer files for each path referenced by this one - to be updated. This has to happen after removing the info file - to preserve the invariant (see registerValidPath()). - - The referrer files are updated lazily in flushDelayedUpdates() - to prevent quadratic performance in the garbage collector - (i.e., when N referrers to some path X are deleted, we have to - rewrite the referrers file for X N times, causing O(N^2) I/O). - - What happens if we die before the referrer file can be updated? - That's not a problem, because stale (invalid) entries in the - referrer file are ignored by queryReferrers(). Thus a referrer - file is allowed to have stale entries; removing them is just an - optimisation. verifyStore() gets rid of them eventually. - */ - foreach (PathSet::iterator, i, info.references) - if (*i != path) delayedUpdates.insert(*i); +#endif } @@ -1130,6 +1017,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed, unsigned long long & blocksFreed) { +#if 0 bytesFreed = 0; assertStorePath(path); @@ -1149,11 +1037,13 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr } deletePathWrapped(path, bytesFreed, blocksFreed); +#endif } void LocalStore::verifyStore(bool checkContents) { +#if 0 /* Check whether all valid paths actually exist. */ printMsg(lvlInfo, "checking path existence"); @@ -1279,6 +1169,64 @@ void LocalStore::verifyStore(bool checkContents) if (update) rewriteReferrers(from, false, referrersNew); } +#endif +} + + +/* Functions for upgrading from the pre-SQLite database. */ + +static Path infoFileFor(const Path & path) +{ + string baseName = baseNameOf(path); + return (format("%1%/info/%2%") % nixDBPath % baseName).str(); +} + + +PathSet LocalStore::queryValidPathsOld() +{ + PathSet paths; + Strings entries = readDirectory(nixDBPath + "/info"); + foreach (Strings::iterator, i, entries) + if (i->at(0) != '.') paths.insert(nixStore + "/" + *i); + return paths; +} + + +ValidPathInfo LocalStore::queryPathInfoOld(const Path & path) +{ + ValidPathInfo res; + res.path = path; + + /* Read the info file. */ + Path infoFile = infoFileFor(path); + if (!pathExists(infoFile)) + throw Error(format("path `%1%' is not valid") % path); + string info = readFile(infoFile); + + /* Parse it. */ + Strings lines = tokenizeString(info, "\n"); + + foreach (Strings::iterator, i, lines) { + string::size_type p = i->find(':'); + if (p == string::npos) + throw Error(format("corrupt line in `%1%': %2%") % infoFile % *i); + string name(*i, 0, p); + string value(*i, p + 2); + if (name == "References") { + Strings refs = tokenizeString(value, " "); + res.references = PathSet(refs.begin(), refs.end()); + } else if (name == "Deriver") { + res.deriver = value; + } else if (name == "Hash") { + res.hash = parseHashField(path, value); + } else if (name == "Registered-At") { + int n = 0; + string2Int(value, n); + res.registrationTime = n; + } + } + + return res; } @@ -1294,17 +1242,16 @@ void LocalStore::upgradeStore6() initSchema(); - PathSet validPaths = queryValidPaths(); + PathSet validPaths = queryValidPathsOld(); SQLiteTxn txn(db); std::map pathToId; foreach (PathSet::iterator, i, validPaths) { - ValidPathInfo info = queryPathInfo(*i, true); + ValidPathInfo info = queryPathInfoOld(*i); - if (sqlite3_reset(stmtRegisterValidPath) != SQLITE_OK) - throw SQLiteError(db, "resetting statement"); + stmtRegisterValidPath.reset(); if (sqlite3_bind_text(stmtRegisterValidPath, 1, i->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throw SQLiteError(db, "binding argument 1"); string h = "sha256:" + printHash(info.hash); @@ -1330,11 +1277,10 @@ void LocalStore::upgradeStore6() std::cerr << "|"; foreach (PathSet::iterator, i, validPaths) { - ValidPathInfo info = queryPathInfo(*i, true); + ValidPathInfo info = queryPathInfoOld(*i); foreach (PathSet::iterator, j, info.references) { - if (sqlite3_reset(stmtAddReference) != SQLITE_OK) - throw SQLiteError(db, "resetting statement"); + stmtAddReference.reset(); if (sqlite3_bind_int(stmtAddReference, 1, pathToId[*i]) != SQLITE_OK) throw SQLiteError(db, "binding argument 1"); if (pathToId.find(*j) == pathToId.end()) diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 3637a5d07..fe4ed035d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -63,6 +63,7 @@ struct SQLiteStmt sqlite3_stmt * stmt; SQLiteStmt() { stmt = 0; } void create(sqlite3 * db, const string & s); + void reset(); ~SQLiteStmt(); operator sqlite3_stmt * () { return stmt; } }; @@ -178,13 +179,6 @@ private: /* Lock file used for upgrading. */ AutoCloseFD globalLock; - /* !!! The cache can grow very big. Maybe it should be pruned - every once in a while. */ - std::map pathInfoCache; - - /* Store paths for which the referrers file must be purged. */ - PathSet delayedUpdates; - /* Whether to do an fsync() after writing Nix metadata. */ bool doFsync; @@ -194,6 +188,7 @@ private: /* Some precompiled SQLite statements. */ SQLiteStmt stmtRegisterValidPath; SQLiteStmt stmtAddReference; + SQLiteStmt stmtIsValidPath; int getSchema(); @@ -209,13 +204,13 @@ private: void rewriteReferrers(const Path & path, bool purge, PathSet referrers); - void flushDelayedUpdates(); - bool queryReferrersInternal(const Path & path, PathSet & referrers); void invalidatePath(const Path & path); - + void upgradeStore6(); + PathSet queryValidPathsOld(); + ValidPathInfo queryPathInfoOld(const Path & path); struct GCState; -- cgit v1.2.3 From 77cb9e3fb19005a40724cb42773d2b11e2528858 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 15:52:57 +0000 Subject: * Implement queryPathInfo(). --- src/libstore/local-store.cc | 95 ++++++++++++++++++++++----------------------- src/libstore/local-store.hh | 5 ++- 2 files changed, 49 insertions(+), 51 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 05133ba55..76911da67 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -236,7 +236,10 @@ void LocalStore::prepareStatements() "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); stmtAddReference.create(db, "insert into Refs (referrer, reference) values (?, ?);"); - stmtIsValidPath.create(db, "select 1 from ValidPaths where path = ?;"); + stmtQueryPathInfo.create(db, + "select id, hash, registrationTime, deriver from ValidPaths where path = ?;"); + stmtQueryReferences.create(db, + "select path from Refs join ValidPaths on reference = id where referrer = ?;"); } @@ -418,65 +421,59 @@ Hash parseHashField(const Path & path, const string & s) } -ValidPathInfo LocalStore::queryPathInfo(const Path & path, bool ignoreErrors) +ValidPathInfo LocalStore::queryPathInfo(const Path & path) { -#if 0 ValidPathInfo res; res.path = path; assertStorePath(path); - if (!isValidPath(path)) - throw Error(format("path `%1%' is not valid") % path); + /* Get the path info. */ + stmtQueryPathInfo.reset(); + + if (sqlite3_bind_text(stmtQueryPathInfo, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + + int r = sqlite3_step(stmtQueryPathInfo); + if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); + if (r != SQLITE_ROW) throw SQLiteError(db, "querying path in database"); - /* Read the info file. */ - Path infoFile = infoFileFor(path); - if (!pathExists(infoFile)) - throw Error(format("path `%1%' is not valid") % path); - string info = readFile(infoFile); + unsigned int id = sqlite3_column_int(stmtQueryPathInfo, 0); - /* Parse it. */ - Strings lines = tokenizeString(info, "\n"); + const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); + assert(s); + res.hash = parseHashField(path, s); + + res.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); - foreach (Strings::iterator, i, lines) { - string::size_type p = i->find(':'); - if (p == string::npos) { - if (!ignoreErrors) - throw Error(format("corrupt line in `%1%': %2%") % infoFile % *i); - continue; /* bad line */ - } - string name(*i, 0, p); - string value(*i, p + 2); - if (name == "References") { - Strings refs = tokenizeString(value, " "); - res.references = PathSet(refs.begin(), refs.end()); - } else if (name == "Deriver") { - res.deriver = value; - } else if (name == "Hash") { - try { - res.hash = parseHashField(path, value); - } catch (Error & e) { - if (!ignoreErrors) throw; - printMsg(lvlError, format("cannot parse hash field in `%1%': %2%") % infoFile % e.msg()); - } - } else if (name == "Registered-At") { - int n = 0; - string2Int(value, n); - res.registrationTime = n; - } + s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); + if (s) res.deriver = s; + + /* Get the references. */ + stmtQueryReferences.reset(); + + if (sqlite3_bind_int(stmtQueryReferences, 1, id) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + + while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { + s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); + assert(s); + res.references.insert(s); } + if (r != SQLITE_DONE) + throw Error(format("error getting references of `%1%'") % path); + return res; -#endif } bool LocalStore::isValidPath(const Path & path) { - stmtIsValidPath.reset(); - if (sqlite3_bind_text(stmtIsValidPath, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + stmtQueryPathInfo.reset(); + if (sqlite3_bind_text(stmtQueryPathInfo, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throw SQLiteError(db, "binding argument"); - int res = sqlite3_step(stmtIsValidPath); + int res = sqlite3_step(stmtQueryPathInfo); if (res != SQLITE_DONE && res != SQLITE_ROW) throw SQLiteError(db, "querying path in database"); return res == SQLITE_ROW; @@ -1253,18 +1250,18 @@ void LocalStore::upgradeStore6() stmtRegisterValidPath.reset(); if (sqlite3_bind_text(stmtRegisterValidPath, 1, i->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument 1"); + throw SQLiteError(db, "binding argument"); string h = "sha256:" + printHash(info.hash); if (sqlite3_bind_text(stmtRegisterValidPath, 2, h.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument 2"); + throw SQLiteError(db, "binding argument"); if (sqlite3_bind_int(stmtRegisterValidPath, 3, info.registrationTime) != SQLITE_OK) - throw SQLiteError(db, "binding argument 3"); + throw SQLiteError(db, "binding argument"); if (info.deriver != "") { if (sqlite3_bind_text(stmtRegisterValidPath, 4, info.deriver.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument 4"); + throw SQLiteError(db, "binding argument"); } else { if (sqlite3_bind_null(stmtRegisterValidPath, 4) != SQLITE_OK) - throw SQLiteError(db, "binding argument 4"); + throw SQLiteError(db, "binding argument"); } if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) throw SQLiteError(db, "registering valid path in database"); @@ -1282,11 +1279,11 @@ void LocalStore::upgradeStore6() foreach (PathSet::iterator, j, info.references) { stmtAddReference.reset(); if (sqlite3_bind_int(stmtAddReference, 1, pathToId[*i]) != SQLITE_OK) - throw SQLiteError(db, "binding argument 1"); + throw SQLiteError(db, "binding argument"); if (pathToId.find(*j) == pathToId.end()) throw Error(format("path `%1%' referenced by `%2%' is invalid") % *j % *i); if (sqlite3_bind_int(stmtAddReference, 2, pathToId[*j]) != SQLITE_OK) - throw SQLiteError(db, "binding argument 2"); + throw SQLiteError(db, "binding argument"); if (sqlite3_step(stmtAddReference) != SQLITE_DONE) throw SQLiteError(db, "adding reference to database"); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index fe4ed035d..af1cd26a3 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -188,7 +188,8 @@ private: /* Some precompiled SQLite statements. */ SQLiteStmt stmtRegisterValidPath; SQLiteStmt stmtAddReference; - SQLiteStmt stmtIsValidPath; + SQLiteStmt stmtQueryPathInfo; + SQLiteStmt stmtQueryReferences; int getSchema(); @@ -198,7 +199,7 @@ private: void registerValidPath(const ValidPathInfo & info, bool ignoreValidity = false); - ValidPathInfo queryPathInfo(const Path & path, bool ignoreErrors = false); + ValidPathInfo queryPathInfo(const Path & path); void appendReferrer(const Path & from, const Path & to, bool lock); -- cgit v1.2.3 From 836e5b6f5741c845da08490a6fd8ad9a3f37726e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 16:21:59 +0000 Subject: * Implemented queryReferrers(). --- src/libstore/local-store.cc | 48 +++++++++++++++++---------------------------- src/libstore/local-store.hh | 3 +-- 2 files changed, 19 insertions(+), 32 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 76911da67..f1deb2eae 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -167,6 +167,9 @@ LocalStore::LocalStore() if (sqlite3_busy_timeout(db, 60000) != SQLITE_OK) throw SQLiteError(db, "setting timeout"); + + /* !!! check whether sqlite has been built with foreign key + support */ /* Check the current database schema and if necessary do an upgrade. !!! Race condition: several processes could start @@ -240,6 +243,8 @@ void LocalStore::prepareStatements() "select id, hash, registrationTime, deriver from ValidPaths where path = ?;"); stmtQueryReferences.create(db, "select path from Refs join ValidPaths on reference = id where referrer = ?;"); + stmtQueryReferrers.create(db, + "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); } @@ -498,41 +503,24 @@ void LocalStore::queryReferences(const Path & path, } -bool LocalStore::queryReferrersInternal(const Path & path, PathSet & referrers) +void LocalStore::queryReferrers(const Path & path, PathSet & referrers) { -#if 0 - bool allValid = true; - - if (!isValidPath(path)) - throw Error(format("path `%1%' is not valid") % path); - - /* No locking is necessary here: updates are only done by - appending or by atomically replacing the file. When appending, - there is a possibility that we see a partial entry, but it will - just be filtered out below (the partially written path will not - be valid, so it will be ignored). */ - - Path referrersFile = referrersFileFor(path); - if (!pathExists(referrersFile)) return true; - - AutoCloseFD fd = open(referrersFile.c_str(), O_RDONLY); - if (fd == -1) throw SysError(format("opening file `%1%'") % referrersFile); - - Paths refs = tokenizeString(readFile(fd), " "); + assertStorePath(path); - foreach (Paths::iterator, i, refs) - /* Referrers can be invalid (see registerValidPath() for the - invariant), so we only return one if it is valid. */ - if (isStorePath(*i) && isValidPath(*i)) referrers.insert(*i); else allValid = false; + stmtQueryReferrers.reset(); - return allValid; -#endif -} + if (sqlite3_bind_text(stmtQueryReferrers, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + int r; + while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryReferrers, 0); + assert(s); + referrers.insert(s); + } -void LocalStore::queryReferrers(const Path & path, PathSet & referrers) -{ - queryReferrersInternal(path, referrers); + if (r != SQLITE_DONE) + throw Error(format("error getting references of `%1%'") % path); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index af1cd26a3..9eae443d1 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -190,6 +190,7 @@ private: SQLiteStmt stmtAddReference; SQLiteStmt stmtQueryPathInfo; SQLiteStmt stmtQueryReferences; + SQLiteStmt stmtQueryReferrers; int getSchema(); @@ -205,8 +206,6 @@ private: void rewriteReferrers(const Path & path, bool purge, PathSet referrers); - bool queryReferrersInternal(const Path & path, PathSet & referrers); - void invalidatePath(const Path & path); void upgradeStore6(); -- cgit v1.2.3 From 268f9aaf286294c05f34be338ac4ec75e20d1442 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 18 Feb 2010 16:51:27 +0000 Subject: * Implemented queryValidPaths() and verifyStore(). --- src/libstore/local-store.cc | 151 ++++++++++---------------------------------- 1 file changed, 34 insertions(+), 117 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f1deb2eae..7d6fc5e33 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -56,6 +56,7 @@ void SQLiteStmt::create(sqlite3 * db, const string & s) void SQLiteStmt::reset() { + checkInterrupt(); assert(stmt); if (sqlite3_reset(stmt) != SQLITE_OK) throw SQLiteError(db, "resetting statement"); @@ -337,6 +338,7 @@ void LocalStore::registerValidPath(const Path & path, void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidity) { + abort(); #if 0 Path infoFile = infoFileFor(info.path); @@ -396,19 +398,13 @@ void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidi void LocalStore::registerFailedPath(const Path & path) { -#if 0 - /* Write an empty file in the .../failed directory to denote the - failure of the builder for `path'. */ - writeFile(failedFileFor(path), ""); -#endif + abort(); } bool LocalStore::hasPathFailed(const Path & path) { -#if 0 - return pathExists(failedFileFor(path)); -#endif + abort(); } @@ -467,7 +463,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) } if (r != SQLITE_DONE) - throw Error(format("error getting references of `%1%'") % path); + throw SQLiteError(db, format("error getting references of `%1%'") % path); return res; } @@ -487,11 +483,22 @@ bool LocalStore::isValidPath(const Path & path) PathSet LocalStore::queryValidPaths() { - PathSet paths; - Strings entries = readDirectory(nixDBPath + "/info"); - foreach (Strings::iterator, i, entries) - if (i->at(0) != '.') paths.insert(nixStore + "/" + *i); - return paths; + SQLiteStmt stmt; + stmt.create(db, "select path from ValidPaths"); + + PathSet res; + + int r; + while ((r = sqlite3_step(stmt)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmt, 0); + assert(s); + res.insert(s); + } + + if (r != SQLITE_DONE) + throw SQLiteError(db, "error getting valid paths"); + + return res; } @@ -520,7 +527,7 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers) } if (r != SQLITE_DONE) - throw Error(format("error getting references of `%1%'") % path); + throw SQLiteError(db, format("error getting references of `%1%'") % path); } @@ -680,6 +687,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) there are no referrers. */ void LocalStore::invalidatePath(const Path & path) { + abort(); #if 0 debug(format("invalidating path `%1%'") % path); @@ -1002,6 +1010,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed, unsigned long long & blocksFreed) { + abort(); #if 0 bytesFreed = 0; @@ -1028,7 +1037,6 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr void LocalStore::verifyStore(bool checkContents) { -#if 0 /* Check whether all valid paths actually exist. */ printMsg(lvlInfo, "checking path existence"); @@ -1036,68 +1044,26 @@ void LocalStore::verifyStore(bool checkContents) foreach (PathSet::iterator, i, validPaths2) { checkInterrupt(); + /* !!! invalidatePath() will probably fail due to the foreign + key constraints on the Ref table. */ if (!isStorePath(*i)) { printMsg(lvlError, format("path `%1%' is not in the Nix store") % *i); invalidatePath(*i); } else if (!pathExists(*i)) { printMsg(lvlError, format("path `%1%' disappeared") % *i); invalidatePath(*i); - } else { - Path infoFile = infoFileFor(*i); - struct stat st; - if (lstat(infoFile.c_str(), &st)) - throw SysError(format("getting status of `%1%'") % infoFile); - if (st.st_size == 0) { - printMsg(lvlError, format("removing corrupt info file `%1%'") % infoFile); - if (unlink(infoFile.c_str()) == -1) - throw SysError(format("unlinking `%1%'") % infoFile); - } - else validPaths.insert(*i); - } + } else validPaths.insert(*i); } + /* Optionally, check the content hashes (slow). */ + if (checkContents) { + printMsg(lvlInfo, "checking hashes"); - /* Check the store path meta-information. */ - printMsg(lvlInfo, "checking path meta-information"); - - std::map referrersCache; - - foreach (PathSet::iterator, i, validPaths) { - bool update = false; - ValidPathInfo info = queryPathInfo(*i, true); - - /* Check the references: each reference should be valid, and - it should have a matching referrer. */ - foreach (PathSet::iterator, j, info.references) { - if (validPaths.find(*j) == validPaths.end()) { - printMsg(lvlError, format("incomplete closure: `%1%' needs missing `%2%'") - % *i % *j); - /* nothing we can do about it... */ - } else { - if (referrersCache.find(*j) == referrersCache.end()) - queryReferrers(*j, referrersCache[*j]); - if (referrersCache[*j].find(*i) == referrersCache[*j].end()) { - printMsg(lvlError, format("adding missing referrer mapping from `%1%' to `%2%'") - % *j % *i); - appendReferrer(*j, *i, true); - } - } - } + foreach (PathSet::iterator, i, validPaths) { + ValidPathInfo info = queryPathInfo(*i); - /* Check the deriver. (Note that the deriver doesn't have to - be a valid path.) */ - if (!info.deriver.empty() && !isStorePath(info.deriver)) { - info.deriver = ""; - update = true; - } - - /* Check the content hash (optionally - slow). */ - if (info.hash.hashSize == 0) { - printMsg(lvlError, format("re-hashing `%1%'") % *i); - info.hash = hashPath(htSHA256, *i); - update = true; - } else if (checkContents) { - debug(format("checking contents of `%1%'") % *i); + /* Check the content hash (optionally - slow). */ + printMsg(lvlTalkative, format("checking contents of `%1%'") % *i); Hash current = hashPath(info.hash.type, *i); if (current != info.hash) { printMsg(lvlError, format("path `%1%' was modified! " @@ -1105,56 +1071,7 @@ void LocalStore::verifyStore(bool checkContents) % *i % printHash(info.hash) % printHash(current)); } } - - if (update) registerValidPath(info); - } - - referrersCache.clear(); - - - /* Check the referrers. */ - printMsg(lvlInfo, "checking referrers"); - - std::map referencesCache; - - Strings entries = readDirectory(nixDBPath + "/referrer"); - foreach (Strings::iterator, i, entries) { - Path from = nixStore + "/" + *i; - - if (validPaths.find(from) == validPaths.end()) { - /* !!! This removes lock files as well. Need to check - whether that's okay. */ - printMsg(lvlError, format("removing referrers file for invalid `%1%'") % from); - Path p = referrersFileFor(from); - if (unlink(p.c_str()) == -1) - throw SysError(format("unlinking `%1%'") % p); - continue; - } - - PathSet referrers; - bool allValid = queryReferrersInternal(from, referrers); - bool update = false; - - if (!allValid) { - printMsg(lvlError, format("removing some stale referrers for `%1%'") % from); - update = true; - } - - /* Each referrer should have a matching reference. */ - PathSet referrersNew; - foreach (PathSet::iterator, j, referrers) { - if (referencesCache.find(*j) == referencesCache.end()) - queryReferences(*j, referencesCache[*j]); - if (referencesCache[*j].find(from) == referencesCache[*j].end()) { - printMsg(lvlError, format("removing unexpected referrer mapping from `%1%' to `%2%'") - % from % *j); - update = true; - } else referrersNew.insert(*j); - } - - if (update) rewriteReferrers(from, false, referrersNew); } -#endif } -- cgit v1.2.3 From 762cee72ccd860e72c7b639a1dd542ac0f298bb2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Feb 2010 16:04:51 +0000 Subject: * Implement registerValidPath(). --- src/libstore/local-store.cc | 201 ++++++++++++++++++++------------------------ src/libstore/local-store.hh | 6 +- src/libstore/store-api.hh | 1 + 3 files changed, 96 insertions(+), 112 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7d6fc5e33..af240b61c 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -74,6 +74,24 @@ SQLiteStmt::~SQLiteStmt() } +/* Helper class to ensure that prepared statements are reset when + leaving the scope that uses them. Unfinished prepared statements + prevent transactions from being aborted, and can cause locks to be + kept when they should be released. */ +struct SQLiteStmtUse +{ + SQLiteStmt & stmt; + SQLiteStmtUse(SQLiteStmt & stmt) : stmt(stmt) + { + stmt.reset(); + } + ~SQLiteStmtUse() + { + stmt.reset(); + } +}; + + struct SQLiteTxn { bool active; @@ -223,10 +241,10 @@ int LocalStore::getSchema() } -#include "schema.sql.hh" - void LocalStore::initSchema() { +#include "schema.sql.hh" + if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "initialising database schema"); @@ -237,9 +255,9 @@ void LocalStore::initSchema() void LocalStore::prepareStatements() { stmtRegisterValidPath.create(db, - "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); + "insert or replace into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); stmtAddReference.create(db, - "insert into Refs (referrer, reference) values (?, ?);"); + "insert or replace into Refs (referrer, reference) values (?, ?);"); stmtQueryPathInfo.create(db, "select id, hash, registrationTime, deriver from ValidPaths where path = ?;"); stmtQueryReferences.create(db, @@ -336,75 +354,69 @@ void LocalStore::registerValidPath(const Path & path, } -void LocalStore::registerValidPath(const ValidPathInfo & info, bool ignoreValidity) +unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) { - abort(); -#if 0 - Path infoFile = infoFileFor(info.path); - - ValidPathInfo oldInfo; - if (pathExists(infoFile)) oldInfo = queryPathInfo(info.path); - - /* Note that it's possible for infoFile to already exist. */ - - /* Acquire a lock on each referrer file. This prevents those - paths from being invalidated. (It would be a violation of the - store invariants if we registered info.path as valid while some - of its references are invalid.) NB: there can be no deadlock - here since we're acquiring the locks in sorted order. */ - PathSet lockNames; - foreach (PathSet::const_iterator, i, info.references) - if (*i != info.path) lockNames.insert(referrersFileFor(*i)); - PathLocks referrerLocks(lockNames); - referrerLocks.setDeletion(true); - - string refs; - foreach (PathSet::const_iterator, i, info.references) { - if (!refs.empty()) refs += " "; - refs += *i; - - if (!ignoreValidity && *i != info.path && !isValidPath(*i)) - throw Error(format("cannot register `%1%' as valid, because its reference `%2%' isn't valid") - % info.path % *i); - - /* Update the referrer mapping for *i. This must be done - before the info file is written to maintain the invariant - that if `path' is a valid path, then all its references - have referrer mappings back to `path'. A " " is prefixed - to separate it from the previous entry. It's not suffixed - to deal with interrupted partial writes to this file. */ - if (oldInfo.references.find(*i) == oldInfo.references.end()) - appendReferrer(*i, info.path, false); + SQLiteStmtUse use(stmtRegisterValidPath); + if (sqlite3_bind_text(stmtRegisterValidPath, 1, info.path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + string h = "sha256:" + printHash(info.hash); + if (sqlite3_bind_text(stmtRegisterValidPath, 2, h.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + if (sqlite3_bind_int(stmtRegisterValidPath, 3, info.registrationTime) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + if (info.deriver != "") { + if (sqlite3_bind_text(stmtRegisterValidPath, 4, info.deriver.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + } else { + if (sqlite3_bind_null(stmtRegisterValidPath, 4) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); } + if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) + throw SQLiteError(db, format("registering valid path `%1%' in database") % info.path); + return sqlite3_last_insert_rowid(db); +} + +void LocalStore::addReference(unsigned long long referrer, unsigned long long reference) +{ + SQLiteStmtUse use(stmtAddReference); + if (sqlite3_bind_int(stmtAddReference, 1, referrer) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + if (sqlite3_bind_int(stmtAddReference, 2, reference) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); + if (sqlite3_step(stmtAddReference) != SQLITE_DONE) + throw SQLiteError(db, "adding reference to database"); +} + + +void LocalStore::registerValidPath(const ValidPathInfo & info) +{ assert(info.hash.type == htSHA256); + ValidPathInfo info2(info); + if (info2.registrationTime == 0) info2.registrationTime = time(0); + + SQLiteTxn txn(db); + + unsigned long long id = addValidPath(info2); - string s = (format( - "Hash: sha256:%1%\n" - "References: %2%\n" - "Deriver: %3%\n" - "Registered-At: %4%\n") - % printHash(info.hash) % refs % info.deriver % - (oldInfo.registrationTime ? oldInfo.registrationTime : time(0))).str(); - - /* Atomically rewrite the info file. */ - Path tmpFile = tmpFileForAtomicUpdate(infoFile); - writeFile(tmpFile, s, doFsync); - if (rename(tmpFile.c_str(), infoFile.c_str()) == -1) - throw SysError(format("cannot rename `%1%' to `%2%'") % tmpFile % infoFile); -#endif + foreach (PathSet::const_iterator, i, info2.references) { + ValidPathInfo ref = queryPathInfo(*i); + addReference(id, ref.id); + } + + txn.commit(); } void LocalStore::registerFailedPath(const Path & path) { - abort(); + throw Error("not implemented"); } bool LocalStore::hasPathFailed(const Path & path) { - abort(); + throw Error("not implemented"); } @@ -424,13 +436,13 @@ Hash parseHashField(const Path & path, const string & s) ValidPathInfo LocalStore::queryPathInfo(const Path & path) { - ValidPathInfo res; - res.path = path; + ValidPathInfo info; + info.path = path; assertStorePath(path); /* Get the path info. */ - stmtQueryPathInfo.reset(); + SQLiteStmtUse use1(stmtQueryPathInfo); if (sqlite3_bind_text(stmtQueryPathInfo, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throw SQLiteError(db, "binding argument"); @@ -439,39 +451,39 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); if (r != SQLITE_ROW) throw SQLiteError(db, "querying path in database"); - unsigned int id = sqlite3_column_int(stmtQueryPathInfo, 0); + info.id = sqlite3_column_int(stmtQueryPathInfo, 0); const char * s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 1); assert(s); - res.hash = parseHashField(path, s); + info.hash = parseHashField(path, s); - res.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); + info.registrationTime = sqlite3_column_int(stmtQueryPathInfo, 2); s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); - if (s) res.deriver = s; + if (s) info.deriver = s; /* Get the references. */ - stmtQueryReferences.reset(); + SQLiteStmtUse use2(stmtQueryReferences); - if (sqlite3_bind_int(stmtQueryReferences, 1, id) != SQLITE_OK) + if (sqlite3_bind_int(stmtQueryReferences, 1, info.id) != SQLITE_OK) throw SQLiteError(db, "binding argument"); while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); assert(s); - res.references.insert(s); + info.references.insert(s); } if (r != SQLITE_DONE) throw SQLiteError(db, format("error getting references of `%1%'") % path); - return res; + return info; } bool LocalStore::isValidPath(const Path & path) { - stmtQueryPathInfo.reset(); + SQLiteStmtUse use(stmtQueryPathInfo); if (sqlite3_bind_text(stmtQueryPathInfo, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throw SQLiteError(db, "binding argument"); int res = sqlite3_step(stmtQueryPathInfo); @@ -514,7 +526,7 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers) { assertStorePath(path); - stmtQueryReferrers.reset(); + SQLiteStmtUse use(stmtQueryReferrers); if (sqlite3_bind_text(stmtQueryReferrers, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) throw SQLiteError(db, "binding argument"); @@ -687,7 +699,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) there are no referrers. */ void LocalStore::invalidatePath(const Path & path) { - abort(); + throw Error("not implemented"); #if 0 debug(format("invalidating path `%1%'") % path); @@ -1010,7 +1022,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source) void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed, unsigned long long & blocksFreed) { - abort(); + throw Error("not implemented"); + #if 0 bytesFreed = 0; @@ -1077,13 +1090,6 @@ void LocalStore::verifyStore(bool checkContents) /* Functions for upgrading from the pre-SQLite database. */ -static Path infoFileFor(const Path & path) -{ - string baseName = baseNameOf(path); - return (format("%1%/info/%2%") % nixDBPath % baseName).str(); -} - - PathSet LocalStore::queryValidPathsOld() { PathSet paths; @@ -1100,7 +1106,8 @@ ValidPathInfo LocalStore::queryPathInfoOld(const Path & path) res.path = path; /* Read the info file. */ - Path infoFile = infoFileFor(path); + string baseName = baseNameOf(path); + Path infoFile = (format("%1%/info/%2%") % nixDBPath % baseName).str(); if (!pathExists(infoFile)) throw Error(format("path `%1%' is not valid") % path); string info = readFile(infoFile); @@ -1152,27 +1159,7 @@ void LocalStore::upgradeStore6() foreach (PathSet::iterator, i, validPaths) { ValidPathInfo info = queryPathInfoOld(*i); - - stmtRegisterValidPath.reset(); - if (sqlite3_bind_text(stmtRegisterValidPath, 1, i->c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - string h = "sha256:" + printHash(info.hash); - if (sqlite3_bind_text(stmtRegisterValidPath, 2, h.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - if (sqlite3_bind_int(stmtRegisterValidPath, 3, info.registrationTime) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - if (info.deriver != "") { - if (sqlite3_bind_text(stmtRegisterValidPath, 4, info.deriver.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - } else { - if (sqlite3_bind_null(stmtRegisterValidPath, 4) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - } - if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) - throw SQLiteError(db, "registering valid path in database"); - - pathToId[*i] = sqlite3_last_insert_rowid(db); - + pathToId[*i] = addValidPath(info); std::cerr << "."; } @@ -1180,19 +1167,11 @@ void LocalStore::upgradeStore6() foreach (PathSet::iterator, i, validPaths) { ValidPathInfo info = queryPathInfoOld(*i); - foreach (PathSet::iterator, j, info.references) { - stmtAddReference.reset(); - if (sqlite3_bind_int(stmtAddReference, 1, pathToId[*i]) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); if (pathToId.find(*j) == pathToId.end()) throw Error(format("path `%1%' referenced by `%2%' is invalid") % *j % *i); - if (sqlite3_bind_int(stmtAddReference, 2, pathToId[*j]) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - if (sqlite3_step(stmtAddReference) != SQLITE_DONE) - throw SQLiteError(db, "adding reference to database"); + addReference(pathToId[*i], pathToId[*j]); } - std::cerr << "."; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 9eae443d1..cfe26039e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -198,7 +198,11 @@ private: void prepareStatements(); - void registerValidPath(const ValidPathInfo & info, bool ignoreValidity = false); + unsigned long long addValidPath(const ValidPathInfo & info); + + void addReference(unsigned long long referrer, unsigned long long reference); + + void registerValidPath(const ValidPathInfo & info); ValidPathInfo queryPathInfo(const Path & path); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 8506d47e3..23a3ab25e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -333,6 +333,7 @@ struct ValidPathInfo Hash hash; PathSet references; time_t registrationTime; + unsigned long long id; // internal use only ValidPathInfo() : registrationTime(0) { } }; -- cgit v1.2.3 From 9c9a88e9e25bdc4456368a40691e61acf5d3b330 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Feb 2010 16:43:25 +0000 Subject: * Implement more stuff. --- src/libstore/local-store.cc | 97 +++++++++++++++++++++------------------------ src/libstore/local-store.hh | 5 +++ 2 files changed, 50 insertions(+), 52 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index af240b61c..cd25ded72 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -60,6 +60,7 @@ void SQLiteStmt::reset() assert(stmt); if (sqlite3_reset(stmt) != SQLITE_OK) throw SQLiteError(db, "resetting statement"); + curArg = 1; } @@ -74,6 +75,27 @@ SQLiteStmt::~SQLiteStmt() } +void SQLiteStmt::bind(const string & value) +{ + if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); +} + + +void SQLiteStmt::bind(int value) +{ + if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); +} + + +void SQLiteStmt::bind() +{ + if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); +} + + /* Helper class to ensure that prepared statements are reset when leaving the scope that uses them. Unfinished prepared statements prevent transactions from being aborted, and can cause locks to be @@ -264,6 +286,8 @@ void LocalStore::prepareStatements() "select path from Refs join ValidPaths on reference = id where referrer = ?;"); stmtQueryReferrers.create(db, "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); + stmtInvalidatePath.create(db, + "delete from ValidPaths where path = ?;"); } @@ -357,20 +381,13 @@ void LocalStore::registerValidPath(const Path & path, unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) { SQLiteStmtUse use(stmtRegisterValidPath); - if (sqlite3_bind_text(stmtRegisterValidPath, 1, info.path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - string h = "sha256:" + printHash(info.hash); - if (sqlite3_bind_text(stmtRegisterValidPath, 2, h.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - if (sqlite3_bind_int(stmtRegisterValidPath, 3, info.registrationTime) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - if (info.deriver != "") { - if (sqlite3_bind_text(stmtRegisterValidPath, 4, info.deriver.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - } else { - if (sqlite3_bind_null(stmtRegisterValidPath, 4) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - } + stmtRegisterValidPath.bind(info.path); + stmtRegisterValidPath.bind("sha256:" + printHash(info.hash)); + stmtRegisterValidPath.bind(info.registrationTime); + if (info.deriver != "") + stmtRegisterValidPath.bind(info.deriver); + else + stmtRegisterValidPath.bind(); // null if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) throw SQLiteError(db, format("registering valid path `%1%' in database") % info.path); return sqlite3_last_insert_rowid(db); @@ -380,10 +397,8 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) void LocalStore::addReference(unsigned long long referrer, unsigned long long reference) { SQLiteStmtUse use(stmtAddReference); - if (sqlite3_bind_int(stmtAddReference, 1, referrer) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); - if (sqlite3_bind_int(stmtAddReference, 2, reference) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + stmtAddReference.bind(referrer); + stmtAddReference.bind(reference); if (sqlite3_step(stmtAddReference) != SQLITE_DONE) throw SQLiteError(db, "adding reference to database"); } @@ -443,9 +458,8 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) /* Get the path info. */ SQLiteStmtUse use1(stmtQueryPathInfo); - - if (sqlite3_bind_text(stmtQueryPathInfo, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + + stmtQueryPathInfo.bind(path); int r = sqlite3_step(stmtQueryPathInfo); if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); @@ -465,8 +479,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) /* Get the references. */ SQLiteStmtUse use2(stmtQueryReferences); - if (sqlite3_bind_int(stmtQueryReferences, 1, info.id) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + stmtQueryReferences.bind(info.id); while ((r = sqlite3_step(stmtQueryReferences)) == SQLITE_ROW) { s = (const char *) sqlite3_column_text(stmtQueryReferences, 0); @@ -484,8 +497,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) bool LocalStore::isValidPath(const Path & path) { SQLiteStmtUse use(stmtQueryPathInfo); - if (sqlite3_bind_text(stmtQueryPathInfo, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + stmtQueryPathInfo.bind(path); int res = sqlite3_step(stmtQueryPathInfo); if (res != SQLITE_DONE && res != SQLITE_ROW) throw SQLiteError(db, "querying path in database"); @@ -528,8 +540,7 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers) SQLiteStmtUse use(stmtQueryReferrers); - if (sqlite3_bind_text(stmtQueryReferrers, 1, path.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + stmtQueryReferrers.bind(path); int r; while ((r = sqlite3_step(stmtQueryReferrers)) == SQLITE_ROW) { @@ -699,26 +710,17 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) there are no referrers. */ void LocalStore::invalidatePath(const Path & path) { - throw Error("not implemented"); -#if 0 debug(format("invalidating path `%1%'") % path); + + SQLiteStmtUse use(stmtInvalidatePath); - ValidPathInfo info; - - if (pathExists(infoFileFor(path))) { - info = queryPathInfo(path); + stmtInvalidatePath.bind(path); - /* Remove the info file. */ - Path p = infoFileFor(path); - if (unlink(p.c_str()) == -1) - throw SysError(format("unlinking `%1%'") % p); - } + if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE) + throw SQLiteError(db, format("invalidating path `%1%' in database") % path); - /* Remove the referrers file for `path'. */ - Path p = referrersFileFor(path); - if (pathExists(p) && unlink(p.c_str()) == -1) - throw SysError(format("unlinking `%1%'") % p); -#endif + /* Note that the foreign key constraints on the Refs table take + care of deleting the references entries for `path'. */ } @@ -1022,19 +1024,11 @@ Path LocalStore::importPath(bool requireSignature, Source & source) void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFreed, unsigned long long & blocksFreed) { - throw Error("not implemented"); - -#if 0 bytesFreed = 0; assertStorePath(path); if (isValidPath(path)) { - /* Acquire a lock on the referrers file to prevent new - referrers to this path from appearing while we're deleting - it. */ - PathLocks referrersLock(singleton(referrersFileFor(path))); - referrersLock.setDeletion(true); PathSet referrers; queryReferrers(path, referrers); referrers.erase(path); /* ignore self-references */ if (!referrers.empty()) @@ -1044,7 +1038,6 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr } deletePathWrapped(path, bytesFreed, blocksFreed); -#endif } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index cfe26039e..34aa55fc8 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -61,11 +61,15 @@ struct SQLiteStmt { sqlite3 * db; sqlite3_stmt * stmt; + unsigned int curArg; SQLiteStmt() { stmt = 0; } void create(sqlite3 * db, const string & s); void reset(); ~SQLiteStmt(); operator sqlite3_stmt * () { return stmt; } + void bind(const string & value); + void bind(int value); + void bind(); }; @@ -191,6 +195,7 @@ private: SQLiteStmt stmtQueryPathInfo; SQLiteStmt stmtQueryReferences; SQLiteStmt stmtQueryReferrers; + SQLiteStmt stmtInvalidatePath; int getSchema(); -- cgit v1.2.3 From 1930570ad96e47de9e8557a7734c7bfd9f36f942 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 19 Feb 2010 17:15:22 +0000 Subject: * Foreign key support in SQLite is not a persistent setting, so enable it at startup. * Implement negative caching. Now `make check' passes. --- src/libstore/local-store.cc | 21 +++++++++++++++++++-- src/libstore/local-store.hh | 2 ++ src/libstore/schema.sql | 4 +--- 3 files changed, 22 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index cd25ded72..4c1bbb708 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -209,6 +209,9 @@ LocalStore::LocalStore() if (sqlite3_busy_timeout(db, 60000) != SQLITE_OK) throw SQLiteError(db, "setting timeout"); + if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "enabling foreign keys"); + /* !!! check whether sqlite has been built with foreign key support */ @@ -288,6 +291,10 @@ void LocalStore::prepareStatements() "select path from Refs join ValidPaths on referrer = id where reference = (select id from ValidPaths where path = ?);"); stmtInvalidatePath.create(db, "delete from ValidPaths where path = ?;"); + stmtRegisterFailedPath.create(db, + "insert into FailedPaths (path, time) values (?, ?);"); + stmtHasPathFailed.create(db, + "select time from FailedPaths where path = ?;"); } @@ -425,13 +432,23 @@ void LocalStore::registerValidPath(const ValidPathInfo & info) void LocalStore::registerFailedPath(const Path & path) { - throw Error("not implemented"); + if (hasPathFailed(path)) return; + SQLiteStmtUse use(stmtRegisterFailedPath); + stmtRegisterFailedPath.bind(path); + stmtRegisterFailedPath.bind(time(0)); + if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) + throw SQLiteError(db, format("registering failed path `%1%'") % path); } bool LocalStore::hasPathFailed(const Path & path) { - throw Error("not implemented"); + SQLiteStmtUse use(stmtHasPathFailed); + stmtHasPathFailed.bind(path); + int res = sqlite3_step(stmtHasPathFailed); + if (res != SQLITE_DONE && res != SQLITE_ROW) + throw SQLiteError(db, "querying whether path failed"); + return res == SQLITE_ROW; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 34aa55fc8..e1859e68e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -196,6 +196,8 @@ private: SQLiteStmt stmtQueryReferences; SQLiteStmt stmtQueryReferrers; SQLiteStmt stmtInvalidatePath; + SQLiteStmt stmtRegisterFailedPath; + SQLiteStmt stmtHasPathFailed; int getSchema(); diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index dc53f452c..1e707ce1f 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -1,5 +1,3 @@ -pragma foreign_keys = on; - create table if not exists ValidPaths ( id integer primary key autoincrement not null, path text unique not null, @@ -19,7 +17,7 @@ create table if not exists Refs ( create index if not exists IndexReferrer on Refs(referrer); create index if not exists IndexReference on Refs(reference); -create table if not exists FailedDerivations ( +create table if not exists FailedPaths ( path text primary key not null, time integer not null ); -- cgit v1.2.3 From 299ff64812ce166d230f1b630f794be226c7a178 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Feb 2010 11:15:50 +0000 Subject: * Put the derivation outputs in the database. This is useful for the garbage collector. --- src/libstore/local-store.cc | 26 ++++++++++++++++++++++++-- src/libstore/local-store.hh | 1 + src/libstore/schema.sql | 10 ++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4c1bbb708..2f12256db 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -3,9 +3,9 @@ #include "globals.hh" #include "archive.hh" #include "pathlocks.hh" -#include "aterm.hh" #include "derivations-ast.hh" #include "worker-protocol.hh" +#include "derivations.hh" #include #include @@ -295,6 +295,8 @@ void LocalStore::prepareStatements() "insert into FailedPaths (path, time) values (?, ?);"); stmtHasPathFailed.create(db, "select time from FailedPaths where path = ?;"); + stmtAddDerivationOutput.create(db, + "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); } @@ -397,7 +399,27 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) stmtRegisterValidPath.bind(); // null if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) throw SQLiteError(db, format("registering valid path `%1%' in database") % info.path); - return sqlite3_last_insert_rowid(db); + unsigned long long id = sqlite3_last_insert_rowid(db); + + /* If this is a derivation, then store the derivation outputs in + the database. This is useful for the garbage collector: it can + efficiently query whether a path is an output of some + derivation. */ + if (isDerivation(info.path)) { + ATerm t = ATreadFromNamedFile(info.path.c_str()); + if (!t) throw Error(format("cannot read derivation `%1%'") % info.path); + Derivation drv = parseDerivation(t); + foreach (DerivationOutputs::iterator, i, drv.outputs) { + SQLiteStmtUse use(stmtAddDerivationOutput); + stmtAddDerivationOutput.bind(id); + stmtAddDerivationOutput.bind(i->first); + stmtAddDerivationOutput.bind(i->second.path); + if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE) + throw SQLiteError(db, format("adding derivation output for `%1%' in database") % info.path); + } + } + + return id; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index e1859e68e..f10ba18d5 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -198,6 +198,7 @@ private: SQLiteStmt stmtInvalidatePath; SQLiteStmt stmtRegisterFailedPath; SQLiteStmt stmtHasPathFailed; + SQLiteStmt stmtAddDerivationOutput; int getSchema(); diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index 1e707ce1f..682ce5ed7 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -17,6 +17,16 @@ create table if not exists Refs ( create index if not exists IndexReferrer on Refs(referrer); create index if not exists IndexReference on Refs(reference); +create table if not exists DerivationOutputs ( + drv integer not null, + id text not null, -- symbolic output id, usually "out" + path text not null, + primary key (drv, id), + foreign key (drv) references ValidPaths(id) on delete cascade +); + +create index if not exists IndexDerivationOutputs on DerivationOutputs(path); + create table if not exists FailedPaths ( path text primary key not null, time integer not null -- cgit v1.2.3 From 103cfee056cbc8f002929fd5958e305c1a75fe45 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Feb 2010 11:44:17 +0000 Subject: * Revert r19650 (implement gc-keep-outputs by looking for derivations with the same name as the output) and instead use the DerivationOutputs table in the database, which is the correct way to to do things. --- src/libstore/gc.cc | 56 +++------------------------------------------ src/libstore/local-store.cc | 24 +++++++++++++++++++ src/libstore/local-store.hh | 9 ++++++-- src/libstore/store-api.cc | 13 ----------- src/libstore/store-api.hh | 6 ----- 5 files changed, 34 insertions(+), 74 deletions(-) (limited to 'src') diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index f58f691c9..659c636e3 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -416,12 +416,7 @@ struct LocalStore::GCState PathSet busy; bool gcKeepOutputs; bool gcKeepDerivations; - - bool drvsIndexed; - typedef std::multimap DrvsByName; - DrvsByName drvsByName; // derivation paths hashed by name attribute - - GCState(GCResults & results_) : results(results_), drvsIndexed(false) + GCState(GCResults & results_) : results(results_) { } }; @@ -441,42 +436,6 @@ bool LocalStore::isActiveTempFile(const GCState & state, && state.tempRoots.find(string(path, 0, path.size() - suffix.size())) != state.tempRoots.end(); } - -/* Return all the derivations in the Nix store that have `path' as an - output. This function assumes that derivations have the same name - as their outputs. */ -PathSet LocalStore::findDerivers(GCState & state, const Path & path) -{ - PathSet derivers; - - Path deriver = queryDeriver(path); - if (deriver != "") derivers.insert(deriver); - - if (!state.drvsIndexed) { - Paths entries = readDirectory(nixStore); - foreach (Paths::iterator, i, entries) - if (isDerivation(*i)) - state.drvsByName.insert(std::pair( - getNameOfStorePath(*i), nixStore + "/" + *i)); - state.drvsIndexed = true; - } - - string name = getNameOfStorePath(path); - - // Urgh, I should have used Haskell... - std::pair range = - state.drvsByName.equal_range(name); - - for (GCState::DrvsByName::iterator i = range.first; i != range.second; ++i) - if (isValidPath(i->second)) { - Derivation drv = derivationFromPath(i->second); - foreach (DerivationOutputs::iterator, j, drv.outputs) - if (j->second.path == path) derivers.insert(i->second); - } - - return derivers; -} - bool LocalStore::tryToDelete(GCState & state, const Path & path) { @@ -522,18 +481,9 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path) if (!pathExists(path)) return true; /* If gc-keep-outputs is set, then don't delete this path if - its deriver is not garbage. !!! Nix does not reliably - store derivers, so we have to look at all derivations to - determine which of them derive `path'. Since this makes - the garbage collector very slow to start on large Nix - stores, here we just look for all derivations that have the - same name as `path' (where the name is the part of the - filename after the hash, i.e. the `name' attribute of the - derivation). This is somewhat hacky: currently, the - deriver of a path always has the same name as the output, - but this might change in the future. */ + there are derivers of this path that are not garbage. */ if (state.gcKeepOutputs) { - PathSet derivers = findDerivers(state, path); + PathSet derivers = queryValidDerivers(path); foreach (PathSet::iterator, deriver, derivers) { /* Break an infinite recursion if gc-keep-derivations and gc-keep-outputs are both set by tentatively diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 2f12256db..0590b294b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -297,6 +297,8 @@ void LocalStore::prepareStatements() "select time from FailedPaths where path = ?;"); stmtAddDerivationOutput.create(db, "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); + stmtQueryValidDerivers.create(db, + "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); } @@ -599,6 +601,28 @@ Path LocalStore::queryDeriver(const Path & path) } +PathSet LocalStore::queryValidDerivers(const Path & path) +{ + assertStorePath(path); + + SQLiteStmtUse use(stmtQueryValidDerivers); + stmtQueryValidDerivers.bind(path); + + PathSet derivers; + int r; + while ((r = sqlite3_step(stmtQueryValidDerivers)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryValidDerivers, 1); + assert(s); + derivers.insert(s); + } + + if (r != SQLITE_DONE) + throw SQLiteError(db, format("error getting valid derivers of `%1%'") % path); + + return derivers; +} + + void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run) { if (run.pid != -1) return; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index f10ba18d5..6bd47b305 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -103,6 +103,12 @@ public: void queryReferrers(const Path & path, PathSet & referrers); Path queryDeriver(const Path & path); + + /* Return all currently valid derivations that have `path' as an + output. (Note that the result of `queryDeriver()' is the + derivation that was actually used to produce `path', which may + not exist anymore.) */ + PathSet queryValidDerivers(const Path & path); PathSet querySubstitutablePaths(); @@ -199,6 +205,7 @@ private: SQLiteStmt stmtRegisterFailedPath; SQLiteStmt stmtHasPathFailed; SQLiteStmt stmtAddDerivationOutput; + SQLiteStmt stmtQueryValidDerivers; int getSchema(); @@ -228,8 +235,6 @@ private: bool tryToDelete(GCState & state, const Path & path); - PathSet findDerivers(GCState & state, const Path & path); - bool isActiveTempFile(const GCState & state, const Path & path, const string & suffix); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index f0abe61ad..01dd51621 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,7 +1,6 @@ #include "store-api.hh" #include "globals.hh" #include "util.hh" -#include "derivations.hh" #include @@ -53,18 +52,6 @@ Path toStorePath(const Path & path) } -string getNameOfStorePath(const Path & path) -{ - Path::size_type slash = path.rfind('/'); - string p = slash == Path::npos ? path : string(path, slash + 1); - Path::size_type dash = p.find('-'); - assert(dash != Path::npos); - string p2 = string(p, dash + 1); - if (isDerivation(p2)) p2 = string(p2, 0, p2.size() - 4); - return p2; -} - - Path followLinksToStore(const Path & _path) { Path path = absPath(_path); diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 23a3ab25e..d85ae0c9a 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -243,12 +243,6 @@ void checkStoreName(const string & name); Path toStorePath(const Path & path); -/* Get the "name" part of a store path, that is, the part after the - hash and the dash, and with any ".drv" suffix removed - (e.g. /nix/store/-foo-1.2.3.drv => foo-1.2.3). */ -string getNameOfStorePath(const Path & path); - - /* Follow symlinks until we end up with a path in the Nix store. */ Path followLinksToStore(const Path & path); -- cgit v1.2.3 From c4d388add4942f6f99a8df12f4e49149005047e2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Feb 2010 12:44:36 +0000 Subject: * Get derivation outputs from the database instead of the .drv file, which requires more I/O. --- src/libstore/gc.cc | 8 ++++---- src/libstore/local-store.cc | 24 ++++++++++++++++++++++++ src/libstore/local-store.hh | 3 +++ src/libstore/misc.cc | 8 ++++---- src/libstore/remote-store.cc | 6 ++++++ src/libstore/remote-store.hh | 2 ++ src/libstore/store-api.hh | 3 +++ 7 files changed, 46 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 659c636e3..cf073c5d9 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -467,10 +467,10 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path) then don't delete the derivation if any of the outputs are live. */ if (state.gcKeepDerivations && isDerivation(path)) { - Derivation drv = derivationFromPath(path); - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (!tryToDelete(state, i->second.path)) { - printMsg(lvlDebug, format("cannot delete derivation `%1%' because its output is alive") % path); + PathSet outputs = queryDerivationOutputs(path); + foreach (PathSet::iterator, i, outputs) + if (!tryToDelete(state, *i)) { + printMsg(lvlDebug, format("cannot delete derivation `%1%' because its output `%2%' is alive") % path % *i); goto isLive; } } diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0590b294b..0823e785b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -299,6 +299,8 @@ void LocalStore::prepareStatements() "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); stmtQueryValidDerivers.create(db, "select v.id, v.path from DerivationOutputs d join ValidPaths v on d.drv = v.id where d.path = ?;"); + stmtQueryDerivationOutputs.create(db, + "select id, path from DerivationOutputs where drv = ?;"); } @@ -623,6 +625,28 @@ PathSet LocalStore::queryValidDerivers(const Path & path) } +PathSet LocalStore::queryDerivationOutputs(const Path & path) +{ + SQLiteTxn txn(db); + + SQLiteStmtUse use(stmtQueryDerivationOutputs); + stmtQueryDerivationOutputs.bind(queryPathInfo(path).id); + + PathSet outputs; + int r; + while ((r = sqlite3_step(stmtQueryDerivationOutputs)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryDerivationOutputs, 1); + assert(s); + outputs.insert(s); + } + + if (r != SQLITE_DONE) + throw SQLiteError(db, format("error getting outputs of `%1%'") % path); + + return outputs; +} + + void LocalStore::startSubstituter(const Path & substituter, RunningSubstituter & run) { if (run.pid != -1) return; diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 6bd47b305..1a4acbe0e 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -109,6 +109,8 @@ public: derivation that was actually used to produce `path', which may not exist anymore.) */ PathSet queryValidDerivers(const Path & path); + + PathSet queryDerivationOutputs(const Path & path); PathSet querySubstitutablePaths(); @@ -206,6 +208,7 @@ private: SQLiteStmt stmtHasPathFailed; SQLiteStmt stmtAddDerivationOutput; SQLiteStmt stmtQueryValidDerivers; + SQLiteStmt stmtQueryDerivationOutputs; int getSchema(); diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index 2d7d13a0e..f79cb11cc 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -31,10 +31,10 @@ void computeFSClosure(const Path & storePath, store->queryReferences(storePath, references); if (includeOutputs && isDerivation(storePath)) { - Derivation drv = derivationFromPath(storePath); - foreach (DerivationOutputs::iterator, i, drv.outputs) - if (store->isValidPath(i->second.path)) - computeFSClosure(i->second.path, paths, flipDirection, true); + PathSet outputs = store->queryDerivationOutputs(storePath); + foreach (PathSet::iterator, i, outputs) + if (store->isValidPath(*i)) + computeFSClosure(*i, paths, flipDirection, true); } foreach (PathSet::iterator, i, references) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5143143f5..07cb62dc8 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -294,6 +294,12 @@ Path RemoteStore::queryDeriver(const Path & path) } +PathSet RemoteStore::queryDerivationOutputs(const Path & path) +{ + throw Error("not yet implemented"); +} + + Path RemoteStore::addToStore(const Path & _srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) { diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 3d55d23d9..8bab1d8c4 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -37,6 +37,8 @@ public: Path queryDeriver(const Path & path); + PathSet queryDerivationOutputs(const Path & path); + bool hasSubstitutes(const Path & path); bool querySubstitutablePathInfo(const Path & path, diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index d85ae0c9a..b6a8ff40e 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -139,6 +139,9 @@ public: no deriver has been set. */ virtual Path queryDeriver(const Path & path) = 0; + /* Query the outputs of the derivation denoted by `path'. */ + virtual PathSet queryDerivationOutputs(const Path & path) = 0; + /* Query whether a path has substitutes. */ virtual bool hasSubstitutes(const Path & path) = 0; -- cgit v1.2.3 From 9cda61694957f2f0428779319f85f626578d0cf0 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Feb 2010 14:18:55 +0000 Subject: * The database needs a trigger to get rid of self-references to prevent a foreign key constraint violation on the Refs table when deleting a path. --- src/libstore/local-store.cc | 12 ++++++++++-- src/libstore/schema.sql | 10 ++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 0823e785b..7df67555e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -109,7 +109,11 @@ struct SQLiteStmtUse } ~SQLiteStmtUse() { - stmt.reset(); + try { + stmt.reset(); + } catch (...) { + ignoreException(); + } } }; @@ -798,6 +802,8 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) void LocalStore::invalidatePath(const Path & path) { debug(format("invalidating path `%1%'") % path); + + SQLiteTxn txn(db); SQLiteStmtUse use(stmtInvalidatePath); @@ -807,7 +813,9 @@ void LocalStore::invalidatePath(const Path & path) throw SQLiteError(db, format("invalidating path `%1%' in database") % path); /* Note that the foreign key constraints on the Refs table take - care of deleting the references entries for `path'. */ + care of deleting the references entries for `path'. */ + + txn.commit(); } diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index 682ce5ed7..4adad34b7 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -17,6 +17,16 @@ create table if not exists Refs ( create index if not exists IndexReferrer on Refs(referrer); create index if not exists IndexReference on Refs(reference); +-- Paths can refer to themselves, causing a tuple (N, N) in the Refs +-- table. This causes a deletion of the corresponding row in +-- ValidPaths to cause a foreign key constraint violation (due to `on +-- delete restrict' on the `reference' column). Therefore, explicitly +-- get rid of self-references. +create trigger DeleteSelfRefs before delete on ValidPaths + begin + delete from Refs where referrer = old.id and reference = old.id; + end; + create table if not exists DerivationOutputs ( drv integer not null, id text not null, -- symbolic output id, usually "out" -- cgit v1.2.3 From 2b20318b0e968432438a7528b2d11d05585877c2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 22 Feb 2010 14:24:37 +0000 Subject: --- src/libstore/local-store.cc | 4 ---- 1 file changed, 4 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7df67555e..7db2aabe4 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -803,8 +803,6 @@ void LocalStore::invalidatePath(const Path & path) { debug(format("invalidating path `%1%'") % path); - SQLiteTxn txn(db); - SQLiteStmtUse use(stmtInvalidatePath); stmtInvalidatePath.bind(path); @@ -814,8 +812,6 @@ void LocalStore::invalidatePath(const Path & path) /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ - - txn.commit(); } -- cgit v1.2.3 From b4e6d98fc33df055c8230903bb7d9633042a6374 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Feb 2010 22:12:46 +0000 Subject: * configure: flag --with-sqlite. --- src/libstore/Makefile.am | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/libstore/Makefile.am b/src/libstore/Makefile.am index 0140857b8..2f8bb72da 100644 --- a/src/libstore/Makefile.am +++ b/src/libstore/Makefile.am @@ -10,14 +10,15 @@ pkginclude_HEADERS = \ globals.hh references.hh pathlocks.hh \ worker-protocol.hh -libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la @ADDITIONAL_NETWORK_LIBS@ +libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la \ + ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ BUILT_SOURCES = derivations-ast.cc derivations-ast.hh EXTRA_DIST = derivations-ast.def derivations-ast.cc AM_CXXFLAGS = -Wall \ - -I$(srcdir)/.. ${aterm_include} -I$(srcdir)/../libutil + -I$(srcdir)/.. ${aterm_include} ${sqlite_include} -I$(srcdir)/../libutil local-store.lo: schema.sql.hh -- cgit v1.2.3 From 63b09c5e414f9592f52d4f3a19185df6f8c00e42 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 23 Feb 2010 22:31:38 +0000 Subject: --- src/libstore/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/libstore/Makefile.am b/src/libstore/Makefile.am index 2f8bb72da..e7d683499 100644 --- a/src/libstore/Makefile.am +++ b/src/libstore/Makefile.am @@ -15,7 +15,7 @@ libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la \ BUILT_SOURCES = derivations-ast.cc derivations-ast.hh -EXTRA_DIST = derivations-ast.def derivations-ast.cc +EXTRA_DIST = derivations-ast.def derivations-ast.cc schema.sql AM_CXXFLAGS = -Wall \ -I$(srcdir)/.. ${aterm_include} ${sqlite_include} -I$(srcdir)/../libutil -- cgit v1.2.3 From 462bd50aefca3d70e76ccf17f42c2aef4714c95b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 10:57:57 +0000 Subject: * Use normal (rather than full) synchronous mode, which I gather from the description at http://www.sqlite.org/atomiccommit.html should be safe enough. --- src/libstore/local-store.cc | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7db2aabe4..8e5448446 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -219,6 +219,10 @@ LocalStore::LocalStore() /* !!! check whether sqlite has been built with foreign key support */ + /* "Normal" synchronous mode should be safe enough. */ + if (sqlite3_exec(db, "pragma synchronous = normal;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "changing synchronous mode to normal"); + /* Check the current database schema and if necessary do an upgrade. !!! Race condition: several processes could start the upgrade at the same time. */ -- cgit v1.2.3 From 5954eadf6741e0b4cdab23c414d7a6b470cd06db Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 12:16:50 +0000 Subject: * Remove the fdatasync check since it's no longer needed. --- src/libutil/util.cc | 3 +-- src/libutil/util.hh | 6 +----- 2 files changed, 2 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 8c52625a2..3f76be670 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -227,13 +227,12 @@ string readFile(const Path & path) } -void writeFile(const Path & path, const string & s, bool doFsync) +void writeFile(const Path & path, const string & s) { AutoCloseFD fd = open(path.c_str(), O_WRONLY | O_TRUNC | O_CREAT, 0666); if (fd == -1) throw SysError(format("opening file `%1%'") % path); writeFull(fd, (unsigned char *) s.c_str(), s.size()); - if (doFsync) fdatasync(fd); } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 90132da73..a3580b547 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -12,10 +12,6 @@ #include -#ifndef HAVE_FDATASYNC -#define fdatasync fsync -#endif - namespace nix { @@ -66,7 +62,7 @@ string readFile(int fd); string readFile(const Path & path); /* Write a string to a file. */ -void writeFile(const Path & path, const string & s, bool doFsync = false); +void writeFile(const Path & path, const string & s); /* Read a line from a file descriptor. */ string readLine(int fd); -- cgit v1.2.3 From fa6a4fcb11fa6db1c35cd1199ce8d1dc794e6cd4 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 12:18:48 +0000 Subject: * Add ${sqlite_lib} everywhere. Just adding it in `libstore' doesn't work on x86_64 when sqlite is compiled statically. --- src/libstore/Makefile.am | 2 +- src/nix-env/Makefile.am | 2 +- src/nix-hash/Makefile.am | 2 +- src/nix-instantiate/Makefile.am | 2 +- src/nix-store/Makefile.am | 2 +- src/nix-worker/Makefile.am | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/libstore/Makefile.am b/src/libstore/Makefile.am index e7d683499..b19238e7d 100644 --- a/src/libstore/Makefile.am +++ b/src/libstore/Makefile.am @@ -11,7 +11,7 @@ pkginclude_HEADERS = \ worker-protocol.hh libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la \ - ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ + @ADDITIONAL_NETWORK_LIBS@ BUILT_SOURCES = derivations-ast.cc derivations-ast.hh diff --git a/src/nix-env/Makefile.am b/src/nix-env/Makefile.am index 900524f76..35585602e 100644 --- a/src/nix-env/Makefile.am +++ b/src/nix-env/Makefile.am @@ -3,7 +3,7 @@ bin_PROGRAMS = nix-env nix_env_SOURCES = nix-env.cc profiles.cc profiles.hh help.txt nix_env_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \ ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ nix-env.o: help.txt.hh diff --git a/src/nix-hash/Makefile.am b/src/nix-hash/Makefile.am index 350aa8ebd..11d2a2d8d 100644 --- a/src/nix-hash/Makefile.am +++ b/src/nix-hash/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-hash nix_hash_SOURCES = nix-hash.cc help.txt nix_hash_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ nix-hash.o: help.txt.hh diff --git a/src/nix-instantiate/Makefile.am b/src/nix-instantiate/Makefile.am index 3f6671719..6b9a95943 100644 --- a/src/nix-instantiate/Makefile.am +++ b/src/nix-instantiate/Makefile.am @@ -3,7 +3,7 @@ bin_PROGRAMS = nix-instantiate nix_instantiate_SOURCES = nix-instantiate.cc help.txt nix_instantiate_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \ ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ nix-instantiate.o: help.txt.hh diff --git a/src/nix-store/Makefile.am b/src/nix-store/Makefile.am index ca0fec570..d604920dd 100644 --- a/src/nix-store/Makefile.am +++ b/src/nix-store/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-store nix_store_SOURCES = nix-store.cc dotgraph.cc dotgraph.hh help.txt nix_store_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ nix-store.o: help.txt.hh diff --git a/src/nix-worker/Makefile.am b/src/nix-worker/Makefile.am index d1163ce37..a54e5ec21 100644 --- a/src/nix-worker/Makefile.am +++ b/src/nix-worker/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-worker nix_worker_SOURCES = nix-worker.cc help.txt nix_worker_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ nix-worker.o: help.txt.hh -- cgit v1.2.3 From fae0427324269e420d8ea5774f2fab2330acda9d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 12:25:48 +0000 Subject: * ADDITIONAL_NETWORK_LIBS -> LIBS. --- src/libstore/Makefile.am | 3 +-- src/nix-env/Makefile.am | 2 +- src/nix-hash/Makefile.am | 2 +- src/nix-instantiate/Makefile.am | 2 +- src/nix-store/Makefile.am | 2 +- src/nix-worker/Makefile.am | 2 +- 6 files changed, 6 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/libstore/Makefile.am b/src/libstore/Makefile.am index b19238e7d..dbfc1474e 100644 --- a/src/libstore/Makefile.am +++ b/src/libstore/Makefile.am @@ -10,8 +10,7 @@ pkginclude_HEADERS = \ globals.hh references.hh pathlocks.hh \ worker-protocol.hh -libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la \ - @ADDITIONAL_NETWORK_LIBS@ +libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la BUILT_SOURCES = derivations-ast.cc derivations-ast.hh diff --git a/src/nix-env/Makefile.am b/src/nix-env/Makefile.am index 35585602e..07b29c186 100644 --- a/src/nix-env/Makefile.am +++ b/src/nix-env/Makefile.am @@ -3,7 +3,7 @@ bin_PROGRAMS = nix-env nix_env_SOURCES = nix-env.cc profiles.cc profiles.hh help.txt nix_env_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \ ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} nix-env.o: help.txt.hh diff --git a/src/nix-hash/Makefile.am b/src/nix-hash/Makefile.am index 11d2a2d8d..d7c569b68 100644 --- a/src/nix-hash/Makefile.am +++ b/src/nix-hash/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-hash nix_hash_SOURCES = nix-hash.cc help.txt nix_hash_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} nix-hash.o: help.txt.hh diff --git a/src/nix-instantiate/Makefile.am b/src/nix-instantiate/Makefile.am index 6b9a95943..f39331805 100644 --- a/src/nix-instantiate/Makefile.am +++ b/src/nix-instantiate/Makefile.am @@ -3,7 +3,7 @@ bin_PROGRAMS = nix-instantiate nix_instantiate_SOURCES = nix-instantiate.cc help.txt nix_instantiate_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \ ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} nix-instantiate.o: help.txt.hh diff --git a/src/nix-store/Makefile.am b/src/nix-store/Makefile.am index d604920dd..0d0d0e033 100644 --- a/src/nix-store/Makefile.am +++ b/src/nix-store/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-store nix_store_SOURCES = nix-store.cc dotgraph.cc dotgraph.hh help.txt nix_store_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} nix-store.o: help.txt.hh diff --git a/src/nix-worker/Makefile.am b/src/nix-worker/Makefile.am index a54e5ec21..82bbaf1f7 100644 --- a/src/nix-worker/Makefile.am +++ b/src/nix-worker/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-worker nix_worker_SOURCES = nix-worker.cc help.txt nix_worker_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} @ADDITIONAL_NETWORK_LIBS@ + ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} nix-worker.o: help.txt.hh -- cgit v1.2.3 From 90b6352d0a5d08dc7feabcfde92653dd1f6e324b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 12:48:00 +0000 Subject: * Do registerValidPaths() in one transaction, which is much faster. E.g. it cuts the runtime of the referrers test from 50s to 23s. --- src/libstore/local-store.cc | 38 +++++++++----------------------------- 1 file changed, 9 insertions(+), 29 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8e5448446..6e8082776 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -765,39 +765,19 @@ Hash LocalStore::queryPathHash(const Path & path) } -static void dfsVisit(std::map & infos, - const Path & path, PathSet & visited, Paths & sorted) +void LocalStore::registerValidPaths(const ValidPathInfos & infos) { - if (visited.find(path) != visited.end()) return; - visited.insert(path); - - ValidPathInfo & info(infos[path]); + SQLiteTxn txn(db); - foreach (PathSet::iterator, i, info.references) - if (infos.find(*i) != infos.end()) - dfsVisit(infos, *i, visited, sorted); - - sorted.push_back(path); -} + foreach (ValidPathInfos::const_iterator, i, infos) addValidPath(*i); + foreach (ValidPathInfos::const_iterator, i, infos) { + unsigned long long referrer = queryPathInfo(i->path).id; + foreach (PathSet::iterator, j, i->references) + addReference(referrer, queryPathInfo(*j).id); + } -void LocalStore::registerValidPaths(const ValidPathInfos & infos) -{ - std::map infosMap; - - /* Sort the paths topologically under the references relation, so - that if path A is referenced by B, then A is registered before - B. */ - foreach (ValidPathInfos::const_iterator, i, infos) - infosMap[i->path] = *i; - - PathSet visited; - Paths sorted; - foreach (ValidPathInfos::const_iterator, i, infos) - dfsVisit(infosMap, i->path, visited, sorted); - - foreach (Paths::iterator, i, sorted) - registerValidPath(infosMap[*i]); + txn.commit(); } -- cgit v1.2.3 From a3c63d0d6c2570ce3218be809f67dddc5239cdff Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 13:12:57 +0000 Subject: * Disable fsync() in SQLite if the fsync-metadata option is set to false. * Change the default for `fsync-metadata' to true. * Disable `fsync-metadata' in `make check'. --- src/libstore/local-store.cc | 12 +++++++----- src/libstore/local-store.hh | 3 --- 2 files changed, 7 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6e8082776..67a91bb49 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -219,9 +219,13 @@ LocalStore::LocalStore() /* !!! check whether sqlite has been built with foreign key support */ - /* "Normal" synchronous mode should be safe enough. */ - if (sqlite3_exec(db, "pragma synchronous = normal;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "changing synchronous mode to normal"); + /* Whether SQLite should fsync(). "Normal" synchronous mode + should be safe enough. If the user asks for it, don't sync at + all. This can cause database corruption if the system + crashes. */ + string syncMode = queryBoolSetting("fsync-metadata", true) ? "normal" : "off"; + if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "setting synchronous mode"); /* Check the current database schema and if necessary do an upgrade. !!! Race condition: several processes could start @@ -243,8 +247,6 @@ LocalStore::LocalStore() "please upgrade Nix to version 0.12 first."); else if (curSchema < 6) upgradeStore6(); else prepareStatements(); - - doFsync = queryBoolSetting("fsync-metadata", false); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1a4acbe0e..ec0b482ea 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -191,9 +191,6 @@ private: /* Lock file used for upgrading. */ AutoCloseFD globalLock; - /* Whether to do an fsync() after writing Nix metadata. */ - bool doFsync; - /* The SQLite database object. */ SQLite db; -- cgit v1.2.3 From fefd467539683e45e3682a54cefc67ead33eb346 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 13:24:27 +0000 Subject: * `helpText' is now zero-terminated. --- src/nix-env/nix-env.cc | 2 +- src/nix-hash/nix-hash.cc | 2 +- src/nix-instantiate/nix-instantiate.cc | 2 +- src/nix-store/nix-store.cc | 2 +- src/nix-worker/nix-worker.cc | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index 35caf687b..d11be3808 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -70,7 +70,7 @@ typedef void (* Operation) (Globals & globals, void printHelp() { - cout << string((char *) helpText, sizeof helpText); + cout << string((char *) helpText); } diff --git a/src/nix-hash/nix-hash.cc b/src/nix-hash/nix-hash.cc index 1282af070..a70f59e0c 100644 --- a/src/nix-hash/nix-hash.cc +++ b/src/nix-hash/nix-hash.cc @@ -10,7 +10,7 @@ using namespace nix; void printHelp() { - std::cout << string((char *) helpText, sizeof helpText); + std::cout << string((char *) helpText); } diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 3822de5c6..22df2fbe5 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -19,7 +19,7 @@ using namespace nix; void printHelp() { - std::cout << string((char *) helpText, sizeof helpText); + std::cout << string((char *) helpText); } diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 22effc65d..ddf2062c2 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -21,7 +21,7 @@ typedef void (* Operation) (Strings opFlags, Strings opArgs); void printHelp() { - cout << string((char *) helpText, sizeof helpText); + cout << string((char *) helpText); } diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index bd2209c6d..362b9b928 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -765,7 +765,7 @@ void run(Strings args) void printHelp() { - std::cout << string((char *) helpText, sizeof helpText); + std::cout << string((char *) helpText); } -- cgit v1.2.3 From 9fd85c94de90ec91ccf8a4bf04e8911e6e3e483b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 14:22:34 +0000 Subject: * Use `truncate' journal mode, which should be a bit faster. --- src/libstore/local-store.cc | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 67a91bb49..6963ba5a1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -227,6 +227,10 @@ LocalStore::LocalStore() if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "setting synchronous mode"); + /* Use `truncate' journal mode, which should be a bit faster. */ + if (sqlite3_exec(db, "pragma main.journal_mode = truncate;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "setting journal mode"); + /* Check the current database schema and if necessary do an upgrade. !!! Race condition: several processes could start the upgrade at the same time. */ -- cgit v1.2.3 From 6baa2a2f5ef5b11859b95f453338efaf7c5b2724 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 14:39:52 +0000 Subject: --- src/libstore/schema.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index 4adad34b7..7438632ed 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -22,7 +22,7 @@ create index if not exists IndexReference on Refs(reference); -- ValidPaths to cause a foreign key constraint violation (due to `on -- delete restrict' on the `reference' column). Therefore, explicitly -- get rid of self-references. -create trigger DeleteSelfRefs before delete on ValidPaths +create trigger if not exists DeleteSelfRefs before delete on ValidPaths begin delete from Refs where referrer = old.id and reference = old.id; end; -- cgit v1.2.3 From cfe742cfc50e40b590e75200179013dd62b07bde Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 15:07:23 +0000 Subject: * A function to query just the database id of a valid path. --- src/libstore/local-store.cc | 36 ++++++++++++++++++++---------------- src/libstore/local-store.hh | 2 ++ 2 files changed, 22 insertions(+), 16 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6963ba5a1..bfb253bc1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -461,10 +461,8 @@ void LocalStore::registerValidPath(const ValidPathInfo & info) unsigned long long id = addValidPath(info2); - foreach (PathSet::const_iterator, i, info2.references) { - ValidPathInfo ref = queryPathInfo(*i); - addReference(id, ref.id); - } + foreach (PathSet::const_iterator, i, info2.references) + addReference(id, queryValidPathId(*i)); txn.commit(); } @@ -551,6 +549,17 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) } +unsigned long long LocalStore::queryValidPathId(const Path & path) +{ + SQLiteStmtUse use(stmtQueryPathInfo); + stmtQueryPathInfo.bind(path); + int res = sqlite3_step(stmtQueryPathInfo); + if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0); + if (res == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); + throw SQLiteError(db, "querying path in database"); +} + + bool LocalStore::isValidPath(const Path & path) { SQLiteStmtUse use(stmtQueryPathInfo); @@ -644,7 +653,7 @@ PathSet LocalStore::queryDerivationOutputs(const Path & path) SQLiteTxn txn(db); SQLiteStmtUse use(stmtQueryDerivationOutputs); - stmtQueryDerivationOutputs.bind(queryPathInfo(path).id); + stmtQueryDerivationOutputs.bind(queryValidPathId(path)); PathSet outputs; int r; @@ -778,9 +787,9 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) foreach (ValidPathInfos::const_iterator, i, infos) addValidPath(*i); foreach (ValidPathInfos::const_iterator, i, infos) { - unsigned long long referrer = queryPathInfo(i->path).id; + unsigned long long referrer = queryValidPathId(i->path); foreach (PathSet::iterator, j, i->references) - addReference(referrer, queryPathInfo(*j).id); + addReference(referrer, queryValidPathId(*j)); } txn.commit(); @@ -1229,11 +1238,8 @@ void LocalStore::upgradeStore6() SQLiteTxn txn(db); - std::map pathToId; - foreach (PathSet::iterator, i, validPaths) { - ValidPathInfo info = queryPathInfoOld(*i); - pathToId[*i] = addValidPath(info); + addValidPath(queryPathInfoOld(*i)); std::cerr << "."; } @@ -1241,11 +1247,9 @@ void LocalStore::upgradeStore6() foreach (PathSet::iterator, i, validPaths) { ValidPathInfo info = queryPathInfoOld(*i); - foreach (PathSet::iterator, j, info.references) { - if (pathToId.find(*j) == pathToId.end()) - throw Error(format("path `%1%' referenced by `%2%' is invalid") % *j % *i); - addReference(pathToId[*i], pathToId[*j]); - } + unsigned long long referrer = queryValidPathId(*i); + foreach (PathSet::iterator, j, info.references) + addReference(referrer, queryValidPathId(*j)); std::cerr << "."; } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index ec0b482ea..1e4080bec 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -213,6 +213,8 @@ private: void prepareStatements(); + unsigned long long queryValidPathId(const Path & path); + unsigned long long addValidPath(const ValidPathInfo & info); void addReference(unsigned long long referrer, unsigned long long reference); -- cgit v1.2.3 From e33f67ff0b02cc8fc2e0024dd87a6d61467df177 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 16:30:20 +0000 Subject: * Refactor the upgrade / database initialisation logic a bit. --- src/libstore/local-store.cc | 117 +++++++++++++++++++++++--------------------- src/libstore/local-store.hh | 4 +- 2 files changed, 62 insertions(+), 59 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index bfb253bc1..7e205a9bf 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -205,52 +205,43 @@ LocalStore::LocalStore() lockFile(globalLock, ltRead, true); } - /* Open the Nix database. */ - if (sqlite3_open_v2((nixDBPath + "/db.sqlite").c_str(), &db.db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, 0) != SQLITE_OK) - throw Error("cannot open SQLite database"); - - if (sqlite3_busy_timeout(db, 60000) != SQLITE_OK) - throw SQLiteError(db, "setting timeout"); - - if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "enabling foreign keys"); - - /* !!! check whether sqlite has been built with foreign key - support */ - - /* Whether SQLite should fsync(). "Normal" synchronous mode - should be safe enough. If the user asks for it, don't sync at - all. This can cause database corruption if the system - crashes. */ - string syncMode = queryBoolSetting("fsync-metadata", true) ? "normal" : "off"; - if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "setting synchronous mode"); - - /* Use `truncate' journal mode, which should be a bit faster. */ - if (sqlite3_exec(db, "pragma main.journal_mode = truncate;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "setting journal mode"); - /* Check the current database schema and if necessary do an - upgrade. !!! Race condition: several processes could start - the upgrade at the same time. */ + upgrade. */ int curSchema = getSchema(); if (curSchema > nixSchemaVersion) throw Error(format("current Nix store schema is version %1%, but I only support %2%") % curSchema % nixSchemaVersion); - if (curSchema == 0) { /* new store */ + + else if (curSchema == 0) { /* new store */ curSchema = nixSchemaVersion; - initSchema(); + openDB(true); + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + } + + else if (curSchema < nixSchemaVersion) { + if (curSchema < 5) + throw Error( + "Your Nix store has a database in Berkeley DB format,\n" + "which is no longer supported. To convert to the new format,\n" + "please upgrade Nix to version 0.12 first."); + + if (!lockFile(globalLock, ltWrite, false)) { + printMsg(lvlError, "waiting for exclusive access to the Nix store..."); + lockFile(globalLock, ltWrite, true); + } + + /* Get the schema version again, because another process may + have performed the upgrade already. */ + curSchema = getSchema(); + + if (curSchema < 6) upgradeStore6(); + writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); + + lockFile(globalLock, ltRead, true); } - else if (curSchema == 1) throw Error("your Nix store is no longer supported"); - else if (curSchema < 5) - throw Error( - "Your Nix store has a database in Berkeley DB format,\n" - "which is no longer supported. To convert to the new format,\n" - "please upgrade Nix to version 0.12 first."); - else if (curSchema < 6) upgradeStore6(); - else prepareStatements(); + + else openDB(false); } @@ -280,19 +271,42 @@ int LocalStore::getSchema() } -void LocalStore::initSchema() +void LocalStore::openDB(bool create) { -#include "schema.sql.hh" + /* Open the Nix database. */ + if (sqlite3_open_v2((nixDBPath + "/db.sqlite").c_str(), &db.db, + SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) + throw Error("cannot open SQLite database"); - if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "initialising database schema"); + if (sqlite3_busy_timeout(db, 60000) != SQLITE_OK) + throw SQLiteError(db, "setting timeout"); - prepareStatements(); -} + if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "enabling foreign keys"); + + /* !!! check whether sqlite has been built with foreign key + support */ + + /* Whether SQLite should fsync(). "Normal" synchronous mode + should be safe enough. If the user asks for it, don't sync at + all. This can cause database corruption if the system + crashes. */ + string syncMode = queryBoolSetting("fsync-metadata", true) ? "normal" : "off"; + if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "setting synchronous mode"); + /* Use `truncate' journal mode, which should be a bit faster. */ + if (sqlite3_exec(db, "pragma main.journal_mode = truncate;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "setting journal mode"); -void LocalStore::prepareStatements() -{ + /* Initialise the database schema, if necessary. */ + if (create) { +#include "schema.sql.hh" + if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "initialising database schema"); + } + + /* Prepare SQL statements. */ stmtRegisterValidPath.create(db, "insert or replace into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); stmtAddReference.create(db, @@ -1225,14 +1239,9 @@ ValidPathInfo LocalStore::queryPathInfoOld(const Path & path) /* Upgrade from schema 5 (Nix 0.12) to schema 6 (Nix >= 0.15). */ void LocalStore::upgradeStore6() { - if (!lockFile(globalLock, ltWrite, false)) { - printMsg(lvlError, "waiting for exclusive access to the Nix store..."); - lockFile(globalLock, ltWrite, true); - } - printMsg(lvlError, "upgrading Nix store to new schema (this may take a while)..."); - initSchema(); + openDB(true); PathSet validPaths = queryValidPathsOld(); @@ -1256,10 +1265,6 @@ void LocalStore::upgradeStore6() std::cerr << "\n"; txn.commit(); - - writeFile(schemaPath, (format("%1%") % nixSchemaVersion).str()); - - lockFile(globalLock, ltRead, true); } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 1e4080bec..0c5f04158 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -209,9 +209,7 @@ private: int getSchema(); - void initSchema(); - - void prepareStatements(); + void openDB(bool create); unsigned long long queryValidPathId(const Path & path); -- cgit v1.2.3 From af565c348a286fb55ad17f8c3c4233465d32a9f6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 24 Feb 2010 16:44:43 +0000 Subject: * Support read-only access to the database. --- src/libstore/local-store.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 7e205a9bf..95e775ce7 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -173,7 +173,10 @@ LocalStore::LocalStore() schemaPath = nixDBPath + "/schema"; - if (readOnlyMode) return; + if (readOnlyMode) { + openDB(false); + return; + } /* Create missing state directories if they don't already exist. */ createDirs(nixStore); @@ -197,6 +200,7 @@ LocalStore::LocalStore() } catch (SysError & e) { if (e.errNo != EACCES) throw; readOnlyMode = true; + openDB(false); return; } -- cgit v1.2.3 From e42401ee7b0c0b166677595d4fd1255176ba33c2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 25 Feb 2010 15:52:22 +0000 Subject: * Implement RemoteStore::queryDerivationOutputs(). --- src/libstore/remote-store.cc | 6 +++++- src/libstore/worker-protocol.hh | 1 + src/nix-worker/nix-worker.cc | 6 ++++-- 3 files changed, 10 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 07cb62dc8..b438c074f 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -296,7 +296,11 @@ Path RemoteStore::queryDeriver(const Path & path) PathSet RemoteStore::queryDerivationOutputs(const Path & path) { - throw Error("not yet implemented"); + openConnection(); + writeInt(wopQueryDerivationOutputs, to); + writeString(path, to); + processStderr(); + return readStorePaths(from); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index e44c1e36b..9e36aab0f 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -34,6 +34,7 @@ typedef enum { wopSetOptions = 19, wopCollectGarbage = 20, wopQuerySubstitutablePathInfo = 21, + wopQueryDerivationOutputs = 22, } WorkerOp; diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 362b9b928..59ab48776 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -313,14 +313,16 @@ static void performOp(unsigned int clientVersion, } case wopQueryReferences: - case wopQueryReferrers: { + case wopQueryReferrers: + case wopQueryDerivationOutputs: { Path path = readStorePath(from); startWork(); PathSet paths; if (op == wopQueryReferences) store->queryReferences(path, paths); - else + else if (op == wopQueryReferrers) store->queryReferrers(path, paths); + else paths = store->queryDerivationOutputs(path); stopWork(); writeStringSet(paths, to); break; -- cgit v1.2.3 From 24035b98b155dd2d35414bdf015b3d7ab9a57af7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 26 Feb 2010 12:05:01 +0000 Subject: * Implement RemoteStore::queryValidPaths(). --- src/libstore/remote-store.cc | 4 +++- src/libstore/worker-protocol.hh | 1 + src/nix-worker/nix-worker.cc | 8 ++++++++ 3 files changed, 12 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index b438c074f..c5d7975b5 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -214,7 +214,9 @@ bool RemoteStore::isValidPath(const Path & path) PathSet RemoteStore::queryValidPaths() { openConnection(); - throw Error("not implemented"); + writeInt(wopQueryValidPaths, to); + processStderr(); + return readStorePaths(from); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 9e36aab0f..c3096010b 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -35,6 +35,7 @@ typedef enum { wopCollectGarbage = 20, wopQuerySubstitutablePathInfo = 21, wopQueryDerivationOutputs = 22, + wopQueryValidPaths = 23, } WorkerOp; diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 59ab48776..a41fb2e15 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -520,6 +520,14 @@ static void performOp(unsigned int clientVersion, break; } + case wopQueryValidPaths: { + startWork(); + PathSet paths = store->queryValidPaths(); + stopWork(); + writeStringSet(paths, to); + break; + } + default: throw Error(format("invalid operation %1%") % op); } -- cgit v1.2.3 From 594eaddd1157db8abe2e1c47cdf2180f027559ad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Mar 2010 15:58:13 +0000 Subject: * When using the included sqlite/aterm libraries, build with --enable-shared. * In libutil/libstore/libexpr etc., link against sqlite and aterm. * Some more header file hygiene. --- src/libexpr/attr-path.cc | 1 + src/libexpr/common-opts.cc | 1 + src/libexpr/eval.cc | 1 + src/libexpr/eval.hh | 3 ++- src/libexpr/expr-to-xml.hh | 1 - src/libexpr/get-drvs.cc | 1 + src/libexpr/primops.cc | 1 + src/libstore/Makefile.am | 4 ++-- src/libstore/derivations.hh | 2 +- src/libutil/Makefile.am | 2 +- src/libutil/aterm-map.hh | 3 ++- src/nix-env/Makefile.am | 5 +++-- src/nix-env/nix-env.cc | 1 + src/nix-hash/Makefile.am | 2 +- src/nix-instantiate/Makefile.am | 2 +- src/nix-instantiate/nix-instantiate.cc | 1 + src/nix-setuid-helper/Makefile.am | 5 ++--- src/nix-store/Makefile.am | 4 ++-- src/nix-worker/Makefile.am | 4 ++-- 19 files changed, 26 insertions(+), 18 deletions(-) (limited to 'src') diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index e8e4c050c..092d9b1c2 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -1,6 +1,7 @@ #include "attr-path.hh" #include "nixexpr-ast.hh" #include "util.hh" +#include "aterm.hh" namespace nix { diff --git a/src/libexpr/common-opts.cc b/src/libexpr/common-opts.cc index 9e3f8f961..0ef488373 100644 --- a/src/libexpr/common-opts.cc +++ b/src/libexpr/common-opts.cc @@ -2,6 +2,7 @@ #include "../libmain/shared.hh" #include "util.hh" #include "parser.hh" +#include "aterm.hh" namespace nix { diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index cd9c64594..5a0e8bc27 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -6,6 +6,7 @@ #include "derivations.hh" #include "nixexpr-ast.hh" #include "globals.hh" +#include "aterm.hh" #define LocalNoInline(f) static f __attribute__((noinline)); f diff --git a/src/libexpr/eval.hh b/src/libexpr/eval.hh index fed6d3472..00d0e3564 100644 --- a/src/libexpr/eval.hh +++ b/src/libexpr/eval.hh @@ -3,9 +3,10 @@ #include -#include "aterm.hh" #include "nixexpr.hh" +typedef union _ATermList * ATermList; + namespace nix { diff --git a/src/libexpr/expr-to-xml.hh b/src/libexpr/expr-to-xml.hh index 36b8e4042..576a46fc3 100644 --- a/src/libexpr/expr-to-xml.hh +++ b/src/libexpr/expr-to-xml.hh @@ -5,7 +5,6 @@ #include #include "nixexpr.hh" -#include "aterm.hh" namespace nix { diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index 1442d7988..cd5a85e5b 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -1,6 +1,7 @@ #include "get-drvs.hh" #include "nixexpr-ast.hh" #include "util.hh" +#include "aterm.hh" namespace nix { diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index 7dddc91a8..bf2752d0d 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -8,6 +8,7 @@ #include "nixexpr-ast.hh" #include "parser.hh" #include "names.hh" +#include "aterm.hh" #include #include diff --git a/src/libstore/Makefile.am b/src/libstore/Makefile.am index dbfc1474e..dd6760554 100644 --- a/src/libstore/Makefile.am +++ b/src/libstore/Makefile.am @@ -10,14 +10,14 @@ pkginclude_HEADERS = \ globals.hh references.hh pathlocks.hh \ worker-protocol.hh -libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la +libstore_la_LIBADD = ../libutil/libutil.la ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} BUILT_SOURCES = derivations-ast.cc derivations-ast.hh EXTRA_DIST = derivations-ast.def derivations-ast.cc schema.sql AM_CXXFLAGS = -Wall \ - -I$(srcdir)/.. ${aterm_include} ${sqlite_include} -I$(srcdir)/../libutil + -I$(srcdir)/.. ${aterm_include} ${sqlite_include} -I$(srcdir)/../libutil -I${top_srcdir}/externals/sqlite-3.6.22/ local-store.lo: schema.sql.hh diff --git a/src/libstore/derivations.hh b/src/libstore/derivations.hh index 042f4738d..c3f579bee 100644 --- a/src/libstore/derivations.hh +++ b/src/libstore/derivations.hh @@ -1,7 +1,7 @@ #ifndef __DERIVATIONS_H #define __DERIVATIONS_H -#include +typedef union _ATerm * ATerm; #include "hash.hh" diff --git a/src/libutil/Makefile.am b/src/libutil/Makefile.am index bd0996543..f17236d29 100644 --- a/src/libutil/Makefile.am +++ b/src/libutil/Makefile.am @@ -3,7 +3,7 @@ pkglib_LTLIBRARIES = libutil.la libutil_la_SOURCES = util.cc hash.cc serialise.cc \ archive.cc aterm.cc aterm-map.cc xml-writer.cc -libutil_la_LIBADD = ../boost/format/libformat.la +libutil_la_LIBADD = ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} pkginclude_HEADERS = util.hh hash.hh serialise.hh \ archive.hh aterm.hh aterm-map.hh xml-writer.hh types.hh diff --git a/src/libutil/aterm-map.hh b/src/libutil/aterm-map.hh index b732453a7..bcfefd9ee 100644 --- a/src/libutil/aterm-map.hh +++ b/src/libutil/aterm-map.hh @@ -1,7 +1,8 @@ #ifndef __ATERM_MAP_H #define __ATERM_MAP_H -#include +typedef union _ATerm * ATerm; + #include diff --git a/src/nix-env/Makefile.am b/src/nix-env/Makefile.am index 07b29c186..53fea3d9d 100644 --- a/src/nix-env/Makefile.am +++ b/src/nix-env/Makefile.am @@ -3,7 +3,7 @@ bin_PROGRAMS = nix-env nix_env_SOURCES = nix-env.cc profiles.cc profiles.hh help.txt nix_env_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \ ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} + ../boost/format/libformat.la nix-env.o: help.txt.hh @@ -11,6 +11,7 @@ nix-env.o: help.txt.hh ../bin2c/bin2c helpText < $< > $@ || (rm $@ && exit 1) AM_CXXFLAGS = \ - -I$(srcdir)/.. ${aterm_include} \ + ${aterm_include} \ + -I$(srcdir)/.. \ -I$(srcdir)/../libutil -I$(srcdir)/../libstore \ -I$(srcdir)/../libexpr -I$(srcdir)/../libmain -I../libexpr diff --git a/src/nix-env/nix-env.cc b/src/nix-env/nix-env.cc index d11be3808..d3f74c7ed 100644 --- a/src/nix-env/nix-env.cc +++ b/src/nix-env/nix-env.cc @@ -14,6 +14,7 @@ #include "xml-writer.hh" #include "store-api.hh" #include "util.hh" +#include "aterm.hh" #include #include diff --git a/src/nix-hash/Makefile.am b/src/nix-hash/Makefile.am index d7c569b68..a4fdb3246 100644 --- a/src/nix-hash/Makefile.am +++ b/src/nix-hash/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-hash nix_hash_SOURCES = nix-hash.cc help.txt nix_hash_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} + ../boost/format/libformat.la nix-hash.o: help.txt.hh diff --git a/src/nix-instantiate/Makefile.am b/src/nix-instantiate/Makefile.am index f39331805..1c6d30873 100644 --- a/src/nix-instantiate/Makefile.am +++ b/src/nix-instantiate/Makefile.am @@ -3,7 +3,7 @@ bin_PROGRAMS = nix-instantiate nix_instantiate_SOURCES = nix-instantiate.cc help.txt nix_instantiate_LDADD = ../libmain/libmain.la ../libexpr/libexpr.la \ ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} + ../boost/format/libformat.la nix-instantiate.o: help.txt.hh diff --git a/src/nix-instantiate/nix-instantiate.cc b/src/nix-instantiate/nix-instantiate.cc index 22df2fbe5..d8b39eca2 100644 --- a/src/nix-instantiate/nix-instantiate.cc +++ b/src/nix-instantiate/nix-instantiate.cc @@ -11,6 +11,7 @@ #include "util.hh" #include "store-api.hh" #include "common-opts.hh" +#include "aterm.hh" #include "help.txt.hh" diff --git a/src/nix-setuid-helper/Makefile.am b/src/nix-setuid-helper/Makefile.am index a0fbdf39d..a04701636 100644 --- a/src/nix-setuid-helper/Makefile.am +++ b/src/nix-setuid-helper/Makefile.am @@ -2,7 +2,6 @@ libexec_PROGRAMS = nix-setuid-helper nix_setuid_helper_SOURCES = nix-setuid-helper.cc nix_setuid_helper_LDADD = ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} + ../boost/format/libformat.la -AM_CXXFLAGS = \ - -I$(srcdir)/.. $(aterm_include) -I$(srcdir)/../libutil +AM_CXXFLAGS = -I$(srcdir)/.. -I$(srcdir)/../libutil diff --git a/src/nix-store/Makefile.am b/src/nix-store/Makefile.am index 0d0d0e033..e0ba809dc 100644 --- a/src/nix-store/Makefile.am +++ b/src/nix-store/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-store nix_store_SOURCES = nix-store.cc dotgraph.cc dotgraph.hh help.txt nix_store_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} + ../boost/format/libformat.la nix-store.o: help.txt.hh @@ -10,5 +10,5 @@ nix-store.o: help.txt.hh ../bin2c/bin2c helpText < $< > $@ || (rm $@ && exit 1) AM_CXXFLAGS = \ - -I$(srcdir)/.. $(aterm_include) -I$(srcdir)/../libutil \ + -I$(srcdir)/.. -I$(srcdir)/../libutil \ -I$(srcdir)/../libstore -I$(srcdir)/../libmain diff --git a/src/nix-worker/Makefile.am b/src/nix-worker/Makefile.am index 82bbaf1f7..b6094a2a0 100644 --- a/src/nix-worker/Makefile.am +++ b/src/nix-worker/Makefile.am @@ -2,7 +2,7 @@ bin_PROGRAMS = nix-worker nix_worker_SOURCES = nix-worker.cc help.txt nix_worker_LDADD = ../libmain/libmain.la ../libstore/libstore.la ../libutil/libutil.la \ - ../boost/format/libformat.la ${aterm_lib} ${sqlite_lib} + ../boost/format/libformat.la nix-worker.o: help.txt.hh @@ -10,5 +10,5 @@ nix-worker.o: help.txt.hh ../bin2c/bin2c helpText < $< > $@ || (rm $@ && exit 1) AM_CXXFLAGS = \ - -I$(srcdir)/.. $(aterm_include) -I$(srcdir)/../libutil \ + -I$(srcdir)/.. -I$(srcdir)/../libutil \ -I$(srcdir)/../libstore -I$(srcdir)/../libmain -- cgit v1.2.3 From 3f9e647ae8c4928a8fc4de0d704119245f58ff45 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Mar 2010 19:04:17 +0000 Subject: * checkInterrupt() shouldn't be called from a destructor. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 95e775ce7..8a690b913 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -47,6 +47,7 @@ SQLite::~SQLite() void SQLiteStmt::create(sqlite3 * db, const string & s) { + checkInterrupt(); assert(!stmt); if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) throw SQLiteError(db, "creating statement"); @@ -56,7 +57,6 @@ void SQLiteStmt::create(sqlite3 * db, const string & s) void SQLiteStmt::reset() { - checkInterrupt(); assert(stmt); if (sqlite3_reset(stmt) != SQLITE_OK) throw SQLiteError(db, "resetting statement"); -- cgit v1.2.3 From 56af8e86e3e00d8417838f582221e60e1d5f5fdb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Mar 2010 20:09:12 +0000 Subject: * Protect the `true', `false' and `__overrides' constants. Without an ATprotect call, these could be garbage collected, leading to weird crashes or wrong results. --- src/aterm-helper.pl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/aterm-helper.pl b/src/aterm-helper.pl index 9b2bde700..f1eb77ee8 100755 --- a/src/aterm-helper.pl +++ b/src/aterm-helper.pl @@ -146,8 +146,9 @@ while () { my $value = $2; print HEADER "extern ATerm $name;\n"; print IMPL "ATerm $name = 0;\n"; - $init .= " $name = $value;\n"; - } + $init .= " $name = $value;\n"; + $init .= " ATprotect(&$name);\n"; + } elsif (/^\s*init\s+(\w+)\s*$/) { $initFun = $1; -- cgit v1.2.3 From e14e2399ed5b1ffc30f08d1f30f19d2ceb24dabb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 2 Mar 2010 20:23:42 +0000 Subject: * Prevent a potential memory corruption problem if an ATerm garbage collection happens during fixAttrs(). --- src/libexpr/parser.y | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index c4afb72ea..8706ce025 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -85,6 +85,10 @@ static Expr fixAttrs(bool recursive, ATermList as) { Tree attrs; + /* This ATermMap is needed to ensure that the `leaf' fields in the + Tree nodes are not garbage collected. */ + ATermMap gcRoots; + for (ATermIterator i(as); i; ++i) { ATermList names, attrPath; Expr src, e; ATerm name, pos; @@ -95,7 +99,9 @@ static Expr fixAttrs(bool recursive, ATermList as) throw ParseError(format("duplicate definition of attribute `%1%' at %2%") % showAttrPath(ATmakeList1(*j)) % showPos(pos)); Tree & t(attrs.children[*j]); - t.leaf = fromScope ? makeVar(*j) : makeSelect(src, *j); + Expr leaf = fromScope ? makeVar(*j) : makeSelect(src, *j); + gcRoots.set(leaf, leaf); + t.leaf = leaf; t.pos = pos; if (recursive && fromScope) t.recursive = false; } -- cgit v1.2.3 From bc6f7fc139b5a72306a54c89db74bf126cca9ca7 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 4 Mar 2010 13:03:26 +0000 Subject: * Remove some unused functions. --- src/libstore/store-api.hh | 18 ------------------ 1 file changed, 18 deletions(-) (limited to 'src') diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b6a8ff40e..095fdd24b 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -112,29 +112,11 @@ public: virtual void queryReferences(const Path & path, PathSet & references) = 0; - /* Like queryReferences, but with self-references filtered out. */ - PathSet queryReferencesNoSelf(const Path & path) - { - PathSet res; - queryReferences(path, res); - res.erase(path); - return res; - } - /* Queries the set of incoming FS references for a store path. The result is not cleared. */ virtual void queryReferrers(const Path & path, PathSet & referrers) = 0; - /* Like queryReferrers, but with self-references filtered out. */ - PathSet queryReferrersNoSelf(const Path & path) - { - PathSet res; - queryReferrers(path, res); - res.erase(path); - return res; - } - /* Query the deriver of a store path. Return the empty string if no deriver has been set. */ virtual Path queryDeriver(const Path & path) = 0; -- cgit v1.2.3 From 04791840f4dd4d6bcc96aea133e9fda7c03897de Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 5 Mar 2010 12:54:58 +0000 Subject: * Emit warning='1' or error='1' attributes for lines marked as warnings or errors with \e[w or \e[e. --- src/nix-log2xml/log2xml.cc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'src') diff --git a/src/nix-log2xml/log2xml.cc b/src/nix-log2xml/log2xml.cc index b2a25eefa..6645dc500 100644 --- a/src/nix-log2xml/log2xml.cc +++ b/src/nix-log2xml/log2xml.cc @@ -18,6 +18,8 @@ struct Decoder int priority; bool ignoreLF; int lineNo, charNo; + bool warning; + bool error; Decoder() { @@ -29,6 +31,8 @@ struct Decoder ignoreLF = false; lineNo = 1; charNo = 0; + warning = false; + error = false; } void pushChar(char c); @@ -95,6 +99,12 @@ void Decoder::pushChar(char c) case 'b': ignoreLF = false; break; + case 'e': + error = true; + break; + case 'w': + warning = true; + break; } } else if (c >= '0' && c <= '9') { int n = 0; @@ -118,6 +128,8 @@ void Decoder::finishLine() string tag = inHeader ? "head" : "line"; cout << "<" << tag; if (priority != 1) cout << " priority='" << priority << "'"; + if (warning) cout << " warning='1'"; + if (error) cout << " error='1'"; cout << ">"; for (unsigned int i = 0; i < line.size(); i++) { @@ -158,6 +170,8 @@ void Decoder::finishLine() line = ""; inHeader = false; priority = 1; + warning = false; + error = false; } -- cgit v1.2.3 From 2e4ef03aa3247782339f3d5af8547b448d38b8d2 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Mar 2010 10:35:45 +0000 Subject: * Increase the sqlite timeout. --- src/libstore/local-store.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 8a690b913..c7232056f 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -282,7 +282,7 @@ void LocalStore::openDB(bool create) SQLITE_OPEN_READWRITE | (create ? SQLITE_OPEN_CREATE : 0), 0) != SQLITE_OK) throw Error("cannot open SQLite database"); - if (sqlite3_busy_timeout(db, 60000) != SQLITE_OK) + if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) throw SQLiteError(db, "setting timeout"); if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK) -- cgit v1.2.3 From 44f6e6de77dd318800775d594b1f33cffa2be9a5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 8 Mar 2010 21:31:42 +0000 Subject: * Set `gc-keep-outputs' or `gc-keep-derivations' to false with `--delete --ignore-liveness'. --- src/libstore/gc.cc | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'src') diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index cf073c5d9..87e0a05bc 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -563,6 +563,15 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) state.gcKeepOutputs = queryBoolSetting("gc-keep-outputs", false); state.gcKeepDerivations = queryBoolSetting("gc-keep-derivations", true); + + /* Using `--ignore-liveness' with `--delete' can have unintended + consequences if `gc-keep-outputs' or `gc-keep-derivations' are + true (the garbage collector will recurse into deleting the + outputs or derivers, respectively). So disable them. */ + if (options.action == GCOptions::gcDeleteSpecific && options.ignoreLiveness) { + state.gcKeepOutputs = false; + state.gcKeepDerivations = false; + } /* Acquire the global GC root. This prevents a) New roots from being added. -- cgit v1.2.3 From 4c356acd044dffbf459ac895b483b49959042931 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 9 Mar 2010 14:32:03 +0000 Subject: * In `nix-store --export', abort if the contents of a path has changed. This prevents corrupt paths from spreading to other machines. Note that checking the hash is cheap because we're hashing anyway (because of the --sign feature). --- src/libstore/local-store.cc | 23 +++++++++++++++++------ src/libutil/hash.cc | 7 +++++++ src/libutil/hash.hh | 3 ++- 3 files changed, 26 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c7232056f..ff7196644 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -930,16 +930,19 @@ struct HashAndWriteSink : Sink { Sink & writeSink; HashSink hashSink; - bool hashing; HashAndWriteSink(Sink & writeSink) : writeSink(writeSink), hashSink(htSHA256) { - hashing = true; } virtual void operator () (const unsigned char * data, unsigned int len) { writeSink(data, len); - if (hashing) hashSink(data, len); + hashSink(data, len); + } + Hash currentHash() + { + HashSink hashSinkClone(hashSink); + return hashSinkClone.finish(); } }; @@ -970,6 +973,15 @@ void LocalStore::exportPath(const Path & path, bool sign, dumpPath(path, hashAndWriteSink); + /* Refuse to export paths that have changed. This prevents + filesystem corruption from spreading to other machines. */ + Hash hash = hashAndWriteSink.currentHash(); + Hash storedHash = queryPathHash(path); + if (hash != storedHash) + throw Error(format("hash of path `%1%' has changed from `%2%' to `%3%'!") % path + % printHash(storedHash) % printHash(hash)); + printMsg(lvlError, printHash(hash)); + writeInt(EXPORT_MAGIC, hashAndWriteSink); writeString(path, hashAndWriteSink); @@ -982,9 +994,8 @@ void LocalStore::exportPath(const Path & path, bool sign, writeString(deriver, hashAndWriteSink); if (sign) { - Hash hash = hashAndWriteSink.hashSink.finish(); - hashAndWriteSink.hashing = false; - + Hash hash = hashAndWriteSink.currentHash(); + writeInt(1, hashAndWriteSink); Path tmpDir = createTempDir(); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index eef01fe4d..bd7e33a48 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -289,6 +289,13 @@ HashSink::HashSink(HashType ht) : ht(ht) start(ht, *ctx); } +HashSink::HashSink(const HashSink & h) +{ + ht = h.ht; + ctx = new Ctx; + *ctx = *h.ctx; +} + HashSink::~HashSink() { delete ctx; diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 062d97254..81425b234 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -96,6 +96,7 @@ private: public: HashSink(HashType ht); + HashSink(const HashSink & h); ~HashSink(); virtual void operator () (const unsigned char * data, unsigned int len); Hash finish(); @@ -104,5 +105,5 @@ public: } - + #endif /* !__HASH_H */ -- cgit v1.2.3 From 1a65142ec40c92b2df9973a1fc8d948e146e05d8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 10 Mar 2010 12:46:25 +0000 Subject: * Remove a debug statement. --- src/libstore/local-store.cc | 1 - 1 file changed, 1 deletion(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index ff7196644..4d82547c6 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -980,7 +980,6 @@ void LocalStore::exportPath(const Path & path, bool sign, if (hash != storedHash) throw Error(format("hash of path `%1%' has changed from `%2%' to `%3%'!") % path % printHash(storedHash) % printHash(hash)); - printMsg(lvlError, printHash(hash)); writeInt(EXPORT_MAGIC, hashAndWriteSink); -- cgit v1.2.3 From 2398af13c53217b5de5821bac0f0c44e9081c23d Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Apr 2010 12:43:42 +0000 Subject: * Add an command `nix-store --query-failed-paths' to list the cached failed paths (when using the `build-cache-failure' option). --- src/libstore/local-store.cc | 21 +++++++++++++++++++++ src/libstore/local-store.hh | 4 ++++ src/nix-store/nix-store.cc | 17 ++++++++++++++--- 3 files changed, 39 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 4d82547c6..f93ba3639 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -327,6 +327,8 @@ void LocalStore::openDB(bool create) "insert into FailedPaths (path, time) values (?, ?);"); stmtHasPathFailed.create(db, "select time from FailedPaths where path = ?;"); + stmtQueryFailedPaths.create(db, + "select path from FailedPaths;"); stmtAddDerivationOutput.create(db, "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); stmtQueryValidDerivers.create(db, @@ -508,6 +510,25 @@ bool LocalStore::hasPathFailed(const Path & path) } +PathSet LocalStore::queryFailedPaths() +{ + SQLiteStmtUse use(stmtQueryFailedPaths); + + PathSet res; + int r; + while ((r = sqlite3_step(stmtQueryFailedPaths)) == SQLITE_ROW) { + const char * s = (const char *) sqlite3_column_text(stmtQueryFailedPaths, 0); + assert(s); + res.insert(s); + } + + if (r != SQLITE_DONE) + throw SQLiteError(db, "error querying failed paths"); + + return res; +} + + Hash parseHashField(const Path & path, const string & s) { string::size_type colon = s.find(':'); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 0c5f04158..70fc64fdc 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -184,6 +184,9 @@ public: /* Query whether `path' previously failed to build. */ bool hasPathFailed(const Path & path); + /* Return the set of paths that have failed to build.*/ + PathSet queryFailedPaths(); + private: Path schemaPath; @@ -203,6 +206,7 @@ private: SQLiteStmt stmtInvalidatePath; SQLiteStmt stmtRegisterFailedPath; SQLiteStmt stmtHasPathFailed; + SQLiteStmt stmtQueryFailedPaths; SQLiteStmt stmtAddDerivationOutput; SQLiteStmt stmtQueryValidDerivers; SQLiteStmt stmtQueryDerivationOutputs; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index ddf2062c2..34bbbcd9f 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -33,7 +33,7 @@ static bool indirectRoot = false; LocalStore & ensureLocalStore() { LocalStore * store2(dynamic_cast(store.get())); - if (!store2) throw Error("you don't have sufficient rights to use --verify"); + if (!store2) throw Error("you don't have sufficient rights to use this command"); return *store2; } @@ -651,8 +651,7 @@ static void opOptimise(Strings opFlags, Strings opArgs) bool dryRun = false; - for (Strings::iterator i = opFlags.begin(); - i != opFlags.end(); ++i) + foreach (Strings::iterator, i, opFlags) if (*i == "--dry-run") dryRun = true; else throw UsageError(format("unknown flag `%1%'") % *i); @@ -667,6 +666,16 @@ static void opOptimise(Strings opFlags, Strings opArgs) } +static void opQueryFailedPaths(Strings opFlags, Strings opArgs) +{ + if (!opArgs.empty() || !opFlags.empty()) + throw UsageError("no arguments expected"); + PathSet failed = ensureLocalStore().queryFailedPaths(); + foreach (PathSet::iterator, i, failed) + cout << format("%1%\n") % *i; +} + + /* Scan the arguments; find the operation, set global flags, put all other flags in a list, and put all other arguments in another list. */ @@ -718,6 +727,8 @@ void run(Strings args) op = opVerify; else if (arg == "--optimise") op = opOptimise; + else if (arg == "--query-failed-paths") + op = opQueryFailedPaths; else if (arg == "--add-root") { if (i == args.end()) throw UsageError("`--add-root requires an argument"); -- cgit v1.2.3 From 6199f9b93ee234139906792c8c1b4908a35df146 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Apr 2010 12:56:42 +0000 Subject: * Added a command `nix-store --clear-failed-paths ' to clear the "failed" status of the given store paths. The special value `*' clears all failed paths. --- src/libstore/local-store.cc | 17 +++++++++++++++++ src/libstore/local-store.hh | 5 +++++ src/nix-store/nix-store.cc | 12 +++++++++++- 3 files changed, 33 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index f93ba3639..633b3599e 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -329,6 +329,8 @@ void LocalStore::openDB(bool create) "select time from FailedPaths where path = ?;"); stmtQueryFailedPaths.create(db, "select path from FailedPaths;"); + stmtClearFailedPath.create(db, + "delete from FailedPaths where ?1 = '*' or path = ?1;"); stmtAddDerivationOutput.create(db, "insert or replace into DerivationOutputs (drv, id, path) values (?, ?, ?);"); stmtQueryValidDerivers.create(db, @@ -529,6 +531,21 @@ PathSet LocalStore::queryFailedPaths() } +void LocalStore::clearFailedPaths(const PathSet & paths) +{ + SQLiteTxn txn(db); + + foreach (PathSet::const_iterator, i, paths) { + SQLiteStmtUse use(stmtClearFailedPath); + stmtClearFailedPath.bind(*i); + if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE) + throw SQLiteError(db, format("clearing failed path `%1%' in database") % *i); + } + + txn.commit(); +} + + Hash parseHashField(const Path & path, const string & s) { string::size_type colon = s.find(':'); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 70fc64fdc..2fd640e39 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -187,6 +187,10 @@ public: /* Return the set of paths that have failed to build.*/ PathSet queryFailedPaths(); + /* Clear the "failed" status of the given paths. The special + value `*' causes all failed paths to be cleared. */ + void clearFailedPaths(const PathSet & paths); + private: Path schemaPath; @@ -207,6 +211,7 @@ private: SQLiteStmt stmtRegisterFailedPath; SQLiteStmt stmtHasPathFailed; SQLiteStmt stmtQueryFailedPaths; + SQLiteStmt stmtClearFailedPath; SQLiteStmt stmtAddDerivationOutput; SQLiteStmt stmtQueryValidDerivers; SQLiteStmt stmtQueryDerivationOutputs; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 34bbbcd9f..148fd6add 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -666,7 +666,7 @@ static void opOptimise(Strings opFlags, Strings opArgs) } -static void opQueryFailedPaths(Strings opFlags, Strings opArgs) +static void opQueryFailedPaths(Strings opFlags, Strings opArgs) { if (!opArgs.empty() || !opFlags.empty()) throw UsageError("no arguments expected"); @@ -676,6 +676,14 @@ static void opQueryFailedPaths(Strings opFlags, Strings opArgs) } +static void opClearFailedPaths(Strings opFlags, Strings opArgs) +{ + if (!opFlags.empty()) + throw UsageError("no flags expected"); + ensureLocalStore().clearFailedPaths(PathSet(opArgs.begin(), opArgs.end())); +} + + /* Scan the arguments; find the operation, set global flags, put all other flags in a list, and put all other arguments in another list. */ @@ -729,6 +737,8 @@ void run(Strings args) op = opOptimise; else if (arg == "--query-failed-paths") op = opQueryFailedPaths; + else if (arg == "--clear-failed-paths") + op = opClearFailedPaths; else if (arg == "--add-root") { if (i == args.end()) throw UsageError("`--add-root requires an argument"); -- cgit v1.2.3 From ef337f7089e484929be92114dac5455d00cebb45 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 26 Apr 2010 12:58:12 +0000 Subject: --- src/nix-store/help.txt | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/nix-store/help.txt b/src/nix-store/help.txt index 7576e3ef5..8022bf7c7 100644 --- a/src/nix-store/help.txt +++ b/src/nix-store/help.txt @@ -27,6 +27,9 @@ Operations: --verify: verify Nix structures --optimise: optimise the Nix store by hard-linking identical files + --query-failed-paths: list paths that failed to build (if enabled) + --clear-failed-paths: clear the failed status of the given paths + --version: output version information --help: display help -- cgit v1.2.3 From f92c9a0ac585d30e245c6667cbce4b035659cb11 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 May 2010 10:45:10 +0000 Subject: * Allow unprivileged users to do `nix-store --clear-failed-paths' and `nix-store --query-failed-paths'. --- src/libstore/local-store.hh | 3 --- src/libstore/remote-store.cc | 19 +++++++++++++++++++ src/libstore/remote-store.hh | 4 ++++ src/libstore/store-api.hh | 7 +++++++ src/libstore/worker-protocol.hh | 2 ++ src/nix-store/nix-store.cc | 4 ++-- src/nix-worker/nix-worker.cc | 17 +++++++++++++++++ 7 files changed, 51 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 2fd640e39..c1e0e335f 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -184,11 +184,8 @@ public: /* Query whether `path' previously failed to build. */ bool hasPathFailed(const Path & path); - /* Return the set of paths that have failed to build.*/ PathSet queryFailedPaths(); - /* Clear the "failed" status of the given paths. The special - value `*' causes all failed paths to be cleared. */ void clearFailedPaths(const PathSet & paths); private: diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index c5d7975b5..334ad95cf 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -451,6 +451,25 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) } +PathSet RemoteStore::queryFailedPaths() +{ + openConnection(); + writeInt(wopQueryFailedPaths, to); + processStderr(); + return readStorePaths(from); +} + + +void RemoteStore::clearFailedPaths(const PathSet & paths) +{ + openConnection(); + writeInt(wopClearFailedPaths, to); + writeStringSet(paths, to); + processStderr(); + readInt(from); +} + + void RemoteStore::processStderr(Sink * sink, Source * source) { unsigned int msg; diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 8bab1d8c4..02a1c4752 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -70,6 +70,10 @@ public: void collectGarbage(const GCOptions & options, GCResults & results); + PathSet queryFailedPaths(); + + void clearFailedPaths(const PathSet & paths); + private: AutoCloseFD fdSocket; FdSink to; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 095fdd24b..fbe0cce81 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -209,6 +209,13 @@ public: /* Perform a garbage collection. */ virtual void collectGarbage(const GCOptions & options, GCResults & results) = 0; + + /* Return the set of paths that have failed to build.*/ + virtual PathSet queryFailedPaths() = 0; + + /* Clear the "failed" status of the given paths. The special + value `*' causes all failed paths to be cleared. */ + virtual void clearFailedPaths(const PathSet & paths) = 0; }; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index c3096010b..392a69acf 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -36,6 +36,8 @@ typedef enum { wopQuerySubstitutablePathInfo = 21, wopQueryDerivationOutputs = 22, wopQueryValidPaths = 23, + wopQueryFailedPaths = 24, + wopClearFailedPaths = 25, } WorkerOp; diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 148fd6add..3b34b9dae 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -670,7 +670,7 @@ static void opQueryFailedPaths(Strings opFlags, Strings opArgs) { if (!opArgs.empty() || !opFlags.empty()) throw UsageError("no arguments expected"); - PathSet failed = ensureLocalStore().queryFailedPaths(); + PathSet failed = store->queryFailedPaths(); foreach (PathSet::iterator, i, failed) cout << format("%1%\n") % *i; } @@ -680,7 +680,7 @@ static void opClearFailedPaths(Strings opFlags, Strings opArgs) { if (!opFlags.empty()) throw UsageError("no flags expected"); - ensureLocalStore().clearFailedPaths(PathSet(opArgs.begin(), opArgs.end())); + store->clearFailedPaths(PathSet(opArgs.begin(), opArgs.end())); } diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index a41fb2e15..d41877e88 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -528,6 +528,23 @@ static void performOp(unsigned int clientVersion, break; } + case wopQueryFailedPaths: { + startWork(); + PathSet paths = store->queryFailedPaths(); + stopWork(); + writeStringSet(paths, to); + break; + } + + case wopClearFailedPaths: { + PathSet paths = readStringSet(from); + startWork(); + store->clearFailedPaths(paths); + stopWork(); + writeInt(1, to); + break; + } + default: throw Error(format("invalid operation %1%") % op); } -- cgit v1.2.3 From a0e3b84fac56cad6377ecd1462058a6b29bb1ea8 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 4 May 2010 12:42:58 +0000 Subject: * Revert r15436. This was a workaround for a bug in btrfs which seems to have been fixed now. --- src/libutil/archive.cc | 8 -------- src/libutil/archive.hh | 1 - 2 files changed, 9 deletions(-) (limited to 'src') diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index 8fde4328c..999b17cd2 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -181,8 +181,6 @@ static void parseContents(ParseSink & sink, Source & source, const Path & path) left -= n; } - sink.finalizeContents(size); - readPadding(size, source); } @@ -317,12 +315,6 @@ struct RestoreSink : ParseSink writeFull(fd, data, len); } - void finalizeContents(unsigned long long size) - { - errno = ftruncate(fd, size); - if (errno) throw SysError(format("truncating file to its allocated length of %1% bytes") % size); - } - void createSymlink(const Path & path, const string & target) { Path p = dstPath + path; diff --git a/src/libutil/archive.hh b/src/libutil/archive.hh index f358a2a6b..fff620313 100644 --- a/src/libutil/archive.hh +++ b/src/libutil/archive.hh @@ -64,7 +64,6 @@ struct ParseSink virtual void isExecutable() { }; virtual void preallocateContents(unsigned long long size) { }; virtual void receiveContents(unsigned char * data, unsigned int len) { }; - virtual void finalizeContents(unsigned long long size) { }; virtual void createSymlink(const Path & path, const string & target) { }; }; -- cgit v1.2.3 From a443c7573b3d76a6db107e6de974205e605a2738 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 1 Jun 2010 10:01:14 +0000 Subject: * Hack to allow derivations to disable chroot builds by setting the attribute "__noChroot = true" (requested by Rob). --- src/libstore/build.cc | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index e068ab0bd..a482ace00 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1545,6 +1545,9 @@ void DerivationGoal::startBuilder() if (fixedOutput) useChroot = false; + /* Hack to allow derivations to disable chroot builds. */ + if (drv.env["__noChroot"] == "1") useChroot = false; + if (useChroot) { #if CHROOT_ENABLED /* Create a temporary directory in which we set up the chroot -- cgit v1.2.3 From d1f6c0cbe39b509545f809f08cbd580859f38e34 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 8 Jun 2010 13:38:28 +0000 Subject: * Replacing ValidPath rows doesn't work because it causes a constraint violation of the Refs table. So don't do that. --- src/libstore/local-store.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 071f8a223..cf0d559bd 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -311,7 +311,7 @@ void LocalStore::openDB(bool create) /* Prepare SQL statements. */ stmtRegisterValidPath.create(db, - "insert or replace into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); + "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); stmtAddReference.create(db, "insert or replace into Refs (referrer, reference) values (?, ?);"); stmtQueryPathInfo.create(db, @@ -837,7 +837,10 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) { SQLiteTxn txn(db); - foreach (ValidPathInfos::const_iterator, i, infos) addValidPath(*i); + foreach (ValidPathInfos::const_iterator, i, infos) + /* !!! Maybe the registration info should be updated if the + path is already valid. */ + if (!isValidPath(i->path)) addValidPath(*i); foreach (ValidPathInfos::const_iterator, i, infos) { unsigned long long referrer = queryValidPathId(i->path); -- cgit v1.2.3 From 3e5e0faf9cf93c01fb621774c0c3c50ce51bdd91 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 21 Jun 2010 11:08:09 +0000 Subject: * Okay, putting a lock on the temporary directory used by importPath() doesn't work because the garbage collector doesn't actually look at locks. So r22253 was stupid. Use addTempRoot() instead. Also, locking the temporary directory in exportPath() was silly because it isn't even in the store. --- src/libstore/local-store.cc | 24 ++++++++++++++++++------ src/libstore/local-store.hh | 2 ++ 2 files changed, 20 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c7530f1b1..6f3d9efa8 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1036,8 +1036,6 @@ void LocalStore::exportPath(const Path & path, bool sign, writeInt(1, hashAndWriteSink); Path tmpDir = createTempDir(); - PathLocks tmpDirLock(singleton(tmpDir)); - tmpDirLock.setDeletion(true); AutoDelete delTmp(tmpDir); Path hashFile = tmpDir + "/hash"; writeFile(hashFile, printHash(hash)); @@ -1079,6 +1077,22 @@ struct HashAndReadSource : Source }; +/* Create a temporary directory in the store that won't be + garbage-collected. */ +Path LocalStore::createTempDirInStore() +{ + Path tmpDir; + do { + /* There is a slight possibility that `tmpDir' gets deleted by + the GC between createTempDir() and addTempRoot(), so repeat + until `tmpDir' exists. */ + tmpDir = createTempDir(nixStore); + addTempRoot(tmpDir); + } while (!pathExists(tmpDir)); + return tmpDir; +} + + Path LocalStore::importPath(bool requireSignature, Source & source) { HashAndReadSource hashAndReadSource(source); @@ -1086,10 +1100,8 @@ Path LocalStore::importPath(bool requireSignature, Source & source) /* We don't yet know what store path this archive contains (the store path follows the archive data proper), and besides, we don't know yet whether the signature is valid. */ - Path tmpDir = createTempDir(nixStore); - PathLocks tmpDirLock(singleton(tmpDir)); - tmpDirLock.setDeletion(true); - AutoDelete delTmp(tmpDir); /* !!! could be GC'ed! */ + Path tmpDir = createTempDirInStore(); + AutoDelete delTmp(tmpDir); Path unpacked = tmpDir + "/unpacked"; restorePath(unpacked, hashAndReadSource); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index c1e0e335f..3ae964f05 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -246,6 +246,8 @@ private: void startSubstituter(const Path & substituter, RunningSubstituter & runningSubstituter); + + Path createTempDirInStore(); }; -- cgit v1.2.3 From 7f893b7a43fdca728fd1f7a72e51d31d2a6e7147 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Aug 2010 12:13:58 +0000 Subject: * Allow derivations to hint that they should not be built remotely using the build hook mechanism, by setting the derivation attribute "preferLocalBuild" to true. This has a few use cases: - The user environment builder. Since it just creates a bunch of symlinks without much computation, there is no reason to do it remotely. In fact, doing it remotely requires the entire closure of the user environment to be copied to the remote machine, which is extremely wasteful. - `fetchurl'. Performing the download on a remote machine and then copying it to the local machine involves twice as much network traffic as performing the download locally, and doesn't save any CPU cycles on the local machine. --- src/libstore/build.cc | 62 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 39 insertions(+), 23 deletions(-) (limited to 'src') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 8ce8c873d..5818aa51d 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -960,6 +960,16 @@ PathSet outputPaths(const DerivationOutputs & outputs) } +static bool canBuildLocally(const string & platform) +{ + return platform == thisSystem +#ifdef CAN_DO_LINUX32_BUILDS + || (platform == "i686-linux" && thisSystem == "x86_64-linux") +#endif + ; +} + + void DerivationGoal::tryToBuild() { trace("trying to build"); @@ -1027,28 +1037,38 @@ void DerivationGoal::tryToBuild() foreach (DerivationOutputs::iterator, i, drv.outputs) if (pathFailed(i->second.path)) return; + /* Don't do a remote build if the derivation has the attribute + `preferLocalBuild' set. */ + bool preferLocalBuild = + drv.env["preferLocalBuild"] == "1" && canBuildLocally(drv.platform); + /* Is the build hook willing to accept this job? */ - usingBuildHook = true; - switch (tryBuildHook()) { - case rpAccept: - /* Yes, it has started doing so. Wait until we get EOF - from the hook. */ - state = &DerivationGoal::buildDone; - return; - case rpPostpone: - /* Not now; wait until at least one child finishes. */ - worker.waitForAWhile(shared_from_this()); - outputLocks.unlock(); - return; - case rpDecline: - /* We should do it ourselves. */ - break; + if (!preferLocalBuild) { + usingBuildHook = true; + switch (tryBuildHook()) { + case rpAccept: + /* Yes, it has started doing so. Wait until we get + EOF from the hook. */ + state = &DerivationGoal::buildDone; + return; + case rpPostpone: + /* Not now; wait until at least one child finishes. */ + worker.waitForAWhile(shared_from_this()); + outputLocks.unlock(); + return; + case rpDecline: + /* We should do it ourselves. */ + break; + } } - + usingBuildHook = false; - /* Make sure that we are allowed to start a build. */ - if (worker.getNrLocalBuilds() >= maxBuildJobs) { + /* Make sure that we are allowed to start a build. If this + derivation prefers to be done locally, do it even if + maxBuildJobs is 0. */ + unsigned int curBuilds = worker.getNrLocalBuilds(); + if (curBuilds >= maxBuildJobs && !(preferLocalBuild && curBuilds == 0)) { worker.waitForBuildSlot(shared_from_this()); outputLocks.unlock(); return; @@ -1379,11 +1399,7 @@ void DerivationGoal::startBuilder() format("building path(s) %1%") % showPaths(outputPaths(drv.outputs))) /* Right platform? */ - if (drv.platform != thisSystem -#ifdef CAN_DO_LINUX32_BUILDS - && !(drv.platform == "i686-linux" && thisSystem == "x86_64-linux") -#endif - ) + if (!canBuildLocally(drv.platform)) throw Error( format("a `%1%' is required to build `%3%', but I am a `%2%'") % drv.platform % thisSystem % drvPath); -- cgit v1.2.3 From fd9c77dfc7b90d447e6bfdb4f0d5b521184aeddb Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 4 Aug 2010 17:35:59 +0000 Subject: * Use SQLite 3.7.0's write-ahead logging (WAL mode). This is a lot faster than the old mode when fsyncs are enabled, because it only performs an fsync() when doing a checkpoint, rather than at every commit. Some timings for doing a "nix-instantiate /etc/nixos/nixos -A system" after modifying the stdenv setup script: 42.5s - SQLite 3.6.23 with truncate mode and fsync 3.4s - SQLite 3.6.23 with truncate mode and no fsync 32.1s - SQLite 3.7.0 with truncate mode and fsync 16.8s - SQLite 3.7.0 with WAL mode and fsync, auto-checkpoint every 1000 pages 8.3s - SQLite 3.7.0 with WAL mode and fsync, auto-checkpoint every 8192 pages 1.7s - SQLite 3.7.0 with WAL mode and no fsync The default is now to use WAL mode with fsyncs. Because WAL doesn't work on remote filesystems such as NFS (as it uses shared memory), truncate mode can be re-enabled by setting the "use-sqlite-wal" option to false. --- src/libstore/local-store.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 6f3d9efa8..bddcbad12 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -298,10 +298,20 @@ void LocalStore::openDB(bool create) if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "setting synchronous mode"); - /* Use `truncate' journal mode, which should be a bit faster. */ - if (sqlite3_exec(db, "pragma main.journal_mode = truncate;", 0, 0, 0) != SQLITE_OK) + /* Set the SQLite journal mode. The default is write-ahead + logging since it's the fastest and supports more concurrency. + The downside is that it doesn't work over NFS, so allow + truncate mode alternatively. */ + string mode = queryBoolSetting("use-sqlite-wal", true) ? "wal" : "truncate"; + if (sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "setting journal mode"); + /* Increase the auto-checkpoint interval to 8192 pages. This + seems enough to ensure that instantiating the NixOS system + derivation is done in a single fsync(). */ + if (sqlite3_exec(db, "pragma wal_autocheckpoint = 8192;", 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "setting autocheckpoint interval"); + /* Initialise the database schema, if necessary. */ if (create) { #include "schema.sql.hh" -- cgit v1.2.3 From f58f51f38007f8bfb1089c3b4c88b6e66da15f39 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 24 Aug 2010 11:45:44 +0000 Subject: * Handle the unlikely case where a derivation has no dependencies at all. --- src/libstore/build.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 6f0276244..5617eccf1 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -887,7 +887,10 @@ void DerivationGoal::outputsSubstituted() foreach (PathSet::iterator, i, drv.inputSrcs) addWaitee(worker.makeSubstitutionGoal(*i)); - state = &DerivationGoal::inputsRealised; + if (waitees.empty()) /* to prevent hang (no wake-up event) */ + inputsRealised(); + else + state = &DerivationGoal::inputsRealised; } -- cgit v1.2.3 From 034f608e004c3206ff99c55c107139c38bfe9408 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 24 Aug 2010 14:25:33 +0000 Subject: * Don't complain if the stored hash of a path is zero (unknown). --- src/libstore/local-store.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index bddcbad12..9254ceee4 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1022,10 +1022,11 @@ void LocalStore::exportPath(const Path & path, bool sign, dumpPath(path, hashAndWriteSink); /* Refuse to export paths that have changed. This prevents - filesystem corruption from spreading to other machines. */ + filesystem corruption from spreading to other machines. + Don't complain if the stored hash is zero (unknown). */ Hash hash = hashAndWriteSink.currentHash(); Hash storedHash = queryPathHash(path); - if (hash != storedHash) + if (hash != storedHash && storedHash != Hash(storedHash.type)) throw Error(format("hash of path `%1%' has changed from `%2%' to `%3%'!") % path % printHash(storedHash) % printHash(hash)); -- cgit v1.2.3 From e437b0825018b1935f9a849382c12b1df0aeae06 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 25 Aug 2010 20:44:28 +0000 Subject: * Made the build hook mechanism more efficient. Rather than starting the hook every time we want to ask whether we can run a remote build (which can be very often), we now reuse a hook process for answering those queries until it accepts a build. So if there are N derivations to be built, at most N hooks will be started. --- src/libstore/build.cc | 390 ++++++++++++++++++++++++-------------------------- 1 file changed, 187 insertions(+), 203 deletions(-) (limited to 'src') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 5617eccf1..f2781776c 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -64,6 +64,7 @@ static const uid_t rootUserId = 0; /* Forward definition. */ class Worker; +class HookInstance; /* A pointer to a goal. */ @@ -215,6 +216,8 @@ public: LocalStore & store; + boost::shared_ptr hook; + Worker(LocalStore & store); ~Worker(); @@ -615,6 +618,94 @@ void deletePathWrapped(const Path & path) ////////////////////////////////////////////////////////////////////// +struct HookInstance +{ + /* Pipes for talking to the build hook. */ + Pipe toHook; + + /* Pipe for the hook's standard output/error. */ + Pipe fromHook; + + /* The process ID of the hook. */ + Pid pid; + + HookInstance(); + + ~HookInstance(); +}; + + +HookInstance::HookInstance() +{ + debug("starting build hook"); + + Path buildHook = absPath(getEnv("NIX_BUILD_HOOK")); + + /* Create a pipe to get the output of the child. */ + fromHook.create(); + + /* Create the communication pipes. */ + toHook.create(); + + /* Fork the hook. */ + pid = fork(); + switch (pid) { + + case -1: + throw SysError("unable to fork"); + + case 0: + try { /* child */ + + commonChildInit(fromHook); + + if (chdir("/") == -1) throw SysError("changing into `/"); + + /* Dup the communication pipes. */ + toHook.writeSide.close(); + if (dup2(toHook.readSide, STDIN_FILENO) == -1) + throw SysError("dupping to-hook read side"); + + execl(buildHook.c_str(), buildHook.c_str(), thisSystem.c_str(), + (format("%1%") % maxSilentTime).str().c_str(), NULL); + + throw SysError(format("executing `%1%'") % buildHook); + + } catch (std::exception & e) { + std::cerr << format("build hook error: %1%") % e.what() << std::endl; + } + quickExit(1); + } + + /* parent */ + pid.setSeparatePG(true); + pid.setKillSignal(SIGTERM); + fromHook.writeSide.close(); + toHook.readSide.close(); +} + + +HookInstance::~HookInstance() +{ + try { + /* Cleanly shut down the hook by closing its stdin if it's not + already building. Otherwise pid's destructor will kill + it. */ + if (pid != -1 && toHook.writeSide != -1) { + toHook.writeSide.close(); + pid.wait(true); + } + } catch (...) { + ignoreException(); + } +} + + +////////////////////////////////////////////////////////////////////// + + +typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; + class DerivationGoal : public Goal { private: @@ -651,12 +742,9 @@ private: /* Pipe for the builder's standard output/error. */ Pipe logPipe; - /* Whether we're building using a build hook. */ - bool usingBuildHook; - - /* Pipes for talking to the build hook (if any). */ - Pipe toHook; - + /* The build hook. */ + boost::shared_ptr hook; + /* Whether we're currently doing a chroot build. */ bool useChroot; @@ -694,12 +782,8 @@ private: void buildDone(); /* Is the build hook willing to perform the build? */ - typedef enum {rpAccept, rpDecline, rpPostpone} HookReply; HookReply tryBuildHook(); - /* Synchronously wait for a build hook to finish. */ - void terminateBuildHook(bool kill = false); - /* Start building a derivation. */ void startBuilder(); @@ -711,10 +795,6 @@ private: /* Open a log file and a pipe to it. */ Path openLogFile(); - /* Common initialisation to be performed in child processes (i.e., - both in builders and in build hooks). */ - void initChild(); - /* Delete the temporary directory, if we have one. */ void deleteTmpDir(bool force); @@ -742,6 +822,7 @@ DerivationGoal::DerivationGoal(const Path & drvPath, Worker & worker) trace("created"); } + DerivationGoal::~DerivationGoal() { /* Careful: we should never ever throw an exception from a @@ -754,6 +835,7 @@ DerivationGoal::~DerivationGoal() } } + void DerivationGoal::killChild() { if (pid != -1) { @@ -778,6 +860,8 @@ void DerivationGoal::killChild() assert(pid == -1); } + + hook.reset(); } @@ -1048,7 +1132,6 @@ void DerivationGoal::tryToBuild() /* Is the build hook willing to accept this job? */ if (!preferLocalBuild) { - usingBuildHook = true; switch (tryBuildHook()) { case rpAccept: /* Yes, it has started doing so. Wait until we get @@ -1056,7 +1139,8 @@ void DerivationGoal::tryToBuild() state = &DerivationGoal::buildDone; return; case rpPostpone: - /* Not now; wait until at least one child finishes. */ + /* Not now; wait until at least one child finishes or + the wake-up timeout expires. */ worker.waitForAWhile(shared_from_this()); outputLocks.unlock(); return; @@ -1066,8 +1150,6 @@ void DerivationGoal::tryToBuild() } } - usingBuildHook = false; - /* Make sure that we are allowed to start a build. If this derivation prefers to be done locally, do it even if maxBuildJobs is 0. */ @@ -1108,16 +1190,23 @@ void DerivationGoal::buildDone() to have terminated. In fact, the builder could also have simply have closed its end of the pipe --- just don't do that :-) */ - /* !!! this could block! security problem! solution: kill the - child */ - pid_t savedPid = pid; - int status = pid.wait(true); + int status; + pid_t savedPid; + if (hook) { + savedPid = hook->pid; + status = hook->pid.wait(true); + } else { + /* !!! this could block! security problem! solution: kill the + child */ + savedPid = pid; + status = pid.wait(true); + } debug(format("builder process for `%1%' finished") % drvPath); /* So the child is gone now. */ worker.childTerminated(savedPid); - + /* Close the read side of the logger pipe. */ logPipe.readSide.close(); @@ -1192,11 +1281,11 @@ void DerivationGoal::buildDone() /* When using a build hook, the hook will return a remote build failure using exit code 100. Anything else is a hook problem. */ - bool hookError = usingBuildHook && + bool hookError = hook && (!WIFEXITED(status) || WEXITSTATUS(status) != 100); if (printBuildTrace) { - if (usingBuildHook && hookError) + if (hook && hookError) printMsg(lvlError, format("@ hook-failed %1% %2% %3% %4%") % drvPath % drv.outputs["out"].path % status % e.msg()); else @@ -1231,162 +1320,74 @@ void DerivationGoal::buildDone() } -DerivationGoal::HookReply DerivationGoal::tryBuildHook() +HookReply DerivationGoal::tryBuildHook() { - if (!useBuildHook) return rpDecline; - Path buildHook = getEnv("NIX_BUILD_HOOK"); - if (buildHook == "") return rpDecline; - buildHook = absPath(buildHook); + if (!useBuildHook || getEnv("NIX_BUILD_HOOK") == "") return rpDecline; - /* Create a directory where we will store files used for - communication between us and the build hook. */ - tmpDir = createTempDir(); - - /* Create the log file and pipe. */ - Path logFile = openLogFile(); + if (!worker.hook) + worker.hook = boost::shared_ptr(new HookInstance); - /* Create the communication pipes. */ - toHook.create(); - - /* Fork the hook. */ - pid = fork(); - switch (pid) { - - case -1: - throw SysError("unable to fork"); - - case 0: - try { /* child */ - - initChild(); - - string s; - foreach (DerivationOutputs::const_iterator, i, drv.outputs) - s += i->second.path + " "; - if (setenv("NIX_HELD_LOCKS", s.c_str(), 1)) - throw SysError("setting an environment variable"); - - execl(buildHook.c_str(), buildHook.c_str(), - (worker.getNrLocalBuilds() < maxBuildJobs ? (string) "1" : "0").c_str(), - thisSystem.c_str(), - drv.platform.c_str(), - drvPath.c_str(), - (format("%1%") % maxSilentTime).str().c_str(), - NULL); - - throw SysError(format("executing `%1%'") % buildHook); - - } catch (std::exception & e) { - std::cerr << format("build hook error: %1%") % e.what() << std::endl; - } - quickExit(1); - } - - /* parent */ - pid.setSeparatePG(true); - pid.setKillSignal(SIGTERM); - logPipe.writeSide.close(); - worker.childStarted(shared_from_this(), - pid, singleton >(logPipe.readSide), false, false); - - toHook.readSide.close(); + writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3%") % + (worker.getNrLocalBuilds() < maxBuildJobs ? "1" : "0") % drv.platform % drvPath).str()); /* Read the first line of input, which should be a word indicating whether the hook wishes to perform the build. */ string reply; - try { - while (true) { - string s = readLine(logPipe.readSide); - if (string(s, 0, 2) == "# ") { - reply = string(s, 2); - break; - } - handleChildOutput(logPipe.readSide, s + "\n"); + while (true) { + string s = readLine(worker.hook->fromHook.readSide); + if (string(s, 0, 2) == "# ") { + reply = string(s, 2); + break; } - } catch (Error & e) { - terminateBuildHook(true); - throw; + handleChildOutput(worker.hook->fromHook.readSide, s + "\n"); } debug(format("hook reply is `%1%'") % reply); - if (reply == "decline" || reply == "postpone") { - /* Clean up the child. !!! hacky / should verify */ - terminateBuildHook(); + if (reply == "decline" || reply == "postpone") return reply == "decline" ? rpDecline : rpPostpone; - } + else if (reply != "accept") + throw Error(format("bad hook reply `%1%'") % reply); - else if (reply == "accept") { + printMsg(lvlTalkative, format("using hook to build path(s) %1%") + % showPaths(outputPaths(drv.outputs))); - printMsg(lvlInfo, format("using hook to build path(s) %1%") - % showPaths(outputPaths(drv.outputs))); - - /* Write the information that the hook needs to perform the - build, i.e., the set of input paths, the set of output - paths, and the references (pointer graph) in the input - paths. */ + hook = worker.hook; + worker.hook.reset(); - Path inputListFN = tmpDir + "/inputs"; - Path outputListFN = tmpDir + "/outputs"; - Path referencesFN = tmpDir + "/references"; - - /* The `inputs' file lists all inputs that have to be copied - to the remote system. This unfortunately has to contain - the entire derivation closure to ensure that the validity - invariant holds on the remote system. (I.e., it's - unfortunate that we have to list it since the remote system - *probably* already has it.) */ - PathSet allInputs; - allInputs.insert(inputPaths.begin(), inputPaths.end()); - computeFSClosure(drvPath, allInputs); + /* Tell the hook all the inputs that have to be copied to the + remote system. This unfortunately has to contain the entire + derivation closure to ensure that the validity invariant holds + on the remote system. (I.e., it's unfortunate that we have to + list it since the remote system *probably* already has it.) */ + PathSet allInputs; + allInputs.insert(inputPaths.begin(), inputPaths.end()); + computeFSClosure(drvPath, allInputs); - string s; - foreach (PathSet::iterator, i, allInputs) s += *i + "\n"; + string s; + foreach (PathSet::iterator, i, allInputs) s += *i + " "; + writeLine(hook->toHook.writeSide, s); - writeFile(inputListFN, s); - - /* The `outputs' file lists all outputs that have to be copied - from the remote system. */ - s = ""; - foreach (DerivationOutputs::iterator, i, drv.outputs) - s += i->second.path + "\n"; - writeFile(outputListFN, s); - - /* The `references' file has exactly the format accepted by - `nix-store --register-validity'. */ - writeFile(referencesFN, - makeValidityRegistration(allInputs, true, false)); - - /* Tell the hook to proceed. */ - writeLine(toHook.writeSide, "okay"); - toHook.writeSide.close(); + /* Tell the hooks the outputs that have to be copied back from the + remote system. */ + s = ""; + foreach (DerivationOutputs::iterator, i, drv.outputs) + s += i->second.path + " "; + writeLine(hook->toHook.writeSide, s); + + hook->toHook.writeSide.close(); - if (printBuildTrace) - printMsg(lvlError, format("@ build-started %1% %2% %3% %4%") - % drvPath % drv.outputs["out"].path % drv.platform % logFile); + /* Create the log file and pipe. */ + Path logFile = openLogFile(); + + worker.childStarted(shared_from_this(), + hook->pid, singleton >(hook->fromHook.readSide), false, false); + + if (printBuildTrace) + printMsg(lvlError, format("@ build-started %1% %2% %3% %4%") + % drvPath % drv.outputs["out"].path % drv.platform % logFile); - return rpAccept; - } - - else throw Error(format("bad hook reply `%1%'") % reply); -} - - -void DerivationGoal::terminateBuildHook(bool kill) -{ - debug("terminating build hook"); - pid_t savedPid = pid; - if (kill) - pid.kill(); - else - pid.wait(true); - /* `false' means don't wake up waiting goals, since we want to - keep this build slot ourselves. */ - worker.childTerminated(savedPid, false); - toHook.writeSide.close(); - fdLogFile.close(); - logPipe.readSide.close(); - deleteTmpDir(true); /* get rid of the hook's temporary directory */ + return rpAccept; } @@ -1667,9 +1668,12 @@ void DerivationGoal::startBuilder() printMsg(lvlChatty, format("executing builder `%1%'") % drv.builder); - /* Create the log file and pipe. */ + /* Create the log file. */ Path logFile = openLogFile(); + /* Create a pipe to get the output of the child. */ + logPipe.create(); + /* Fork a child to build the package. Note that while we currently use forks to run and wait for the children, it shouldn't be hard to use threads for this on systems where @@ -1710,18 +1714,23 @@ void DerivationGoal::startBuilder() throw SysError(format("bind mount from `%1%' to `%2%' failed") % source % target); } - /* Do the chroot(). initChild() will do a chdir() to - the temporary build directory to make sure the - current directory is in the chroot. (Actually the - order doesn't matter, since due to the bind mount - tmpDir and tmpRootDit/tmpDir are the same - directories.) */ + /* Do the chroot(). Below we do a chdir() to the + temporary build directory to make sure the current + directory is in the chroot. (Actually the order + doesn't matter, since due to the bind mount tmpDir + and tmpRootDit/tmpDir are the same directories.) */ if (chroot(chrootRootDir.c_str()) == -1) throw SysError(format("cannot change root directory to `%1%'") % chrootRootDir); } #endif - initChild(); + commonChildInit(logPipe); + + if (chdir(tmpDir.c_str()) == -1) + throw SysError(format("changing into `%1%'") % tmpDir); + + /* Close all other file descriptors. */ + closeMostFDs(set()); #ifdef CAN_DO_LINUX32_BUILDS if (drv.platform == "i686-linux" && thisSystem == "x86_64-linux") { @@ -1742,10 +1751,10 @@ void DerivationGoal::startBuilder() /* If we are running in `build-users' mode, then switch to the user we allocated above. Make sure that we drop - all root privileges. Note that initChild() above has - closed all file descriptors except std*, so that's - safe. Also note that setuid() when run as root sets - the real, effective and saved UIDs. */ + all root privileges. Note that above we have closed + all file descriptors except std*, so that's safe. Also + note that setuid() when run as root sets the real, + effective and saved UIDs. */ if (buildUser.enabled()) { printMsg(lvlChatty, format("switching to user `%1%'") % buildUser.getUser()); @@ -1838,7 +1847,7 @@ void DerivationGoal::computeClosure() /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have to do anything here. */ - if (usingBuildHook) { + if (hook) { bool allValid = true; foreach (DerivationOutputs::iterator, i, drv.outputs) if (!worker.store.isValidPath(i->second.path)) allValid = false; @@ -1966,32 +1975,10 @@ Path DerivationGoal::openLogFile() if (fdLogFile == -1) throw SysError(format("creating log file `%1%'") % logFileName); - /* Create a pipe to get the output of the child. */ - logPipe.create(); - return logFileName; } -void DerivationGoal::initChild() -{ - commonChildInit(logPipe); - - if (chdir(tmpDir.c_str()) == -1) - throw SysError(format("changing into `%1%'") % tmpDir); - - /* When running a hook, dup the communication pipes. */ - if (usingBuildHook) { - toHook.writeSide.close(); - if (dup2(toHook.readSide, STDIN_FILENO) == -1) - throw SysError("dupping to-hook read side"); - } - - /* Close all other file descriptors. */ - closeMostFDs(set()); -} - - void DerivationGoal::deleteTmpDir(bool force) { if (tmpDir != "") { @@ -2011,19 +1998,16 @@ void DerivationGoal::deleteTmpDir(bool force) void DerivationGoal::handleChildOutput(int fd, const string & data) { - if (fd == logPipe.readSide) { - if (verbosity >= buildVerbosity) - writeToStderr((unsigned char *) data.c_str(), data.size()); + if (verbosity >= buildVerbosity) + writeToStderr((unsigned char *) data.c_str(), data.size()); + if (fdLogFile != -1) writeFull(fdLogFile, (unsigned char *) data.c_str(), data.size()); - } - - else abort(); } void DerivationGoal::handleEOF(int fd) { - if (fd == logPipe.readSide) worker.wakeUp(shared_from_this()); + worker.wakeUp(shared_from_this()); } -- cgit v1.2.3 From 766f7084188d8c0dd0595bbbfc50fbc4f9d67a50 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 27 Aug 2010 13:18:13 +0000 Subject: * Experimental feature: allow a derivation to tell the build hook that it requires a certain feature on the build machine, e.g. requiredSystemFeatures = [ "kvm" ]; We need this in Hydra to make sure that builds that require KVM support are forwarded to machines that have KVM support. Probably this should also be enforced for local builds. --- src/libstore/build.cc | 12 ++++++++++-- src/libutil/util.cc | 11 +++++++++++ src/libutil/util.hh | 5 +++++ 3 files changed, 26 insertions(+), 2 deletions(-) (limited to 'src') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f2781776c..f9c9a0a1e 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1327,8 +1327,16 @@ HookReply DerivationGoal::tryBuildHook() if (!worker.hook) worker.hook = boost::shared_ptr(new HookInstance); - writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3%") % - (worker.getNrLocalBuilds() < maxBuildJobs ? "1" : "0") % drv.platform % drvPath).str()); + /* Tell the hook about system features (beyond the system type) + required from the build machine. (The hook could parse the + drv file itself, but this is easier.) */ + Strings features = tokenizeString(drv.env["requiredSystemFeatures"]); + foreach (Strings::iterator, i, features) checkStoreName(*i); /* !!! abuse */ + + /* Send the request to the hook. */ + writeLine(worker.hook->toHook.writeSide, (format("%1% %2% %3% %4%") + % (worker.getNrLocalBuilds() < maxBuildJobs ? "1" : "0") + % drv.platform % drvPath % concatStringsSep(",", features)).str()); /* Read the first line of input, which should be a word indicating whether the hook wishes to perform the build. */ diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 9540720fe..2d26fc66d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -966,6 +966,17 @@ Strings tokenizeString(const string & s, const string & separators) } +string concatStringsSep(const string & sep, const Strings & ss) +{ + string s; + foreach (Strings::const_iterator, i, ss) { + if (s.size() != 0) s += sep; + s += *i; + } + return s; +} + + string statusToString(int status) { if (!WIFEXITED(status) || WEXITSTATUS(status) != 0) { diff --git a/src/libutil/util.hh b/src/libutil/util.hh index c45ebc062..27ad46904 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -280,6 +280,11 @@ MakeError(Interrupted, BaseError) Strings tokenizeString(const string & s, const string & separators = " \t\n\r"); +/* Concatenate the given strings with a separator between the + elements. */ +string concatStringsSep(const string & sep, const Strings & ss); + + /* Convert the exit status of a child as returned by wait() into an error string. */ string statusToString(int status); -- cgit v1.2.3 From 20acd43c25a388f5c31c2ee601f1cac88cf12f7b Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Aug 2010 14:11:57 +0000 Subject: * Disable the GC reachability check for now (when creating new roots), as it's hopelessly inefficient. --- src/libstore/gc.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 87e0a05bc..b94442923 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -127,7 +127,7 @@ Path addPermRoot(const Path & _storePath, const Path & _gcRoot, Instead of reading all the roots, it would be more efficient to check if the root is in a directory in or linked from the gcroots directory. */ - if (queryBoolSetting("gc-check-reachability", true)) { + if (queryBoolSetting("gc-check-reachability", false)) { Roots roots = store->findRoots(); if (roots.find(gcRoot) == roots.end()) printMsg(lvlError, -- cgit v1.2.3 From 80e722278ca03bf303961e2f27487dc98d042803 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 30 Aug 2010 14:53:03 +0000 Subject: * When using the build hook, distinguish between the stderr of the hook script proper, and the stdout/stderr of the builder. Only the latter should be saved in /nix/var/log/nix/drvs. * Allow the verbosity to be set through an option. * Added a flag --quiet to lower the verbosity level. --- src/libmain/shared.cc | 10 ++++++---- src/libstore/build.cc | 40 +++++++++++++++++++++++++++++----------- src/libstore/gc.cc | 2 +- src/libstore/globals.cc | 2 +- src/libstore/remote-store.cc | 9 +-------- src/libutil/types.hh | 2 +- 6 files changed, 39 insertions(+), 26 deletions(-) (limited to 'src') diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index eddc4e64b..f58def403 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -196,17 +196,16 @@ static void initAndRun(int argc, char * * argv) remaining.clear(); /* Process default options. */ + int verbosityDelta = 0; for (Strings::iterator i = args.begin(); i != args.end(); ++i) { string arg = *i; - if (arg == "--verbose" || arg == "-v") - verbosity = (Verbosity) ((int) verbosity + 1); + if (arg == "--verbose" || arg == "-v") verbosityDelta++; + else if (arg == "--quiet") verbosityDelta--; else if (arg == "--log-type") { ++i; if (i == args.end()) throw UsageError("`--log-type' requires an argument"); setLogType(*i); } - else if (arg == "--build-output" || arg == "-B") - ; /* !!! obsolete - remove eventually */ else if (arg == "--no-build-output" || arg == "-Q") buildVerbosity = lvlVomit; else if (arg == "--print-build-trace") @@ -247,6 +246,9 @@ static void initAndRun(int argc, char * * argv) else remaining.push_back(arg); } + verbosityDelta += queryIntSetting("verbosity", lvlInfo); + verbosity = (Verbosity) (verbosityDelta < 0 ? 0 : verbosityDelta); + /* Automatically clean up the temporary roots file when we exit. */ RemoveTempRoots removeTempRoots __attribute__((unused)); diff --git a/src/libstore/build.cc b/src/libstore/build.cc index f9c9a0a1e..30e37e4e9 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -626,6 +626,9 @@ struct HookInstance /* Pipe for the hook's standard output/error. */ Pipe fromHook; + /* Pipe for the builder's standard output/error. */ + Pipe builderOut; + /* The process ID of the hook. */ Pid pid; @@ -647,6 +650,9 @@ HookInstance::HookInstance() /* Create the communication pipes. */ toHook.create(); + /* Create a pipe to get the output of the builder. */ + builderOut.create(); + /* Fork the hook. */ pid = fork(); switch (pid) { @@ -666,6 +672,11 @@ HookInstance::HookInstance() if (dup2(toHook.readSide, STDIN_FILENO) == -1) throw SysError("dupping to-hook read side"); + /* Use fd 4 for the builder's stdout/stderr. */ + builderOut.readSide.close(); + if (dup2(builderOut.writeSide, 4) == -1) + throw SysError("dupping builder's stdout/stderr"); + execl(buildHook.c_str(), buildHook.c_str(), thisSystem.c_str(), (format("%1%") % maxSilentTime).str().c_str(), NULL); @@ -740,7 +751,7 @@ private: AutoCloseFD fdLogFile; /* Pipe for the builder's standard output/error. */ - Pipe logPipe; + Pipe builderOut; /* The build hook. */ boost::shared_ptr hook; @@ -1208,7 +1219,11 @@ void DerivationGoal::buildDone() worker.childTerminated(savedPid); /* Close the read side of the logger pipe. */ - logPipe.readSide.close(); + if (hook) { + hook->builderOut.readSide.close(); + hook->fromHook.readSide.close(); + } + else builderOut.readSide.close(); /* Close the log file. */ fdLogFile.close(); @@ -1387,9 +1402,11 @@ HookReply DerivationGoal::tryBuildHook() /* Create the log file and pipe. */ Path logFile = openLogFile(); - - worker.childStarted(shared_from_this(), - hook->pid, singleton >(hook->fromHook.readSide), false, false); + + set fds; + fds.insert(hook->fromHook.readSide); + fds.insert(hook->builderOut.readSide); + worker.childStarted(shared_from_this(), hook->pid, fds, false, false); if (printBuildTrace) printMsg(lvlError, format("@ build-started %1% %2% %3% %4%") @@ -1679,8 +1696,8 @@ void DerivationGoal::startBuilder() /* Create the log file. */ Path logFile = openLogFile(); - /* Create a pipe to get the output of the child. */ - logPipe.create(); + /* Create a pipe to get the output of the builder. */ + builderOut.create(); /* Fork a child to build the package. Note that while we currently use forks to run and wait for the children, it @@ -1732,7 +1749,7 @@ void DerivationGoal::startBuilder() } #endif - commonChildInit(logPipe); + commonChildInit(builderOut); if (chdir(tmpDir.c_str()) == -1) throw SysError(format("changing into `%1%'") % tmpDir); @@ -1816,9 +1833,9 @@ void DerivationGoal::startBuilder() /* parent */ pid.setSeparatePG(true); - logPipe.writeSide.close(); + builderOut.writeSide.close(); worker.childStarted(shared_from_this(), pid, - singleton >(logPipe.readSide), true, true); + singleton >(builderOut.readSide), true, true); if (printBuildTrace) { printMsg(lvlError, format("@ build-started %1% %2% %3% %4%") @@ -2008,7 +2025,8 @@ void DerivationGoal::handleChildOutput(int fd, const string & data) { if (verbosity >= buildVerbosity) writeToStderr((unsigned char *) data.c_str(), data.size()); - if (fdLogFile != -1) + if ((hook && fd == hook->builderOut.readSide) || + (!hook && fd == builderOut.readSide)) writeFull(fdLogFile, (unsigned char *) data.c_str(), data.size()); } diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index b94442923..ea784bcd9 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -136,7 +136,7 @@ Path addPermRoot(const Path & _storePath, const Path & _gcRoot, "therefore, `%2%' might be removed by the garbage collector") % gcRoot % storePath); } - + /* Grab the global GC root, causing us to block while a GC is in progress. This prevents the set of permanent roots from increasing while a GC is in progress. */ diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 75d2f69c2..7069d104a 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -20,7 +20,7 @@ string nixBinDir = "/UNINIT"; bool keepFailed = false; bool keepGoing = false; bool tryFallback = false; -Verbosity buildVerbosity = lvlInfo; +Verbosity buildVerbosity = lvlError; unsigned int maxBuildJobs = 1; unsigned int buildCores = 1; bool readOnlyMode = false; diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 8f162daee..92d517bbb 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -97,10 +97,6 @@ void RemoteStore::forkSlave() if (worker == "") worker = nixBinDir + "/nix-worker"; - string verbosityArg = "-"; - for (int i = 1; i < verbosity; ++i) - verbosityArg += "v"; - child = fork(); switch (child) { @@ -120,10 +116,7 @@ void RemoteStore::forkSlave() close(fdSocket); close(fdChild); - execlp(worker.c_str(), worker.c_str(), "--slave", - /* hacky - must be at the end */ - verbosityArg == "-" ? NULL : verbosityArg.c_str(), - NULL); + execlp(worker.c_str(), worker.c_str(), "--slave", NULL); throw SysError(format("executing `%1%'") % worker); diff --git a/src/libutil/types.hh b/src/libutil/types.hh index f110188da..86801342f 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -63,7 +63,7 @@ typedef set PathSet; typedef enum { - lvlError, + lvlError = 0, lvlInfo, lvlTalkative, lvlChatty, -- cgit v1.2.3 From e2e168f7c27f5239badf6e8705264bd907d6b82c Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 Aug 2010 11:47:31 +0000 Subject: `nix-store --verify' improvements: * If a path has disappeared, check its referrers first, and don't try to invalidate paths that have valid referrers. Otherwise we get a foreign key constraint violation. * Read the whole Nix store directory instead of statting each valid path, which is slower. * Acquire the global GC lock. --- src/libstore/gc.cc | 3 +- src/libstore/local-store.cc | 69 ++++++++++++++++++++++++++++++++++----------- src/libstore/local-store.hh | 10 +++++-- 3 files changed, 62 insertions(+), 20 deletions(-) (limited to 'src') diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index ea784bcd9..7d58cd50a 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,6 +1,5 @@ #include "globals.hh" #include "misc.hh" -#include "pathlocks.hh" #include "local-store.hh" #include @@ -31,7 +30,7 @@ static const int defaultGcLevel = 1000; read. To be precise: when they try to create a new temporary root file, they will block until the garbage collector has finished / yielded the GC lock. */ -static int openGCLock(LockType lockType) +int LocalStore::openGCLock(LockType lockType) { Path fnGCLock = (format("%1%/%2%") % nixStateDir % gcLockName).str(); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9254ceee4..fd6ad6464 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -1226,27 +1226,25 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr void LocalStore::verifyStore(bool checkContents) { - /* Check whether all valid paths actually exist. */ - printMsg(lvlInfo, "checking path existence"); + printMsg(lvlError, format("reading the Nix store...")); - PathSet validPaths2 = queryValidPaths(), validPaths; + /* Acquire the global GC lock to prevent a garbage collection. */ + AutoCloseFD fdGCLock = openGCLock(ltWrite); - foreach (PathSet::iterator, i, validPaths2) { - checkInterrupt(); - /* !!! invalidatePath() will probably fail due to the foreign - key constraints on the Ref table. */ - if (!isStorePath(*i)) { - printMsg(lvlError, format("path `%1%' is not in the Nix store") % *i); - invalidatePath(*i); - } else if (!pathExists(*i)) { - printMsg(lvlError, format("path `%1%' disappeared") % *i); - invalidatePath(*i); - } else validPaths.insert(*i); - } + Paths entries = readDirectory(nixStore); + PathSet store(entries.begin(), entries.end()); + + /* Check whether all valid paths actually exist. */ + printMsg(lvlInfo, "checking path existence..."); + + PathSet validPaths2 = queryValidPaths(), validPaths, done; + + foreach (PathSet::iterator, i, validPaths2) + verifyPath(*i, store, done, validPaths); /* Optionally, check the content hashes (slow). */ if (checkContents) { - printMsg(lvlInfo, "checking hashes"); + printMsg(lvlInfo, "checking hashes..."); foreach (PathSet::iterator, i, validPaths) { ValidPathInfo info = queryPathInfo(*i); @@ -1264,6 +1262,45 @@ void LocalStore::verifyStore(bool checkContents) } +void LocalStore::verifyPath(const Path & path, const PathSet & store, + PathSet & done, PathSet & validPaths) +{ + checkInterrupt(); + + if (done.find(path) != done.end()) return; + done.insert(path); + + if (!isStorePath(path)) { + printMsg(lvlError, format("path `%1%' is not in the Nix store") % path); + invalidatePath(path); + return; + } + + if (store.find(baseNameOf(path)) == store.end()) { + /* Check any referrers first. If we can invalidate them + first, then we can invalidate this path as well. */ + bool canInvalidate = true; + PathSet referrers; queryReferrers(path, referrers); + foreach (PathSet::iterator, i, referrers) + if (*i != path) { + verifyPath(*i, store, done, validPaths); + if (validPaths.find(*i) != validPaths.end()) + canInvalidate = false; + } + + if (canInvalidate) { + printMsg(lvlError, format("path `%1%' disappeared, removing from database...") % path); + invalidatePath(path); + } else + printMsg(lvlError, format("path `%1%' disappeared, but it still has valid referrers!") % path); + + return; + } + + validPaths.insert(path); +} + + /* Functions for upgrading from the pre-SQLite database. */ PathSet LocalStore::queryValidPathsOld() diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 3ae964f05..0d7ec1f49 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -5,6 +5,7 @@ #include "store-api.hh" #include "util.hh" +#include "pathlocks.hh" class sqlite3; @@ -16,8 +17,8 @@ namespace nix { /* Nix store and database schema version. Version 1 (or 0) was Nix <= 0.7. Version 2 was Nix 0.8 and 0.9. Version 3 is Nix 0.10. - Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.14. Version 6 is - Nix 0.15. */ + Version 4 is Nix 0.11. Version 5 is Nix 0.12-0.16. Version 6 is + Nix 1.0. */ const int nixSchemaVersion = 6; @@ -233,6 +234,9 @@ private: void invalidatePath(const Path & path); + void verifyPath(const Path & path, const PathSet & store, + PathSet & done, PathSet & validPaths); + void upgradeStore6(); PathSet queryValidPathsOld(); ValidPathInfo queryPathInfoOld(const Path & path); @@ -244,6 +248,8 @@ private: bool isActiveTempFile(const GCState & state, const Path & path, const string & suffix); + int openGCLock(LockType lockType); + void startSubstituter(const Path & substituter, RunningSubstituter & runningSubstituter); -- cgit v1.2.3 From bf0dde959771661c6893001a7e0779b65d7be490 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 31 Aug 2010 12:36:24 +0000 Subject: * Always print hook output on stderr, even if --no-build-output is set. * In the build hook, print a trace message to allow Hydra to pick up the name of the remote machine used for the build. --- src/libstore/build.cc | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 30e37e4e9..210486fbc 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -678,7 +678,9 @@ HookInstance::HookInstance() throw SysError("dupping builder's stdout/stderr"); execl(buildHook.c_str(), buildHook.c_str(), thisSystem.c_str(), - (format("%1%") % maxSilentTime).str().c_str(), NULL); + (format("%1%") % maxSilentTime).str().c_str(), + (format("%1%") % printBuildTrace).str().c_str(), + NULL); throw SysError(format("executing `%1%'") % buildHook); @@ -1362,7 +1364,8 @@ HookReply DerivationGoal::tryBuildHook() reply = string(s, 2); break; } - handleChildOutput(worker.hook->fromHook.readSide, s + "\n"); + s += "\n"; + writeToStderr((unsigned char *) s.c_str(), s.size()); } debug(format("hook reply is `%1%'") % reply); @@ -2023,11 +2026,16 @@ void DerivationGoal::deleteTmpDir(bool force) void DerivationGoal::handleChildOutput(int fd, const string & data) { - if (verbosity >= buildVerbosity) - writeToStderr((unsigned char *) data.c_str(), data.size()); if ((hook && fd == hook->builderOut.readSide) || (!hook && fd == builderOut.readSide)) + { + if (verbosity >= buildVerbosity) + writeToStderr((unsigned char *) data.c_str(), data.size()); writeFull(fdLogFile, (unsigned char *) data.c_str(), data.size()); + } + + if (hook && fd == hook->fromHook.readSide) + writeToStderr((unsigned char *) data.c_str(), data.size()); } -- cgit v1.2.3 From e4907411c2d902215d1a18456ce8b0c653650461 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 1 Sep 2010 11:36:22 +0000 Subject: * Only do "pragma journal_mode = ..." if the current journal mode differs from the desired mode. There is an open SQLite ticket `Executing "PRAGMA journal_mode" may delete journal file while it is in use.' --- src/libstore/local-store.cc | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index fd6ad6464..c0c1461b4 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -303,7 +303,16 @@ void LocalStore::openDB(bool create) The downside is that it doesn't work over NFS, so allow truncate mode alternatively. */ string mode = queryBoolSetting("use-sqlite-wal", true) ? "wal" : "truncate"; - if (sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) + string prevMode; + { + SQLiteStmt stmt; + stmt.create(db, "pragma main.journal_mode;"); + if (sqlite3_step(stmt) != SQLITE_ROW) + throw SQLiteError(db, "querying journal mode"); + prevMode = string((const char *) sqlite3_column_text(stmt, 0)); + } + if (prevMode != mode && + sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "setting journal mode"); /* Increase the auto-checkpoint interval to 8192 pages. This -- cgit v1.2.3 From 64fd29855a8ae49cacdaff424679821b4fd3bf57 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Thu, 14 Oct 2010 15:55:51 +0000 Subject: * Wrap deleteFromStore() in a transaction. Otherwise there might be a race with other processes that add new referrers to a path, resulting in the garbage collector crashing with "foreign key constraint failed". (Nix/4) * Make --gc --print-dead etc. interruptible. --- src/libstore/gc.cc | 8 +++++--- src/libstore/local-store.cc | 13 ++++++++----- 2 files changed, 13 insertions(+), 8 deletions(-) (limited to 'src') diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 7d58cd50a..b8395bfc4 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -421,7 +421,7 @@ struct LocalStore::GCState }; -static bool doDelete(GCOptions::GCAction action) +static bool shouldDelete(GCOptions::GCAction action) { return action == GCOptions::gcDeleteDead || action == GCOptions::gcDeleteSpecific; @@ -438,6 +438,8 @@ bool LocalStore::isActiveTempFile(const GCState & state, bool LocalStore::tryToDelete(GCState & state, const Path & path) { + checkInterrupt(); + if (!pathExists(path)) return true; if (state.deleted.find(path) != state.deleted.end()) return true; if (state.live.find(path) != state.live.end()) return false; @@ -516,7 +518,7 @@ bool LocalStore::tryToDelete(GCState & state, const Path & path) } /* The path is garbage, so delete it. */ - if (doDelete(state.options.action)) { + if (shouldDelete(state.options.action)) { printMsg(lvlInfo, format("deleting `%1%'") % path); unsigned long long bytesFreed, blocksFreed; @@ -625,7 +627,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) vector entries_(entries.begin(), entries.end()); random_shuffle(entries_.begin(), entries_.end()); - if (doDelete(state.options.action)) + if (shouldDelete(state.options.action)) printMsg(lvlError, format("deleting garbage...")); else printMsg(lvlError, format("determining live/dead paths...")); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c0c1461b4..c0c75e34d 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -298,11 +298,10 @@ void LocalStore::openDB(bool create) if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) throw SQLiteError(db, "setting synchronous mode"); - /* Set the SQLite journal mode. The default is write-ahead - logging since it's the fastest and supports more concurrency. - The downside is that it doesn't work over NFS, so allow - truncate mode alternatively. */ - string mode = queryBoolSetting("use-sqlite-wal", true) ? "wal" : "truncate"; + /* Set the SQLite journal mode. WAL mode is fastest, but doesn't + seem entirely stable at the moment (Oct. 2010). Thus, use + truncate mode by default. */ + string mode = queryBoolSetting("use-sqlite-wal", false) ? "wal" : "truncate"; string prevMode; { SQLiteStmt stmt; @@ -1220,6 +1219,8 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr assertStorePath(path); + SQLiteTxn txn(db); + if (isValidPath(path)) { PathSet referrers; queryReferrers(path, referrers); referrers.erase(path); /* ignore self-references */ @@ -1230,6 +1231,8 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr } deletePathWrapped(path, bytesFreed, blocksFreed); + + txn.commit(); } -- cgit v1.2.3 From a3883cbd28057a3dd2573f77dcda9a26faaac555 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 16 Nov 2010 17:11:46 +0000 Subject: * Store the size of a store path in the database (to be precise, the size of the NAR serialisation of the path, i.e., `nix-store --dump PATH'). This is useful for Hydra. --- src/libstore/build.cc | 41 ++++++++++++++---------- src/libstore/local-store.cc | 72 +++++++++++++++++++++++++++--------------- src/libstore/local-store.hh | 9 ++---- src/libstore/optimise-store.cc | 2 +- src/libstore/references.cc | 2 +- src/libstore/references.hh | 2 +- src/libstore/remote-store.cc | 6 ++++ src/libstore/remote-store.hh | 2 ++ src/libstore/schema.sql | 3 +- src/libstore/store-api.cc | 23 ++++++++------ src/libstore/store-api.hh | 40 +++++++++++++---------- src/libutil/hash.cc | 10 ++++-- src/libutil/hash.hh | 7 ++-- src/nix-hash/nix-hash.cc | 2 +- src/nix-store/nix-store.cc | 12 ++++--- 15 files changed, 143 insertions(+), 90 deletions(-) (limited to 'src') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 210486fbc..8b8be3e80 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1547,7 +1547,7 @@ void DerivationGoal::startBuilder() /* Write closure info to `fileName'. */ writeFile(tmpDir + "/" + fileName, - makeValidityRegistration(paths, false, false)); + worker.store.makeValidityRegistration(paths, false, false)); } @@ -1870,7 +1870,7 @@ PathSet parseReferenceSpecifiers(const Derivation & drv, string attr) void DerivationGoal::computeClosure() { map allReferences; - map contentHashes; + map contentHashes; /* When using a build hook, the build hook can register the output as valid (by doing `nix-store --import'). If so we don't have @@ -1927,7 +1927,7 @@ void DerivationGoal::computeClosure() if (ht == htUnknown) throw BuildError(format("unknown hash algorithm `%1%'") % algo); Hash h = parseHash(ht, i->second.hash); - Hash h2 = recursive ? hashPath(ht, path) : hashFile(ht, path); + Hash h2 = recursive ? hashPath(ht, path).first : hashFile(ht, path); if (h != h2) throw BuildError( format("output path `%1%' should have %2% hash `%3%', instead has `%4%'") @@ -1941,7 +1941,7 @@ void DerivationGoal::computeClosure() contained in it. Compute the SHA-256 NAR hash at the same time. The hash is stored in the database so that we can verify later on whether nobody has messed with the store. */ - Hash hash; + HashResult hash; PathSet references = scanForReferences(path, allPaths, hash); contentHashes[path] = hash; @@ -1970,14 +1970,18 @@ void DerivationGoal::computeClosure() } /* Register each output path as valid, and register the sets of - paths referenced by each of them. !!! this should be - atomic so that either all paths are registered as valid, or - none are. */ - foreach (DerivationOutputs::iterator, i, drv.outputs) - worker.store.registerValidPath(i->second.path, - contentHashes[i->second.path], - allReferences[i->second.path], - drvPath); + paths referenced by each of them. */ + ValidPathInfos infos; + foreach (DerivationOutputs::iterator, i, drv.outputs) { + ValidPathInfo info; + info.path = i->second.path; + info.hash = contentHashes[i->second.path].first; + info.narSize = contentHashes[i->second.path].second; + info.references = allReferences[i->second.path]; + info.deriver = drvPath; + infos.push_back(info); + } + worker.store.registerValidPaths(infos); /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will not @@ -2385,10 +2389,15 @@ void SubstitutionGoal::finished() canonicalisePathMetaData(storePath); - Hash contentHash = hashPath(htSHA256, storePath); - - worker.store.registerValidPath(storePath, contentHash, - info.references, info.deriver); + HashResult hash = hashPath(htSHA256, storePath); + + ValidPathInfo info2; + info2.path = storePath; + info2.hash = hash.first; + info2.narSize = hash.second; + info2.references = info.references; + info2.deriver = info.deriver; + worker.store.registerValidPath(info2); outputLock->setDeletion(true); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index c0c75e34d..9595561bc 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -327,9 +327,16 @@ void LocalStore::openDB(bool create) throw SQLiteError(db, "initialising database schema"); } + /* Backwards compatibility with old (pre-release) databases. Can + remove this eventually. */ + if (sqlite3_table_column_metadata(db, 0, "ValidPaths", "narSize", 0, 0, 0, 0, 0) != SQLITE_OK) { + if (sqlite3_exec(db, "alter table ValidPaths add column narSize integer" , 0, 0, 0) != SQLITE_OK) + throw SQLiteError(db, "adding column narSize"); + } + /* Prepare SQL statements. */ stmtRegisterValidPath.create(db, - "insert into ValidPaths (path, hash, registrationTime, deriver) values (?, ?, ?, ?);"); + "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);"); stmtAddReference.create(db, "insert or replace into Refs (referrer, reference) values (?, ?);"); stmtQueryPathInfo.create(db, @@ -431,19 +438,6 @@ void canonicalisePathMetaData(const Path & path) } -void LocalStore::registerValidPath(const Path & path, - const Hash & hash, const PathSet & references, - const Path & deriver) -{ - ValidPathInfo info; - info.path = path; - info.hash = hash; - info.references = references; - info.deriver = deriver; - registerValidPath(info); -} - - unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) { SQLiteStmtUse use(stmtRegisterValidPath); @@ -454,6 +448,10 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) stmtRegisterValidPath.bind(info.deriver); else stmtRegisterValidPath.bind(); // null + if (info.narSize != 0) + stmtRegisterValidPath.bind(info.narSize); + else + stmtRegisterValidPath.bind(); // null if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) throw SQLiteError(db, format("registering valid path `%1%' in database") % info.path); unsigned long long id = sqlite3_last_insert_rowid(db); @@ -920,10 +918,18 @@ Path LocalStore::addToStoreFromDump(const string & dump, const string & name, the path in the database. We may just have computed it above (if called with recursive == true and hashAlgo == sha256); otherwise, compute it here. */ - registerValidPath(dstPath, - (recursive && hashAlgo == htSHA256) ? h : - (recursive ? hashString(htSHA256, dump) : hashPath(htSHA256, dstPath)), - PathSet(), ""); + HashResult hash; + if (recursive) { + hash.first = hashAlgo == htSHA256 ? h : hashString(htSHA256, dump); + hash.second = dump.size(); + } else + hash = hashPath(htSHA256, dstPath); + + ValidPathInfo info; + info.path = dstPath; + info.hash = hash.first; + info.narSize = hash.second; + registerValidPath(info); } outputLock.setDeletion(true); @@ -970,9 +976,15 @@ Path LocalStore::addTextToStore(const string & name, const string & s, writeFile(dstPath, s); canonicalisePathMetaData(dstPath); + + HashResult hash = hashPath(htSHA256, dstPath); - registerValidPath(dstPath, - hashPath(htSHA256, dstPath), references, ""); + ValidPathInfo info; + info.path = dstPath; + info.hash = hash.first; + info.narSize = hash.second; + info.references = references; + registerValidPath(info); } outputLock.setDeletion(true); @@ -998,7 +1010,7 @@ struct HashAndWriteSink : Sink Hash currentHash() { HashSink hashSinkClone(hashSink); - return hashSinkClone.finish(); + return hashSinkClone.finish().first; } }; @@ -1136,7 +1148,7 @@ Path LocalStore::importPath(bool requireSignature, Source & source) Path deriver = readString(hashAndReadSource); if (deriver != "") assertStorePath(deriver); - Hash hash = hashAndReadSource.hashSink.finish(); + Hash hash = hashAndReadSource.hashSink.finish().first; hashAndReadSource.hashing = false; bool haveSignature = readInt(hashAndReadSource) == 1; @@ -1200,9 +1212,15 @@ Path LocalStore::importPath(bool requireSignature, Source & source) /* !!! if we were clever, we could prevent the hashPath() here. */ - if (deriver != "" && !isValidPath(deriver)) deriver = ""; - registerValidPath(dstPath, - hashPath(htSHA256, dstPath), references, deriver); + HashResult hash = hashPath(htSHA256, dstPath); + + ValidPathInfo info; + info.path = dstPath; + info.hash = hash.first; + info.narSize = hash.second; + info.references = references; + info.deriver = deriver != "" && isValidPath(deriver) ? deriver : ""; + registerValidPath(info); } outputLock.setDeletion(true); @@ -1263,12 +1281,14 @@ void LocalStore::verifyStore(bool checkContents) /* Check the content hash (optionally - slow). */ printMsg(lvlTalkative, format("checking contents of `%1%'") % *i); - Hash current = hashPath(info.hash.type, *i); + Hash current = hashPath(info.hash.type, *i).first; if (current != info.hash) { printMsg(lvlError, format("path `%1%' was modified! " "expected hash `%2%', got `%3%'") % *i % printHash(info.hash) % printHash(current)); } + + /* !!! Check info.narSize */ } } } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 0d7ec1f49..2fd31c26d 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -97,6 +97,8 @@ public: PathSet queryValidPaths(); + ValidPathInfo queryPathInfo(const Path & path); + Hash queryPathHash(const Path & path); void queryReferences(const Path & path, PathSet & references); @@ -173,8 +175,7 @@ public: execution of the derivation (or something equivalent). Also register the hash of the file system contents of the path. The hash must be a SHA-256 hash. */ - void registerValidPath(const Path & path, - const Hash & hash, const PathSet & references, const Path & deriver); + void registerValidPath(const ValidPathInfo & info); void registerValidPaths(const ValidPathInfos & infos); @@ -224,10 +225,6 @@ private: void addReference(unsigned long long referrer, unsigned long long reference); - void registerValidPath(const ValidPathInfo & info); - - ValidPathInfo queryPathInfo(const Path & path); - void appendReferrer(const Path & from, const Path & to, bool lock); void rewriteReferrers(const Path & path, bool purge, PathSet referrers); diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 3ed54e24d..89be6ac65 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -68,7 +68,7 @@ static void hashAndLink(bool dryRun, HashToPath & hashToPath, the contents of the symlink (i.e. the result of readlink()), not the contents of the target (which may not even exist). */ - Hash hash = hashPath(htSHA256, path); + Hash hash = hashPath(htSHA256, path).first; stats.totalFiles++; printMsg(lvlDebug, format("`%1%' has hash `%2%'") % path % printHash(hash)); diff --git a/src/libstore/references.cc b/src/libstore/references.cc index a6f6e85fc..ade9c9aa2 100644 --- a/src/libstore/references.cc +++ b/src/libstore/references.cc @@ -81,7 +81,7 @@ void RefScanSink::operator () (const unsigned char * data, unsigned int len) PathSet scanForReferences(const string & path, - const PathSet & refs, Hash & hash) + const PathSet & refs, HashResult & hash) { RefScanSink sink; std::map backMap; diff --git a/src/libstore/references.hh b/src/libstore/references.hh index 7d068eb51..158c08a77 100644 --- a/src/libstore/references.hh +++ b/src/libstore/references.hh @@ -7,7 +7,7 @@ namespace nix { PathSet scanForReferences(const Path & path, const PathSet & refs, - Hash & hash); + HashResult & hash); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 92d517bbb..517c886b4 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -247,6 +247,12 @@ bool RemoteStore::querySubstitutablePathInfo(const Path & path, } +ValidPathInfo RemoteStore::queryPathInfo(const Path & path) +{ + throw Error("not implemented"); +} + + Hash RemoteStore::queryPathHash(const Path & path) { openConnection(); diff --git a/src/libstore/remote-store.hh b/src/libstore/remote-store.hh index 02a1c4752..519f46fd1 100644 --- a/src/libstore/remote-store.hh +++ b/src/libstore/remote-store.hh @@ -29,6 +29,8 @@ public: PathSet queryValidPaths(); + ValidPathInfo queryPathInfo(const Path & path); + Hash queryPathHash(const Path & path); void queryReferences(const Path & path, PathSet & references); diff --git a/src/libstore/schema.sql b/src/libstore/schema.sql index 7438632ed..c1b4a689a 100644 --- a/src/libstore/schema.sql +++ b/src/libstore/schema.sql @@ -3,7 +3,8 @@ create table if not exists ValidPaths ( path text unique not null, hash text not null, registrationTime integer not null, - deriver text + deriver text, + narSize integer ); create table if not exists Refs ( diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index 01dd51621..4b04f5751 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -190,7 +190,7 @@ std::pair computeStorePathForPath(const Path & srcPath, bool recursive, HashType hashAlgo, PathFilter & filter) { HashType ht(hashAlgo); - Hash h = recursive ? hashPath(ht, srcPath, filter) : hashFile(ht, srcPath); + Hash h = recursive ? hashPath(ht, srcPath, filter).first : hashFile(ht, srcPath); string name = baseNameOf(srcPath); Path dstPath = makeFixedOutputPath(recursive, hashAlgo, h, name); return std::pair(dstPath, h); @@ -216,7 +216,7 @@ Path computeStorePathForText(const string & name, const string & s, /* Return a string accepted by decodeValidPathInfo() that registers the specified paths as valid. Note: it's the responsibility of the caller to provide a closure. */ -string makeValidityRegistration(const PathSet & paths, +string StoreAPI::makeValidityRegistration(const PathSet & paths, bool showDerivers, bool showHash) { string s = ""; @@ -224,18 +224,19 @@ string makeValidityRegistration(const PathSet & paths, foreach (PathSet::iterator, i, paths) { s += *i + "\n"; - if (showHash) - s += printHash(store->queryPathHash(*i)) + "\n"; + ValidPathInfo info = queryPathInfo(*i); - Path deriver = showDerivers ? store->queryDeriver(*i) : ""; + if (showHash) { + s += printHash(info.hash) + "\n"; + s += (format("%1%\n") % info.narSize).str(); + } + + Path deriver = showDerivers ? info.deriver : ""; s += deriver + "\n"; - PathSet references; - store->queryReferences(*i, references); + s += (format("%1%\n") % info.references.size()).str(); - s += (format("%1%\n") % references.size()).str(); - - foreach (PathSet::iterator, j, references) + foreach (PathSet::iterator, j, info.references) s += *j + "\n"; } @@ -252,6 +253,8 @@ ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven) string s; getline(str, s); info.hash = parseHash(htSHA256, s); + getline(str, s); + if (!string2Int(s, info.narSize)) throw Error("number expected"); } getline(str, info.deriver); string s; int n; diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index 082a9edc4..b0071da83 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -90,6 +90,21 @@ struct SubstitutablePathInfo }; +struct ValidPathInfo +{ + Path path; + Path deriver; + Hash hash; + PathSet references; + time_t registrationTime; + unsigned long long narSize; // 0 = unknown + unsigned long long id; // internal use only + ValidPathInfo() : registrationTime(0), narSize(0) { } +}; + +typedef list ValidPathInfos; + + class StoreAPI { public: @@ -102,6 +117,9 @@ public: /* Query the set of valid paths. */ virtual PathSet queryValidPaths() = 0; + /* Query information about a valid path. */ + virtual ValidPathInfo queryPathInfo(const Path & path) = 0; + /* Queries the hash of a valid path. */ virtual Hash queryPathHash(const Path & path) = 0; @@ -214,6 +232,12 @@ public: /* Clear the "failed" status of the given paths. The special value `*' causes all failed paths to be cleared. */ virtual void clearFailedPaths(const PathSet & paths) = 0; + + /* Return a string representing information about the path that + can be loaded into the database using `nix-store --load-db' or + `nix-store --register-validity'. */ + string makeValidityRegistration(const PathSet & paths, + bool showDerivers, bool showHash); }; @@ -307,22 +331,6 @@ boost::shared_ptr openStore(); string showPaths(const PathSet & paths); -string makeValidityRegistration(const PathSet & paths, - bool showDerivers, bool showHash); - -struct ValidPathInfo -{ - Path path; - Path deriver; - Hash hash; - PathSet references; - time_t registrationTime; - unsigned long long id; // internal use only - ValidPathInfo() : registrationTime(0) { } -}; - -typedef list ValidPathInfos; - ValidPathInfo decodeValidPathInfo(std::istream & str, bool hashGiven = false); diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index bd7e33a48..b9e784699 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -286,12 +286,14 @@ Hash hashFile(HashType ht, const Path & path) HashSink::HashSink(HashType ht) : ht(ht) { ctx = new Ctx; + bytes = 0; start(ht, *ctx); } HashSink::HashSink(const HashSink & h) { ht = h.ht; + bytes = h.bytes; ctx = new Ctx; *ctx = *h.ctx; } @@ -304,18 +306,20 @@ HashSink::~HashSink() void HashSink::operator () (const unsigned char * data, unsigned int len) { + bytes += len; update(ht, *ctx, data, len); } -Hash HashSink::finish() +HashResult HashSink::finish() { Hash hash(ht); nix::finish(ht, *ctx, hash.hash); - return hash; + return HashResult(hash, bytes); } -Hash hashPath(HashType ht, const Path & path, PathFilter & filter) +HashResult hashPath( + HashType ht, const Path & path, PathFilter & filter) { HashSink sink(ht); dumpPath(path, sink, filter); diff --git a/src/libutil/hash.hh b/src/libutil/hash.hh index 81425b234..13740954d 100644 --- a/src/libutil/hash.hh +++ b/src/libutil/hash.hh @@ -40,7 +40,6 @@ struct Hash /* For sorting. */ bool operator < (const Hash & h) const; - }; @@ -72,7 +71,8 @@ Hash hashFile(HashType ht, const Path & path); (essentially) hashString(ht, dumpPath(path)). */ struct PathFilter; extern PathFilter defaultPathFilter; -Hash hashPath(HashType ht, const Path & path, +typedef std::pair HashResult; +HashResult hashPath(HashType ht, const Path & path, PathFilter & filter = defaultPathFilter); /* Compress a hash to the specified number of bytes by cyclically @@ -93,13 +93,14 @@ class HashSink : public Sink private: HashType ht; Ctx * ctx; + unsigned long long bytes; public: HashSink(HashType ht); HashSink(const HashSink & h); ~HashSink(); virtual void operator () (const unsigned char * data, unsigned int len); - Hash finish(); + HashResult finish(); }; diff --git a/src/nix-hash/nix-hash.cc b/src/nix-hash/nix-hash.cc index 8a6655403..4867234bf 100644 --- a/src/nix-hash/nix-hash.cc +++ b/src/nix-hash/nix-hash.cc @@ -44,7 +44,7 @@ void run(Strings args) if (op == opHash) { for (Strings::iterator i = ss.begin(); i != ss.end(); ++i) { - Hash h = flat ? hashFile(ht, *i) : hashPath(ht, *i); + Hash h = flat ? hashFile(ht, *i) : hashPath(ht, *i).first; if (truncate && h.hashSize > 20) h = compressHash(h, 20); std::cout << format("%1%\n") % (base32 ? printHash32(h) : printHash(h)); diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 695eb10dc..120f6ce72 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -393,9 +393,8 @@ static void opDumpDB(Strings opFlags, Strings opArgs) if (!opArgs.empty()) throw UsageError("no arguments expected"); PathSet validPaths = store->queryValidPaths(); - foreach (PathSet::iterator, i, validPaths) { - cout << makeValidityRegistration(singleton(*i), true, true); - } + foreach (PathSet::iterator, i, validPaths) + cout << store->makeValidityRegistration(singleton(*i), true, true); } @@ -410,8 +409,11 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) /* !!! races */ if (canonicalise) canonicalisePathMetaData(info.path); - if (!hashGiven) - info.hash = hashPath(htSHA256, info.path); + if (!hashGiven) { + HashResult hash = hashPath(htSHA256, info.path); + info.hash = hash.first; + info.narSize = hash.second; + } infos.push_back(info); } } -- cgit v1.2.3 From 1db6259076b1b8f667451da8d2e44a55ece19056 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Nov 2010 12:08:01 +0000 Subject: * Implement RemoteStore::queryPathInfo(). --- src/libstore/remote-store.cc | 14 +++++++++++++- src/libstore/worker-protocol.hh | 1 + src/nix-worker/nix-worker.cc | 13 +++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) (limited to 'src') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 517c886b4..08969a623 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -249,7 +249,19 @@ bool RemoteStore::querySubstitutablePathInfo(const Path & path, ValidPathInfo RemoteStore::queryPathInfo(const Path & path) { - throw Error("not implemented"); + openConnection(); + writeInt(wopQueryPathInfo, to); + writeString(path, to); + processStderr(); + ValidPathInfo info; + info.path = path; + info.deriver = readString(from); + if (info.deriver != "") assertStorePath(info.deriver); + info.hash = parseHash(htSHA256, readString(from)); + info.references = readStorePaths(from); + info.registrationTime = readInt(from); + info.narSize = readLongLong(from); + return info; } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index a4dc690b7..2764f82c2 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -38,6 +38,7 @@ typedef enum { wopQueryValidPaths = 23, wopQueryFailedPaths = 24, wopClearFailedPaths = 25, + wopQueryPathInfo = 26, } WorkerOp; diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 9be733d8c..dbcd90be1 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -550,6 +550,19 @@ static void performOp(unsigned int clientVersion, break; } + case wopQueryPathInfo: { + Path path = readStorePath(from); + startWork(); + ValidPathInfo info = store->queryPathInfo(path); + stopWork(); + writeString(info.deriver, to); + writeString(printHash(info.hash), to); + writeStringSet(info.references, to); + writeInt(info.registrationTime, to); + writeLongLong(info.narSize, to); + break; + } + default: throw Error(format("invalid operation %1%") % op); } -- cgit v1.2.3 From e60c962fb8dd3d8be37c1f4ae08d5247901fa129 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Nov 2010 12:40:52 +0000 Subject: * Add an operation `nix-store -q --size'. --- src/libstore/local-store.cc | 14 ++++++++++++-- src/libstore/local-store.hh | 1 + src/nix-store/help.txt | 3 ++- src/nix-store/nix-store.cc | 13 +++++++++---- 4 files changed, 24 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 9595561bc..201e4bae1 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -88,6 +88,13 @@ void SQLiteStmt::bind(int value) } +void SQLiteStmt::bind64(long long value) +{ + if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) + throw SQLiteError(db, "binding argument"); +} + + void SQLiteStmt::bind() { if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) @@ -340,7 +347,7 @@ void LocalStore::openDB(bool create) stmtAddReference.create(db, "insert or replace into Refs (referrer, reference) values (?, ?);"); stmtQueryPathInfo.create(db, - "select id, hash, registrationTime, deriver from ValidPaths where path = ?;"); + "select id, hash, registrationTime, deriver, narSize from ValidPaths where path = ?;"); stmtQueryReferences.create(db, "select path from Refs join ValidPaths on reference = id where referrer = ?;"); stmtQueryReferrers.create(db, @@ -449,7 +456,7 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) else stmtRegisterValidPath.bind(); // null if (info.narSize != 0) - stmtRegisterValidPath.bind(info.narSize); + stmtRegisterValidPath.bind64(info.narSize); else stmtRegisterValidPath.bind(); // null if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) @@ -600,6 +607,9 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) s = (const char *) sqlite3_column_text(stmtQueryPathInfo, 3); if (s) info.deriver = s; + /* Note that narSize = NULL yields 0. */ + info.narSize = sqlite3_column_int64(stmtQueryPathInfo, 4); + /* Get the references. */ SQLiteStmtUse use2(stmtQueryReferences); diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 2fd31c26d..4076e5957 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -70,6 +70,7 @@ struct SQLiteStmt operator sqlite3_stmt * () { return stmt; } void bind(const string & value); void bind(int value); + void bind64(long long value); void bind(); }; diff --git a/src/nix-store/help.txt b/src/nix-store/help.txt index 342281b33..478251851 100644 --- a/src/nix-store/help.txt +++ b/src/nix-store/help.txt @@ -16,7 +16,7 @@ Operations: --gc: run the garbage collector - --dump: dump a path as a Nix archive, forgetting dependencies + --dump: dump a path as a Nix archive (NAR), forgetting dependencies --restore: restore a path from a Nix archive, without registering validity @@ -44,6 +44,7 @@ Query flags: --graph: print a dot graph rooted at given path --xml: emit an XML representation of the graph rooted at the given path --hash: print the SHA-256 hash of the contents of the path + --size: print the size of the NAR dump of the path --roots: print the garbage collector roots that point to the path Query switches (not applicable to all queries): diff --git a/src/nix-store/nix-store.cc b/src/nix-store/nix-store.cc index 120f6ce72..49a705585 100644 --- a/src/nix-store/nix-store.cc +++ b/src/nix-store/nix-store.cc @@ -226,7 +226,7 @@ static void printTree(const Path & path, static void opQuery(Strings opFlags, Strings opArgs) { enum { qOutputs, qRequisites, qReferences, qReferrers - , qReferrersClosure, qDeriver, qBinding, qHash + , qReferrersClosure, qDeriver, qBinding, qHash, qSize , qTree, qGraph, qXml, qResolve, qRoots } query = qOutputs; bool useOutput = false; bool includeOutputs = false; @@ -248,6 +248,7 @@ static void opQuery(Strings opFlags, Strings opArgs) query = qBinding; } else if (*i == "--hash") query = qHash; + else if (*i == "--size") query = qSize; else if (*i == "--tree") query = qTree; else if (*i == "--graph") query = qGraph; else if (*i == "--xml") query = qXml; @@ -310,11 +311,15 @@ static void opQuery(Strings opFlags, Strings opArgs) break; case qHash: + case qSize: foreach (Strings::iterator, i, opArgs) { Path path = maybeUseOutput(followLinksToStorePath(*i), useOutput, forceRealise); - Hash hash = store->queryPathHash(path); - assert(hash.type == htSHA256); - cout << format("sha256:%1%\n") % printHash32(hash); + ValidPathInfo info = store->queryPathInfo(path); + if (query == qHash) { + assert(info.hash.type == htSHA256); + cout << format("sha256:%1%\n") % printHash32(info.hash); + } else if (query == qSize) + cout << format("%1%\n") % info.narSize; } break; -- cgit v1.2.3 From bdf089f46362b8c9defefa0a14e3198582e12818 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 17 Nov 2010 14:31:42 +0000 Subject: * Before a build, show the disk space that the downloaded store paths will approximately require. --- src/libmain/shared.cc | 13 +++++++------ src/libstore/local-store.cc | 1 + src/libstore/misc.cc | 5 +++-- src/libstore/misc.hh | 2 +- src/libstore/remote-store.cc | 4 ++-- src/libstore/store-api.hh | 1 + src/libstore/worker-protocol.hh | 2 +- src/nix-worker/nix-worker.cc | 2 ++ 8 files changed, 18 insertions(+), 12 deletions(-) (limited to 'src') diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index d7879f035..7c2d92030 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -54,25 +54,26 @@ void printGCWarning() void printMissing(const PathSet & paths) { - unsigned long long downloadSize; + unsigned long long downloadSize, narSize; PathSet willBuild, willSubstitute, unknown; - queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize); + queryMissing(paths, willBuild, willSubstitute, unknown, downloadSize, narSize); if (!willBuild.empty()) { - printMsg(lvlInfo, format("the following derivations will be built:")); + printMsg(lvlInfo, format("these derivations will be built:")); foreach (PathSet::iterator, i, willBuild) printMsg(lvlInfo, format(" %1%") % *i); } if (!willSubstitute.empty()) { - printMsg(lvlInfo, format("the following paths will be downloaded/copied (%.2f MiB):") % - (downloadSize / (1024.0 * 1024.0))); + printMsg(lvlInfo, format("these paths will be downloaded/copied (%.2f MiB download, %.2f MiB unpacked):") + % (downloadSize / (1024.0 * 1024.0)) + % (narSize / (1024.0 * 1024.0))); foreach (PathSet::iterator, i, willSubstitute) printMsg(lvlInfo, format(" %1%") % *i); } if (!unknown.empty()) { - printMsg(lvlInfo, format("don't know how to build the following paths%1%:") + printMsg(lvlInfo, format("don't know how to build these paths%1%:") % (readOnlyMode ? " (may be caused by read-only store access)" : "")); foreach (PathSet::iterator, i, unknown) printMsg(lvlInfo, format(" %1%") % *i); diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 201e4bae1..37bbbfdad 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -839,6 +839,7 @@ bool LocalStore::querySubstitutablePathInfo(const Path & substituter, info.references.insert(p); } info.downloadSize = getIntLine(run.from); + info.narSize = getIntLine(run.from); return true; } diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index d52dd6346..01d6a97ae 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -48,9 +48,9 @@ Path findOutput(const Derivation & drv, string id) void queryMissing(const PathSet & targets, PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, - unsigned long long & downloadSize) + unsigned long long & downloadSize, unsigned long long & narSize) { - downloadSize = 0; + downloadSize = narSize = 0; PathSet todo(targets.begin(), targets.end()), done; @@ -88,6 +88,7 @@ void queryMissing(const PathSet & targets, if (store->querySubstitutablePathInfo(p, info)) { willSubstitute.insert(p); downloadSize += info.downloadSize; + narSize += info.narSize; todo.insert(info.references.begin(), info.references.end()); } else unknown.insert(p); diff --git a/src/libstore/misc.hh b/src/libstore/misc.hh index 0bc9a2ee0..abef6237e 100644 --- a/src/libstore/misc.hh +++ b/src/libstore/misc.hh @@ -31,7 +31,7 @@ Path findOutput(const Derivation & drv, string id); will be substituted. */ void queryMissing(const PathSet & targets, PathSet & willBuild, PathSet & willSubstitute, PathSet & unknown, - unsigned long long & downloadSize); + unsigned long long & downloadSize, unsigned long long & narSize); } diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 08969a623..26093a5d3 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -191,9 +191,8 @@ void RemoteStore::setOptions() writeInt(logType, to); writeInt(printBuildTrace, to); } - if (GET_PROTOCOL_MINOR(daemonVersion) >= 6) { + if (GET_PROTOCOL_MINOR(daemonVersion) >= 6) writeInt(buildCores, to); - } processStderr(); } @@ -243,6 +242,7 @@ bool RemoteStore::querySubstitutablePathInfo(const Path & path, if (info.deriver != "") assertStorePath(info.deriver); info.references = readStorePaths(from); info.downloadSize = readLongLong(from); + info.narSize = GET_PROTOCOL_MINOR(daemonVersion) >= 7 ? readLongLong(from) : 0; return true; } diff --git a/src/libstore/store-api.hh b/src/libstore/store-api.hh index b0071da83..40ac88714 100644 --- a/src/libstore/store-api.hh +++ b/src/libstore/store-api.hh @@ -87,6 +87,7 @@ struct SubstitutablePathInfo Path deriver; PathSet references; unsigned long long downloadSize; /* 0 = unknown or inapplicable */ + unsigned long long narSize; /* 0 = unknown */ }; diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index 2764f82c2..d77049bc7 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -8,7 +8,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x106 +#define PROTOCOL_VERSION 0x107 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index dbcd90be1..081b5b2fd 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -521,6 +521,8 @@ static void performOp(unsigned int clientVersion, writeString(info.deriver, to); writeStringSet(info.references, to); writeLongLong(info.downloadSize, to); + if (GET_PROTOCOL_MINOR(clientVersion) >= 7) + writeLongLong(info.narSize, to); } break; } -- cgit v1.2.3 From 77f7a6d591e32a4a475552f3e67e3e67b7f71a10 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 29 Nov 2010 17:07:04 +0000 Subject: * Quick hack around one of the pathological cases in bsdiff (namely, when there is a long region of 0s). On one 86 MiB case, this cut patch generation time from 44m to 2m. --- src/bsdiff-4.3/bsdiff.c | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/bsdiff-4.3/bsdiff.c b/src/bsdiff-4.3/bsdiff.c index 150a7f79c..374ed038f 100644 --- a/src/bsdiff-4.3/bsdiff.c +++ b/src/bsdiff-4.3/bsdiff.c @@ -277,6 +277,7 @@ int main(int argc,char *argv[]) for(scsc=scan+=len;scan 64 * 1024) break; for(;scsc Date: Sun, 5 Dec 2010 18:23:19 +0000 Subject: * Retry a transaction if SQLite returns SQLITE_BUSY. This can happen even with a very long busy timeout, because SQLITE_BUSY is also returned to resolve deadlocks. This should get rid of random "database is locked" errors. This is kind of hard to test though. * Fix a horrible bug in deleteFromStore(): deletePathWrapped() should be called after committing the transaction, not before, because the commit might not succeed. --- src/libstore/local-store.cc | 144 +++++++++++++++++++++++++------------------- 1 file changed, 83 insertions(+), 61 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 37bbbfdad..88548706b 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -22,22 +22,31 @@ namespace nix { - -class SQLiteError : public Error + +MakeError(SQLiteError, Error); +MakeError(SQLiteBusy, SQLiteError); + + +static void throwSQLiteError(sqlite3 * db, const format & f) + __attribute__ ((noreturn)); + +static void throwSQLiteError(sqlite3 * db, const format & f) { -public: - SQLiteError(sqlite3 * db, const format & f) - : Error(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)) - { + int err = sqlite3_errcode(db); + if (err == SQLITE_BUSY) { + printMsg(lvlError, "warning: SQLite database is busy"); + throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); } -}; + else + throw SQLiteError(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); +} SQLite::~SQLite() { try { if (db && sqlite3_close(db) != SQLITE_OK) - throw SQLiteError(db, "closing database"); + throwSQLiteError(db, "closing database"); } catch (...) { ignoreException(); } @@ -49,7 +58,7 @@ void SQLiteStmt::create(sqlite3 * db, const string & s) checkInterrupt(); assert(!stmt); if (sqlite3_prepare_v2(db, s.c_str(), -1, &stmt, 0) != SQLITE_OK) - throw SQLiteError(db, "creating statement"); + throwSQLiteError(db, "creating statement"); this->db = db; } @@ -58,7 +67,7 @@ void SQLiteStmt::reset() { assert(stmt); if (sqlite3_reset(stmt) != SQLITE_OK) - throw SQLiteError(db, "resetting statement"); + throwSQLiteError(db, "resetting statement"); curArg = 1; } @@ -67,7 +76,7 @@ SQLiteStmt::~SQLiteStmt() { try { if (stmt && sqlite3_finalize(stmt) != SQLITE_OK) - throw SQLiteError(db, "finalizing statement"); + throwSQLiteError(db, "finalizing statement"); } catch (...) { ignoreException(); } @@ -77,28 +86,28 @@ SQLiteStmt::~SQLiteStmt() void SQLiteStmt::bind(const string & value) { if (sqlite3_bind_text(stmt, curArg++, value.c_str(), -1, SQLITE_TRANSIENT) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + throwSQLiteError(db, "binding argument"); } void SQLiteStmt::bind(int value) { if (sqlite3_bind_int(stmt, curArg++, value) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + throwSQLiteError(db, "binding argument"); } void SQLiteStmt::bind64(long long value) { if (sqlite3_bind_int64(stmt, curArg++, value) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + throwSQLiteError(db, "binding argument"); } void SQLiteStmt::bind() { if (sqlite3_bind_null(stmt, curArg++) != SQLITE_OK) - throw SQLiteError(db, "binding argument"); + throwSQLiteError(db, "binding argument"); } @@ -132,14 +141,14 @@ struct SQLiteTxn SQLiteTxn(sqlite3 * db) : active(false) { this->db = db; if (sqlite3_exec(db, "begin;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "starting transaction"); + throwSQLiteError(db, "starting transaction"); active = true; } void commit() { if (sqlite3_exec(db, "commit;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "committing transaction"); + throwSQLiteError(db, "committing transaction"); active = false; } @@ -147,7 +156,7 @@ struct SQLiteTxn { try { if (active && sqlite3_exec(db, "rollback;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "aborting transaction"); + throwSQLiteError(db, "aborting transaction"); } catch (...) { ignoreException(); } @@ -289,10 +298,10 @@ void LocalStore::openDB(bool create) throw Error("cannot open SQLite database"); if (sqlite3_busy_timeout(db, 60 * 60 * 1000) != SQLITE_OK) - throw SQLiteError(db, "setting timeout"); + throwSQLiteError(db, "setting timeout"); if (sqlite3_exec(db, "pragma foreign_keys = 1;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "enabling foreign keys"); + throwSQLiteError(db, "enabling foreign keys"); /* !!! check whether sqlite has been built with foreign key support */ @@ -303,7 +312,7 @@ void LocalStore::openDB(bool create) crashes. */ string syncMode = queryBoolSetting("fsync-metadata", true) ? "normal" : "off"; if (sqlite3_exec(db, ("pragma synchronous = " + syncMode + ";").c_str(), 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "setting synchronous mode"); + throwSQLiteError(db, "setting synchronous mode"); /* Set the SQLite journal mode. WAL mode is fastest, but doesn't seem entirely stable at the moment (Oct. 2010). Thus, use @@ -314,31 +323,31 @@ void LocalStore::openDB(bool create) SQLiteStmt stmt; stmt.create(db, "pragma main.journal_mode;"); if (sqlite3_step(stmt) != SQLITE_ROW) - throw SQLiteError(db, "querying journal mode"); + throwSQLiteError(db, "querying journal mode"); prevMode = string((const char *) sqlite3_column_text(stmt, 0)); } if (prevMode != mode && sqlite3_exec(db, ("pragma main.journal_mode = " + mode + ";").c_str(), 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "setting journal mode"); + throwSQLiteError(db, "setting journal mode"); /* Increase the auto-checkpoint interval to 8192 pages. This seems enough to ensure that instantiating the NixOS system derivation is done in a single fsync(). */ if (sqlite3_exec(db, "pragma wal_autocheckpoint = 8192;", 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "setting autocheckpoint interval"); + throwSQLiteError(db, "setting autocheckpoint interval"); /* Initialise the database schema, if necessary. */ if (create) { #include "schema.sql.hh" if (sqlite3_exec(db, (const char *) schema, 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "initialising database schema"); + throwSQLiteError(db, "initialising database schema"); } /* Backwards compatibility with old (pre-release) databases. Can remove this eventually. */ if (sqlite3_table_column_metadata(db, 0, "ValidPaths", "narSize", 0, 0, 0, 0, 0) != SQLITE_OK) { if (sqlite3_exec(db, "alter table ValidPaths add column narSize integer" , 0, 0, 0) != SQLITE_OK) - throw SQLiteError(db, "adding column narSize"); + throwSQLiteError(db, "adding column narSize"); } /* Prepare SQL statements. */ @@ -460,7 +469,7 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) else stmtRegisterValidPath.bind(); // null if (sqlite3_step(stmtRegisterValidPath) != SQLITE_DONE) - throw SQLiteError(db, format("registering valid path `%1%' in database") % info.path); + throwSQLiteError(db, format("registering valid path `%1%' in database") % info.path); unsigned long long id = sqlite3_last_insert_rowid(db); /* If this is a derivation, then store the derivation outputs in @@ -475,7 +484,7 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) stmtAddDerivationOutput.bind(i->first); stmtAddDerivationOutput.bind(i->second.path); if (sqlite3_step(stmtAddDerivationOutput) != SQLITE_DONE) - throw SQLiteError(db, format("adding derivation output for `%1%' in database") % info.path); + throwSQLiteError(db, format("adding derivation output for `%1%' in database") % info.path); } } @@ -489,7 +498,7 @@ void LocalStore::addReference(unsigned long long referrer, unsigned long long re stmtAddReference.bind(referrer); stmtAddReference.bind(reference); if (sqlite3_step(stmtAddReference) != SQLITE_DONE) - throw SQLiteError(db, "adding reference to database"); + throwSQLiteError(db, "adding reference to database"); } @@ -498,15 +507,23 @@ void LocalStore::registerValidPath(const ValidPathInfo & info) assert(info.hash.type == htSHA256); ValidPathInfo info2(info); if (info2.registrationTime == 0) info2.registrationTime = time(0); - - SQLiteTxn txn(db); - - unsigned long long id = addValidPath(info2); - foreach (PathSet::const_iterator, i, info2.references) - addReference(id, queryValidPathId(*i)); - - txn.commit(); + while (1) { + try { + SQLiteTxn txn(db); + + unsigned long long id = addValidPath(info2); + + foreach (PathSet::const_iterator, i, info2.references) + addReference(id, queryValidPathId(*i)); + + txn.commit(); + break; + } catch (SQLiteBusy & e) { + /* Retry; the `txn' destructor will roll back the current + transaction. */ + } + } } @@ -517,7 +534,7 @@ void LocalStore::registerFailedPath(const Path & path) stmtRegisterFailedPath.bind(path); stmtRegisterFailedPath.bind(time(0)); if (sqlite3_step(stmtRegisterFailedPath) != SQLITE_DONE) - throw SQLiteError(db, format("registering failed path `%1%'") % path); + throwSQLiteError(db, format("registering failed path `%1%'") % path); } @@ -527,7 +544,7 @@ bool LocalStore::hasPathFailed(const Path & path) stmtHasPathFailed.bind(path); int res = sqlite3_step(stmtHasPathFailed); if (res != SQLITE_DONE && res != SQLITE_ROW) - throw SQLiteError(db, "querying whether path failed"); + throwSQLiteError(db, "querying whether path failed"); return res == SQLITE_ROW; } @@ -545,7 +562,7 @@ PathSet LocalStore::queryFailedPaths() } if (r != SQLITE_DONE) - throw SQLiteError(db, "error querying failed paths"); + throwSQLiteError(db, "error querying failed paths"); return res; } @@ -559,7 +576,7 @@ void LocalStore::clearFailedPaths(const PathSet & paths) SQLiteStmtUse use(stmtClearFailedPath); stmtClearFailedPath.bind(*i); if (sqlite3_step(stmtClearFailedPath) != SQLITE_DONE) - throw SQLiteError(db, format("clearing failed path `%1%' in database") % *i); + throwSQLiteError(db, format("clearing failed path `%1%' in database") % *i); } txn.commit(); @@ -594,7 +611,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) int r = sqlite3_step(stmtQueryPathInfo); if (r == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); - if (r != SQLITE_ROW) throw SQLiteError(db, "querying path in database"); + if (r != SQLITE_ROW) throwSQLiteError(db, "querying path in database"); info.id = sqlite3_column_int(stmtQueryPathInfo, 0); @@ -622,7 +639,7 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) } if (r != SQLITE_DONE) - throw SQLiteError(db, format("error getting references of `%1%'") % path); + throwSQLiteError(db, format("error getting references of `%1%'") % path); return info; } @@ -635,7 +652,7 @@ unsigned long long LocalStore::queryValidPathId(const Path & path) int res = sqlite3_step(stmtQueryPathInfo); if (res == SQLITE_ROW) return sqlite3_column_int(stmtQueryPathInfo, 0); if (res == SQLITE_DONE) throw Error(format("path `%1%' is not valid") % path); - throw SQLiteError(db, "querying path in database"); + throwSQLiteError(db, "querying path in database"); } @@ -645,7 +662,7 @@ bool LocalStore::isValidPath(const Path & path) stmtQueryPathInfo.bind(path); int res = sqlite3_step(stmtQueryPathInfo); if (res != SQLITE_DONE && res != SQLITE_ROW) - throw SQLiteError(db, "querying path in database"); + throwSQLiteError(db, "querying path in database"); return res == SQLITE_ROW; } @@ -665,7 +682,7 @@ PathSet LocalStore::queryValidPaths() } if (r != SQLITE_DONE) - throw SQLiteError(db, "error getting valid paths"); + throwSQLiteError(db, "error getting valid paths"); return res; } @@ -695,7 +712,7 @@ void LocalStore::queryReferrers(const Path & path, PathSet & referrers) } if (r != SQLITE_DONE) - throw SQLiteError(db, format("error getting references of `%1%'") % path); + throwSQLiteError(db, format("error getting references of `%1%'") % path); } @@ -721,7 +738,7 @@ PathSet LocalStore::queryValidDerivers(const Path & path) } if (r != SQLITE_DONE) - throw SQLiteError(db, format("error getting valid derivers of `%1%'") % path); + throwSQLiteError(db, format("error getting valid derivers of `%1%'") % path); return derivers; } @@ -743,7 +760,7 @@ PathSet LocalStore::queryDerivationOutputs(const Path & path) } if (r != SQLITE_DONE) - throw SQLiteError(db, format("error getting outputs of `%1%'") % path); + throwSQLiteError(db, format("error getting outputs of `%1%'") % path); return outputs; } @@ -890,7 +907,7 @@ void LocalStore::invalidatePath(const Path & path) stmtInvalidatePath.bind(path); if (sqlite3_step(stmtInvalidatePath) != SQLITE_DONE) - throw SQLiteError(db, format("invalidating path `%1%' in database") % path); + throwSQLiteError(db, format("invalidating path `%1%' in database") % path); /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ @@ -1248,20 +1265,25 @@ void LocalStore::deleteFromStore(const Path & path, unsigned long long & bytesFr assertStorePath(path); - SQLiteTxn txn(db); + while (1) { + try { + SQLiteTxn txn(db); + + if (isValidPath(path)) { + PathSet referrers; queryReferrers(path, referrers); + referrers.erase(path); /* ignore self-references */ + if (!referrers.empty()) + throw PathInUse(format("cannot delete path `%1%' because it is in use by `%2%'") + % path % showPaths(referrers)); + invalidatePath(path); + } - if (isValidPath(path)) { - PathSet referrers; queryReferrers(path, referrers); - referrers.erase(path); /* ignore self-references */ - if (!referrers.empty()) - throw PathInUse(format("cannot delete path `%1%' because it is in use by `%2%'") - % path % showPaths(referrers)); - invalidatePath(path); + txn.commit(); + break; + } catch (SQLiteBusy & e) { }; } - + deletePathWrapped(path, bytesFreed, blocksFreed); - - txn.commit(); } -- cgit v1.2.3 From 8062d3af30b27eb4d617c14856d4f3a173f8012e Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 6 Dec 2010 15:29:38 +0000 Subject: * `nix-store --verify --check-contents': don't hold the global GC lock while checking the contents, since this operation can take a very long time to finish. Also, fill in missing narSize fields in the DB while doing this. --- src/libstore/local-store.cc | 57 ++++++++++++++++++++++++++++++++++++--------- src/libstore/local-store.hh | 3 +++ 2 files changed, 49 insertions(+), 11 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 88548706b..3782d1b16 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -353,6 +353,8 @@ void LocalStore::openDB(bool create) /* Prepare SQL statements. */ stmtRegisterValidPath.create(db, "insert into ValidPaths (path, hash, registrationTime, deriver, narSize) values (?, ?, ?, ?, ?);"); + stmtUpdatePathInfo.create(db, + "update ValidPaths set narSize = ? where path = ?;"); stmtAddReference.create(db, "insert or replace into Refs (referrer, reference) values (?, ?);"); stmtQueryPathInfo.create(db, @@ -645,6 +647,21 @@ ValidPathInfo LocalStore::queryPathInfo(const Path & path) } +/* Update path info in the database. Currently only updated the + narSize field. */ +void LocalStore::updatePathInfo(const ValidPathInfo & info) +{ + SQLiteStmtUse use(stmtUpdatePathInfo); + if (info.narSize != 0) + stmtUpdatePathInfo.bind64(info.narSize); + else + stmtUpdatePathInfo.bind(); // null + stmtUpdatePathInfo.bind(info.path); + if (sqlite3_step(stmtUpdatePathInfo) != SQLITE_DONE) + throwSQLiteError(db, format("updating info of path `%1%' in database") % info.path); +} + + unsigned long long LocalStore::queryValidPathId(const Path & path) { SQLiteStmtUse use(stmtQueryPathInfo); @@ -1305,23 +1322,41 @@ void LocalStore::verifyStore(bool checkContents) foreach (PathSet::iterator, i, validPaths2) verifyPath(*i, store, done, validPaths); + /* Release the GC lock so that checking content hashes (which can + take ages) doesn't block the GC or builds. */ + fdGCLock.close(); + /* Optionally, check the content hashes (slow). */ if (checkContents) { printMsg(lvlInfo, "checking hashes..."); foreach (PathSet::iterator, i, validPaths) { - ValidPathInfo info = queryPathInfo(*i); - - /* Check the content hash (optionally - slow). */ - printMsg(lvlTalkative, format("checking contents of `%1%'") % *i); - Hash current = hashPath(info.hash.type, *i).first; - if (current != info.hash) { - printMsg(lvlError, format("path `%1%' was modified! " - "expected hash `%2%', got `%3%'") - % *i % printHash(info.hash) % printHash(current)); - } + try { + ValidPathInfo info = queryPathInfo(*i); + + /* Check the content hash (optionally - slow). */ + printMsg(lvlTalkative, format("checking contents of `%1%'") % *i); + HashResult current = hashPath(info.hash.type, *i); + + if (current.first != info.hash) { + printMsg(lvlError, format("path `%1%' was modified! " + "expected hash `%2%', got `%3%'") + % *i % printHash(info.hash) % printHash(current.first)); + } else { + /* Fill in missing narSize fields (from old stores). */ + if (info.narSize == 0) { + printMsg(lvlError, format("updating size field on `%1%' to %2%") % *i % current.second); + info.narSize = current.second; + updatePathInfo(info); + } + } - /* !!! Check info.narSize */ + } catch (Error & e) { + /* It's possible that the path got GC'ed, so ignore + errors on invalid paths. */ + if (isValidPath(*i)) throw; + printMsg(lvlError, format("warning: %1%") % e.msg()); + } } } } diff --git a/src/libstore/local-store.hh b/src/libstore/local-store.hh index 4076e5957..f270fb723 100644 --- a/src/libstore/local-store.hh +++ b/src/libstore/local-store.hh @@ -203,6 +203,7 @@ private: /* Some precompiled SQLite statements. */ SQLiteStmt stmtRegisterValidPath; + SQLiteStmt stmtUpdatePathInfo; SQLiteStmt stmtAddReference; SQLiteStmt stmtQueryPathInfo; SQLiteStmt stmtQueryReferences; @@ -235,6 +236,8 @@ private: void verifyPath(const Path & path, const PathSet & store, PathSet & done, PathSet & validPaths); + void updatePathInfo(const ValidPathInfo & info); + void upgradeStore6(); PathSet queryValidPathsOld(); ValidPathInfo queryPathInfoOld(const Path & path); -- cgit v1.2.3 From e4720b1a79f549b3bcb0e33fe999d02d12e3ed23 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Wed, 8 Dec 2010 18:19:15 +0000 Subject: * Ignore the result of sqlite3_reset(). --- src/libstore/local-store.cc | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 3782d1b16..b80f66412 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -66,8 +66,9 @@ void SQLiteStmt::create(sqlite3 * db, const string & s) void SQLiteStmt::reset() { assert(stmt); - if (sqlite3_reset(stmt) != SQLITE_OK) - throwSQLiteError(db, "resetting statement"); + /* Note: sqlite3_reset() returns the error code for the most + recent call to sqlite3_step(). So ignore it. */ + sqlite3_reset(stmt); curArg = 1; } @@ -333,7 +334,7 @@ void LocalStore::openDB(bool create) /* Increase the auto-checkpoint interval to 8192 pages. This seems enough to ensure that instantiating the NixOS system derivation is done in a single fsync(). */ - if (sqlite3_exec(db, "pragma wal_autocheckpoint = 8192;", 0, 0, 0) != SQLITE_OK) + if (mode == "wal" && sqlite3_exec(db, "pragma wal_autocheckpoint = 8192;", 0, 0, 0) != SQLITE_OK) throwSQLiteError(db, "setting autocheckpoint interval"); /* Initialise the database schema, if necessary. */ -- cgit v1.2.3 From d7ca6f44eb506ec315db2ba59216de59de2758b6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2010 13:19:46 +0000 Subject: * Update some comments. --- src/libstore/build.cc | 2 +- src/nix-worker/nix-worker.cc | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 8b8be3e80..0be9c42e0 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -1715,7 +1715,7 @@ void DerivationGoal::startBuilder() case 0: /* Warning: in the child we should absolutely not make any - Berkeley DB calls! */ + SQLite calls! */ try { /* child */ diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index 081b5b2fd..ec5ca58aa 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -712,9 +712,8 @@ static void daemonLoop() while (1) { try { - /* Important: the server process *cannot* open the - Berkeley DB environment, because it doesn't like forks - very much. */ + /* Important: the server process *cannot* open the SQLite + database, because it doesn't like forks very much. */ assert(!store); /* Accept a connection. */ -- cgit v1.2.3 From 100becf8d1b95055c73a024f1a2b318690de72f6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2010 13:32:58 +0000 Subject: * createDirs(path): if path already exists, make sure it's a directory. * Provide a C++ wrapper around lstat(). --- src/libutil/util.cc | 42 ++++++++++++++++++++++-------------------- src/libutil/util.hh | 4 ++++ 2 files changed, 26 insertions(+), 20 deletions(-) (limited to 'src') diff --git a/src/libutil/util.cc b/src/libutil/util.cc index e7c0700cf..990962763 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -7,11 +7,8 @@ #include #include -#include #include -#include #include -#include #include #include "util.hh" @@ -149,6 +146,15 @@ string baseNameOf(const Path & path) } +struct stat lstat(const Path & path) +{ + struct stat st; + if (lstat(path.c_str(), &st)) + throw SysError(format("getting status of `%1%'") % path); + return st; +} + + bool pathExists(const Path & path) { int res; @@ -164,9 +170,7 @@ bool pathExists(const Path & path) Path readLink(const Path & path) { checkInterrupt(); - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of `%1%'") % path); + struct stat st = lstat(path); if (!S_ISLNK(st.st_mode)) throw Error(format("`%1%' is not a symlink") % path); char buf[st.st_size]; @@ -178,9 +182,7 @@ Path readLink(const Path & path) bool isLink(const Path & path) { - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting status of `%1%'") % path); + struct stat st = lstat(path); return S_ISLNK(st.st_mode); } @@ -269,9 +271,7 @@ static void _computePathSize(const Path & path, { checkInterrupt(); - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); + struct stat st = lstat(path); bytes += st.st_size; blocks += st.st_blocks; @@ -301,9 +301,7 @@ static void _deletePath(const Path & path, unsigned long long & bytesFreed, printMsg(lvlVomit, format("%1%") % path); - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); + struct stat st = lstat(path); if (!S_ISDIR(st.st_mode) && st.st_nlink == 1) { bytesFreed += st.st_size; @@ -350,9 +348,7 @@ void makePathReadOnly(const Path & path) { checkInterrupt(); - struct stat st; - if (lstat(path.c_str(), &st)) - throw SysError(format("getting attributes of path `%1%'") % path); + struct stat st = lstat(path); if (!S_ISLNK(st.st_mode) && (st.st_mode & S_IWUSR)) { if (chmod(path.c_str(), st.st_mode & ~S_IWUSR) == -1) @@ -411,12 +407,18 @@ Paths createDirs(const Path & path) { Paths created; if (path == "/") return created; - if (!pathExists(path)) { + + struct stat st; + if (lstat(path.c_str(), &st) == -1) { created = createDirs(dirOf(path)); - if (mkdir(path.c_str(), 0777) == -1) + if (mkdir(path.c_str(), 0777) == -1 && errno != EEXIST) throw SysError(format("creating directory `%1%'") % path); + st = lstat(path); created.push_back(path); } + + if (!S_ISDIR(st.st_mode)) throw Error(format("`%1%' is not a directory") % path); + return created; } diff --git a/src/libutil/util.hh b/src/libutil/util.hh index 27ad46904..f86290f31 100644 --- a/src/libutil/util.hh +++ b/src/libutil/util.hh @@ -4,6 +4,7 @@ #include "types.hh" #include +#include #include #include #include @@ -42,6 +43,9 @@ Path dirOf(const Path & path); following the final `/'. */ string baseNameOf(const Path & path); +/* Get status of `path'. */ +struct stat lstat(const Path & path); + /* Return true iff the given path exists. */ bool pathExists(const Path & path); -- cgit v1.2.3 From 5833243c92f28759ff0fc1ff9266535a3230e2d6 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2010 13:42:34 +0000 Subject: * Create /nix/var/nix/db if it's missing. --- src/libstore/local-store.cc | 1 + 1 file changed, 1 insertion(+) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index b80f66412..32c7a5330 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -199,6 +199,7 @@ LocalStore::LocalStore() Path profilesDir = nixStateDir + "/profiles"; createDirs(nixStateDir + "/profiles"); createDirs(nixStateDir + "/temproots"); + createDirs(nixDBPath); Path gcRootsDir = nixStateDir + "/gcroots"; if (!pathExists(gcRootsDir)) { createDirs(gcRootsDir); -- cgit v1.2.3 From d787285af997a607bb678f39f340e663fafd3122 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Mon, 13 Dec 2010 16:53:23 +0000 Subject: * nix-instantiate: return exit status 100 to denote a permanent build failure. The build hook can use this to distinguish between transient and permanent failures on the remote side. --- src/libmain/shared.cc | 2 +- src/libstore/build.cc | 19 ++++++++++++++++--- src/libutil/types.hh | 5 +++-- src/libutil/util.cc | 3 ++- 4 files changed, 22 insertions(+), 7 deletions(-) (limited to 'src') diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 7c2d92030..68f145820 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -393,7 +393,7 @@ int main(int argc, char * * argv) printMsg(lvlError, format("error: %1%%2%") % (showTrace ? e.prefix() : "") % e.msg()); if (e.prefix() != "" && !showTrace) printMsg(lvlError, "(use `--show-trace' to show detailed location information)"); - return 1; + return e.status; } catch (std::exception & e) { printMsg(lvlError, format("error: %1%") % e.what()); return 1; diff --git a/src/libstore/build.cc b/src/libstore/build.cc index 0be9c42e0..83bd6754a 100644 --- a/src/libstore/build.cc +++ b/src/libstore/build.cc @@ -214,6 +214,10 @@ public: bool cacheFailure; + /* Set if at least one derivation had a BuildError (i.e. permanent + failure). */ + bool permanentFailure; + LocalStore & store; boost::shared_ptr hook; @@ -266,7 +270,8 @@ public: /* Wait for input to become available. */ void waitForInput(); - + + unsigned int exitStatus(); }; @@ -1185,6 +1190,7 @@ void DerivationGoal::tryToBuild() if (printBuildTrace) printMsg(lvlError, format("@ build-failed %1% %2% %3% %4%") % drvPath % drv.outputs["out"].path % 0 % e.msg()); + worker.permanentFailure = true; amDone(ecFailed); return; } @@ -1321,6 +1327,7 @@ void DerivationGoal::buildDone() foreach (DerivationOutputs::iterator, i, drv.outputs) worker.store.registerFailedPath(i->second.path); + worker.permanentFailure = !hookError && !fixedOutput; amDone(ecFailed); return; } @@ -2444,6 +2451,7 @@ Worker::Worker(LocalStore & store) nrLocalBuilds = 0; lastWokenUp = 0; cacheFailure = queryBoolSetting("build-cache-failure", false); + permanentFailure = false; } @@ -2770,6 +2778,11 @@ void Worker::waitForInput() } +unsigned int Worker::exitStatus() +{ + return permanentFailure ? 100 : 1; +} + ////////////////////////////////////////////////////////////////////// @@ -2796,7 +2809,7 @@ void LocalStore::buildDerivations(const PathSet & drvPaths) } if (!failed.empty()) - throw Error(format("build of %1% failed") % showPaths(failed)); + throw Error(format("build of %1% failed") % showPaths(failed), worker.exitStatus()); } @@ -2812,7 +2825,7 @@ void LocalStore::ensurePath(const Path & path) worker.run(goals); if (goal->getExitCode() != Goal::ecSuccess) - throw Error(format("path `%1%' does not exist and cannot be created") % path); + throw Error(format("path `%1%' does not exist and cannot be created") % path, worker.exitStatus()); } diff --git a/src/libutil/types.hh b/src/libutil/types.hh index 854a0f689..533fcca22 100644 --- a/src/libutil/types.hh +++ b/src/libutil/types.hh @@ -29,7 +29,8 @@ protected: string prefix_; // used for location traces etc. string err; public: - BaseError(const format & f); + unsigned int status; // exit status + BaseError(const format & f, unsigned int status = 1); ~BaseError() throw () { }; const char * what() const throw () { return err.c_str(); } const string & msg() const throw () { return err; } @@ -41,7 +42,7 @@ public: class newClass : public superClass \ { \ public: \ - newClass(const format & f) : superClass(f) { }; \ + newClass(const format & f, unsigned int status = 1) : superClass(f, status) { }; \ }; MakeError(Error, BaseError) diff --git a/src/libutil/util.cc b/src/libutil/util.cc index 990962763..9adaac40d 100644 --- a/src/libutil/util.cc +++ b/src/libutil/util.cc @@ -20,7 +20,8 @@ extern char * * environ; namespace nix { -BaseError::BaseError(const format & f) +BaseError::BaseError(const format & f, unsigned int status) + : status(status) { err = f.str(); } -- cgit v1.2.3 From 3dd02580e324d04ebfe999779978b4aa0f999ccd Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Tue, 14 Dec 2010 13:25:20 +0000 Subject: * I forgot to catch SQLiteBusy in registerValidPaths(). So registerValidPaths() now handles busy errors and registerValidPath() is simply a wrapper around it. --- src/libstore/local-store.cc | 69 ++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 38 deletions(-) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 32c7a5330..56d05c7bb 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -463,7 +463,7 @@ unsigned long long LocalStore::addValidPath(const ValidPathInfo & info) SQLiteStmtUse use(stmtRegisterValidPath); stmtRegisterValidPath.bind(info.path); stmtRegisterValidPath.bind("sha256:" + printHash(info.hash)); - stmtRegisterValidPath.bind(info.registrationTime); + stmtRegisterValidPath.bind(info.registrationTime == 0 ? time(0) : info.registrationTime); if (info.deriver != "") stmtRegisterValidPath.bind(info.deriver); else @@ -506,31 +506,6 @@ void LocalStore::addReference(unsigned long long referrer, unsigned long long re } -void LocalStore::registerValidPath(const ValidPathInfo & info) -{ - assert(info.hash.type == htSHA256); - ValidPathInfo info2(info); - if (info2.registrationTime == 0) info2.registrationTime = time(0); - - while (1) { - try { - SQLiteTxn txn(db); - - unsigned long long id = addValidPath(info2); - - foreach (PathSet::const_iterator, i, info2.references) - addReference(id, queryValidPathId(*i)); - - txn.commit(); - break; - } catch (SQLiteBusy & e) { - /* Retry; the `txn' destructor will roll back the current - transaction. */ - } - } -} - - void LocalStore::registerFailedPath(const Path & path) { if (hasPathFailed(path)) return; @@ -896,22 +871,40 @@ Hash LocalStore::queryPathHash(const Path & path) } +void LocalStore::registerValidPath(const ValidPathInfo & info) +{ + ValidPathInfos infos; + infos.push_back(info); + registerValidPaths(infos); +} + + void LocalStore::registerValidPaths(const ValidPathInfos & infos) { - SQLiteTxn txn(db); + while (1) { + try { + SQLiteTxn txn(db); - foreach (ValidPathInfos::const_iterator, i, infos) - /* !!! Maybe the registration info should be updated if the - path is already valid. */ - if (!isValidPath(i->path)) addValidPath(*i); - - foreach (ValidPathInfos::const_iterator, i, infos) { - unsigned long long referrer = queryValidPathId(i->path); - foreach (PathSet::iterator, j, i->references) - addReference(referrer, queryValidPathId(*j)); - } + foreach (ValidPathInfos::const_iterator, i, infos) { + assert(i->hash.type == htSHA256); + /* !!! Maybe the registration info should be updated if the + path is already valid. */ + if (!isValidPath(i->path)) addValidPath(*i); + } - txn.commit(); + foreach (ValidPathInfos::const_iterator, i, infos) { + unsigned long long referrer = queryValidPathId(i->path); + foreach (PathSet::iterator, j, i->references) + addReference(referrer, queryValidPathId(*j)); + } + + txn.commit(); + break; + } catch (SQLiteBusy & e) { + /* Retry; the `txn' destructor will roll back the current + transaction. */ + } + } } -- cgit v1.2.3 From b1eb25217217087cb70a730da5311bd0890cf6ad Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Dec 2010 11:28:26 +0000 Subject: * Propagate the "100" exit status for failed builds through the Nix daemon. --- src/libstore/remote-store.cc | 7 +++++-- src/libstore/worker-protocol.hh | 2 +- src/nix-worker/nix-worker.cc | 5 +++-- 3 files changed, 9 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 26093a5d3..ae99846a3 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -509,8 +509,11 @@ void RemoteStore::processStderr(Sink * sink, Source * source) writeToStderr((const unsigned char *) s.c_str(), s.size()); } } - if (msg == STDERR_ERROR) - throw Error(readString(from)); + if (msg == STDERR_ERROR) { + string error = readString(from); + unsigned int status = GET_PROTOCOL_MINOR(daemonVersion) >= 8 ? readInt(from) : 1; + throw Error(error, status); + } else if (msg != STDERR_LAST) throw Error("protocol error processing standard error"); } diff --git a/src/libstore/worker-protocol.hh b/src/libstore/worker-protocol.hh index d77049bc7..acb8bc8b2 100644 --- a/src/libstore/worker-protocol.hh +++ b/src/libstore/worker-protocol.hh @@ -8,7 +8,7 @@ namespace nix { #define WORKER_MAGIC_1 0x6e697863 #define WORKER_MAGIC_2 0x6478696f -#define PROTOCOL_VERSION 0x107 +#define PROTOCOL_VERSION 0x108 #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) diff --git a/src/nix-worker/nix-worker.cc b/src/nix-worker/nix-worker.cc index ec5ca58aa..0fa1b40ae 100644 --- a/src/nix-worker/nix-worker.cc +++ b/src/nix-worker/nix-worker.cc @@ -178,7 +178,7 @@ static void startWork() /* stopWork() means that we're done; stop sending stderr to the client. */ -static void stopWork(bool success = true, const string & msg = "") +static void stopWork(bool success = true, const string & msg = "", unsigned int status = 0) { /* Stop handling async client death; we're going to a state where we're either sending or receiving from the client, so we'll be @@ -192,6 +192,7 @@ static void stopWork(bool success = true, const string & msg = "") else { writeInt(STDERR_ERROR, to); writeString(msg, to); + if (status != 0) writeInt(status, to); } } @@ -637,7 +638,7 @@ static void processConnection() try { performOp(clientVersion, from, to, op); } catch (Error & e) { - stopWork(false, e.msg()); + stopWork(false, e.msg(), GET_PROTOCOL_MINOR(clientVersion) >= 8 ? e.status : 0); } assert(!canSendStderr); -- cgit v1.2.3 From c931a7aec5ccb2209d3c4bcf4452c807fe894d94 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 17 Dec 2010 17:23:15 +0000 Subject: * Do a short sleep after SQLITE_BUSY. --- src/libstore/local-store.cc | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'src') diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index 56d05c7bb..6af34cc77 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include @@ -35,6 +36,16 @@ static void throwSQLiteError(sqlite3 * db, const format & f) int err = sqlite3_errcode(db); if (err == SQLITE_BUSY) { printMsg(lvlError, "warning: SQLite database is busy"); + /* Sleep for a while since retrying the transaction right away + is likely to fail again. */ +#if HAVE_NANOSLEEP + struct timespec t; + t.tv_sec = 0; + t.tv_nsec = 100 * 1000 * 1000; /* 0.1s */ + nanosleep(&t, 0); +#else + sleep(1); +#endif throw SQLiteBusy(format("%1%: %2%") % f.str() % sqlite3_errmsg(db)); } else -- cgit v1.2.3 From 9db190eb31d4adc412d50bc03574951f9a1f9dae Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Fri, 14 Jan 2011 12:47:10 +0000 Subject: * builtins.substring: if "start" is beyond the end of the string, return the empty string. --- src/libexpr/primops.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index a4812de06..3e955ea3f 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -965,7 +965,7 @@ static void prim_substring(EvalState & state, Value * * args, Value & v) if (start < 0) throw EvalError("negative start position in `substring'"); - mkString(v, string(s, start, len), context); + mkString(v, start >= s.size() ? "" : string(s, start, len), context); } -- cgit v1.2.3 From aeae0beba49e98655861c5c50e573bd7d9cfb5c5 Mon Sep 17 00:00:00 2001 From: Eelco Dolstra Date: Sat, 5 Feb 2011 16:29:10 +0000 Subject: * Clang compatibility fix. Clang actually checks template definitions when they are defined --- src/libmain/shared.hh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src') diff --git a/src/libmain/shared.hh b/src/libmain/shared.hh index f70f6893b..c99810c78 100644 --- a/src/libmain/shared.hh +++ b/src/libmain/shared.hh @@ -1,7 +1,7 @@ #ifndef __SHARED_H #define __SHARED_H -#include "types.hh" +#include "util.hh" #include -- cgit v1.2.3